From 274f0c82d31b0e7c406ed06e15b771f34aca42ca Mon Sep 17 00:00:00 2001 From: bruno Date: Sun, 24 May 2026 10:35:02 +0200 Subject: [PATCH 001/132] refactor StepGLM.dml from a script into a function and resolve a few nested blocks --- scripts/builtin/StepGLM.dml | 1202 +++++++++++++++++++++++++++++++++++ 1 file changed, 1202 insertions(+) create mode 100644 scripts/builtin/StepGLM.dml diff --git a/scripts/builtin/StepGLM.dml b/scripts/builtin/StepGLM.dml new file mode 100644 index 00000000000..3bbaf361650 --- /dev/null +++ b/scripts/builtin/StepGLM.dml @@ -0,0 +1,1202 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# +# THIS SCRIPT CHOOSES A GLM REGRESSION MODEL IN A STEPWISE ALGIRITHM USING AIC +# EACH GLM REGRESSION IS SOLVED USING NEWTON/FISHER SCORING WITH TRUST REGIONS +# +# INPUT PARAMETERS: +# --------------------------------------------------------------------------------------------- +# NAME TYPE DEFAULT MEANING +# --------------------------------------------------------------------------------------------- +# X Matrix --- Matrix X of feature vectors +# Y Matrix --- Response Matrix Y with 1 column +# link Int 2 Link function code: 1 = log, 2 = Logit, 3 = Probit, 4 = Cloglog +# yneg Double 0.0 Response value for Bernoulli "No" label, usually 0.0 or -1.0 +# icpt Int 0 Intercept presence, X columns shifting and rescaling: +# 0 = no intercept, no shifting, no rescaling; +# 1 = add intercept, but neither shift nor rescale X; +# 2 = add intercept, shift & rescale X columns to mean = 0, variance = 1 +# tol Double 0.000001 Tolerance (epsilon) +# disp Double 0.0 (Over-)dispersion value, or 0.0 to estimate it from data +# moi Int 200 Maximum number of outer (Newton / Fisher Scoring) iterations +# mii Int 0 Maximum number of inner (Conjugate Gradient) iterations, 0 = no maximum +# thr Double 0.01 Threshold to stop the algorithm: if the decrease in the value of AIC falls below thr +# no further features are being checked and the algorithm stops +# --------------------------------------------------------------------------------------------- +# OUTPUT: Matrix beta, whose size depends on icpt: +# icpt=0: ncol(X) x 1; icpt=1: (ncol(X) + 1) x 1; icpt=2: (ncol(X) + 1) x 2 +# +# B Matrix --- Estimated regression parameters (betas) +# S Matrix --- The selected features ordered as computed by the algorithm +# O String --- The statistics +# --------------------------------------------------------------------------------------------- + +# THE StepGLM SCRIPT CURRENTLY SUPPORTS BERNOULLI DISTRIBUTION FAMILY AND THE FOLLOWING LINK FUNCTIONS ONLY! +# - LOG +# - LOGIT +# - PROBIT +# - CLOGLOG + +# In addition, in the last run of GLM some statistics are provided in CSV format, one comma-separated name-value +# pair per each line, as follows: +# +# NAME MEANING +# ------------------------------------------------------------------------------------------- +# TERMINATION_CODE A positive integer indicating success/failure as follows: +# 1 = Converged successfully; 2 = Maximum number of iterations reached; +# 3 = Input (X, Y) out of range; 4 = Distribution/link is not supported +# BETA_MIN Smallest beta value (regression coefficient), excluding the intercept +# BETA_MIN_INDEX Column index for the smallest beta value +# BETA_MAX Largest beta value (regression coefficient), excluding the intercept +# BETA_MAX_INDEX Column index for the largest beta value +# INTERCEPT Intercept value, or NaN if there is no intercept (if icpt=0) +# DISPERSION Dispersion used to scale deviance, provided as "disp" input parameter +# or estimated (same as DISPERSION_EST) if the "disp" parameter is <= 0 +# DISPERSION_EST Dispersion estimated from the dataset +# DEVIANCE_UNSCALED Deviance from the saturated model, assuming dispersion == 1.0 +# DEVIANCE_SCALED Deviance from the saturated model, scaled by the DISPERSION value +# ------------------------------------------------------------------------------------------- + + +stepGLM = function ( + Matrix[Double] X, + Matrix[Double] Y, + Int link = 2, + Double yneg = 0.0, + Int icpt = 0, + Double tol = 0.000001, + Double disp = 0.0, + Int moi = 200, + Int mii = 0, + Double thr = 0.01, +) + return ( + Double AIC, + Matrix[Double] B, + Matrix[Double] S, + Matrix[Double] O, + ) + { + intercept_status = icpt; + bernoulli_No_label = yneg; + distribution_type = 2; + + # currently only the forward selection strategy in supported: start from one feature and iteratively add + # features until AIC improves + dir = "forward"; + + if (distribution_type == 2 & ncol(Y) == 1) { + is_Y_negative = (Y == bernoulli_No_label); + Y = cbind (1 - is_Y_negative, is_Y_negative); + count_Y_negative = sum (is_Y_negative); + if (count_Y_negative == 0) { + stop ("StepGLM Input Error: all Y-values encode Bernoulli YES-label, none encode NO-label"); + } + if (count_Y_negative == nrow(Y)) { + stop ("StepGLM Input Error: all Y-values encode Bernoulli NO-label, none encode YES-label"); + } + } + + X_orig = X; + num_records = nrow (X_orig); + num_features = ncol (X_orig); + + # BEGIN STEPWISE GENERALIZED LINEAR MODELS + + if (dir != "forward") { + stop ("Currently only forward selection strategy is supported!"); + } + + continue = TRUE; + columns_fixed = matrix (0, rows = 1, cols = num_features); + columns_fixed_ordered = matrix (0, rows = 1, cols = 1); + + # X_global stores the best model found at each step + X_global = matrix (0, rows = num_records, cols = 1); + + if (intercept_status == 0) { + # Compute AIC of an empty model with no features and no intercept (all Ys are zero) + [AIC_best, _beta, _S, _O] = glm_fit (X_global, Y, 0, num_features, columns_fixed_ordered); + } else { + # compute AIC of an empty model with only intercept (all Ys are constant) + all_ones = matrix (1, rows = num_records, cols = 1); + [AIC_best, _beta, _S, _O] = glm_fit (all_ones, Y, 0, num_features, columns_fixed_ordered); + } + #print ("Best AIC without any features: " + AIC_best); + + # First pass to examine single features + AICs = matrix (AIC_best, rows = 1, cols = num_features); + parfor (i in 1:num_features) { + [AIC_1, _beta, _S, _O] = glm_fit (X_orig[,i], Y, intercept_status, num_features, columns_fixed_ordered); + AICs[1,i] = AIC_1; + } + + # Determine the best AIC + column_best = 0; + for (k in 1:num_features) { + AIC_cur = as.scalar (AICs[1,k]); + if ( (AIC_cur < AIC_best) & ((AIC_best - AIC_cur) > abs (thr * AIC_best)) ) { + column_best = k; + AIC_best = as.scalar(AICs[1,k]); + } + } + + if (column_best == 0) { + #print ("AIC of an empty model is " + AIC_best + " and adding no feature achieves more than " + (thr * 100) + "% decrease in AIC!"); + if (intercept_status == 0) { + # Compute AIC of an empty model with no features and no intercept (all Ys are zero) + [AIC_best, _beta, _S, _O] = glm_fit (X_global, Y, 0, num_features, columns_fixed_ordered); + } else { + # compute AIC of an empty model with only intercept (all Ys are constant) + ###all_ones = matrix (1, rows = num_records, cols = 1); + [AIC_best, _beta, _S, _O] = glm_fit (all_ones, Y, 0, num_features, columns_fixed_ordered); + } + }; + + # print ("Best AIC " + AIC_best + " achieved with feature: " + column_best); + columns_fixed[1,column_best] = 1; + columns_fixed_ordered[1,1] = column_best; + X_global = X_orig[,column_best]; + + while (continue) { + # Subsequent passes over the features + parfor (i in 1:num_features) { + if (as.scalar(columns_fixed[1,i]) == 0) { + + # Construct the feature matrix + X = cbind (X_global, X_orig[,i]); + + [AIC_2, _beta, _S, _O] = glm_fit (X, Y, intercept_status, num_features, columns_fixed_ordered); + AICs[1,i] = AIC_2; + } + } + + # Determine the best AIC + for (k in 1:num_features) { + AIC_cur = as.scalar (AICs[1,k]); + if ( (AIC_cur < AIC_best) & ((AIC_best - AIC_cur) > abs (thr * AIC_best)) & (as.scalar(columns_fixed[1,k]) == 0) ) { + column_best = k; + AIC_best = as.scalar(AICs[1,k]); + } + } + + # cbind best found features (i.e., columns) to X_global + if (as.scalar(columns_fixed[1,column_best]) == 0) { # new best feature found + #print ("Best AIC " + AIC_best + " achieved with feature: " + column_best); + columns_fixed[1,column_best] = 1; + columns_fixed_ordered = cbind (columns_fixed_ordered, as.matrix(column_best)); + if (ncol(columns_fixed_ordered) == num_features) { # all features examined + X_global = cbind (X_global, X_orig[,column_best]); + continue = FALSE; + } else { + X_global = cbind (X_global, X_orig[,column_best]); + } + } else { + continue = FALSE; + } + } + + # run GLM with selected set of features + print ("Running GLM with selected features..."); + [AIC, beta, S, O] = glm_fit (X_global, Y, intercept_status, num_features, columns_fixed_ordered); + return (AIC, beta, S, O) + } + + +################### UDFS USED IN THIS SCRIPT ################## + +glm_fit = function ( + Matrix[Double] X, + Matrix[Double] Y, + Int intercept_status, + Double num_features_orig, + Matrix[Double] Selected, + Int link = 2, + Double disp = 0.0, + Double tol = 0.000001, + Int moi = 200, + Int mii = 0, +) + return ( + Double AIC, + Matrix[Double] beta, + Matrix[Double] S, + String O, + ) + { + + # distribution family code: 1 = Power, 2 = Bernoulli/Binomial; currently only Bernouli distribution family is supported! + distribution_type = 2; # $dfam = 2; + variance_as_power_of_the_mean = 0.0; # $vpow = 0.0; + # link function code: 0 = canonical (depends on distribution), 1 = Power, 2 = Logit, 3 = Probit, 4 = Cloglog, 5 = Cauchit; + # currently only log (link = 1), logit (link = 2), probit (link = 3), and cloglog (link = 4) are supported! + link_type = link; + link_as_power_of_the_mean = 0.0; # $lpow = 0.0; + + dispersion = disp; + eps = tol; + max_iteration_IRLS = moi; + max_iteration_CG = mii; + + variance_as_power_of_the_mean = as.double (variance_as_power_of_the_mean); + link_as_power_of_the_mean = as.double (link_as_power_of_the_mean); + + dispersion = as.double (dispersion); + eps = as.double (eps); + + # Default values for output statistics: + regularization = 0.0; + termination_code = 0.0; + min_beta = NaN; + i_min_beta = NaN; + max_beta = NaN; + i_max_beta = NaN; + intercept_value = NaN; + dispersion = NaN; + estimated_dispersion = NaN; + deviance_nodisp = NaN; + deviance = NaN; + + ##### INITIALIZE THE PARAMETERS ##### + + num_records = nrow (X); + num_features = ncol (X); + zeros_r = matrix (0, rows = num_records, cols = 1); + ones_r = 1 + zeros_r; + + # Introduce the intercept, shift and rescale the columns of X if needed + + if (intercept_status == 1 | intercept_status == 2) { # add the intercept column + X = cbind (X, ones_r); + num_features = ncol (X); + } + + scale_lambda = matrix (1, rows = num_features, cols = 1); + if (intercept_status == 1 | intercept_status == 2) { + scale_lambda [num_features, 1] = 0; + } + + if (intercept_status == 2) { # scale-&-shift X columns to mean 0, variance 1 + # Important assumption: X [, num_features] = ones_r + avg_X_cols = t(colSums(X)) / num_records; + var_X_cols = (t(colSums (X ^ 2)) - num_records * (avg_X_cols ^ 2)) / (num_records - 1); + is_unsafe = (var_X_cols <= 0); + scale_X = 1.0 / sqrt (var_X_cols * (1 - is_unsafe) + is_unsafe); + scale_X [num_features, 1] = 1; + shift_X = - avg_X_cols * scale_X; + shift_X [num_features, 1] = 0; + rowSums_X_sq = (X ^ 2) %*% (scale_X ^ 2) + X %*% (2 * scale_X * shift_X) + sum (shift_X ^ 2); + } else { + scale_X = matrix (1, rows = num_features, cols = 1); + shift_X = matrix (0, rows = num_features, cols = 1); + rowSums_X_sq = rowSums (X ^ 2); + } + + # Henceforth we replace "X" with "X %*% (SHIFT/SCALE TRANSFORM)" and rowSums(X ^ 2) + # with "rowSums_X_sq" in order to preserve the sparsity of X under shift and scale. + # The transform is then associatively applied to the other side of the expression, + # and is rewritten via "scale_X" and "shift_X" as follows: + # + # ssX_A = (SHIFT/SCALE TRANSFORM) %*% A --- is rewritten as: + # ssX_A = diag (scale_X) %*% A; + # ssX_A [num_features, ] = ssX_A [num_features, ] + t(shift_X) %*% A; + # + # tssX_A = t(SHIFT/SCALE TRANSFORM) %*% A --- is rewritten as: + # tssX_A = diag (scale_X) %*% A + shift_X %*% A [num_features, ]; + + # Initialize other input-dependent parameters + + lambda = scale_lambda * regularization; + if (max_iteration_CG == 0) { + max_iteration_CG = num_features; + } + + # Set up the canonical link, if requested [Then we have: Var(mu) * (d link / d mu) = const] + + if (link_type == 0) { + if (distribution_type == 1) { + link_type = 1; + link_as_power_of_the_mean = 1.0 - variance_as_power_of_the_mean; + } else { + if (distribution_type == 2) { + link_type = 2; + } + } + } + + # For power distributions and/or links, we use two constants, + # "variance as power of the mean" and "link_as_power_of_the_mean", + # to specify the variance and the link as arbitrary powers of the + # mean. However, the variance-powers of 1.0 (Poisson family) and + # 2.0 (Gamma family) have to be treated as special cases, because + # these values integrate into logarithms. The link-power of 0.0 + # is also special as it represents the logarithm link. + + num_response_columns = ncol (Y); + is_supported = 0; + if (num_response_columns == 2 & distribution_type == 2 & link_type >= 1 & link_type <= 4) { # BERNOULLI DISTRIBUTION + is_supported = 1; + } + if (num_response_columns == 1 & distribution_type == 2) { + print ("Error: Bernoulli response matrix has not been converted into two-column format."); + } + + if (is_supported != 1) { + stop ("Response matrix with " + num_response_columns + " columns, distribution family (" + distribution_type + ", " + variance_as_power_of_the_mean + + ") and link family (" + link_type + ", " + link_as_power_of_the_mean + ") are NOT supported together."); + } + + ##### INITIALIZE THE BETAS ##### + + [beta, saturated_log_l, isNaN] = + glm_initialize (X, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean, intercept_status, max_iteration_CG); + + # print(" --- saturated logLik " + saturated_log_l); + + if (isNaN != 0) { + stop ("Input matrices X and/or Y are out of range!"); + } + + ##### START OF THE MAIN PART ##### + + sum_X_sq = sum (rowSums_X_sq); + trust_delta = 0.5 * sqrt (num_features) / max (sqrt (rowSums_X_sq)); + ### max_trust_delta = trust_delta * 10000.0; + log_l = 0.0; + deviance_nodisp = 0.0; + new_deviance_nodisp = 0.0; + isNaN_log_l = 2; + newbeta = beta; + g = matrix (0.0, rows = num_features, cols = 1); + g_norm = sqrt (sum ((g + lambda * beta) ^ 2)); + accept_new_beta = 1; + reached_trust_boundary = 0; + neg_log_l_change_predicted = 0.0; + i_IRLS = 0; + + # print ("BEGIN IRLS ITERATIONS..."); + + ssX_newbeta = diag (scale_X) %*% newbeta; + ssX_newbeta [num_features, ] = ssX_newbeta [num_features, ] + t(shift_X) %*% newbeta; + all_linear_terms = X %*% ssX_newbeta; + + [new_log_l, isNaN_new_log_l] = glm_log_likelihood_part + (all_linear_terms, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean); + + if (isNaN_new_log_l == 0) { + new_deviance_nodisp = 2.0 * (saturated_log_l - new_log_l); + new_log_l = new_log_l - 0.5 * sum (lambda * newbeta ^ 2); + } + + while (termination_code == 0) { + accept_new_beta = 1; + + if (i_IRLS > 0) { + if (isNaN_log_l == 0) { + accept_new_beta = 0; + } + + # Decide whether to accept a new iteration point and update the trust region + # See Alg. 4.1 on p. 69 of "Numerical Optimization" 2nd ed. by Nocedal and Wright + + rho = (- new_log_l + log_l) / neg_log_l_change_predicted; + if (rho < 0.25 | isNaN_new_log_l == 1) { + trust_delta = 0.25 * trust_delta; + } + if (rho > 0.75 & isNaN_new_log_l == 0 & reached_trust_boundary == 1) { + trust_delta = 2 * trust_delta; + + ### if (trust_delta > max_trust_delta) { + ### trust_delta = max_trust_delta; + ### } + } + if (rho > 0.1 & isNaN_new_log_l == 0) { + accept_new_beta = 1; + } + } + + if (accept_new_beta == 1) { + beta = newbeta; log_l = new_log_l; deviance_nodisp = new_deviance_nodisp; isNaN_log_l = isNaN_new_log_l; + + [g_Y, w] = glm_dist (all_linear_terms, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean); + + # We introduced these variables to avoid roundoff errors: + # g_Y = y_residual / (y_var * link_grad); + # w = 1.0 / (y_var * link_grad * link_grad); + + gXY = - t(X) %*% g_Y; + g = diag (scale_X) %*% gXY + shift_X %*% gXY [num_features, ]; + g_norm = sqrt (sum ((g + lambda * beta) ^ 2)); + } + + [z, neg_log_l_change_predicted, num_CG_iters, reached_trust_boundary] = + get_CG_Steihaug_point (X, scale_X, shift_X, w, g, beta, lambda, trust_delta, max_iteration_CG); + + newbeta = beta + z; + + ssX_newbeta = diag (scale_X) %*% newbeta; + ssX_newbeta [num_features, ] = ssX_newbeta [num_features, ] + t(shift_X) %*% newbeta; + all_linear_terms = X %*% ssX_newbeta; + + [new_log_l, isNaN_new_log_l] = glm_log_likelihood_part + (all_linear_terms, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean); + + if (isNaN_new_log_l == 0) { + new_deviance_nodisp = 2.0 * (saturated_log_l - new_log_l); + new_log_l = new_log_l - 0.5 * sum (lambda * newbeta ^ 2); + } + + log_l_change = new_log_l - log_l; # R's criterion for termination: |dev - devold|/(|dev| + 0.1) < eps + + if (reached_trust_boundary == 0 & isNaN_new_log_l == 0 & + (2.0 * abs (log_l_change) < eps * (deviance_nodisp + 0.1) | abs (log_l_change) < (abs (log_l) + abs (new_log_l)) * 0.00000000000001) ) { + termination_code = 1; + } + rho = - log_l_change / neg_log_l_change_predicted; + z_norm = sqrt (sum (z * z)); + + i_IRLS = i_IRLS + 1; + + if (i_IRLS == max_iteration_IRLS) { + termination_code = 2; + } + } + + beta = newbeta; + log_l = new_log_l; + deviance_nodisp = new_deviance_nodisp; + + #---------------------------- last part + + if (termination_code != 1) { + print ("One of the runs of GLM did not converged in " + i_IRLS + " steps!"); + } + + ##### COMPUTE AIC ##### + + if (distribution_type == 2 & link_type >= 1 & link_type <= 4) { + AIC = -2 * log_l; + if (sum (X) != 0) { + AIC = AIC + 2 * num_features; + } + } else { + stop ("Currently only the Bernoulli distribution family the following link functions are supported: log, logit, probit, and cloglog!"); + } + + # Output which features give the best AIC and are being used for linear regression + S = Selected; + + ssX_beta = diag (scale_X) %*% beta; + ssX_beta [num_features, ] = ssX_beta [num_features, ] + t(shift_X) %*% beta; + if (intercept_status == 2) { + beta_out = cbind (ssX_beta, beta); + } else { + beta_out = ssX_beta; + } + + if (intercept_status == 0 & num_features == 1) { + p = sum (X == 1); + if (p == num_records) { + beta_out = beta_out[1,]; + } + } + + + if (intercept_status == 1 | intercept_status == 2) { + intercept_value = as.scalar (beta_out [num_features, 1]); + beta_noicept = beta_out [1 : (num_features - 1), 1]; + } else { + beta_noicept = beta_out [1 : num_features, 1]; + } + min_beta = min (beta_noicept); + max_beta = max (beta_noicept); + tmp_i_min_beta = rowIndexMin (t(beta_noicept)) + i_min_beta = as.scalar (tmp_i_min_beta [1, 1]); + tmp_i_max_beta = rowIndexMax (t(beta_noicept)) + i_max_beta = as.scalar (tmp_i_max_beta [1, 1]); + + ##### OVER-DISPERSION PART ##### + + all_linear_terms = X %*% ssX_beta; + [g_Y, w] = glm_dist (all_linear_terms, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean); + + pearson_residual_sq = g_Y ^ 2 / w; + pearson_residual_sq = replace (target = pearson_residual_sq, pattern = NaN, replacement = 0); + # pearson_residual_sq = (y_residual ^ 2) / y_var; + + if (num_records > num_features) { + estimated_dispersion = sum (pearson_residual_sq) / (num_records - num_features); + } + if (dispersion <= 0) { + dispersion = estimated_dispersion; + } + deviance = deviance_nodisp / dispersion; + + ##### END OF THE MAIN PART ##### + + str = "BETA_MIN," + min_beta; + str = append (str, "BETA_MIN_INDEX," + i_min_beta); + str = append (str, "BETA_MAX," + max_beta); + str = append (str, "BETA_MAX_INDEX," + i_max_beta); + str = append (str, "INTERCEPT," + intercept_value); + str = append (str, "DISPERSION," + dispersion); + str = append (str, "DISPERSION_EST," + estimated_dispersion); + str = append (str, "DEVIANCE_UNSCALED," + deviance_nodisp); + str = append (str, "DEVIANCE_SCALED," + deviance); + O = str + + # Prepare the output matrix + print ("Writing the output matrix..."); + if (intercept_status == 0 & num_features == 1) { + if (p == num_records) { + beta_out_tmp = matrix (0, rows = num_features_orig + 1, cols = 1); + beta_out_tmp[num_features_orig + 1,] = beta_out; + beta_out = beta_out_tmp; + return AIC, beta_out, S, O; + } else if (sum (X) == 0){ + beta_out = matrix (0, rows = num_features_orig, cols = 1); + return AIC, beta_out, S, O; + } + } + + no_selected = ncol (Selected); + max_selected = max (Selected); + last = max_selected + 1; + + if (intercept_status != 0) { + + Selected_ext = cbind (Selected, as.matrix (last)); + P1 = table (seq (1, ncol (Selected_ext)), t(Selected_ext)); + + if (intercept_status == 2) { + + P1_ssX_beta = P1 * ssX_beta; + P2_ssX_beta = colSums (P1_ssX_beta); + P1_beta = P1 * beta; + P2_beta = colSums (P1_beta); + + if (max_selected < num_features_orig) { + + P2_ssX_beta = cbind (P2_ssX_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); + P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); + + P2_ssX_beta[1, num_features_orig+1] = P2_ssX_beta[1, max_selected + 1]; + P2_ssX_beta[1, max_selected + 1] = 0; + + P2_beta[1, num_features_orig+1] = P2_beta[1, max_selected + 1]; + P2_beta[1, max_selected + 1] = 0; + + } + beta_out = cbind (t(P2_ssX_beta), t(P2_beta)); + + } else { + + P1_beta = P1 * beta; + P2_beta = colSums (P1_beta); + + if (max_selected < num_features_orig) { + P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); + P2_beta[1, num_features_orig+1] = P2_beta[1, max_selected + 1] ; + P2_beta[1, max_selected + 1] = 0; + } + beta_out = t(P2_beta); + + } + } else { + + P1 = table (seq (1, no_selected), t(Selected)); + P1_beta = P1 * beta; + P2_beta = colSums (P1_beta); + + if (max_selected < num_features_orig) { + P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); + } + + beta_out = t(P2_beta); + } + + return AIC, beta_out, S, O; + } + +glm_initialize = function (Matrix[double] X, Matrix[double] Y, int dist_type, double var_power, int link_type, double link_power, int icept_status, int max_iter_CG) + return (Matrix[double] beta, double saturated_log_l, int isNaN) + { + saturated_log_l = 0.0; + isNaN = 0; + y_corr = Y [, 1]; + if (dist_type == 2) { + n_corr = rowSums (Y); + is_n_zero = (n_corr == 0); + y_corr = Y [, 1] / (n_corr + is_n_zero) + (0.5 - Y [, 1]) * is_n_zero; + } + linear_terms = y_corr; + if (dist_type == 1 & link_type == 1) { # POWER DISTRIBUTION + if (link_power == 0) { + if (sum (y_corr < 0) == 0) { + is_zero_y_corr = (y_corr == 0); + linear_terms = log (y_corr + is_zero_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); + } else { isNaN = 1; } + } else { if (link_power == 1.0) { + linear_terms = y_corr; + } else { if (link_power == -1.0) { + linear_terms = 1.0 / y_corr; + } else { if (link_power == 0.5) { + if (sum (y_corr < 0) == 0) { + linear_terms = sqrt (y_corr); + } else { isNaN = 1; } + } else { if (link_power > 0) { + if (sum (y_corr < 0) == 0) { + is_zero_y_corr = (y_corr == 0); + linear_terms = (y_corr + is_zero_y_corr) ^ link_power - is_zero_y_corr; + } else { isNaN = 1; } + } else { + if (sum (y_corr <= 0) == 0) { + linear_terms = y_corr ^ link_power; + } else { isNaN = 1; } + }}}}} + } + if (dist_type == 2 & link_type >= 1 & link_type <= 5) + { # BINOMIAL/BERNOULLI DISTRIBUTION + if (link_type == 1 & link_power == 0) { # Binomial.log + if (sum (y_corr < 0) == 0) { + is_zero_y_corr = (y_corr == 0); + linear_terms = log (y_corr + is_zero_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); + } else { isNaN = 1; } + } else { if (link_type == 1 & link_power > 0) { # Binomial.power_nonlog pos + if (sum (y_corr < 0) == 0) { + is_zero_y_corr = (y_corr == 0); + linear_terms = (y_corr + is_zero_y_corr) ^ link_power - is_zero_y_corr; + } else { isNaN = 1; } + } else { if (link_type == 1) { # Binomial.power_nonlog neg + if (sum (y_corr <= 0) == 0) { + linear_terms = y_corr ^ link_power; + } else { isNaN = 1; } + } else { + is_zero_y_corr = (y_corr <= 0); + is_one_y_corr = (y_corr >= 1.0); + y_corr = y_corr * (1.0 - is_zero_y_corr) * (1.0 - is_one_y_corr) + 0.5 * (is_zero_y_corr + is_one_y_corr); + if (link_type == 2) { # Binomial.logit + linear_terms = log (y_corr / (1.0 - y_corr)) + + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); + } else { if (link_type == 3) { # Binomial.probit + y_below_half = y_corr + (1.0 - 2.0 * y_corr) * (y_corr > 0.5); + t = sqrt (- 2.0 * log (y_below_half)); + approx_inv_Gauss_CDF = - t + (2.515517 + t * (0.802853 + t * 0.010328)) / (1.0 + t * (1.432788 + t * (0.189269 + t * 0.001308))); + linear_terms = approx_inv_Gauss_CDF * (1.0 - 2.0 * (y_corr > 0.5)) + + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); + } else { if (link_type == 4) { # Binomial.cloglog + linear_terms = log (- log (1.0 - y_corr)) + - log (- log (0.5)) * (is_zero_y_corr + is_one_y_corr) + + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); + } else { if (link_type == 5) { # Binomial.cauchit + linear_terms = tan ((y_corr - 0.5) * pi) + + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); + }} }}}}} + } + + if (isNaN == 0) { + [saturated_log_l, isNaN] = + glm_log_likelihood_part (linear_terms, Y, dist_type, var_power, link_type, link_power); + } + + if ((dist_type == 1 & link_type == 1 & link_power == 0) | + (dist_type == 2 & link_type >= 2)) + { + desired_eta = 0.0; + } else { if (link_type == 1 & link_power == 0) { + desired_eta = log (0.5); + } else { if (link_type == 1) { + desired_eta = 0.5 ^ link_power; + } else { + desired_eta = 0.5; + }}} + + beta = matrix (0.0, rows = ncol(X), cols = 1); + + if (desired_eta != 0) { + if (icept_status == 1 | icept_status == 2) { + beta [nrow(beta), 1] = desired_eta; + } else { + # We want: avg (X %*% ssX_transform %*% beta) = desired_eta + # Note that "ssX_transform" is trivial here, hence ignored + + beta = straightenX (X, 0.000001, max_iter_CG); + beta = beta * desired_eta; + } } } + + +glm_dist = function (Matrix[double] linear_terms, Matrix[double] Y, + int dist_type, double var_power, int link_type, double link_power) + return (Matrix[double] g_Y, Matrix[double] w) +# ORIGINALLY we returned more meaningful vectors, namely: +# Matrix[double] y_residual : y - y_mean, i.e. y observed - y predicted +# Matrix[double] link_gradient : derivative of the link function +# Matrix[double] var_function : variance without dispersion, i.e. the V(mu) function +# BUT, this caused roundoff errors, so we had to compute "directly useful" vectors +# and skip over the "meaningful intermediaries". Now we output these two variables: +# g_Y = y_residual / (var_function * link_gradient); +# w = 1.0 / (var_function * link_gradient ^ 2); + { + num_records = nrow (linear_terms); + zeros_r = matrix (0.0, rows = num_records, cols = 1); + ones_r = 1 + zeros_r; + g_Y = zeros_r; + w = zeros_r; + + # Some constants + + one_over_sqrt_two_pi = 0.39894228040143267793994605993438; + ones_2 = matrix (1.0, rows = 1, cols = 2); + p_one_m_one = ones_2; + p_one_m_one [1, 2] = -1.0; + m_one_p_one = ones_2; + m_one_p_one [1, 1] = -1.0; + zero_one = ones_2; + zero_one [1, 1] = 0.0; + one_zero = ones_2; + one_zero [1, 2] = 0.0; + flip_pos = matrix (0, rows = 2, cols = 2); + flip_neg = flip_pos; + flip_pos [1, 2] = 1; + flip_pos [2, 1] = 1; + flip_neg [1, 2] = -1; + flip_neg [2, 1] = 1; + + if (dist_type == 1 & link_type == 1) { # POWER DISTRIBUTION + y_mean = zeros_r; + if (link_power == 0) { + y_mean = exp (linear_terms); + y_mean_pow = y_mean ^ (1 - var_power); + w = y_mean_pow * y_mean; + g_Y = y_mean_pow * (Y - y_mean); + } else { if (link_power == 1.0) { + y_mean = linear_terms; + w = y_mean ^ (- var_power); + g_Y = w * (Y - y_mean); + } else { + y_mean = linear_terms ^ (1.0 / link_power); + c1 = (1 - var_power) / link_power - 1; + c2 = (2 - var_power) / link_power - 2; + g_Y = (linear_terms ^ c1) * (Y - y_mean) / link_power; + w = (linear_terms ^ c2) / (link_power ^ 2); + } }} + if (dist_type == 2 & link_type >= 1 & link_type <= 5) + { # BINOMIAL/BERNOULLI DISTRIBUTION + if (link_type == 1) { # BINOMIAL.POWER LINKS + if (link_power == 0) { # Binomial.log + vec1 = 1 / (exp (- linear_terms) - 1); + g_Y = Y [, 1] - Y [, 2] * vec1; + w = rowSums (Y) * vec1; + } else { # Binomial.nonlog + vec1 = zeros_r; + if (link_power == 0.5) { + vec1 = 1 / (1 - linear_terms ^ 2); + } else { if (sum (linear_terms < 0) == 0) { + vec1 = linear_terms ^ (- 2 + 1 / link_power) / (1 - linear_terms ^ (1 / link_power)); + } else {isNaN = 1;}} + # We want a "zero-protected" version of + # vec2 = Y [, 1] / linear_terms; + is_y_0 = (Y [, 1] == 0); + vec2 = (Y [, 1] + is_y_0) / (linear_terms * (1 - is_y_0) + is_y_0) - is_y_0; + g_Y = (vec2 - Y [, 2] * vec1 * linear_terms) / link_power; + w = rowSums (Y) * vec1 / link_power ^ 2; + } + } else { + is_LT_pos_infinite = (linear_terms == Inf); + is_LT_neg_infinite = (linear_terms == -Inf); + is_LT_infinite = is_LT_pos_infinite %*% one_zero + is_LT_neg_infinite %*% zero_one; + finite_linear_terms = replace (target = linear_terms, pattern = Inf, replacement = 0); + finite_linear_terms = replace (target = finite_linear_terms, pattern = -Inf, replacement = 0); + if (link_type == 2) { # Binomial.logit + Y_prob = exp (finite_linear_terms) %*% one_zero + ones_r %*% zero_one; + Y_prob = Y_prob / (rowSums (Y_prob) %*% ones_2); + Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; + g_Y = rowSums (Y * (Y_prob %*% flip_neg)); ### = y_residual; + w = rowSums (Y * (Y_prob %*% flip_pos) * Y_prob); ### = y_variance; + } else { if (link_type == 3) { # Binomial.probit + is_lt_pos = (linear_terms >= 0); + t_gp = 1.0 / (1.0 + abs (finite_linear_terms) * 0.231641888); # 0.231641888 = 0.3275911 / sqrt (2.0) + pt_gp = t_gp * ( 0.254829592 + + t_gp * (-0.284496736 # "Handbook of Mathematical Functions", ed. by M. Abramowitz and I.A. Stegun, + + t_gp * ( 1.421413741 # U.S. Nat-l Bureau of Standards, 10th print (Dec 1972), Sec. 7.1.26, p. 299 + + t_gp * (-1.453152027 + + t_gp * 1.061405429)))); + the_gauss_exp = exp (- (linear_terms ^ 2) / 2.0); + vec1 = 0.25 * pt_gp * (2 - the_gauss_exp * pt_gp); + vec2 = Y [, 1] - rowSums (Y) * is_lt_pos + the_gauss_exp * pt_gp * rowSums (Y) * (is_lt_pos - 0.5); + w = the_gauss_exp * (one_over_sqrt_two_pi ^ 2) * rowSums (Y) / vec1; + g_Y = one_over_sqrt_two_pi * vec2 / vec1; + } else { if (link_type == 4) { # Binomial.cloglog + the_exp = exp (linear_terms) + the_exp_exp = exp (- the_exp); + is_too_small = ((10000000 + the_exp) == 10000000); + the_exp_ratio = (1 - is_too_small) * (1 - the_exp_exp) / (the_exp + is_too_small) + is_too_small * (1 - the_exp / 2); + g_Y = (rowSums (Y) * the_exp_exp - Y [, 2]) / the_exp_ratio; + w = the_exp_exp * the_exp * rowSums (Y) / the_exp_ratio; + } else { if (link_type == 5) { # Binomial.cauchit + Y_prob = 0.5 + (atan (finite_linear_terms) %*% p_one_m_one) / pi; + Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; + y_residual = Y [, 1] * Y_prob [, 2] - Y [, 2] * Y_prob [, 1]; + var_function = rowSums (Y) * Y_prob [, 1] * Y_prob [, 2]; + link_gradient_normalized = (1 + linear_terms ^ 2) * pi; + g_Y = rowSums (Y) * y_residual / (var_function * link_gradient_normalized); + w = (rowSums (Y) ^ 2) / (var_function * link_gradient_normalized ^ 2); + }}}} + } + } + } + + +glm_log_likelihood_part = function (Matrix[double] linear_terms, Matrix[double] Y, + int dist_type, double var_power, int link_type, double link_power) + return (double log_l, int isNaN) + { + isNaN = 0; + log_l = 0.0; + num_records = nrow (Y); + zeros_r = matrix (0.0, rows = num_records, cols = 1); + + if (dist_type == 1 & link_type == 1) + { # POWER DISTRIBUTION + b_cumulant = zeros_r; + natural_parameters = zeros_r; + is_natural_parameter_log_zero = zeros_r; + if (var_power == 1.0 & link_power == 0) { # Poisson.log + b_cumulant = exp (linear_terms); + is_natural_parameter_log_zero = (linear_terms == -Inf); + natural_parameters = replace (target = linear_terms, pattern = -Inf, replacement = 0); + } else { if (var_power == 1.0 & link_power == 1.0) { # Poisson.id + if (sum (linear_terms < 0) == 0) { + b_cumulant = linear_terms; + is_natural_parameter_log_zero = (linear_terms == 0); + natural_parameters = log (linear_terms + is_natural_parameter_log_zero); + } else {isNaN = 1;} + } else { if (var_power == 1.0 & link_power == 0.5) { # Poisson.sqrt + if (sum (linear_terms < 0) == 0) { + b_cumulant = linear_terms ^ 2; + is_natural_parameter_log_zero = (linear_terms == 0); + natural_parameters = 2.0 * log (linear_terms + is_natural_parameter_log_zero); + } else {isNaN = 1;} + } else { if (var_power == 1.0 & link_power > 0) { # Poisson.power_nonlog, pos + if (sum (linear_terms < 0) == 0) { + is_natural_parameter_log_zero = (linear_terms == 0); + b_cumulant = (linear_terms + is_natural_parameter_log_zero) ^ (1.0 / link_power) - is_natural_parameter_log_zero; + natural_parameters = log (linear_terms + is_natural_parameter_log_zero) / link_power; + } else {isNaN = 1;} + } else { if (var_power == 1.0) { # Poisson.power_nonlog, neg + if (sum (linear_terms <= 0) == 0) { + b_cumulant = linear_terms ^ (1.0 / link_power); + natural_parameters = log (linear_terms) / link_power; + } else {isNaN = 1;} + } else { if (var_power == 2.0 & link_power == -1.0) { # Gamma.inverse + if (sum (linear_terms <= 0) == 0) { + b_cumulant = - log (linear_terms); + natural_parameters = - linear_terms; + } else {isNaN = 1;} + } else { if (var_power == 2.0 & link_power == 1.0) { # Gamma.id + if (sum (linear_terms <= 0) == 0) { + b_cumulant = log (linear_terms); + natural_parameters = - 1.0 / linear_terms; + } else {isNaN = 1;} + } else { if (var_power == 2.0 & link_power == 0) { # Gamma.log + b_cumulant = linear_terms; + natural_parameters = - exp (- linear_terms); + } else { if (var_power == 2.0) { # Gamma.power_nonlog + if (sum (linear_terms <= 0) == 0) { + b_cumulant = log (linear_terms) / link_power; + natural_parameters = - linear_terms ^ (- 1.0 / link_power); + } else {isNaN = 1;} + } else { if (link_power == 0) { # PowerDist.log + natural_parameters = exp (linear_terms * (1.0 - var_power)) / (1.0 - var_power); + b_cumulant = exp (linear_terms * (2.0 - var_power)) / (2.0 - var_power); + } else { # PowerDist.power_nonlog + if (-2 * link_power == 1.0 - var_power) { + natural_parameters = 1.0 / (linear_terms ^ 2) / (1.0 - var_power); + } else { if (-1 * link_power == 1.0 - var_power) { + natural_parameters = 1.0 / linear_terms / (1.0 - var_power); + } else { if ( link_power == 1.0 - var_power) { + natural_parameters = linear_terms / (1.0 - var_power); + } else { if ( 2 * link_power == 1.0 - var_power) { + natural_parameters = linear_terms ^ 2 / (1.0 - var_power); + } else { + if (sum (linear_terms <= 0) == 0) { + power = (1.0 - var_power) / link_power; + natural_parameters = (linear_terms ^ power) / (1.0 - var_power); + } else {isNaN = 1;} + }}}} + if (-2 * link_power == 2.0 - var_power) { + b_cumulant = 1.0 / (linear_terms ^ 2) / (2.0 - var_power); + } else { if (-1 * link_power == 2.0 - var_power) { + b_cumulant = 1.0 / linear_terms / (2.0 - var_power); + } else { if ( link_power == 2.0 - var_power) { + b_cumulant = linear_terms / (2.0 - var_power); + } else { if ( 2 * link_power == 2.0 - var_power) { + b_cumulant = linear_terms ^ 2 / (2.0 - var_power); + } else { + if (sum (linear_terms <= 0) == 0) { + power = (2.0 - var_power) / link_power; + b_cumulant = (linear_terms ^ power) / (2.0 - var_power); + } else {isNaN = 1;} + }}}} + }}}}} }}}}} + if (sum (is_natural_parameter_log_zero * abs (Y)) > 0) { + log_l = -Inf; + isNaN = 1; + } + if (isNaN == 0) + { + log_l = sum (Y * natural_parameters - b_cumulant); + if (log_l != log_l | (log_l == log_l + 1.0 & log_l == log_l * 2.0)) { + log_l = -Inf; + isNaN = 1; + } } } + + if (dist_type == 2 & link_type >= 1 & link_type <= 5) + { # BINOMIAL/BERNOULLI DISTRIBUTION + + [Y_prob, isNaN] = binomial_probability_two_column (linear_terms, link_type, link_power); + + if (isNaN == 0) { + does_prob_contradict = (Y_prob <= 0); + if (sum (does_prob_contradict * abs (Y)) == 0) { + log_l = sum (Y * log (Y_prob * (1 - does_prob_contradict) + does_prob_contradict)); + if (log_l != log_l | (log_l == log_l + 1.0 & log_l == log_l * 2.0)) { + isNaN = 1; + } + } else { + log_l = -Inf; + isNaN = 1; + } } } + + if (isNaN == 1) { + log_l = - Inf; + } + } + + + +binomial_probability_two_column = + function (Matrix[double] linear_terms, int link_type, double link_power) + return (Matrix[double] Y_prob, int isNaN) + { + isNaN = 0; + num_records = nrow (linear_terms); + + # Define some auxiliary matrices + + ones_2 = matrix (1.0, rows = 1, cols = 2); + p_one_m_one = ones_2; + p_one_m_one [1, 2] = -1.0; + m_one_p_one = ones_2; + m_one_p_one [1, 1] = -1.0; + zero_one = ones_2; + zero_one [1, 1] = 0.0; + one_zero = ones_2; + one_zero [1, 2] = 0.0; + + zeros_r = matrix (0.0, rows = num_records, cols = 1); + ones_r = 1.0 + zeros_r; + + # Begin the function body + + Y_prob = zeros_r %*% ones_2; + if (link_type == 1) { # Binomial.power + if (link_power == 0) { # Binomial.log + Y_prob = exp (linear_terms) %*% p_one_m_one + ones_r %*% zero_one; + } else { if (link_power == 0.5) { # Binomial.sqrt + Y_prob = (linear_terms ^ 2) %*% p_one_m_one + ones_r %*% zero_one; + } else { # Binomial.power_nonlog + if (sum (linear_terms < 0) == 0) { + Y_prob = (linear_terms ^ (1.0 / link_power)) %*% p_one_m_one + ones_r %*% zero_one; + } else {isNaN = 1;} + }} + } else { # Binomial.non_power + is_LT_pos_infinite = (linear_terms == Inf); + is_LT_neg_infinite = (linear_terms == -Inf); + is_LT_infinite = is_LT_pos_infinite %*% one_zero + is_LT_neg_infinite %*% zero_one; + finite_linear_terms = replace (target = linear_terms, pattern = Inf, replacement = 0); + finite_linear_terms = replace (target = finite_linear_terms, pattern = -Inf, replacement = 0); + if (link_type == 2) { # Binomial.logit + Y_prob = exp (finite_linear_terms) %*% one_zero + ones_r %*% zero_one; + Y_prob = Y_prob / (rowSums (Y_prob) %*% ones_2); + } else { if (link_type == 3) { # Binomial.probit + lt_pos_neg = (finite_linear_terms >= 0) %*% p_one_m_one + ones_r %*% zero_one; + t_gp = 1.0 / (1.0 + abs (finite_linear_terms) * 0.231641888); # 0.231641888 = 0.3275911 / sqrt (2.0) + pt_gp = t_gp * ( 0.254829592 + + t_gp * (-0.284496736 # "Handbook of Mathematical Functions", ed. by M. Abramowitz and I.A. Stegun, + + t_gp * ( 1.421413741 # U.S. Nat-l Bureau of Standards, 10th print (Dec 1972), Sec. 7.1.26, p. 299 + + t_gp * (-1.453152027 + + t_gp * 1.061405429)))); + the_gauss_exp = exp (- (finite_linear_terms ^ 2) / 2.0); + Y_prob = lt_pos_neg + ((the_gauss_exp * pt_gp) %*% ones_2) * (0.5 - lt_pos_neg); + } else { if (link_type == 4) { # Binomial.cloglog + the_exp = exp (finite_linear_terms); + the_exp_exp = exp (- the_exp); + is_too_small = ((10000000 + the_exp) == 10000000); + Y_prob [, 1] = (1 - is_too_small) * (1 - the_exp_exp) + is_too_small * the_exp * (1 - the_exp / 2); + Y_prob [, 2] = the_exp_exp; + } else { if (link_type == 5) { # Binomial.cauchit + Y_prob = 0.5 + (atan (finite_linear_terms) %*% p_one_m_one) / pi; + } else { + isNaN = 1; + }}}} + Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; + } } + + +# THE CG-STEIHAUG PROCEDURE SCRIPT + +# Apply Conjugate Gradient - Steihaug algorithm in order to approximately minimize +# 0.5 z^T (X^T diag(w) X + diag (lambda)) z + (g + lambda * beta)^T z +# under constraint: ||z|| <= trust_delta. +# See Alg. 7.2 on p. 171 of "Numerical Optimization" 2nd ed. by Nocedal and Wright +# IN THE ABOVE, "X" IS UNDERSTOOD TO BE "X %*% (SHIFT/SCALE TRANSFORM)"; this transform +# is given separately because sparse "X" may become dense after applying the transform. +# +get_CG_Steihaug_point = + function (Matrix[double] X, Matrix[double] scale_X, Matrix[double] shift_X, Matrix[double] w, + Matrix[double] g, Matrix[double] beta, Matrix[double] lambda, double trust_delta, int max_iter_CG) + return (Matrix[double] z, double neg_log_l_change, int i_CG, int reached_trust_boundary) + { + trust_delta_sq = trust_delta ^ 2; + size_CG = nrow (g); + z = matrix (0.0, rows = size_CG, cols = 1); + neg_log_l_change = 0.0; + reached_trust_boundary = 0; + g_reg = g + lambda * beta; + r_CG = g_reg; + p_CG = -r_CG; + rr_CG = sum(r_CG * r_CG); + eps_CG = rr_CG * min (0.25, sqrt (rr_CG)); + converged_CG = 0; + if (rr_CG < eps_CG) { + converged_CG = 1; + } + + max_iteration_CG = max_iter_CG; + if (max_iteration_CG <= 0) { + max_iteration_CG = size_CG; + } + i_CG = 0; + while (converged_CG == 0) + { + i_CG = i_CG + 1; + ssX_p_CG = diag (scale_X) %*% p_CG; + ssX_p_CG [size_CG, ] = ssX_p_CG [size_CG, ] + t(shift_X) %*% p_CG; + temp_CG = t(X) %*% (w * (X %*% ssX_p_CG)); + q_CG = (lambda * p_CG) + diag (scale_X) %*% temp_CG + shift_X %*% temp_CG [size_CG, ]; + pq_CG = sum (p_CG * q_CG); + if (pq_CG <= 0) { + pp_CG = sum (p_CG * p_CG); + if (pp_CG > 0) { + [z, neg_log_l_change] = + get_trust_boundary_point (g_reg, z, p_CG, q_CG, r_CG, pp_CG, pq_CG, trust_delta_sq); + reached_trust_boundary = 1; + } else { + neg_log_l_change = 0.5 * sum (z * (r_CG + g_reg)); + } + converged_CG = 1; + } + if (converged_CG == 0) { + alpha_CG = rr_CG / pq_CG; + new_z = z + alpha_CG * p_CG; + if (sum(new_z * new_z) >= trust_delta_sq) { + pp_CG = sum (p_CG * p_CG); + [z, neg_log_l_change] = + get_trust_boundary_point (g_reg, z, p_CG, q_CG, r_CG, pp_CG, pq_CG, trust_delta_sq); + reached_trust_boundary = 1; + converged_CG = 1; + } + if (converged_CG == 0) { + z = new_z; + old_rr_CG = rr_CG; + r_CG = r_CG + alpha_CG * q_CG; + rr_CG = sum(r_CG * r_CG); + if (i_CG == max_iteration_CG | rr_CG < eps_CG) { + neg_log_l_change = 0.5 * sum (z * (r_CG + g_reg)); + reached_trust_boundary = 0; + converged_CG = 1; + } + if (converged_CG == 0) { + p_CG = -r_CG + (rr_CG / old_rr_CG) * p_CG; + } } } } } + + +# An auxiliary function used twice inside the CG-STEIHAUG loop: +get_trust_boundary_point = + function (Matrix[double] g, Matrix[double] z, Matrix[double] p, + Matrix[double] q, Matrix[double] r, double pp, double pq, + double trust_delta_sq) + return (Matrix[double] new_z, double f_change) + { + zz = sum (z * z); pz = sum (p * z); + sq_root_d = sqrt (pz * pz - pp * (zz - trust_delta_sq)); + tau_1 = (- pz + sq_root_d) / pp; + tau_2 = (- pz - sq_root_d) / pp; + zq = sum (z * q); gp = sum (g * p); + f_extra = 0.5 * sum (z * (r + g)); + f_change_1 = f_extra + (0.5 * tau_1 * pq + zq + gp) * tau_1; + f_change_2 = f_extra + (0.5 * tau_2 * pq + zq + gp) * tau_2; + ind1 = as.integer(f_change_1 < f_change_2); + ind2 = as.integer(f_change_1 >= f_change_2); + new_z = z + ((ind1 * tau_1 + ind2 * tau_2) * p); + f_change = ind1 * f_change_1 + ind2 * f_change_2; + } + + +# Computes vector w such that ||X %*% w - 1|| -> MIN given avg(X %*% w) = 1 +# We find z_LS such that ||X %*% z_LS - 1|| -> MIN unconditionally, then scale +# it to compute w = c * z_LS such that sum(X %*% w) = nrow(X). +straightenX = + function (Matrix[double] X, double eps, int max_iter_CG) + return (Matrix[double] w) + { + w_X = t(colSums(X)); + lambda_LS = 0.000001 * sum(X ^ 2) / ncol(X); + eps_LS = eps * nrow(X); + + # BEGIN LEAST SQUARES + + r_LS = - w_X; + z_LS = matrix (0.0, rows = ncol(X), cols = 1); + p_LS = - r_LS; + norm_r2_LS = sum (r_LS ^ 2); + i_LS = 0; + while (i_LS < max_iter_CG & i_LS < ncol(X) & norm_r2_LS >= eps_LS) + { + q_LS = t(X) %*% X %*% p_LS; + q_LS = q_LS + lambda_LS * p_LS; + alpha_LS = norm_r2_LS / sum (p_LS * q_LS); + z_LS = z_LS + alpha_LS * p_LS; + old_norm_r2_LS = norm_r2_LS; + r_LS = r_LS + alpha_LS * q_LS; + norm_r2_LS = sum (r_LS ^ 2); + p_LS = -r_LS + (norm_r2_LS / old_norm_r2_LS) * p_LS; + i_LS = i_LS + 1; + } + + # END LEAST SQUARES + + w = (nrow(X) / sum (w_X * z_LS)) * z_LS; + } + + From 78018f9b09bd80e7c47f24fac83e4c1955f13f7a Mon Sep 17 00:00:00 2001 From: bruno Date: Fri, 26 Jun 2026 01:16:07 +0200 Subject: [PATCH 002/132] rename builtin StepGLM.dml to stepglm.dml --- scripts/builtin/{StepGLM.dml => stepglm.dml} | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) rename scripts/builtin/{StepGLM.dml => stepglm.dml} (99%) diff --git a/scripts/builtin/StepGLM.dml b/scripts/builtin/stepglm.dml similarity index 99% rename from scripts/builtin/StepGLM.dml rename to scripts/builtin/stepglm.dml index 3bbaf361650..cd2d881ecba 100644 --- a/scripts/builtin/StepGLM.dml +++ b/scripts/builtin/stepglm.dml @@ -88,12 +88,11 @@ stepGLM = function ( Int moi = 200, Int mii = 0, Double thr = 0.01, -) - return ( - Double AIC, - Matrix[Double] B, - Matrix[Double] S, - Matrix[Double] O, +) return ( + Double AIC, + Matrix[Double] B, + Matrix[Double] S, + Matrix[Double] O, ) { intercept_status = icpt; From 739c5d283a0c82622e42bced0c43079c3e1ce153 Mon Sep 17 00:00:00 2001 From: bruno Date: Fri, 26 Jun 2026 01:18:10 +0200 Subject: [PATCH 003/132] rename builtin stepglm.dml to stepGLM.dml --- scripts/builtin/{stepglm.dml => stepGLM.dml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/builtin/{stepglm.dml => stepGLM.dml} (100%) diff --git a/scripts/builtin/stepglm.dml b/scripts/builtin/stepGLM.dml similarity index 100% rename from scripts/builtin/stepglm.dml rename to scripts/builtin/stepGLM.dml From 9dc956bd0b6728a43d08e6eb8563490484755cdc Mon Sep 17 00:00:00 2001 From: bruno Date: Fri, 26 Jun 2026 05:44:12 +0200 Subject: [PATCH 004/132] fix syntax errors in stepGLM --- scripts/algorithms/TestBuiltinStepGLM.dml | 21 + scripts/builtin/stepGLM.dml | 1102 ++++++++++----------- 2 files changed, 570 insertions(+), 553 deletions(-) create mode 100644 scripts/algorithms/TestBuiltinStepGLM.dml diff --git a/scripts/algorithms/TestBuiltinStepGLM.dml b/scripts/algorithms/TestBuiltinStepGLM.dml new file mode 100644 index 00000000000..ab64960b35a --- /dev/null +++ b/scripts/algorithms/TestBuiltinStepGLM.dml @@ -0,0 +1,21 @@ +source("scripts/builtin/stepGLM.dml") as stepGLM; + +N = 1000; +P = 10; +X = rand(rows=N, cols=P, min=-1.0, max=1.0, pdf="uniform", seed=123); + +beta_true = matrix(0, rows=P, cols=1); +beta_true[2,1] = 3.5; +beta_true[5,1] = -2.0; +beta_true[8,1] = 1.5; + +Z = X %*% beta_true; +P_y = 1.0 / (1.0 + exp(-Z)); +Y = (rand(rows=N, cols=1, min=0.0, max=1.0, seed=456) < P_y) * 1.0; + +[AIC, B, S, O] = stepGLM::stepGLM(X=X, Y=Y, link=2, yneg=0.0, icpt=0, tol=1e-6, disp=0.0, moi=200, mii=0, thr=0.01); + +print("Optimal AIC: " + AIC); +print("Selected Feature Indices:\n" + toString(S)); +print("Estimated Coefficients:\n" + toString(B)); + diff --git a/scripts/builtin/stepGLM.dml b/scripts/builtin/stepGLM.dml index cd2d881ecba..c8be7e582b1 100644 --- a/scripts/builtin/stepGLM.dml +++ b/scripts/builtin/stepGLM.dml @@ -87,12 +87,12 @@ stepGLM = function ( Double disp = 0.0, Int moi = 200, Int mii = 0, - Double thr = 0.01, + Double thr = 0.01 ) return ( Double AIC, Matrix[Double] B, Matrix[Double] S, - Matrix[Double] O, + String O ) { intercept_status = icpt; @@ -134,18 +134,18 @@ stepGLM = function ( if (intercept_status == 0) { # Compute AIC of an empty model with no features and no intercept (all Ys are zero) - [AIC_best, _beta, _S, _O] = glm_fit (X_global, Y, 0, num_features, columns_fixed_ordered); + [AIC_best, ignore_B1, ignore_S1, ignore_O1] = glm_fit (X_global, Y, 0, num_features, columns_fixed_ordered); } else { # compute AIC of an empty model with only intercept (all Ys are constant) all_ones = matrix (1, rows = num_records, cols = 1); - [AIC_best, _beta, _S, _O] = glm_fit (all_ones, Y, 0, num_features, columns_fixed_ordered); + [AIC_best, ignore_beta2, ignore_S2, ignore_O2] = glm_fit (all_ones, Y, 0, num_features, columns_fixed_ordered); } #print ("Best AIC without any features: " + AIC_best); # First pass to examine single features AICs = matrix (AIC_best, rows = 1, cols = num_features); parfor (i in 1:num_features) { - [AIC_1, _beta, _S, _O] = glm_fit (X_orig[,i], Y, intercept_status, num_features, columns_fixed_ordered); + [AIC_1, ignore_beta3, ignore_S3, ignore_O3] = glm_fit (X_orig[,i], Y, intercept_status, num_features, columns_fixed_ordered); AICs[1,i] = AIC_1; } @@ -163,11 +163,11 @@ stepGLM = function ( #print ("AIC of an empty model is " + AIC_best + " and adding no feature achieves more than " + (thr * 100) + "% decrease in AIC!"); if (intercept_status == 0) { # Compute AIC of an empty model with no features and no intercept (all Ys are zero) - [AIC_best, _beta, _S, _O] = glm_fit (X_global, Y, 0, num_features, columns_fixed_ordered); + [AIC_best, ignore_beta4, ignore_S4, ignore_O4] = glm_fit (X_global, Y, 0, num_features, columns_fixed_ordered); } else { # compute AIC of an empty model with only intercept (all Ys are constant) ###all_ones = matrix (1, rows = num_records, cols = 1); - [AIC_best, _beta, _S, _O] = glm_fit (all_ones, Y, 0, num_features, columns_fixed_ordered); + [AIC_best, ignore_beta5, ignore_S5, ignore_O5] = glm_fit (all_ones, Y, 0, num_features, columns_fixed_ordered); } }; @@ -182,9 +182,9 @@ stepGLM = function ( if (as.scalar(columns_fixed[1,i]) == 0) { # Construct the feature matrix - X = cbind (X_global, X_orig[,i]); + X_loop = cbind (X_global, X_orig[,i]); - [AIC_2, _beta, _S, _O] = glm_fit (X, Y, intercept_status, num_features, columns_fixed_ordered); + [AIC_2, ignore_beta6, ignore_S6, ignore_O6] = glm_fit (X_loop, Y, intercept_status, num_features, columns_fixed_ordered); AICs[1,i] = AIC_2; } } @@ -216,8 +216,7 @@ stepGLM = function ( # run GLM with selected set of features print ("Running GLM with selected features..."); - [AIC, beta, S, O] = glm_fit (X_global, Y, intercept_status, num_features, columns_fixed_ordered); - return (AIC, beta, S, O) + [AIC, B, S, O] = glm_fit (X_global, Y, intercept_status, num_features, columns_fixed_ordered); } @@ -233,13 +232,13 @@ glm_fit = function ( Double disp = 0.0, Double tol = 0.000001, Int moi = 200, - Int mii = 0, + Int mii = 0 ) return ( Double AIC, Matrix[Double] beta, Matrix[Double] S, - String O, + String O ) { @@ -563,189 +562,186 @@ glm_fit = function ( str = append (str, "DEVIANCE_SCALED," + deviance); O = str + do_return = 0; # Prepare the output matrix - print ("Writing the output matrix..."); if (intercept_status == 0 & num_features == 1) { if (p == num_records) { beta_out_tmp = matrix (0, rows = num_features_orig + 1, cols = 1); beta_out_tmp[num_features_orig + 1,] = beta_out; - beta_out = beta_out_tmp; - return AIC, beta_out, S, O; + beta = beta_out_tmp; + do_return = 1; } else if (sum (X) == 0){ - beta_out = matrix (0, rows = num_features_orig, cols = 1); - return AIC, beta_out, S, O; + beta = matrix (0, rows = num_features_orig, cols = 1); + do_return = 1; } } + if (do_return != 0) { + no_selected = ncol (Selected); + max_selected = max (Selected); + last = max_selected + 1; - no_selected = ncol (Selected); - max_selected = max (Selected); - last = max_selected + 1; - - if (intercept_status != 0) { + if (intercept_status != 0) { Selected_ext = cbind (Selected, as.matrix (last)); P1 = table (seq (1, ncol (Selected_ext)), t(Selected_ext)); if (intercept_status == 2) { - P1_ssX_beta = P1 * ssX_beta; - P2_ssX_beta = colSums (P1_ssX_beta); - P1_beta = P1 * beta; - P2_beta = colSums (P1_beta); + P1_ssX_beta = P1 * ssX_beta; + P2_ssX_beta = colSums (P1_ssX_beta); + P1_beta = P1 * beta; + P2_beta = colSums (P1_beta); - if (max_selected < num_features_orig) { + if (max_selected < num_features_orig) { - P2_ssX_beta = cbind (P2_ssX_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); - P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); + P2_ssX_beta = cbind (P2_ssX_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); + P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); - P2_ssX_beta[1, num_features_orig+1] = P2_ssX_beta[1, max_selected + 1]; - P2_ssX_beta[1, max_selected + 1] = 0; + P2_ssX_beta[1, num_features_orig+1] = P2_ssX_beta[1, max_selected + 1]; + P2_ssX_beta[1, max_selected + 1] = 0; - P2_beta[1, num_features_orig+1] = P2_beta[1, max_selected + 1]; - P2_beta[1, max_selected + 1] = 0; + P2_beta[1, num_features_orig+1] = P2_beta[1, max_selected + 1]; + P2_beta[1, max_selected + 1] = 0; - } - beta_out = cbind (t(P2_ssX_beta), t(P2_beta)); + } + beta = cbind (t(P2_ssX_beta), t(P2_beta)); } else { - P1_beta = P1 * beta; - P2_beta = colSums (P1_beta); - - if (max_selected < num_features_orig) { - P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); - P2_beta[1, num_features_orig+1] = P2_beta[1, max_selected + 1] ; - P2_beta[1, max_selected + 1] = 0; - } - beta_out = t(P2_beta); + P1_beta = P1 * beta; + P2_beta = colSums (P1_beta); + if (max_selected < num_features_orig) { + P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); + P2_beta[1, num_features_orig+1] = P2_beta[1, max_selected + 1] ; + P2_beta[1, max_selected + 1] = 0; + } + beta = t(P2_beta); } - } else { - + } else { P1 = table (seq (1, no_selected), t(Selected)); P1_beta = P1 * beta; P2_beta = colSums (P1_beta); if (max_selected < num_features_orig) { - P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); + P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); } - beta_out = t(P2_beta); + beta = t(P2_beta); + } } - - return AIC, beta_out, S, O; } glm_initialize = function (Matrix[double] X, Matrix[double] Y, int dist_type, double var_power, int link_type, double link_power, int icept_status, int max_iter_CG) - return (Matrix[double] beta, double saturated_log_l, int isNaN) - { - saturated_log_l = 0.0; - isNaN = 0; - y_corr = Y [, 1]; - if (dist_type == 2) { - n_corr = rowSums (Y); - is_n_zero = (n_corr == 0); - y_corr = Y [, 1] / (n_corr + is_n_zero) + (0.5 - Y [, 1]) * is_n_zero; - } - linear_terms = y_corr; - if (dist_type == 1 & link_type == 1) { # POWER DISTRIBUTION - if (link_power == 0) { + return (Matrix[double] beta, double saturated_log_l, int isNaN) + { + saturated_log_l = 0.0; + isNaN = 0; + y_corr = Y [, 1]; + if (dist_type == 2) { + n_corr = rowSums (Y); + is_n_zero = (n_corr == 0); + y_corr = Y [, 1] / (n_corr + is_n_zero) + (0.5 - Y [, 1]) * is_n_zero; + } + linear_terms = y_corr; + if (dist_type == 1 & link_type == 1) { # POWER DISTRIBUTION + if (link_power == 0) { + if (sum (y_corr < 0) == 0) { + is_zero_y_corr = (y_corr == 0); + linear_terms = log (y_corr + is_zero_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); + } else { isNaN = 1; } + } else { if (link_power == 1.0) { + linear_terms = y_corr; + } else { if (link_power == -1.0) { + linear_terms = 1.0 / y_corr; + } else { if (link_power == 0.5) { + if (sum (y_corr < 0) == 0) { + linear_terms = sqrt (y_corr); + } else { isNaN = 1; } + } else { if (link_power > 0) { if (sum (y_corr < 0) == 0) { - is_zero_y_corr = (y_corr == 0); - linear_terms = log (y_corr + is_zero_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); + is_zero_y_corr = (y_corr == 0); + linear_terms = (y_corr + is_zero_y_corr) ^ link_power - is_zero_y_corr; } else { isNaN = 1; } - } else { if (link_power == 1.0) { - linear_terms = y_corr; - } else { if (link_power == -1.0) { - linear_terms = 1.0 / y_corr; - } else { if (link_power == 0.5) { - if (sum (y_corr < 0) == 0) { - linear_terms = sqrt (y_corr); - } else { isNaN = 1; } - } else { if (link_power > 0) { - if (sum (y_corr < 0) == 0) { - is_zero_y_corr = (y_corr == 0); - linear_terms = (y_corr + is_zero_y_corr) ^ link_power - is_zero_y_corr; - } else { isNaN = 1; } - } else { - if (sum (y_corr <= 0) == 0) { - linear_terms = y_corr ^ link_power; - } else { isNaN = 1; } - }}}}} - } - if (dist_type == 2 & link_type >= 1 & link_type <= 5) - { # BINOMIAL/BERNOULLI DISTRIBUTION - if (link_type == 1 & link_power == 0) { # Binomial.log - if (sum (y_corr < 0) == 0) { - is_zero_y_corr = (y_corr == 0); - linear_terms = log (y_corr + is_zero_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); + } else { + if (sum (y_corr <= 0) == 0) { + linear_terms = y_corr ^ link_power; } else { isNaN = 1; } - } else { if (link_type == 1 & link_power > 0) { # Binomial.power_nonlog pos - if (sum (y_corr < 0) == 0) { - is_zero_y_corr = (y_corr == 0); - linear_terms = (y_corr + is_zero_y_corr) ^ link_power - is_zero_y_corr; - } else { isNaN = 1; } - } else { if (link_type == 1) { # Binomial.power_nonlog neg - if (sum (y_corr <= 0) == 0) { - linear_terms = y_corr ^ link_power; - } else { isNaN = 1; } - } else { - is_zero_y_corr = (y_corr <= 0); - is_one_y_corr = (y_corr >= 1.0); - y_corr = y_corr * (1.0 - is_zero_y_corr) * (1.0 - is_one_y_corr) + 0.5 * (is_zero_y_corr + is_one_y_corr); - if (link_type == 2) { # Binomial.logit - linear_terms = log (y_corr / (1.0 - y_corr)) - + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { if (link_type == 3) { # Binomial.probit - y_below_half = y_corr + (1.0 - 2.0 * y_corr) * (y_corr > 0.5); - t = sqrt (- 2.0 * log (y_below_half)); - approx_inv_Gauss_CDF = - t + (2.515517 + t * (0.802853 + t * 0.010328)) / (1.0 + t * (1.432788 + t * (0.189269 + t * 0.001308))); - linear_terms = approx_inv_Gauss_CDF * (1.0 - 2.0 * (y_corr > 0.5)) - + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { if (link_type == 4) { # Binomial.cloglog - linear_terms = log (- log (1.0 - y_corr)) - - log (- log (0.5)) * (is_zero_y_corr + is_one_y_corr) - + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { if (link_type == 5) { # Binomial.cauchit - linear_terms = tan ((y_corr - 0.5) * pi) - + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - }} }}}}} - } + }}}}} + } + if (dist_type == 2 & link_type >= 1 & link_type <= 5) + { # BINOMIAL/BERNOULLI DISTRIBUTION + if (link_type == 1 & link_power == 0) { # Binomial.log + if (sum (y_corr < 0) == 0) { + is_zero_y_corr = (y_corr == 0); + linear_terms = log (y_corr + is_zero_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); + } else { isNaN = 1; } + } else { if (link_type == 1 & link_power > 0) { # Binomial.power_nonlog pos + if (sum (y_corr < 0) == 0) { + is_zero_y_corr = (y_corr == 0); + linear_terms = (y_corr + is_zero_y_corr) ^ link_power - is_zero_y_corr; + } else { isNaN = 1; } + } else { if (link_type == 1) { # Binomial.power_nonlog neg + if (sum (y_corr <= 0) == 0) { + linear_terms = y_corr ^ link_power; + } else { isNaN = 1; } + } else { + is_zero_y_corr = (y_corr <= 0); + is_one_y_corr = (y_corr >= 1.0); + y_corr = y_corr * (1.0 - is_zero_y_corr) * (1.0 - is_one_y_corr) + 0.5 * (is_zero_y_corr + is_one_y_corr); + if (link_type == 2) { # Binomial.logit + linear_terms = log (y_corr / (1.0 - y_corr)) + + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); + } else { if (link_type == 3) { # Binomial.probit + y_below_half = y_corr + (1.0 - 2.0 * y_corr) * (y_corr > 0.5); + t = sqrt (- 2.0 * log (y_below_half)); + approx_inv_Gauss_CDF = - t + (2.515517 + t * (0.802853 + t * 0.010328)) / (1.0 + t * (1.432788 + t * (0.189269 + t * 0.001308))); + linear_terms = approx_inv_Gauss_CDF * (1.0 - 2.0 * (y_corr > 0.5)) + + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); + } else { if (link_type == 4) { # Binomial.cloglog + linear_terms = log (- log (1.0 - y_corr)) + - log (- log (0.5)) * (is_zero_y_corr + is_one_y_corr) + + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); + } else { if (link_type == 5) { # Binomial.cauchit + linear_terms = tan ((y_corr - 0.5) * pi) + + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); + }} }}}}} + } - if (isNaN == 0) { - [saturated_log_l, isNaN] = - glm_log_likelihood_part (linear_terms, Y, dist_type, var_power, link_type, link_power); - } + if (isNaN == 0) { + [saturated_log_l, isNaN] = + glm_log_likelihood_part (linear_terms, Y, dist_type, var_power, link_type, link_power); + } - if ((dist_type == 1 & link_type == 1 & link_power == 0) | - (dist_type == 2 & link_type >= 2)) - { - desired_eta = 0.0; - } else { if (link_type == 1 & link_power == 0) { - desired_eta = log (0.5); - } else { if (link_type == 1) { - desired_eta = 0.5 ^ link_power; - } else { - desired_eta = 0.5; - }}} + if ((dist_type == 1 & link_type == 1 & link_power == 0) | + (dist_type == 2 & link_type >= 2)) + { + desired_eta = 0.0; + } else { if (link_type == 1 & link_power == 0) { + desired_eta = log (0.5); + } else { if (link_type == 1) { + desired_eta = 0.5 ^ link_power; + } else { + desired_eta = 0.5; + }}} - beta = matrix (0.0, rows = ncol(X), cols = 1); + beta = matrix (0.0, rows = ncol(X), cols = 1); - if (desired_eta != 0) { - if (icept_status == 1 | icept_status == 2) { - beta [nrow(beta), 1] = desired_eta; - } else { - # We want: avg (X %*% ssX_transform %*% beta) = desired_eta - # Note that "ssX_transform" is trivial here, hence ignored + if (desired_eta != 0) { + if (icept_status == 1 | icept_status == 2) { + beta [nrow(beta), 1] = desired_eta; + } else { + # We want: avg (X %*% ssX_transform %*% beta) = desired_eta + # Note that "ssX_transform" is trivial here, hence ignored - beta = straightenX (X, 0.000001, max_iter_CG); - beta = beta * desired_eta; - } } } + beta = straightenX (X, 0.000001, max_iter_CG); + beta = beta * desired_eta; + } } } glm_dist = function (Matrix[double] linear_terms, Matrix[double] Y, int dist_type, double var_power, int link_type, double link_power) - return (Matrix[double] g_Y, Matrix[double] w) + return (Matrix[double] g_Y, Matrix[double] w) # ORIGINALLY we returned more meaningful vectors, namely: # Matrix[double] y_residual : y - y_mean, i.e. y observed - y predicted # Matrix[double] link_gradient : derivative of the link function @@ -754,312 +750,312 @@ glm_dist = function (Matrix[double] linear_terms, Matrix[double] Y, # and skip over the "meaningful intermediaries". Now we output these two variables: # g_Y = y_residual / (var_function * link_gradient); # w = 1.0 / (var_function * link_gradient ^ 2); - { - num_records = nrow (linear_terms); - zeros_r = matrix (0.0, rows = num_records, cols = 1); - ones_r = 1 + zeros_r; - g_Y = zeros_r; - w = zeros_r; - - # Some constants - - one_over_sqrt_two_pi = 0.39894228040143267793994605993438; - ones_2 = matrix (1.0, rows = 1, cols = 2); - p_one_m_one = ones_2; - p_one_m_one [1, 2] = -1.0; - m_one_p_one = ones_2; - m_one_p_one [1, 1] = -1.0; - zero_one = ones_2; - zero_one [1, 1] = 0.0; - one_zero = ones_2; - one_zero [1, 2] = 0.0; - flip_pos = matrix (0, rows = 2, cols = 2); - flip_neg = flip_pos; - flip_pos [1, 2] = 1; - flip_pos [2, 1] = 1; - flip_neg [1, 2] = -1; - flip_neg [2, 1] = 1; - - if (dist_type == 1 & link_type == 1) { # POWER DISTRIBUTION - y_mean = zeros_r; - if (link_power == 0) { - y_mean = exp (linear_terms); - y_mean_pow = y_mean ^ (1 - var_power); - w = y_mean_pow * y_mean; - g_Y = y_mean_pow * (Y - y_mean); - } else { if (link_power == 1.0) { - y_mean = linear_terms; - w = y_mean ^ (- var_power); - g_Y = w * (Y - y_mean); - } else { - y_mean = linear_terms ^ (1.0 / link_power); - c1 = (1 - var_power) / link_power - 1; - c2 = (2 - var_power) / link_power - 2; - g_Y = (linear_terms ^ c1) * (Y - y_mean) / link_power; - w = (linear_terms ^ c2) / (link_power ^ 2); - } }} - if (dist_type == 2 & link_type >= 1 & link_type <= 5) - { # BINOMIAL/BERNOULLI DISTRIBUTION - if (link_type == 1) { # BINOMIAL.POWER LINKS - if (link_power == 0) { # Binomial.log - vec1 = 1 / (exp (- linear_terms) - 1); - g_Y = Y [, 1] - Y [, 2] * vec1; - w = rowSums (Y) * vec1; - } else { # Binomial.nonlog - vec1 = zeros_r; - if (link_power == 0.5) { - vec1 = 1 / (1 - linear_terms ^ 2); - } else { if (sum (linear_terms < 0) == 0) { - vec1 = linear_terms ^ (- 2 + 1 / link_power) / (1 - linear_terms ^ (1 / link_power)); - } else {isNaN = 1;}} - # We want a "zero-protected" version of - # vec2 = Y [, 1] / linear_terms; - is_y_0 = (Y [, 1] == 0); - vec2 = (Y [, 1] + is_y_0) / (linear_terms * (1 - is_y_0) + is_y_0) - is_y_0; - g_Y = (vec2 - Y [, 2] * vec1 * linear_terms) / link_power; - w = rowSums (Y) * vec1 / link_power ^ 2; - } - } else { - is_LT_pos_infinite = (linear_terms == Inf); - is_LT_neg_infinite = (linear_terms == -Inf); - is_LT_infinite = is_LT_pos_infinite %*% one_zero + is_LT_neg_infinite %*% zero_one; - finite_linear_terms = replace (target = linear_terms, pattern = Inf, replacement = 0); - finite_linear_terms = replace (target = finite_linear_terms, pattern = -Inf, replacement = 0); - if (link_type == 2) { # Binomial.logit - Y_prob = exp (finite_linear_terms) %*% one_zero + ones_r %*% zero_one; - Y_prob = Y_prob / (rowSums (Y_prob) %*% ones_2); - Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; - g_Y = rowSums (Y * (Y_prob %*% flip_neg)); ### = y_residual; - w = rowSums (Y * (Y_prob %*% flip_pos) * Y_prob); ### = y_variance; - } else { if (link_type == 3) { # Binomial.probit - is_lt_pos = (linear_terms >= 0); - t_gp = 1.0 / (1.0 + abs (finite_linear_terms) * 0.231641888); # 0.231641888 = 0.3275911 / sqrt (2.0) - pt_gp = t_gp * ( 0.254829592 - + t_gp * (-0.284496736 # "Handbook of Mathematical Functions", ed. by M. Abramowitz and I.A. Stegun, - + t_gp * ( 1.421413741 # U.S. Nat-l Bureau of Standards, 10th print (Dec 1972), Sec. 7.1.26, p. 299 - + t_gp * (-1.453152027 - + t_gp * 1.061405429)))); - the_gauss_exp = exp (- (linear_terms ^ 2) / 2.0); - vec1 = 0.25 * pt_gp * (2 - the_gauss_exp * pt_gp); - vec2 = Y [, 1] - rowSums (Y) * is_lt_pos + the_gauss_exp * pt_gp * rowSums (Y) * (is_lt_pos - 0.5); - w = the_gauss_exp * (one_over_sqrt_two_pi ^ 2) * rowSums (Y) / vec1; - g_Y = one_over_sqrt_two_pi * vec2 / vec1; - } else { if (link_type == 4) { # Binomial.cloglog - the_exp = exp (linear_terms) - the_exp_exp = exp (- the_exp); - is_too_small = ((10000000 + the_exp) == 10000000); - the_exp_ratio = (1 - is_too_small) * (1 - the_exp_exp) / (the_exp + is_too_small) + is_too_small * (1 - the_exp / 2); - g_Y = (rowSums (Y) * the_exp_exp - Y [, 2]) / the_exp_ratio; - w = the_exp_exp * the_exp * rowSums (Y) / the_exp_ratio; - } else { if (link_type == 5) { # Binomial.cauchit - Y_prob = 0.5 + (atan (finite_linear_terms) %*% p_one_m_one) / pi; - Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; - y_residual = Y [, 1] * Y_prob [, 2] - Y [, 2] * Y_prob [, 1]; - var_function = rowSums (Y) * Y_prob [, 1] * Y_prob [, 2]; - link_gradient_normalized = (1 + linear_terms ^ 2) * pi; - g_Y = rowSums (Y) * y_residual / (var_function * link_gradient_normalized); - w = (rowSums (Y) ^ 2) / (var_function * link_gradient_normalized ^ 2); - }}}} - } + { + num_records = nrow (linear_terms); + zeros_r = matrix (0.0, rows = num_records, cols = 1); + ones_r = 1 + zeros_r; + g_Y = zeros_r; + w = zeros_r; + + # Some constants + + one_over_sqrt_two_pi = 0.39894228040143267793994605993438; + ones_2 = matrix (1.0, rows = 1, cols = 2); + p_one_m_one = ones_2; + p_one_m_one [1, 2] = -1.0; + m_one_p_one = ones_2; + m_one_p_one [1, 1] = -1.0; + zero_one = ones_2; + zero_one [1, 1] = 0.0; + one_zero = ones_2; + one_zero [1, 2] = 0.0; + flip_pos = matrix (0, rows = 2, cols = 2); + flip_neg = flip_pos; + flip_pos [1, 2] = 1; + flip_pos [2, 1] = 1; + flip_neg [1, 2] = -1; + flip_neg [2, 1] = 1; + + if (dist_type == 1 & link_type == 1) { # POWER DISTRIBUTION + y_mean = zeros_r; + if (link_power == 0) { + y_mean = exp (linear_terms); + y_mean_pow = y_mean ^ (1 - var_power); + w = y_mean_pow * y_mean; + g_Y = y_mean_pow * (Y - y_mean); + } else { if (link_power == 1.0) { + y_mean = linear_terms; + w = y_mean ^ (- var_power); + g_Y = w * (Y - y_mean); + } else { + y_mean = linear_terms ^ (1.0 / link_power); + c1 = (1 - var_power) / link_power - 1; + c2 = (2 - var_power) / link_power - 2; + g_Y = (linear_terms ^ c1) * (Y - y_mean) / link_power; + w = (linear_terms ^ c2) / (link_power ^ 2); + } }} + if (dist_type == 2 & link_type >= 1 & link_type <= 5) + { # BINOMIAL/BERNOULLI DISTRIBUTION + if (link_type == 1) { # BINOMIAL.POWER LINKS + if (link_power == 0) { # Binomial.log + vec1 = 1 / (exp (- linear_terms) - 1); + g_Y = Y [, 1] - Y [, 2] * vec1; + w = rowSums (Y) * vec1; + } else { # Binomial.nonlog + vec1 = zeros_r; + if (link_power == 0.5) { + vec1 = 1 / (1 - linear_terms ^ 2); + } else { if (sum (linear_terms < 0) == 0) { + vec1 = linear_terms ^ (- 2 + 1 / link_power) / (1 - linear_terms ^ (1 / link_power)); + } else {isNaN = 1;}} + # We want a "zero-protected" version of + # vec2 = Y [, 1] / linear_terms; + is_y_0 = (Y [, 1] == 0); + vec2 = (Y [, 1] + is_y_0) / (linear_terms * (1 - is_y_0) + is_y_0) - is_y_0; + g_Y = (vec2 - Y [, 2] * vec1 * linear_terms) / link_power; + w = rowSums (Y) * vec1 / link_power ^ 2; } + } else { + is_LT_pos_infinite = (linear_terms == Inf); + is_LT_neg_infinite = (linear_terms == -Inf); + is_LT_infinite = is_LT_pos_infinite %*% one_zero + is_LT_neg_infinite %*% zero_one; + finite_linear_terms = replace (target = linear_terms, pattern = Inf, replacement = 0); + finite_linear_terms = replace (target = finite_linear_terms, pattern = -Inf, replacement = 0); + if (link_type == 2) { # Binomial.logit + Y_prob = exp (finite_linear_terms) %*% one_zero + ones_r %*% zero_one; + Y_prob = Y_prob / (rowSums (Y_prob) %*% ones_2); + Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; + g_Y = rowSums (Y * (Y_prob %*% flip_neg)); ### = y_residual; + w = rowSums (Y * (Y_prob %*% flip_pos) * Y_prob); ### = y_variance; + } else { if (link_type == 3) { # Binomial.probit + is_lt_pos = (linear_terms >= 0); + t_gp = 1.0 / (1.0 + abs (finite_linear_terms) * 0.231641888); # 0.231641888 = 0.3275911 / sqrt (2.0) + pt_gp = t_gp * ( 0.254829592 + + t_gp * (-0.284496736 # "Handbook of Mathematical Functions", ed. by M. Abramowitz and I.A. Stegun, + + t_gp * ( 1.421413741 # U.S. Nat-l Bureau of Standards, 10th print (Dec 1972), Sec. 7.1.26, p. 299 + + t_gp * (-1.453152027 + + t_gp * 1.061405429)))); + the_gauss_exp = exp (- (linear_terms ^ 2) / 2.0); + vec1 = 0.25 * pt_gp * (2 - the_gauss_exp * pt_gp); + vec2 = Y [, 1] - rowSums (Y) * is_lt_pos + the_gauss_exp * pt_gp * rowSums (Y) * (is_lt_pos - 0.5); + w = the_gauss_exp * (one_over_sqrt_two_pi ^ 2) * rowSums (Y) / vec1; + g_Y = one_over_sqrt_two_pi * vec2 / vec1; + } else { if (link_type == 4) { # Binomial.cloglog + the_exp = exp (linear_terms) + the_exp_exp = exp (- the_exp); + is_too_small = ((10000000 + the_exp) == 10000000); + the_exp_ratio = (1 - is_too_small) * (1 - the_exp_exp) / (the_exp + is_too_small) + is_too_small * (1 - the_exp / 2); + g_Y = (rowSums (Y) * the_exp_exp - Y [, 2]) / the_exp_ratio; + w = the_exp_exp * the_exp * rowSums (Y) / the_exp_ratio; + } else { if (link_type == 5) { # Binomial.cauchit + Y_prob = 0.5 + (atan (finite_linear_terms) %*% p_one_m_one) / pi; + Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; + y_residual = Y [, 1] * Y_prob [, 2] - Y [, 2] * Y_prob [, 1]; + var_function = rowSums (Y) * Y_prob [, 1] * Y_prob [, 2]; + link_gradient_normalized = (1 + linear_terms ^ 2) * pi; + g_Y = rowSums (Y) * y_residual / (var_function * link_gradient_normalized); + w = (rowSums (Y) ^ 2) / (var_function * link_gradient_normalized ^ 2); + }}}} + } } + } glm_log_likelihood_part = function (Matrix[double] linear_terms, Matrix[double] Y, int dist_type, double var_power, int link_type, double link_power) - return (double log_l, int isNaN) - { - isNaN = 0; - log_l = 0.0; - num_records = nrow (Y); - zeros_r = matrix (0.0, rows = num_records, cols = 1); - - if (dist_type == 1 & link_type == 1) - { # POWER DISTRIBUTION - b_cumulant = zeros_r; - natural_parameters = zeros_r; - is_natural_parameter_log_zero = zeros_r; - if (var_power == 1.0 & link_power == 0) { # Poisson.log - b_cumulant = exp (linear_terms); - is_natural_parameter_log_zero = (linear_terms == -Inf); - natural_parameters = replace (target = linear_terms, pattern = -Inf, replacement = 0); - } else { if (var_power == 1.0 & link_power == 1.0) { # Poisson.id - if (sum (linear_terms < 0) == 0) { - b_cumulant = linear_terms; - is_natural_parameter_log_zero = (linear_terms == 0); - natural_parameters = log (linear_terms + is_natural_parameter_log_zero); + return (double log_l, int isNaN) + { + isNaN = 0; + log_l = 0.0; + num_records = nrow (Y); + zeros_r = matrix (0.0, rows = num_records, cols = 1); + + if (dist_type == 1 & link_type == 1) + { # POWER DISTRIBUTION + b_cumulant = zeros_r; + natural_parameters = zeros_r; + is_natural_parameter_log_zero = zeros_r; + if (var_power == 1.0 & link_power == 0) { # Poisson.log + b_cumulant = exp (linear_terms); + is_natural_parameter_log_zero = (linear_terms == -Inf); + natural_parameters = replace (target = linear_terms, pattern = -Inf, replacement = 0); + } else { if (var_power == 1.0 & link_power == 1.0) { # Poisson.id + if (sum (linear_terms < 0) == 0) { + b_cumulant = linear_terms; + is_natural_parameter_log_zero = (linear_terms == 0); + natural_parameters = log (linear_terms + is_natural_parameter_log_zero); + } else {isNaN = 1;} + } else { if (var_power == 1.0 & link_power == 0.5) { # Poisson.sqrt + if (sum (linear_terms < 0) == 0) { + b_cumulant = linear_terms ^ 2; + is_natural_parameter_log_zero = (linear_terms == 0); + natural_parameters = 2.0 * log (linear_terms + is_natural_parameter_log_zero); + } else {isNaN = 1;} + } else { if (var_power == 1.0 & link_power > 0) { # Poisson.power_nonlog, pos + if (sum (linear_terms < 0) == 0) { + is_natural_parameter_log_zero = (linear_terms == 0); + b_cumulant = (linear_terms + is_natural_parameter_log_zero) ^ (1.0 / link_power) - is_natural_parameter_log_zero; + natural_parameters = log (linear_terms + is_natural_parameter_log_zero) / link_power; + } else {isNaN = 1;} + } else { if (var_power == 1.0) { # Poisson.power_nonlog, neg + if (sum (linear_terms <= 0) == 0) { + b_cumulant = linear_terms ^ (1.0 / link_power); + natural_parameters = log (linear_terms) / link_power; + } else {isNaN = 1;} + } else { if (var_power == 2.0 & link_power == -1.0) { # Gamma.inverse + if (sum (linear_terms <= 0) == 0) { + b_cumulant = - log (linear_terms); + natural_parameters = - linear_terms; + } else {isNaN = 1;} + } else { if (var_power == 2.0 & link_power == 1.0) { # Gamma.id + if (sum (linear_terms <= 0) == 0) { + b_cumulant = log (linear_terms); + natural_parameters = - 1.0 / linear_terms; } else {isNaN = 1;} - } else { if (var_power == 1.0 & link_power == 0.5) { # Poisson.sqrt - if (sum (linear_terms < 0) == 0) { - b_cumulant = linear_terms ^ 2; - is_natural_parameter_log_zero = (linear_terms == 0); - natural_parameters = 2.0 * log (linear_terms + is_natural_parameter_log_zero); + } else { if (var_power == 2.0 & link_power == 0) { # Gamma.log + b_cumulant = linear_terms; + natural_parameters = - exp (- linear_terms); + } else { if (var_power == 2.0) { # Gamma.power_nonlog + if (sum (linear_terms <= 0) == 0) { + b_cumulant = log (linear_terms) / link_power; + natural_parameters = - linear_terms ^ (- 1.0 / link_power); } else {isNaN = 1;} - } else { if (var_power == 1.0 & link_power > 0) { # Poisson.power_nonlog, pos - if (sum (linear_terms < 0) == 0) { - is_natural_parameter_log_zero = (linear_terms == 0); - b_cumulant = (linear_terms + is_natural_parameter_log_zero) ^ (1.0 / link_power) - is_natural_parameter_log_zero; - natural_parameters = log (linear_terms + is_natural_parameter_log_zero) / link_power; - } else {isNaN = 1;} - } else { if (var_power == 1.0) { # Poisson.power_nonlog, neg - if (sum (linear_terms <= 0) == 0) { - b_cumulant = linear_terms ^ (1.0 / link_power); - natural_parameters = log (linear_terms) / link_power; - } else {isNaN = 1;} - } else { if (var_power == 2.0 & link_power == -1.0) { # Gamma.inverse - if (sum (linear_terms <= 0) == 0) { - b_cumulant = - log (linear_terms); - natural_parameters = - linear_terms; - } else {isNaN = 1;} - } else { if (var_power == 2.0 & link_power == 1.0) { # Gamma.id - if (sum (linear_terms <= 0) == 0) { - b_cumulant = log (linear_terms); - natural_parameters = - 1.0 / linear_terms; - } else {isNaN = 1;} - } else { if (var_power == 2.0 & link_power == 0) { # Gamma.log - b_cumulant = linear_terms; - natural_parameters = - exp (- linear_terms); - } else { if (var_power == 2.0) { # Gamma.power_nonlog - if (sum (linear_terms <= 0) == 0) { - b_cumulant = log (linear_terms) / link_power; - natural_parameters = - linear_terms ^ (- 1.0 / link_power); - } else {isNaN = 1;} - } else { if (link_power == 0) { # PowerDist.log - natural_parameters = exp (linear_terms * (1.0 - var_power)) / (1.0 - var_power); - b_cumulant = exp (linear_terms * (2.0 - var_power)) / (2.0 - var_power); - } else { # PowerDist.power_nonlog - if (-2 * link_power == 1.0 - var_power) { - natural_parameters = 1.0 / (linear_terms ^ 2) / (1.0 - var_power); - } else { if (-1 * link_power == 1.0 - var_power) { - natural_parameters = 1.0 / linear_terms / (1.0 - var_power); - } else { if ( link_power == 1.0 - var_power) { - natural_parameters = linear_terms / (1.0 - var_power); - } else { if ( 2 * link_power == 1.0 - var_power) { - natural_parameters = linear_terms ^ 2 / (1.0 - var_power); - } else { - if (sum (linear_terms <= 0) == 0) { - power = (1.0 - var_power) / link_power; - natural_parameters = (linear_terms ^ power) / (1.0 - var_power); - } else {isNaN = 1;} - }}}} - if (-2 * link_power == 2.0 - var_power) { - b_cumulant = 1.0 / (linear_terms ^ 2) / (2.0 - var_power); - } else { if (-1 * link_power == 2.0 - var_power) { - b_cumulant = 1.0 / linear_terms / (2.0 - var_power); - } else { if ( link_power == 2.0 - var_power) { - b_cumulant = linear_terms / (2.0 - var_power); - } else { if ( 2 * link_power == 2.0 - var_power) { - b_cumulant = linear_terms ^ 2 / (2.0 - var_power); - } else { - if (sum (linear_terms <= 0) == 0) { - power = (2.0 - var_power) / link_power; - b_cumulant = (linear_terms ^ power) / (2.0 - var_power); - } else {isNaN = 1;} - }}}} - }}}}} }}}}} - if (sum (is_natural_parameter_log_zero * abs (Y)) > 0) { - log_l = -Inf; - isNaN = 1; - } - if (isNaN == 0) - { - log_l = sum (Y * natural_parameters - b_cumulant); - if (log_l != log_l | (log_l == log_l + 1.0 & log_l == log_l * 2.0)) { - log_l = -Inf; - isNaN = 1; - } } } - - if (dist_type == 2 & link_type >= 1 & link_type <= 5) - { # BINOMIAL/BERNOULLI DISTRIBUTION - - [Y_prob, isNaN] = binomial_probability_two_column (linear_terms, link_type, link_power); - - if (isNaN == 0) { - does_prob_contradict = (Y_prob <= 0); - if (sum (does_prob_contradict * abs (Y)) == 0) { - log_l = sum (Y * log (Y_prob * (1 - does_prob_contradict) + does_prob_contradict)); - if (log_l != log_l | (log_l == log_l + 1.0 & log_l == log_l * 2.0)) { - isNaN = 1; - } - } else { - log_l = -Inf; - isNaN = 1; - } } } + } else { if (link_power == 0) { # PowerDist.log + natural_parameters = exp (linear_terms * (1.0 - var_power)) / (1.0 - var_power); + b_cumulant = exp (linear_terms * (2.0 - var_power)) / (2.0 - var_power); + } else { # PowerDist.power_nonlog + if (-2 * link_power == 1.0 - var_power) { + natural_parameters = 1.0 / (linear_terms ^ 2) / (1.0 - var_power); + } else { if (-1 * link_power == 1.0 - var_power) { + natural_parameters = 1.0 / linear_terms / (1.0 - var_power); + } else { if ( link_power == 1.0 - var_power) { + natural_parameters = linear_terms / (1.0 - var_power); + } else { if ( 2 * link_power == 1.0 - var_power) { + natural_parameters = linear_terms ^ 2 / (1.0 - var_power); + } else { + if (sum (linear_terms <= 0) == 0) { + power = (1.0 - var_power) / link_power; + natural_parameters = (linear_terms ^ power) / (1.0 - var_power); + } else {isNaN = 1;} + }}}} + if (-2 * link_power == 2.0 - var_power) { + b_cumulant = 1.0 / (linear_terms ^ 2) / (2.0 - var_power); + } else { if (-1 * link_power == 2.0 - var_power) { + b_cumulant = 1.0 / linear_terms / (2.0 - var_power); + } else { if ( link_power == 2.0 - var_power) { + b_cumulant = linear_terms / (2.0 - var_power); + } else { if ( 2 * link_power == 2.0 - var_power) { + b_cumulant = linear_terms ^ 2 / (2.0 - var_power); + } else { + if (sum (linear_terms <= 0) == 0) { + power = (2.0 - var_power) / link_power; + b_cumulant = (linear_terms ^ power) / (2.0 - var_power); + } else {isNaN = 1;} + }}}} + }}}}} }}}}} + if (sum (is_natural_parameter_log_zero * abs (Y)) > 0) { + log_l = -Inf; + isNaN = 1; + } + if (isNaN == 0) + { + log_l = sum (Y * natural_parameters - b_cumulant); + if (log_l != log_l | (log_l == log_l + 1.0 & log_l == log_l * 2.0)) { + log_l = -Inf; + isNaN = 1; + } } } - if (isNaN == 1) { - log_l = - Inf; - } + if (dist_type == 2 & link_type >= 1 & link_type <= 5) + { # BINOMIAL/BERNOULLI DISTRIBUTION + + [Y_prob, isNaN] = binomial_probability_two_column (linear_terms, link_type, link_power); + + if (isNaN == 0) { + does_prob_contradict = (Y_prob <= 0); + if (sum (does_prob_contradict * abs (Y)) == 0) { + log_l = sum (Y * log (Y_prob * (1 - does_prob_contradict) + does_prob_contradict)); + if (log_l != log_l | (log_l == log_l + 1.0 & log_l == log_l * 2.0)) { + isNaN = 1; + } + } else { + log_l = -Inf; + isNaN = 1; + } } } + + if (isNaN == 1) { + log_l = - Inf; } + } binomial_probability_two_column = - function (Matrix[double] linear_terms, int link_type, double link_power) - return (Matrix[double] Y_prob, int isNaN) - { - isNaN = 0; - num_records = nrow (linear_terms); - - # Define some auxiliary matrices - - ones_2 = matrix (1.0, rows = 1, cols = 2); - p_one_m_one = ones_2; - p_one_m_one [1, 2] = -1.0; - m_one_p_one = ones_2; - m_one_p_one [1, 1] = -1.0; - zero_one = ones_2; - zero_one [1, 1] = 0.0; - one_zero = ones_2; - one_zero [1, 2] = 0.0; - - zeros_r = matrix (0.0, rows = num_records, cols = 1); - ones_r = 1.0 + zeros_r; - - # Begin the function body - - Y_prob = zeros_r %*% ones_2; - if (link_type == 1) { # Binomial.power - if (link_power == 0) { # Binomial.log - Y_prob = exp (linear_terms) %*% p_one_m_one + ones_r %*% zero_one; - } else { if (link_power == 0.5) { # Binomial.sqrt - Y_prob = (linear_terms ^ 2) %*% p_one_m_one + ones_r %*% zero_one; - } else { # Binomial.power_nonlog - if (sum (linear_terms < 0) == 0) { - Y_prob = (linear_terms ^ (1.0 / link_power)) %*% p_one_m_one + ones_r %*% zero_one; - } else {isNaN = 1;} - }} - } else { # Binomial.non_power - is_LT_pos_infinite = (linear_terms == Inf); - is_LT_neg_infinite = (linear_terms == -Inf); - is_LT_infinite = is_LT_pos_infinite %*% one_zero + is_LT_neg_infinite %*% zero_one; - finite_linear_terms = replace (target = linear_terms, pattern = Inf, replacement = 0); - finite_linear_terms = replace (target = finite_linear_terms, pattern = -Inf, replacement = 0); - if (link_type == 2) { # Binomial.logit - Y_prob = exp (finite_linear_terms) %*% one_zero + ones_r %*% zero_one; - Y_prob = Y_prob / (rowSums (Y_prob) %*% ones_2); - } else { if (link_type == 3) { # Binomial.probit - lt_pos_neg = (finite_linear_terms >= 0) %*% p_one_m_one + ones_r %*% zero_one; - t_gp = 1.0 / (1.0 + abs (finite_linear_terms) * 0.231641888); # 0.231641888 = 0.3275911 / sqrt (2.0) - pt_gp = t_gp * ( 0.254829592 - + t_gp * (-0.284496736 # "Handbook of Mathematical Functions", ed. by M. Abramowitz and I.A. Stegun, - + t_gp * ( 1.421413741 # U.S. Nat-l Bureau of Standards, 10th print (Dec 1972), Sec. 7.1.26, p. 299 - + t_gp * (-1.453152027 - + t_gp * 1.061405429)))); - the_gauss_exp = exp (- (finite_linear_terms ^ 2) / 2.0); - Y_prob = lt_pos_neg + ((the_gauss_exp * pt_gp) %*% ones_2) * (0.5 - lt_pos_neg); - } else { if (link_type == 4) { # Binomial.cloglog - the_exp = exp (finite_linear_terms); - the_exp_exp = exp (- the_exp); - is_too_small = ((10000000 + the_exp) == 10000000); - Y_prob [, 1] = (1 - is_too_small) * (1 - the_exp_exp) + is_too_small * the_exp * (1 - the_exp / 2); - Y_prob [, 2] = the_exp_exp; - } else { if (link_type == 5) { # Binomial.cauchit - Y_prob = 0.5 + (atan (finite_linear_terms) %*% p_one_m_one) / pi; - } else { - isNaN = 1; - }}}} - Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; - } } + function (Matrix[double] linear_terms, int link_type, double link_power) + return (Matrix[double] Y_prob, int isNaN) + { + isNaN = 0; + num_records = nrow (linear_terms); + + # Define some auxiliary matrices + + ones_2 = matrix (1.0, rows = 1, cols = 2); + p_one_m_one = ones_2; + p_one_m_one [1, 2] = -1.0; + m_one_p_one = ones_2; + m_one_p_one [1, 1] = -1.0; + zero_one = ones_2; + zero_one [1, 1] = 0.0; + one_zero = ones_2; + one_zero [1, 2] = 0.0; + + zeros_r = matrix (0.0, rows = num_records, cols = 1); + ones_r = 1.0 + zeros_r; + + # Begin the function body + + Y_prob = zeros_r %*% ones_2; + if (link_type == 1) { # Binomial.power + if (link_power == 0) { # Binomial.log + Y_prob = exp (linear_terms) %*% p_one_m_one + ones_r %*% zero_one; + } else { if (link_power == 0.5) { # Binomial.sqrt + Y_prob = (linear_terms ^ 2) %*% p_one_m_one + ones_r %*% zero_one; + } else { # Binomial.power_nonlog + if (sum (linear_terms < 0) == 0) { + Y_prob = (linear_terms ^ (1.0 / link_power)) %*% p_one_m_one + ones_r %*% zero_one; + } else {isNaN = 1;} + }} + } else { # Binomial.non_power + is_LT_pos_infinite = (linear_terms == Inf); + is_LT_neg_infinite = (linear_terms == -Inf); + is_LT_infinite = is_LT_pos_infinite %*% one_zero + is_LT_neg_infinite %*% zero_one; + finite_linear_terms = replace (target = linear_terms, pattern = Inf, replacement = 0); + finite_linear_terms = replace (target = finite_linear_terms, pattern = -Inf, replacement = 0); + if (link_type == 2) { # Binomial.logit + Y_prob = exp (finite_linear_terms) %*% one_zero + ones_r %*% zero_one; + Y_prob = Y_prob / (rowSums (Y_prob) %*% ones_2); + } else { if (link_type == 3) { # Binomial.probit + lt_pos_neg = (finite_linear_terms >= 0) %*% p_one_m_one + ones_r %*% zero_one; + t_gp = 1.0 / (1.0 + abs (finite_linear_terms) * 0.231641888); # 0.231641888 = 0.3275911 / sqrt (2.0) + pt_gp = t_gp * ( 0.254829592 + + t_gp * (-0.284496736 # "Handbook of Mathematical Functions", ed. by M. Abramowitz and I.A. Stegun, + + t_gp * ( 1.421413741 # U.S. Nat-l Bureau of Standards, 10th print (Dec 1972), Sec. 7.1.26, p. 299 + + t_gp * (-1.453152027 + + t_gp * 1.061405429)))); + the_gauss_exp = exp (- (finite_linear_terms ^ 2) / 2.0); + Y_prob = lt_pos_neg + ((the_gauss_exp * pt_gp) %*% ones_2) * (0.5 - lt_pos_neg); + } else { if (link_type == 4) { # Binomial.cloglog + the_exp = exp (finite_linear_terms); + the_exp_exp = exp (- the_exp); + is_too_small = ((10000000 + the_exp) == 10000000); + Y_prob [, 1] = (1 - is_too_small) * (1 - the_exp_exp) + is_too_small * the_exp * (1 - the_exp / 2); + Y_prob [, 2] = the_exp_exp; + } else { if (link_type == 5) { # Binomial.cauchit + Y_prob = 0.5 + (atan (finite_linear_terms) %*% p_one_m_one) / pi; + } else { + isNaN = 1; + }}}} + Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; + } } # THE CG-STEIHAUG PROCEDURE SCRIPT @@ -1072,130 +1068,130 @@ binomial_probability_two_column = # is given separately because sparse "X" may become dense after applying the transform. # get_CG_Steihaug_point = - function (Matrix[double] X, Matrix[double] scale_X, Matrix[double] shift_X, Matrix[double] w, - Matrix[double] g, Matrix[double] beta, Matrix[double] lambda, double trust_delta, int max_iter_CG) - return (Matrix[double] z, double neg_log_l_change, int i_CG, int reached_trust_boundary) - { - trust_delta_sq = trust_delta ^ 2; - size_CG = nrow (g); - z = matrix (0.0, rows = size_CG, cols = 1); - neg_log_l_change = 0.0; - reached_trust_boundary = 0; - g_reg = g + lambda * beta; - r_CG = g_reg; - p_CG = -r_CG; + function (Matrix[double] X, Matrix[double] scale_X, Matrix[double] shift_X, Matrix[double] w, + Matrix[double] g, Matrix[double] beta, Matrix[double] lambda, double trust_delta, int max_iter_CG) + return (Matrix[double] z, double neg_log_l_change, int i_CG, int reached_trust_boundary) + { + trust_delta_sq = trust_delta ^ 2; + size_CG = nrow (g); + z = matrix (0.0, rows = size_CG, cols = 1); + neg_log_l_change = 0.0; + reached_trust_boundary = 0; + g_reg = g + lambda * beta; + r_CG = g_reg; + p_CG = -r_CG; + rr_CG = sum(r_CG * r_CG); + eps_CG = rr_CG * min (0.25, sqrt (rr_CG)); + converged_CG = 0; + if (rr_CG < eps_CG) { + converged_CG = 1; + } + + max_iteration_CG = max_iter_CG; + if (max_iteration_CG <= 0) { + max_iteration_CG = size_CG; + } + i_CG = 0; + while (converged_CG == 0) + { + i_CG = i_CG + 1; + ssX_p_CG = diag (scale_X) %*% p_CG; + ssX_p_CG [size_CG, ] = ssX_p_CG [size_CG, ] + t(shift_X) %*% p_CG; + temp_CG = t(X) %*% (w * (X %*% ssX_p_CG)); + q_CG = (lambda * p_CG) + diag (scale_X) %*% temp_CG + shift_X %*% temp_CG [size_CG, ]; + pq_CG = sum (p_CG * q_CG); + if (pq_CG <= 0) { + pp_CG = sum (p_CG * p_CG); + if (pp_CG > 0) { + [z, neg_log_l_change] = + get_trust_boundary_point (g_reg, z, p_CG, q_CG, r_CG, pp_CG, pq_CG, trust_delta_sq); + reached_trust_boundary = 1; + } else { + neg_log_l_change = 0.5 * sum (z * (r_CG + g_reg)); + } + converged_CG = 1; + } + if (converged_CG == 0) { + alpha_CG = rr_CG / pq_CG; + new_z = z + alpha_CG * p_CG; + if (sum(new_z * new_z) >= trust_delta_sq) { + pp_CG = sum (p_CG * p_CG); + [z, neg_log_l_change] = + get_trust_boundary_point (g_reg, z, p_CG, q_CG, r_CG, pp_CG, pq_CG, trust_delta_sq); + reached_trust_boundary = 1; + converged_CG = 1; + } + if (converged_CG == 0) { + z = new_z; + old_rr_CG = rr_CG; + r_CG = r_CG + alpha_CG * q_CG; rr_CG = sum(r_CG * r_CG); - eps_CG = rr_CG * min (0.25, sqrt (rr_CG)); - converged_CG = 0; - if (rr_CG < eps_CG) { - converged_CG = 1; - } - - max_iteration_CG = max_iter_CG; - if (max_iteration_CG <= 0) { - max_iteration_CG = size_CG; + if (i_CG == max_iteration_CG | rr_CG < eps_CG) { + neg_log_l_change = 0.5 * sum (z * (r_CG + g_reg)); + reached_trust_boundary = 0; + converged_CG = 1; } - i_CG = 0; - while (converged_CG == 0) - { - i_CG = i_CG + 1; - ssX_p_CG = diag (scale_X) %*% p_CG; - ssX_p_CG [size_CG, ] = ssX_p_CG [size_CG, ] + t(shift_X) %*% p_CG; - temp_CG = t(X) %*% (w * (X %*% ssX_p_CG)); - q_CG = (lambda * p_CG) + diag (scale_X) %*% temp_CG + shift_X %*% temp_CG [size_CG, ]; - pq_CG = sum (p_CG * q_CG); - if (pq_CG <= 0) { - pp_CG = sum (p_CG * p_CG); - if (pp_CG > 0) { - [z, neg_log_l_change] = - get_trust_boundary_point (g_reg, z, p_CG, q_CG, r_CG, pp_CG, pq_CG, trust_delta_sq); - reached_trust_boundary = 1; - } else { - neg_log_l_change = 0.5 * sum (z * (r_CG + g_reg)); - } - converged_CG = 1; - } - if (converged_CG == 0) { - alpha_CG = rr_CG / pq_CG; - new_z = z + alpha_CG * p_CG; - if (sum(new_z * new_z) >= trust_delta_sq) { - pp_CG = sum (p_CG * p_CG); - [z, neg_log_l_change] = - get_trust_boundary_point (g_reg, z, p_CG, q_CG, r_CG, pp_CG, pq_CG, trust_delta_sq); - reached_trust_boundary = 1; - converged_CG = 1; - } - if (converged_CG == 0) { - z = new_z; - old_rr_CG = rr_CG; - r_CG = r_CG + alpha_CG * q_CG; - rr_CG = sum(r_CG * r_CG); - if (i_CG == max_iteration_CG | rr_CG < eps_CG) { - neg_log_l_change = 0.5 * sum (z * (r_CG + g_reg)); - reached_trust_boundary = 0; - converged_CG = 1; - } - if (converged_CG == 0) { - p_CG = -r_CG + (rr_CG / old_rr_CG) * p_CG; - } } } } } + if (converged_CG == 0) { + p_CG = -r_CG + (rr_CG / old_rr_CG) * p_CG; + } } } } } # An auxiliary function used twice inside the CG-STEIHAUG loop: get_trust_boundary_point = - function (Matrix[double] g, Matrix[double] z, Matrix[double] p, - Matrix[double] q, Matrix[double] r, double pp, double pq, - double trust_delta_sq) - return (Matrix[double] new_z, double f_change) - { - zz = sum (z * z); pz = sum (p * z); - sq_root_d = sqrt (pz * pz - pp * (zz - trust_delta_sq)); - tau_1 = (- pz + sq_root_d) / pp; - tau_2 = (- pz - sq_root_d) / pp; - zq = sum (z * q); gp = sum (g * p); - f_extra = 0.5 * sum (z * (r + g)); - f_change_1 = f_extra + (0.5 * tau_1 * pq + zq + gp) * tau_1; - f_change_2 = f_extra + (0.5 * tau_2 * pq + zq + gp) * tau_2; - ind1 = as.integer(f_change_1 < f_change_2); - ind2 = as.integer(f_change_1 >= f_change_2); - new_z = z + ((ind1 * tau_1 + ind2 * tau_2) * p); - f_change = ind1 * f_change_1 + ind2 * f_change_2; - } + function (Matrix[double] g, Matrix[double] z, Matrix[double] p, + Matrix[double] q, Matrix[double] r, double pp, double pq, + double trust_delta_sq) + return (Matrix[double] new_z, double f_change) + { + zz = sum (z * z); pz = sum (p * z); + sq_root_d = sqrt (pz * pz - pp * (zz - trust_delta_sq)); + tau_1 = (- pz + sq_root_d) / pp; + tau_2 = (- pz - sq_root_d) / pp; + zq = sum (z * q); gp = sum (g * p); + f_extra = 0.5 * sum (z * (r + g)); + f_change_1 = f_extra + (0.5 * tau_1 * pq + zq + gp) * tau_1; + f_change_2 = f_extra + (0.5 * tau_2 * pq + zq + gp) * tau_2; + ind1 = as.integer(f_change_1 < f_change_2); + ind2 = as.integer(f_change_1 >= f_change_2); + new_z = z + ((ind1 * tau_1 + ind2 * tau_2) * p); + f_change = ind1 * f_change_1 + ind2 * f_change_2; + } # Computes vector w such that ||X %*% w - 1|| -> MIN given avg(X %*% w) = 1 # We find z_LS such that ||X %*% z_LS - 1|| -> MIN unconditionally, then scale # it to compute w = c * z_LS such that sum(X %*% w) = nrow(X). straightenX = - function (Matrix[double] X, double eps, int max_iter_CG) - return (Matrix[double] w) - { - w_X = t(colSums(X)); - lambda_LS = 0.000001 * sum(X ^ 2) / ncol(X); - eps_LS = eps * nrow(X); - - # BEGIN LEAST SQUARES - - r_LS = - w_X; - z_LS = matrix (0.0, rows = ncol(X), cols = 1); - p_LS = - r_LS; - norm_r2_LS = sum (r_LS ^ 2); - i_LS = 0; - while (i_LS < max_iter_CG & i_LS < ncol(X) & norm_r2_LS >= eps_LS) - { - q_LS = t(X) %*% X %*% p_LS; - q_LS = q_LS + lambda_LS * p_LS; - alpha_LS = norm_r2_LS / sum (p_LS * q_LS); - z_LS = z_LS + alpha_LS * p_LS; - old_norm_r2_LS = norm_r2_LS; - r_LS = r_LS + alpha_LS * q_LS; - norm_r2_LS = sum (r_LS ^ 2); - p_LS = -r_LS + (norm_r2_LS / old_norm_r2_LS) * p_LS; - i_LS = i_LS + 1; - } - - # END LEAST SQUARES - - w = (nrow(X) / sum (w_X * z_LS)) * z_LS; - } + function (Matrix[double] X, double eps, int max_iter_CG) + return (Matrix[double] w) + { + w_X = t(colSums(X)); + lambda_LS = 0.000001 * sum(X ^ 2) / ncol(X); + eps_LS = eps * nrow(X); + + # BEGIN LEAST SQUARES + + r_LS = - w_X; + z_LS = matrix (0.0, rows = ncol(X), cols = 1); + p_LS = - r_LS; + norm_r2_LS = sum (r_LS ^ 2); + i_LS = 0; + while (i_LS < max_iter_CG & i_LS < ncol(X) & norm_r2_LS >= eps_LS) + { + q_LS = t(X) %*% X %*% p_LS; + q_LS = q_LS + lambda_LS * p_LS; + alpha_LS = norm_r2_LS / sum (p_LS * q_LS); + z_LS = z_LS + alpha_LS * p_LS; + old_norm_r2_LS = norm_r2_LS; + r_LS = r_LS + alpha_LS * q_LS; + norm_r2_LS = sum (r_LS ^ 2); + p_LS = -r_LS + (norm_r2_LS / old_norm_r2_LS) * p_LS; + i_LS = i_LS + 1; + } + + # END LEAST SQUARES + + w = (nrow(X) / sum (w_X * z_LS)) * z_LS; + } From e60c2dd1ab3c1b52f947463cd6a2515db21de8c1 Mon Sep 17 00:00:00 2001 From: bruno Date: Fri, 26 Jun 2026 05:48:34 +0200 Subject: [PATCH 005/132] fix glm_fit call syntax --- scripts/builtin/stepGLM.dml | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/scripts/builtin/stepGLM.dml b/scripts/builtin/stepGLM.dml index c8be7e582b1..0431b27ddde 100644 --- a/scripts/builtin/stepGLM.dml +++ b/scripts/builtin/stepGLM.dml @@ -134,18 +134,18 @@ stepGLM = function ( if (intercept_status == 0) { # Compute AIC of an empty model with no features and no intercept (all Ys are zero) - [AIC_best, ignore_B1, ignore_S1, ignore_O1] = glm_fit (X_global, Y, 0, num_features, columns_fixed_ordered); + [AIC_best, ignore_B1, ignore_S1, ignore_O1] = glm_fit (X=X_global, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } else { # compute AIC of an empty model with only intercept (all Ys are constant) all_ones = matrix (1, rows = num_records, cols = 1); - [AIC_best, ignore_beta2, ignore_S2, ignore_O2] = glm_fit (all_ones, Y, 0, num_features, columns_fixed_ordered); + [AIC_best, ignore_beta2, ignore_S2, ignore_O2] = glm_fit (X=all_ones, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } #print ("Best AIC without any features: " + AIC_best); # First pass to examine single features AICs = matrix (AIC_best, rows = 1, cols = num_features); parfor (i in 1:num_features) { - [AIC_1, ignore_beta3, ignore_S3, ignore_O3] = glm_fit (X_orig[,i], Y, intercept_status, num_features, columns_fixed_ordered); + [AIC_1, ignore_beta3, ignore_S3, ignore_O3] = glm_fit (X=X_orig[,i], Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); AICs[1,i] = AIC_1; } @@ -163,11 +163,11 @@ stepGLM = function ( #print ("AIC of an empty model is " + AIC_best + " and adding no feature achieves more than " + (thr * 100) + "% decrease in AIC!"); if (intercept_status == 0) { # Compute AIC of an empty model with no features and no intercept (all Ys are zero) - [AIC_best, ignore_beta4, ignore_S4, ignore_O4] = glm_fit (X_global, Y, 0, num_features, columns_fixed_ordered); + [AIC_best, ignore_beta4, ignore_S4, ignore_O4] = glm_fit (X=X_global, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } else { # compute AIC of an empty model with only intercept (all Ys are constant) ###all_ones = matrix (1, rows = num_records, cols = 1); - [AIC_best, ignore_beta5, ignore_S5, ignore_O5] = glm_fit (all_ones, Y, 0, num_features, columns_fixed_ordered); + [AIC_best, ignore_beta5, ignore_S5, ignore_O5] = glm_fit (X=all_ones, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } }; @@ -184,7 +184,7 @@ stepGLM = function ( # Construct the feature matrix X_loop = cbind (X_global, X_orig[,i]); - [AIC_2, ignore_beta6, ignore_S6, ignore_O6] = glm_fit (X_loop, Y, intercept_status, num_features, columns_fixed_ordered); + [AIC_2, ignore_beta6, ignore_S6, ignore_O6] = glm_fit (X=X_loop, Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); AICs[1,i] = AIC_2; } } @@ -216,7 +216,7 @@ stepGLM = function ( # run GLM with selected set of features print ("Running GLM with selected features..."); - [AIC, B, S, O] = glm_fit (X_global, Y, intercept_status, num_features, columns_fixed_ordered); + [AIC, B, S, O] = glm_fit (X=X_global, Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } @@ -834,7 +834,7 @@ glm_dist = function (Matrix[double] linear_terms, Matrix[double] Y, + t_gp * (-0.284496736 # "Handbook of Mathematical Functions", ed. by M. Abramowitz and I.A. Stegun, + t_gp * ( 1.421413741 # U.S. Nat-l Bureau of Standards, 10th print (Dec 1972), Sec. 7.1.26, p. 299 + t_gp * (-1.453152027 - + t_gp * 1.061405429)))); + + t_gp * 1.061405429)))); the_gauss_exp = exp (- (linear_terms ^ 2) / 2.0); vec1 = 0.25 * pt_gp * (2 - the_gauss_exp * pt_gp); vec2 = Y [, 1] - rowSums (Y) * is_lt_pos + the_gauss_exp * pt_gp * rowSums (Y) * (is_lt_pos - 0.5); @@ -1040,7 +1040,7 @@ binomial_probability_two_column = + t_gp * (-0.284496736 # "Handbook of Mathematical Functions", ed. by M. Abramowitz and I.A. Stegun, + t_gp * ( 1.421413741 # U.S. Nat-l Bureau of Standards, 10th print (Dec 1972), Sec. 7.1.26, p. 299 + t_gp * (-1.453152027 - + t_gp * 1.061405429)))); + + t_gp * 1.061405429)))); the_gauss_exp = exp (- (finite_linear_terms ^ 2) / 2.0); Y_prob = lt_pos_neg + ((the_gauss_exp * pt_gp) %*% ones_2) * (0.5 - lt_pos_neg); } else { if (link_type == 4) { # Binomial.cloglog @@ -1193,5 +1193,3 @@ straightenX = w = (nrow(X) / sum (w_X * z_LS)) * z_LS; } - - From 9de039d912e842bdeaec7b8a2231550893e6258a Mon Sep 17 00:00:00 2001 From: bruno Date: Fri, 26 Jun 2026 06:20:55 +0200 Subject: [PATCH 006/132] remove fit_glm --- scripts/builtin/stepGLM.dml | 973 ------------------------------------ 1 file changed, 973 deletions(-) diff --git a/scripts/builtin/stepGLM.dml b/scripts/builtin/stepGLM.dml index 0431b27ddde..3ec1cffcb7f 100644 --- a/scripts/builtin/stepGLM.dml +++ b/scripts/builtin/stepGLM.dml @@ -220,976 +220,3 @@ stepGLM = function ( } -################### UDFS USED IN THIS SCRIPT ################## - -glm_fit = function ( - Matrix[Double] X, - Matrix[Double] Y, - Int intercept_status, - Double num_features_orig, - Matrix[Double] Selected, - Int link = 2, - Double disp = 0.0, - Double tol = 0.000001, - Int moi = 200, - Int mii = 0 -) - return ( - Double AIC, - Matrix[Double] beta, - Matrix[Double] S, - String O - ) - { - - # distribution family code: 1 = Power, 2 = Bernoulli/Binomial; currently only Bernouli distribution family is supported! - distribution_type = 2; # $dfam = 2; - variance_as_power_of_the_mean = 0.0; # $vpow = 0.0; - # link function code: 0 = canonical (depends on distribution), 1 = Power, 2 = Logit, 3 = Probit, 4 = Cloglog, 5 = Cauchit; - # currently only log (link = 1), logit (link = 2), probit (link = 3), and cloglog (link = 4) are supported! - link_type = link; - link_as_power_of_the_mean = 0.0; # $lpow = 0.0; - - dispersion = disp; - eps = tol; - max_iteration_IRLS = moi; - max_iteration_CG = mii; - - variance_as_power_of_the_mean = as.double (variance_as_power_of_the_mean); - link_as_power_of_the_mean = as.double (link_as_power_of_the_mean); - - dispersion = as.double (dispersion); - eps = as.double (eps); - - # Default values for output statistics: - regularization = 0.0; - termination_code = 0.0; - min_beta = NaN; - i_min_beta = NaN; - max_beta = NaN; - i_max_beta = NaN; - intercept_value = NaN; - dispersion = NaN; - estimated_dispersion = NaN; - deviance_nodisp = NaN; - deviance = NaN; - - ##### INITIALIZE THE PARAMETERS ##### - - num_records = nrow (X); - num_features = ncol (X); - zeros_r = matrix (0, rows = num_records, cols = 1); - ones_r = 1 + zeros_r; - - # Introduce the intercept, shift and rescale the columns of X if needed - - if (intercept_status == 1 | intercept_status == 2) { # add the intercept column - X = cbind (X, ones_r); - num_features = ncol (X); - } - - scale_lambda = matrix (1, rows = num_features, cols = 1); - if (intercept_status == 1 | intercept_status == 2) { - scale_lambda [num_features, 1] = 0; - } - - if (intercept_status == 2) { # scale-&-shift X columns to mean 0, variance 1 - # Important assumption: X [, num_features] = ones_r - avg_X_cols = t(colSums(X)) / num_records; - var_X_cols = (t(colSums (X ^ 2)) - num_records * (avg_X_cols ^ 2)) / (num_records - 1); - is_unsafe = (var_X_cols <= 0); - scale_X = 1.0 / sqrt (var_X_cols * (1 - is_unsafe) + is_unsafe); - scale_X [num_features, 1] = 1; - shift_X = - avg_X_cols * scale_X; - shift_X [num_features, 1] = 0; - rowSums_X_sq = (X ^ 2) %*% (scale_X ^ 2) + X %*% (2 * scale_X * shift_X) + sum (shift_X ^ 2); - } else { - scale_X = matrix (1, rows = num_features, cols = 1); - shift_X = matrix (0, rows = num_features, cols = 1); - rowSums_X_sq = rowSums (X ^ 2); - } - - # Henceforth we replace "X" with "X %*% (SHIFT/SCALE TRANSFORM)" and rowSums(X ^ 2) - # with "rowSums_X_sq" in order to preserve the sparsity of X under shift and scale. - # The transform is then associatively applied to the other side of the expression, - # and is rewritten via "scale_X" and "shift_X" as follows: - # - # ssX_A = (SHIFT/SCALE TRANSFORM) %*% A --- is rewritten as: - # ssX_A = diag (scale_X) %*% A; - # ssX_A [num_features, ] = ssX_A [num_features, ] + t(shift_X) %*% A; - # - # tssX_A = t(SHIFT/SCALE TRANSFORM) %*% A --- is rewritten as: - # tssX_A = diag (scale_X) %*% A + shift_X %*% A [num_features, ]; - - # Initialize other input-dependent parameters - - lambda = scale_lambda * regularization; - if (max_iteration_CG == 0) { - max_iteration_CG = num_features; - } - - # Set up the canonical link, if requested [Then we have: Var(mu) * (d link / d mu) = const] - - if (link_type == 0) { - if (distribution_type == 1) { - link_type = 1; - link_as_power_of_the_mean = 1.0 - variance_as_power_of_the_mean; - } else { - if (distribution_type == 2) { - link_type = 2; - } - } - } - - # For power distributions and/or links, we use two constants, - # "variance as power of the mean" and "link_as_power_of_the_mean", - # to specify the variance and the link as arbitrary powers of the - # mean. However, the variance-powers of 1.0 (Poisson family) and - # 2.0 (Gamma family) have to be treated as special cases, because - # these values integrate into logarithms. The link-power of 0.0 - # is also special as it represents the logarithm link. - - num_response_columns = ncol (Y); - is_supported = 0; - if (num_response_columns == 2 & distribution_type == 2 & link_type >= 1 & link_type <= 4) { # BERNOULLI DISTRIBUTION - is_supported = 1; - } - if (num_response_columns == 1 & distribution_type == 2) { - print ("Error: Bernoulli response matrix has not been converted into two-column format."); - } - - if (is_supported != 1) { - stop ("Response matrix with " + num_response_columns + " columns, distribution family (" + distribution_type + ", " + variance_as_power_of_the_mean - + ") and link family (" + link_type + ", " + link_as_power_of_the_mean + ") are NOT supported together."); - } - - ##### INITIALIZE THE BETAS ##### - - [beta, saturated_log_l, isNaN] = - glm_initialize (X, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean, intercept_status, max_iteration_CG); - - # print(" --- saturated logLik " + saturated_log_l); - - if (isNaN != 0) { - stop ("Input matrices X and/or Y are out of range!"); - } - - ##### START OF THE MAIN PART ##### - - sum_X_sq = sum (rowSums_X_sq); - trust_delta = 0.5 * sqrt (num_features) / max (sqrt (rowSums_X_sq)); - ### max_trust_delta = trust_delta * 10000.0; - log_l = 0.0; - deviance_nodisp = 0.0; - new_deviance_nodisp = 0.0; - isNaN_log_l = 2; - newbeta = beta; - g = matrix (0.0, rows = num_features, cols = 1); - g_norm = sqrt (sum ((g + lambda * beta) ^ 2)); - accept_new_beta = 1; - reached_trust_boundary = 0; - neg_log_l_change_predicted = 0.0; - i_IRLS = 0; - - # print ("BEGIN IRLS ITERATIONS..."); - - ssX_newbeta = diag (scale_X) %*% newbeta; - ssX_newbeta [num_features, ] = ssX_newbeta [num_features, ] + t(shift_X) %*% newbeta; - all_linear_terms = X %*% ssX_newbeta; - - [new_log_l, isNaN_new_log_l] = glm_log_likelihood_part - (all_linear_terms, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean); - - if (isNaN_new_log_l == 0) { - new_deviance_nodisp = 2.0 * (saturated_log_l - new_log_l); - new_log_l = new_log_l - 0.5 * sum (lambda * newbeta ^ 2); - } - - while (termination_code == 0) { - accept_new_beta = 1; - - if (i_IRLS > 0) { - if (isNaN_log_l == 0) { - accept_new_beta = 0; - } - - # Decide whether to accept a new iteration point and update the trust region - # See Alg. 4.1 on p. 69 of "Numerical Optimization" 2nd ed. by Nocedal and Wright - - rho = (- new_log_l + log_l) / neg_log_l_change_predicted; - if (rho < 0.25 | isNaN_new_log_l == 1) { - trust_delta = 0.25 * trust_delta; - } - if (rho > 0.75 & isNaN_new_log_l == 0 & reached_trust_boundary == 1) { - trust_delta = 2 * trust_delta; - - ### if (trust_delta > max_trust_delta) { - ### trust_delta = max_trust_delta; - ### } - } - if (rho > 0.1 & isNaN_new_log_l == 0) { - accept_new_beta = 1; - } - } - - if (accept_new_beta == 1) { - beta = newbeta; log_l = new_log_l; deviance_nodisp = new_deviance_nodisp; isNaN_log_l = isNaN_new_log_l; - - [g_Y, w] = glm_dist (all_linear_terms, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean); - - # We introduced these variables to avoid roundoff errors: - # g_Y = y_residual / (y_var * link_grad); - # w = 1.0 / (y_var * link_grad * link_grad); - - gXY = - t(X) %*% g_Y; - g = diag (scale_X) %*% gXY + shift_X %*% gXY [num_features, ]; - g_norm = sqrt (sum ((g + lambda * beta) ^ 2)); - } - - [z, neg_log_l_change_predicted, num_CG_iters, reached_trust_boundary] = - get_CG_Steihaug_point (X, scale_X, shift_X, w, g, beta, lambda, trust_delta, max_iteration_CG); - - newbeta = beta + z; - - ssX_newbeta = diag (scale_X) %*% newbeta; - ssX_newbeta [num_features, ] = ssX_newbeta [num_features, ] + t(shift_X) %*% newbeta; - all_linear_terms = X %*% ssX_newbeta; - - [new_log_l, isNaN_new_log_l] = glm_log_likelihood_part - (all_linear_terms, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean); - - if (isNaN_new_log_l == 0) { - new_deviance_nodisp = 2.0 * (saturated_log_l - new_log_l); - new_log_l = new_log_l - 0.5 * sum (lambda * newbeta ^ 2); - } - - log_l_change = new_log_l - log_l; # R's criterion for termination: |dev - devold|/(|dev| + 0.1) < eps - - if (reached_trust_boundary == 0 & isNaN_new_log_l == 0 & - (2.0 * abs (log_l_change) < eps * (deviance_nodisp + 0.1) | abs (log_l_change) < (abs (log_l) + abs (new_log_l)) * 0.00000000000001) ) { - termination_code = 1; - } - rho = - log_l_change / neg_log_l_change_predicted; - z_norm = sqrt (sum (z * z)); - - i_IRLS = i_IRLS + 1; - - if (i_IRLS == max_iteration_IRLS) { - termination_code = 2; - } - } - - beta = newbeta; - log_l = new_log_l; - deviance_nodisp = new_deviance_nodisp; - - #---------------------------- last part - - if (termination_code != 1) { - print ("One of the runs of GLM did not converged in " + i_IRLS + " steps!"); - } - - ##### COMPUTE AIC ##### - - if (distribution_type == 2 & link_type >= 1 & link_type <= 4) { - AIC = -2 * log_l; - if (sum (X) != 0) { - AIC = AIC + 2 * num_features; - } - } else { - stop ("Currently only the Bernoulli distribution family the following link functions are supported: log, logit, probit, and cloglog!"); - } - - # Output which features give the best AIC and are being used for linear regression - S = Selected; - - ssX_beta = diag (scale_X) %*% beta; - ssX_beta [num_features, ] = ssX_beta [num_features, ] + t(shift_X) %*% beta; - if (intercept_status == 2) { - beta_out = cbind (ssX_beta, beta); - } else { - beta_out = ssX_beta; - } - - if (intercept_status == 0 & num_features == 1) { - p = sum (X == 1); - if (p == num_records) { - beta_out = beta_out[1,]; - } - } - - - if (intercept_status == 1 | intercept_status == 2) { - intercept_value = as.scalar (beta_out [num_features, 1]); - beta_noicept = beta_out [1 : (num_features - 1), 1]; - } else { - beta_noicept = beta_out [1 : num_features, 1]; - } - min_beta = min (beta_noicept); - max_beta = max (beta_noicept); - tmp_i_min_beta = rowIndexMin (t(beta_noicept)) - i_min_beta = as.scalar (tmp_i_min_beta [1, 1]); - tmp_i_max_beta = rowIndexMax (t(beta_noicept)) - i_max_beta = as.scalar (tmp_i_max_beta [1, 1]); - - ##### OVER-DISPERSION PART ##### - - all_linear_terms = X %*% ssX_beta; - [g_Y, w] = glm_dist (all_linear_terms, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean); - - pearson_residual_sq = g_Y ^ 2 / w; - pearson_residual_sq = replace (target = pearson_residual_sq, pattern = NaN, replacement = 0); - # pearson_residual_sq = (y_residual ^ 2) / y_var; - - if (num_records > num_features) { - estimated_dispersion = sum (pearson_residual_sq) / (num_records - num_features); - } - if (dispersion <= 0) { - dispersion = estimated_dispersion; - } - deviance = deviance_nodisp / dispersion; - - ##### END OF THE MAIN PART ##### - - str = "BETA_MIN," + min_beta; - str = append (str, "BETA_MIN_INDEX," + i_min_beta); - str = append (str, "BETA_MAX," + max_beta); - str = append (str, "BETA_MAX_INDEX," + i_max_beta); - str = append (str, "INTERCEPT," + intercept_value); - str = append (str, "DISPERSION," + dispersion); - str = append (str, "DISPERSION_EST," + estimated_dispersion); - str = append (str, "DEVIANCE_UNSCALED," + deviance_nodisp); - str = append (str, "DEVIANCE_SCALED," + deviance); - O = str - - do_return = 0; - # Prepare the output matrix - if (intercept_status == 0 & num_features == 1) { - if (p == num_records) { - beta_out_tmp = matrix (0, rows = num_features_orig + 1, cols = 1); - beta_out_tmp[num_features_orig + 1,] = beta_out; - beta = beta_out_tmp; - do_return = 1; - } else if (sum (X) == 0){ - beta = matrix (0, rows = num_features_orig, cols = 1); - do_return = 1; - } - } - if (do_return != 0) { - no_selected = ncol (Selected); - max_selected = max (Selected); - last = max_selected + 1; - - if (intercept_status != 0) { - - Selected_ext = cbind (Selected, as.matrix (last)); - P1 = table (seq (1, ncol (Selected_ext)), t(Selected_ext)); - - if (intercept_status == 2) { - - P1_ssX_beta = P1 * ssX_beta; - P2_ssX_beta = colSums (P1_ssX_beta); - P1_beta = P1 * beta; - P2_beta = colSums (P1_beta); - - if (max_selected < num_features_orig) { - - P2_ssX_beta = cbind (P2_ssX_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); - P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); - - P2_ssX_beta[1, num_features_orig+1] = P2_ssX_beta[1, max_selected + 1]; - P2_ssX_beta[1, max_selected + 1] = 0; - - P2_beta[1, num_features_orig+1] = P2_beta[1, max_selected + 1]; - P2_beta[1, max_selected + 1] = 0; - - } - beta = cbind (t(P2_ssX_beta), t(P2_beta)); - - } else { - - P1_beta = P1 * beta; - P2_beta = colSums (P1_beta); - - if (max_selected < num_features_orig) { - P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); - P2_beta[1, num_features_orig+1] = P2_beta[1, max_selected + 1] ; - P2_beta[1, max_selected + 1] = 0; - } - beta = t(P2_beta); - } - } else { - P1 = table (seq (1, no_selected), t(Selected)); - P1_beta = P1 * beta; - P2_beta = colSums (P1_beta); - - if (max_selected < num_features_orig) { - P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); - } - - beta = t(P2_beta); - } - } - } - -glm_initialize = function (Matrix[double] X, Matrix[double] Y, int dist_type, double var_power, int link_type, double link_power, int icept_status, int max_iter_CG) - return (Matrix[double] beta, double saturated_log_l, int isNaN) - { - saturated_log_l = 0.0; - isNaN = 0; - y_corr = Y [, 1]; - if (dist_type == 2) { - n_corr = rowSums (Y); - is_n_zero = (n_corr == 0); - y_corr = Y [, 1] / (n_corr + is_n_zero) + (0.5 - Y [, 1]) * is_n_zero; - } - linear_terms = y_corr; - if (dist_type == 1 & link_type == 1) { # POWER DISTRIBUTION - if (link_power == 0) { - if (sum (y_corr < 0) == 0) { - is_zero_y_corr = (y_corr == 0); - linear_terms = log (y_corr + is_zero_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { isNaN = 1; } - } else { if (link_power == 1.0) { - linear_terms = y_corr; - } else { if (link_power == -1.0) { - linear_terms = 1.0 / y_corr; - } else { if (link_power == 0.5) { - if (sum (y_corr < 0) == 0) { - linear_terms = sqrt (y_corr); - } else { isNaN = 1; } - } else { if (link_power > 0) { - if (sum (y_corr < 0) == 0) { - is_zero_y_corr = (y_corr == 0); - linear_terms = (y_corr + is_zero_y_corr) ^ link_power - is_zero_y_corr; - } else { isNaN = 1; } - } else { - if (sum (y_corr <= 0) == 0) { - linear_terms = y_corr ^ link_power; - } else { isNaN = 1; } - }}}}} - } - if (dist_type == 2 & link_type >= 1 & link_type <= 5) - { # BINOMIAL/BERNOULLI DISTRIBUTION - if (link_type == 1 & link_power == 0) { # Binomial.log - if (sum (y_corr < 0) == 0) { - is_zero_y_corr = (y_corr == 0); - linear_terms = log (y_corr + is_zero_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { isNaN = 1; } - } else { if (link_type == 1 & link_power > 0) { # Binomial.power_nonlog pos - if (sum (y_corr < 0) == 0) { - is_zero_y_corr = (y_corr == 0); - linear_terms = (y_corr + is_zero_y_corr) ^ link_power - is_zero_y_corr; - } else { isNaN = 1; } - } else { if (link_type == 1) { # Binomial.power_nonlog neg - if (sum (y_corr <= 0) == 0) { - linear_terms = y_corr ^ link_power; - } else { isNaN = 1; } - } else { - is_zero_y_corr = (y_corr <= 0); - is_one_y_corr = (y_corr >= 1.0); - y_corr = y_corr * (1.0 - is_zero_y_corr) * (1.0 - is_one_y_corr) + 0.5 * (is_zero_y_corr + is_one_y_corr); - if (link_type == 2) { # Binomial.logit - linear_terms = log (y_corr / (1.0 - y_corr)) - + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { if (link_type == 3) { # Binomial.probit - y_below_half = y_corr + (1.0 - 2.0 * y_corr) * (y_corr > 0.5); - t = sqrt (- 2.0 * log (y_below_half)); - approx_inv_Gauss_CDF = - t + (2.515517 + t * (0.802853 + t * 0.010328)) / (1.0 + t * (1.432788 + t * (0.189269 + t * 0.001308))); - linear_terms = approx_inv_Gauss_CDF * (1.0 - 2.0 * (y_corr > 0.5)) - + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { if (link_type == 4) { # Binomial.cloglog - linear_terms = log (- log (1.0 - y_corr)) - - log (- log (0.5)) * (is_zero_y_corr + is_one_y_corr) - + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { if (link_type == 5) { # Binomial.cauchit - linear_terms = tan ((y_corr - 0.5) * pi) - + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - }} }}}}} - } - - if (isNaN == 0) { - [saturated_log_l, isNaN] = - glm_log_likelihood_part (linear_terms, Y, dist_type, var_power, link_type, link_power); - } - - if ((dist_type == 1 & link_type == 1 & link_power == 0) | - (dist_type == 2 & link_type >= 2)) - { - desired_eta = 0.0; - } else { if (link_type == 1 & link_power == 0) { - desired_eta = log (0.5); - } else { if (link_type == 1) { - desired_eta = 0.5 ^ link_power; - } else { - desired_eta = 0.5; - }}} - - beta = matrix (0.0, rows = ncol(X), cols = 1); - - if (desired_eta != 0) { - if (icept_status == 1 | icept_status == 2) { - beta [nrow(beta), 1] = desired_eta; - } else { - # We want: avg (X %*% ssX_transform %*% beta) = desired_eta - # Note that "ssX_transform" is trivial here, hence ignored - - beta = straightenX (X, 0.000001, max_iter_CG); - beta = beta * desired_eta; - } } } - - -glm_dist = function (Matrix[double] linear_terms, Matrix[double] Y, - int dist_type, double var_power, int link_type, double link_power) - return (Matrix[double] g_Y, Matrix[double] w) -# ORIGINALLY we returned more meaningful vectors, namely: -# Matrix[double] y_residual : y - y_mean, i.e. y observed - y predicted -# Matrix[double] link_gradient : derivative of the link function -# Matrix[double] var_function : variance without dispersion, i.e. the V(mu) function -# BUT, this caused roundoff errors, so we had to compute "directly useful" vectors -# and skip over the "meaningful intermediaries". Now we output these two variables: -# g_Y = y_residual / (var_function * link_gradient); -# w = 1.0 / (var_function * link_gradient ^ 2); - { - num_records = nrow (linear_terms); - zeros_r = matrix (0.0, rows = num_records, cols = 1); - ones_r = 1 + zeros_r; - g_Y = zeros_r; - w = zeros_r; - - # Some constants - - one_over_sqrt_two_pi = 0.39894228040143267793994605993438; - ones_2 = matrix (1.0, rows = 1, cols = 2); - p_one_m_one = ones_2; - p_one_m_one [1, 2] = -1.0; - m_one_p_one = ones_2; - m_one_p_one [1, 1] = -1.0; - zero_one = ones_2; - zero_one [1, 1] = 0.0; - one_zero = ones_2; - one_zero [1, 2] = 0.0; - flip_pos = matrix (0, rows = 2, cols = 2); - flip_neg = flip_pos; - flip_pos [1, 2] = 1; - flip_pos [2, 1] = 1; - flip_neg [1, 2] = -1; - flip_neg [2, 1] = 1; - - if (dist_type == 1 & link_type == 1) { # POWER DISTRIBUTION - y_mean = zeros_r; - if (link_power == 0) { - y_mean = exp (linear_terms); - y_mean_pow = y_mean ^ (1 - var_power); - w = y_mean_pow * y_mean; - g_Y = y_mean_pow * (Y - y_mean); - } else { if (link_power == 1.0) { - y_mean = linear_terms; - w = y_mean ^ (- var_power); - g_Y = w * (Y - y_mean); - } else { - y_mean = linear_terms ^ (1.0 / link_power); - c1 = (1 - var_power) / link_power - 1; - c2 = (2 - var_power) / link_power - 2; - g_Y = (linear_terms ^ c1) * (Y - y_mean) / link_power; - w = (linear_terms ^ c2) / (link_power ^ 2); - } }} - if (dist_type == 2 & link_type >= 1 & link_type <= 5) - { # BINOMIAL/BERNOULLI DISTRIBUTION - if (link_type == 1) { # BINOMIAL.POWER LINKS - if (link_power == 0) { # Binomial.log - vec1 = 1 / (exp (- linear_terms) - 1); - g_Y = Y [, 1] - Y [, 2] * vec1; - w = rowSums (Y) * vec1; - } else { # Binomial.nonlog - vec1 = zeros_r; - if (link_power == 0.5) { - vec1 = 1 / (1 - linear_terms ^ 2); - } else { if (sum (linear_terms < 0) == 0) { - vec1 = linear_terms ^ (- 2 + 1 / link_power) / (1 - linear_terms ^ (1 / link_power)); - } else {isNaN = 1;}} - # We want a "zero-protected" version of - # vec2 = Y [, 1] / linear_terms; - is_y_0 = (Y [, 1] == 0); - vec2 = (Y [, 1] + is_y_0) / (linear_terms * (1 - is_y_0) + is_y_0) - is_y_0; - g_Y = (vec2 - Y [, 2] * vec1 * linear_terms) / link_power; - w = rowSums (Y) * vec1 / link_power ^ 2; - } - } else { - is_LT_pos_infinite = (linear_terms == Inf); - is_LT_neg_infinite = (linear_terms == -Inf); - is_LT_infinite = is_LT_pos_infinite %*% one_zero + is_LT_neg_infinite %*% zero_one; - finite_linear_terms = replace (target = linear_terms, pattern = Inf, replacement = 0); - finite_linear_terms = replace (target = finite_linear_terms, pattern = -Inf, replacement = 0); - if (link_type == 2) { # Binomial.logit - Y_prob = exp (finite_linear_terms) %*% one_zero + ones_r %*% zero_one; - Y_prob = Y_prob / (rowSums (Y_prob) %*% ones_2); - Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; - g_Y = rowSums (Y * (Y_prob %*% flip_neg)); ### = y_residual; - w = rowSums (Y * (Y_prob %*% flip_pos) * Y_prob); ### = y_variance; - } else { if (link_type == 3) { # Binomial.probit - is_lt_pos = (linear_terms >= 0); - t_gp = 1.0 / (1.0 + abs (finite_linear_terms) * 0.231641888); # 0.231641888 = 0.3275911 / sqrt (2.0) - pt_gp = t_gp * ( 0.254829592 - + t_gp * (-0.284496736 # "Handbook of Mathematical Functions", ed. by M. Abramowitz and I.A. Stegun, - + t_gp * ( 1.421413741 # U.S. Nat-l Bureau of Standards, 10th print (Dec 1972), Sec. 7.1.26, p. 299 - + t_gp * (-1.453152027 - + t_gp * 1.061405429)))); - the_gauss_exp = exp (- (linear_terms ^ 2) / 2.0); - vec1 = 0.25 * pt_gp * (2 - the_gauss_exp * pt_gp); - vec2 = Y [, 1] - rowSums (Y) * is_lt_pos + the_gauss_exp * pt_gp * rowSums (Y) * (is_lt_pos - 0.5); - w = the_gauss_exp * (one_over_sqrt_two_pi ^ 2) * rowSums (Y) / vec1; - g_Y = one_over_sqrt_two_pi * vec2 / vec1; - } else { if (link_type == 4) { # Binomial.cloglog - the_exp = exp (linear_terms) - the_exp_exp = exp (- the_exp); - is_too_small = ((10000000 + the_exp) == 10000000); - the_exp_ratio = (1 - is_too_small) * (1 - the_exp_exp) / (the_exp + is_too_small) + is_too_small * (1 - the_exp / 2); - g_Y = (rowSums (Y) * the_exp_exp - Y [, 2]) / the_exp_ratio; - w = the_exp_exp * the_exp * rowSums (Y) / the_exp_ratio; - } else { if (link_type == 5) { # Binomial.cauchit - Y_prob = 0.5 + (atan (finite_linear_terms) %*% p_one_m_one) / pi; - Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; - y_residual = Y [, 1] * Y_prob [, 2] - Y [, 2] * Y_prob [, 1]; - var_function = rowSums (Y) * Y_prob [, 1] * Y_prob [, 2]; - link_gradient_normalized = (1 + linear_terms ^ 2) * pi; - g_Y = rowSums (Y) * y_residual / (var_function * link_gradient_normalized); - w = (rowSums (Y) ^ 2) / (var_function * link_gradient_normalized ^ 2); - }}}} - } - } - } - - -glm_log_likelihood_part = function (Matrix[double] linear_terms, Matrix[double] Y, - int dist_type, double var_power, int link_type, double link_power) - return (double log_l, int isNaN) - { - isNaN = 0; - log_l = 0.0; - num_records = nrow (Y); - zeros_r = matrix (0.0, rows = num_records, cols = 1); - - if (dist_type == 1 & link_type == 1) - { # POWER DISTRIBUTION - b_cumulant = zeros_r; - natural_parameters = zeros_r; - is_natural_parameter_log_zero = zeros_r; - if (var_power == 1.0 & link_power == 0) { # Poisson.log - b_cumulant = exp (linear_terms); - is_natural_parameter_log_zero = (linear_terms == -Inf); - natural_parameters = replace (target = linear_terms, pattern = -Inf, replacement = 0); - } else { if (var_power == 1.0 & link_power == 1.0) { # Poisson.id - if (sum (linear_terms < 0) == 0) { - b_cumulant = linear_terms; - is_natural_parameter_log_zero = (linear_terms == 0); - natural_parameters = log (linear_terms + is_natural_parameter_log_zero); - } else {isNaN = 1;} - } else { if (var_power == 1.0 & link_power == 0.5) { # Poisson.sqrt - if (sum (linear_terms < 0) == 0) { - b_cumulant = linear_terms ^ 2; - is_natural_parameter_log_zero = (linear_terms == 0); - natural_parameters = 2.0 * log (linear_terms + is_natural_parameter_log_zero); - } else {isNaN = 1;} - } else { if (var_power == 1.0 & link_power > 0) { # Poisson.power_nonlog, pos - if (sum (linear_terms < 0) == 0) { - is_natural_parameter_log_zero = (linear_terms == 0); - b_cumulant = (linear_terms + is_natural_parameter_log_zero) ^ (1.0 / link_power) - is_natural_parameter_log_zero; - natural_parameters = log (linear_terms + is_natural_parameter_log_zero) / link_power; - } else {isNaN = 1;} - } else { if (var_power == 1.0) { # Poisson.power_nonlog, neg - if (sum (linear_terms <= 0) == 0) { - b_cumulant = linear_terms ^ (1.0 / link_power); - natural_parameters = log (linear_terms) / link_power; - } else {isNaN = 1;} - } else { if (var_power == 2.0 & link_power == -1.0) { # Gamma.inverse - if (sum (linear_terms <= 0) == 0) { - b_cumulant = - log (linear_terms); - natural_parameters = - linear_terms; - } else {isNaN = 1;} - } else { if (var_power == 2.0 & link_power == 1.0) { # Gamma.id - if (sum (linear_terms <= 0) == 0) { - b_cumulant = log (linear_terms); - natural_parameters = - 1.0 / linear_terms; - } else {isNaN = 1;} - } else { if (var_power == 2.0 & link_power == 0) { # Gamma.log - b_cumulant = linear_terms; - natural_parameters = - exp (- linear_terms); - } else { if (var_power == 2.0) { # Gamma.power_nonlog - if (sum (linear_terms <= 0) == 0) { - b_cumulant = log (linear_terms) / link_power; - natural_parameters = - linear_terms ^ (- 1.0 / link_power); - } else {isNaN = 1;} - } else { if (link_power == 0) { # PowerDist.log - natural_parameters = exp (linear_terms * (1.0 - var_power)) / (1.0 - var_power); - b_cumulant = exp (linear_terms * (2.0 - var_power)) / (2.0 - var_power); - } else { # PowerDist.power_nonlog - if (-2 * link_power == 1.0 - var_power) { - natural_parameters = 1.0 / (linear_terms ^ 2) / (1.0 - var_power); - } else { if (-1 * link_power == 1.0 - var_power) { - natural_parameters = 1.0 / linear_terms / (1.0 - var_power); - } else { if ( link_power == 1.0 - var_power) { - natural_parameters = linear_terms / (1.0 - var_power); - } else { if ( 2 * link_power == 1.0 - var_power) { - natural_parameters = linear_terms ^ 2 / (1.0 - var_power); - } else { - if (sum (linear_terms <= 0) == 0) { - power = (1.0 - var_power) / link_power; - natural_parameters = (linear_terms ^ power) / (1.0 - var_power); - } else {isNaN = 1;} - }}}} - if (-2 * link_power == 2.0 - var_power) { - b_cumulant = 1.0 / (linear_terms ^ 2) / (2.0 - var_power); - } else { if (-1 * link_power == 2.0 - var_power) { - b_cumulant = 1.0 / linear_terms / (2.0 - var_power); - } else { if ( link_power == 2.0 - var_power) { - b_cumulant = linear_terms / (2.0 - var_power); - } else { if ( 2 * link_power == 2.0 - var_power) { - b_cumulant = linear_terms ^ 2 / (2.0 - var_power); - } else { - if (sum (linear_terms <= 0) == 0) { - power = (2.0 - var_power) / link_power; - b_cumulant = (linear_terms ^ power) / (2.0 - var_power); - } else {isNaN = 1;} - }}}} - }}}}} }}}}} - if (sum (is_natural_parameter_log_zero * abs (Y)) > 0) { - log_l = -Inf; - isNaN = 1; - } - if (isNaN == 0) - { - log_l = sum (Y * natural_parameters - b_cumulant); - if (log_l != log_l | (log_l == log_l + 1.0 & log_l == log_l * 2.0)) { - log_l = -Inf; - isNaN = 1; - } } } - - if (dist_type == 2 & link_type >= 1 & link_type <= 5) - { # BINOMIAL/BERNOULLI DISTRIBUTION - - [Y_prob, isNaN] = binomial_probability_two_column (linear_terms, link_type, link_power); - - if (isNaN == 0) { - does_prob_contradict = (Y_prob <= 0); - if (sum (does_prob_contradict * abs (Y)) == 0) { - log_l = sum (Y * log (Y_prob * (1 - does_prob_contradict) + does_prob_contradict)); - if (log_l != log_l | (log_l == log_l + 1.0 & log_l == log_l * 2.0)) { - isNaN = 1; - } - } else { - log_l = -Inf; - isNaN = 1; - } } } - - if (isNaN == 1) { - log_l = - Inf; - } - } - - - -binomial_probability_two_column = - function (Matrix[double] linear_terms, int link_type, double link_power) - return (Matrix[double] Y_prob, int isNaN) - { - isNaN = 0; - num_records = nrow (linear_terms); - - # Define some auxiliary matrices - - ones_2 = matrix (1.0, rows = 1, cols = 2); - p_one_m_one = ones_2; - p_one_m_one [1, 2] = -1.0; - m_one_p_one = ones_2; - m_one_p_one [1, 1] = -1.0; - zero_one = ones_2; - zero_one [1, 1] = 0.0; - one_zero = ones_2; - one_zero [1, 2] = 0.0; - - zeros_r = matrix (0.0, rows = num_records, cols = 1); - ones_r = 1.0 + zeros_r; - - # Begin the function body - - Y_prob = zeros_r %*% ones_2; - if (link_type == 1) { # Binomial.power - if (link_power == 0) { # Binomial.log - Y_prob = exp (linear_terms) %*% p_one_m_one + ones_r %*% zero_one; - } else { if (link_power == 0.5) { # Binomial.sqrt - Y_prob = (linear_terms ^ 2) %*% p_one_m_one + ones_r %*% zero_one; - } else { # Binomial.power_nonlog - if (sum (linear_terms < 0) == 0) { - Y_prob = (linear_terms ^ (1.0 / link_power)) %*% p_one_m_one + ones_r %*% zero_one; - } else {isNaN = 1;} - }} - } else { # Binomial.non_power - is_LT_pos_infinite = (linear_terms == Inf); - is_LT_neg_infinite = (linear_terms == -Inf); - is_LT_infinite = is_LT_pos_infinite %*% one_zero + is_LT_neg_infinite %*% zero_one; - finite_linear_terms = replace (target = linear_terms, pattern = Inf, replacement = 0); - finite_linear_terms = replace (target = finite_linear_terms, pattern = -Inf, replacement = 0); - if (link_type == 2) { # Binomial.logit - Y_prob = exp (finite_linear_terms) %*% one_zero + ones_r %*% zero_one; - Y_prob = Y_prob / (rowSums (Y_prob) %*% ones_2); - } else { if (link_type == 3) { # Binomial.probit - lt_pos_neg = (finite_linear_terms >= 0) %*% p_one_m_one + ones_r %*% zero_one; - t_gp = 1.0 / (1.0 + abs (finite_linear_terms) * 0.231641888); # 0.231641888 = 0.3275911 / sqrt (2.0) - pt_gp = t_gp * ( 0.254829592 - + t_gp * (-0.284496736 # "Handbook of Mathematical Functions", ed. by M. Abramowitz and I.A. Stegun, - + t_gp * ( 1.421413741 # U.S. Nat-l Bureau of Standards, 10th print (Dec 1972), Sec. 7.1.26, p. 299 - + t_gp * (-1.453152027 - + t_gp * 1.061405429)))); - the_gauss_exp = exp (- (finite_linear_terms ^ 2) / 2.0); - Y_prob = lt_pos_neg + ((the_gauss_exp * pt_gp) %*% ones_2) * (0.5 - lt_pos_neg); - } else { if (link_type == 4) { # Binomial.cloglog - the_exp = exp (finite_linear_terms); - the_exp_exp = exp (- the_exp); - is_too_small = ((10000000 + the_exp) == 10000000); - Y_prob [, 1] = (1 - is_too_small) * (1 - the_exp_exp) + is_too_small * the_exp * (1 - the_exp / 2); - Y_prob [, 2] = the_exp_exp; - } else { if (link_type == 5) { # Binomial.cauchit - Y_prob = 0.5 + (atan (finite_linear_terms) %*% p_one_m_one) / pi; - } else { - isNaN = 1; - }}}} - Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; - } } - - -# THE CG-STEIHAUG PROCEDURE SCRIPT - -# Apply Conjugate Gradient - Steihaug algorithm in order to approximately minimize -# 0.5 z^T (X^T diag(w) X + diag (lambda)) z + (g + lambda * beta)^T z -# under constraint: ||z|| <= trust_delta. -# See Alg. 7.2 on p. 171 of "Numerical Optimization" 2nd ed. by Nocedal and Wright -# IN THE ABOVE, "X" IS UNDERSTOOD TO BE "X %*% (SHIFT/SCALE TRANSFORM)"; this transform -# is given separately because sparse "X" may become dense after applying the transform. -# -get_CG_Steihaug_point = - function (Matrix[double] X, Matrix[double] scale_X, Matrix[double] shift_X, Matrix[double] w, - Matrix[double] g, Matrix[double] beta, Matrix[double] lambda, double trust_delta, int max_iter_CG) - return (Matrix[double] z, double neg_log_l_change, int i_CG, int reached_trust_boundary) - { - trust_delta_sq = trust_delta ^ 2; - size_CG = nrow (g); - z = matrix (0.0, rows = size_CG, cols = 1); - neg_log_l_change = 0.0; - reached_trust_boundary = 0; - g_reg = g + lambda * beta; - r_CG = g_reg; - p_CG = -r_CG; - rr_CG = sum(r_CG * r_CG); - eps_CG = rr_CG * min (0.25, sqrt (rr_CG)); - converged_CG = 0; - if (rr_CG < eps_CG) { - converged_CG = 1; - } - - max_iteration_CG = max_iter_CG; - if (max_iteration_CG <= 0) { - max_iteration_CG = size_CG; - } - i_CG = 0; - while (converged_CG == 0) - { - i_CG = i_CG + 1; - ssX_p_CG = diag (scale_X) %*% p_CG; - ssX_p_CG [size_CG, ] = ssX_p_CG [size_CG, ] + t(shift_X) %*% p_CG; - temp_CG = t(X) %*% (w * (X %*% ssX_p_CG)); - q_CG = (lambda * p_CG) + diag (scale_X) %*% temp_CG + shift_X %*% temp_CG [size_CG, ]; - pq_CG = sum (p_CG * q_CG); - if (pq_CG <= 0) { - pp_CG = sum (p_CG * p_CG); - if (pp_CG > 0) { - [z, neg_log_l_change] = - get_trust_boundary_point (g_reg, z, p_CG, q_CG, r_CG, pp_CG, pq_CG, trust_delta_sq); - reached_trust_boundary = 1; - } else { - neg_log_l_change = 0.5 * sum (z * (r_CG + g_reg)); - } - converged_CG = 1; - } - if (converged_CG == 0) { - alpha_CG = rr_CG / pq_CG; - new_z = z + alpha_CG * p_CG; - if (sum(new_z * new_z) >= trust_delta_sq) { - pp_CG = sum (p_CG * p_CG); - [z, neg_log_l_change] = - get_trust_boundary_point (g_reg, z, p_CG, q_CG, r_CG, pp_CG, pq_CG, trust_delta_sq); - reached_trust_boundary = 1; - converged_CG = 1; - } - if (converged_CG == 0) { - z = new_z; - old_rr_CG = rr_CG; - r_CG = r_CG + alpha_CG * q_CG; - rr_CG = sum(r_CG * r_CG); - if (i_CG == max_iteration_CG | rr_CG < eps_CG) { - neg_log_l_change = 0.5 * sum (z * (r_CG + g_reg)); - reached_trust_boundary = 0; - converged_CG = 1; - } - if (converged_CG == 0) { - p_CG = -r_CG + (rr_CG / old_rr_CG) * p_CG; - } } } } } - - -# An auxiliary function used twice inside the CG-STEIHAUG loop: -get_trust_boundary_point = - function (Matrix[double] g, Matrix[double] z, Matrix[double] p, - Matrix[double] q, Matrix[double] r, double pp, double pq, - double trust_delta_sq) - return (Matrix[double] new_z, double f_change) - { - zz = sum (z * z); pz = sum (p * z); - sq_root_d = sqrt (pz * pz - pp * (zz - trust_delta_sq)); - tau_1 = (- pz + sq_root_d) / pp; - tau_2 = (- pz - sq_root_d) / pp; - zq = sum (z * q); gp = sum (g * p); - f_extra = 0.5 * sum (z * (r + g)); - f_change_1 = f_extra + (0.5 * tau_1 * pq + zq + gp) * tau_1; - f_change_2 = f_extra + (0.5 * tau_2 * pq + zq + gp) * tau_2; - ind1 = as.integer(f_change_1 < f_change_2); - ind2 = as.integer(f_change_1 >= f_change_2); - new_z = z + ((ind1 * tau_1 + ind2 * tau_2) * p); - f_change = ind1 * f_change_1 + ind2 * f_change_2; - } - - -# Computes vector w such that ||X %*% w - 1|| -> MIN given avg(X %*% w) = 1 -# We find z_LS such that ||X %*% z_LS - 1|| -> MIN unconditionally, then scale -# it to compute w = c * z_LS such that sum(X %*% w) = nrow(X). -straightenX = - function (Matrix[double] X, double eps, int max_iter_CG) - return (Matrix[double] w) - { - w_X = t(colSums(X)); - lambda_LS = 0.000001 * sum(X ^ 2) / ncol(X); - eps_LS = eps * nrow(X); - - # BEGIN LEAST SQUARES - - r_LS = - w_X; - z_LS = matrix (0.0, rows = ncol(X), cols = 1); - p_LS = - r_LS; - norm_r2_LS = sum (r_LS ^ 2); - i_LS = 0; - while (i_LS < max_iter_CG & i_LS < ncol(X) & norm_r2_LS >= eps_LS) - { - q_LS = t(X) %*% X %*% p_LS; - q_LS = q_LS + lambda_LS * p_LS; - alpha_LS = norm_r2_LS / sum (p_LS * q_LS); - z_LS = z_LS + alpha_LS * p_LS; - old_norm_r2_LS = norm_r2_LS; - r_LS = r_LS + alpha_LS * q_LS; - norm_r2_LS = sum (r_LS ^ 2); - p_LS = -r_LS + (norm_r2_LS / old_norm_r2_LS) * p_LS; - i_LS = i_LS + 1; - } - - # END LEAST SQUARES - - w = (nrow(X) / sum (w_X * z_LS)) * z_LS; - } From 482349c03a6ba48197748dd43b74cc8cf40cc49f Mon Sep 17 00:00:00 2001 From: bruno Date: Fri, 26 Jun 2026 07:12:46 +0200 Subject: [PATCH 007/132] replace glm_fit with m_glm --- scripts/builtin/stepGLM.dml | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/scripts/builtin/stepGLM.dml b/scripts/builtin/stepGLM.dml index 3ec1cffcb7f..f7bd72e6b51 100644 --- a/scripts/builtin/stepGLM.dml +++ b/scripts/builtin/stepGLM.dml @@ -47,7 +47,6 @@ # # B Matrix --- Estimated regression parameters (betas) # S Matrix --- The selected features ordered as computed by the algorithm -# O String --- The statistics # --------------------------------------------------------------------------------------------- # THE StepGLM SCRIPT CURRENTLY SUPPORTS BERNOULLI DISTRIBUTION FAMILY AND THE FOLLOWING LINK FUNCTIONS ONLY! @@ -76,6 +75,7 @@ # DEVIANCE_SCALED Deviance from the saturated model, scaled by the DISPERSION value # ------------------------------------------------------------------------------------------- +source("./scripts/builtin/glm.dlm") as glm; stepGLM = function ( Matrix[Double] X, @@ -92,16 +92,12 @@ stepGLM = function ( Double AIC, Matrix[Double] B, Matrix[Double] S, - String O ) { intercept_status = icpt; bernoulli_No_label = yneg; distribution_type = 2; - # currently only the forward selection strategy in supported: start from one feature and iteratively add - # features until AIC improves - dir = "forward"; if (distribution_type == 2 & ncol(Y) == 1) { is_Y_negative = (Y == bernoulli_No_label); @@ -121,10 +117,6 @@ stepGLM = function ( # BEGIN STEPWISE GENERALIZED LINEAR MODELS - if (dir != "forward") { - stop ("Currently only forward selection strategy is supported!"); - } - continue = TRUE; columns_fixed = matrix (0, rows = 1, cols = num_features); columns_fixed_ordered = matrix (0, rows = 1, cols = 1); @@ -134,18 +126,18 @@ stepGLM = function ( if (intercept_status == 0) { # Compute AIC of an empty model with no features and no intercept (all Ys are zero) - [AIC_best, ignore_B1, ignore_S1, ignore_O1] = glm_fit (X=X_global, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + [AIC_best, ignore_B1, ignore_S1, ignore_O1] = glm::m_glm(X=X_global, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } else { # compute AIC of an empty model with only intercept (all Ys are constant) all_ones = matrix (1, rows = num_records, cols = 1); - [AIC_best, ignore_beta2, ignore_S2, ignore_O2] = glm_fit (X=all_ones, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + [AIC_best, ignore_beta2, ignore_S2, ignore_O2] = glm::m_glm(X=all_ones, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } #print ("Best AIC without any features: " + AIC_best); # First pass to examine single features AICs = matrix (AIC_best, rows = 1, cols = num_features); parfor (i in 1:num_features) { - [AIC_1, ignore_beta3, ignore_S3, ignore_O3] = glm_fit (X=X_orig[,i], Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + [AIC_1, ignore_beta3, ignore_S3, ignore_O3] = glm::m_glm(X=X_orig[,i], Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); AICs[1,i] = AIC_1; } @@ -163,11 +155,11 @@ stepGLM = function ( #print ("AIC of an empty model is " + AIC_best + " and adding no feature achieves more than " + (thr * 100) + "% decrease in AIC!"); if (intercept_status == 0) { # Compute AIC of an empty model with no features and no intercept (all Ys are zero) - [AIC_best, ignore_beta4, ignore_S4, ignore_O4] = glm_fit (X=X_global, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + [AIC_best, ignore_beta4, ignore_S4, ignore_O4] = glm::m_glm(X=X_global, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } else { # compute AIC of an empty model with only intercept (all Ys are constant) ###all_ones = matrix (1, rows = num_records, cols = 1); - [AIC_best, ignore_beta5, ignore_S5, ignore_O5] = glm_fit (X=all_ones, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + [AIC_best, ignore_beta5, ignore_S5, ignore_O5] = glm::m_glm(X=all_ones, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } }; @@ -184,7 +176,7 @@ stepGLM = function ( # Construct the feature matrix X_loop = cbind (X_global, X_orig[,i]); - [AIC_2, ignore_beta6, ignore_S6, ignore_O6] = glm_fit (X=X_loop, Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + [AIC_2, ignore_beta6, ignore_S6, ignore_O6] = glm::m_glm(X=X_loop, Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); AICs[1,i] = AIC_2; } } @@ -216,7 +208,7 @@ stepGLM = function ( # run GLM with selected set of features print ("Running GLM with selected features..."); - [AIC, B, S, O] = glm_fit (X=X_global, Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + [AIC, B, S, O] = glm::m_glm(X=X_global, Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } From 8e24e9c4a1aabebbd5c0f6994aef7c57f6b49579 Mon Sep 17 00:00:00 2001 From: bruno Date: Sun, 28 Jun 2026 03:01:15 +0200 Subject: [PATCH 008/132] add missing glm_fit replacment --- scripts/algorithms/TestBuiltinStepGLM.dml | 19 +++- scripts/builtin/stepGLM.dml | 118 ++++++++++++++++------ 2 files changed, 106 insertions(+), 31 deletions(-) diff --git a/scripts/algorithms/TestBuiltinStepGLM.dml b/scripts/algorithms/TestBuiltinStepGLM.dml index ab64960b35a..ffe1a1f64db 100644 --- a/scripts/algorithms/TestBuiltinStepGLM.dml +++ b/scripts/algorithms/TestBuiltinStepGLM.dml @@ -13,9 +13,26 @@ Z = X %*% beta_true; P_y = 1.0 / (1.0 + exp(-Z)); Y = (rand(rows=N, cols=1, min=0.0, max=1.0, seed=456) < P_y) * 1.0; -[AIC, B, S, O] = stepGLM::stepGLM(X=X, Y=Y, link=2, yneg=0.0, icpt=0, tol=1e-6, disp=0.0, moi=200, mii=0, thr=0.01); +[AIC, B, S] = stepGLM::stepGLM(X=X, Y=Y, link=2, yneg=0.0, icpt=0, tol=1e-6, disp=0.0, moi=200, mii=0, thr=0.01); print("Optimal AIC: " + AIC); print("Selected Feature Indices:\n" + toString(S)); print("Estimated Coefficients:\n" + toString(B)); +epsilon = 0.5; +beta_est = matrix(0, rows=P, cols=1); +for (i in 1:nrow(B)) { + idx = as.scalar(S[1, i]); + if (nrow(S) > 1) { idx = as.scalar(S[i, 1]); } + beta_est[idx, 1] = B[i, 1]; +} + +if (nrow(B) != 3 | sum(beta_est != 0 & beta_true == 0) > 0 | sum(beta_est == 0 & beta_true != 0) > 0) { + stop("Test failed: Inexact feature support recovery."); +} + +if (max(abs(beta_est - beta_true)) > epsilon) { + stop("Test failed: Parameter estimates exceed tolerance bound epsilon = " + epsilon + "."); +} + +print("Automated convergence and estimation validation passed."); diff --git a/scripts/builtin/stepGLM.dml b/scripts/builtin/stepGLM.dml index f7bd72e6b51..d4f894a52ac 100644 --- a/scripts/builtin/stepGLM.dml +++ b/scripts/builtin/stepGLM.dml @@ -45,37 +45,18 @@ # OUTPUT: Matrix beta, whose size depends on icpt: # icpt=0: ncol(X) x 1; icpt=1: (ncol(X) + 1) x 1; icpt=2: (ncol(X) + 1) x 2 # +# AIC Double --- AIC value # B Matrix --- Estimated regression parameters (betas) # S Matrix --- The selected features ordered as computed by the algorithm # --------------------------------------------------------------------------------------------- # THE StepGLM SCRIPT CURRENTLY SUPPORTS BERNOULLI DISTRIBUTION FAMILY AND THE FOLLOWING LINK FUNCTIONS ONLY! -# - LOG +# - LOG # - LOGIT # - PROBIT # - CLOGLOG -# In addition, in the last run of GLM some statistics are provided in CSV format, one comma-separated name-value -# pair per each line, as follows: -# -# NAME MEANING -# ------------------------------------------------------------------------------------------- -# TERMINATION_CODE A positive integer indicating success/failure as follows: -# 1 = Converged successfully; 2 = Maximum number of iterations reached; -# 3 = Input (X, Y) out of range; 4 = Distribution/link is not supported -# BETA_MIN Smallest beta value (regression coefficient), excluding the intercept -# BETA_MIN_INDEX Column index for the smallest beta value -# BETA_MAX Largest beta value (regression coefficient), excluding the intercept -# BETA_MAX_INDEX Column index for the largest beta value -# INTERCEPT Intercept value, or NaN if there is no intercept (if icpt=0) -# DISPERSION Dispersion used to scale deviance, provided as "disp" input parameter -# or estimated (same as DISPERSION_EST) if the "disp" parameter is <= 0 -# DISPERSION_EST Dispersion estimated from the dataset -# DEVIANCE_UNSCALED Deviance from the saturated model, assuming dispersion == 1.0 -# DEVIANCE_SCALED Deviance from the saturated model, scaled by the DISPERSION value -# ------------------------------------------------------------------------------------------- - -source("./scripts/builtin/glm.dlm") as glm; +source("./scripts/builtin/glm.dml") as glm; stepGLM = function ( Matrix[Double] X, @@ -91,7 +72,7 @@ stepGLM = function ( ) return ( Double AIC, Matrix[Double] B, - Matrix[Double] S, + Matrix[Double] S ) { intercept_status = icpt; @@ -126,18 +107,18 @@ stepGLM = function ( if (intercept_status == 0) { # Compute AIC of an empty model with no features and no intercept (all Ys are zero) - [AIC_best, ignore_B1, ignore_S1, ignore_O1] = glm::m_glm(X=X_global, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + [AIC_best, ignore_B1, ignore_S1] = internal_glm(X=X_global, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } else { # compute AIC of an empty model with only intercept (all Ys are constant) all_ones = matrix (1, rows = num_records, cols = 1); - [AIC_best, ignore_beta2, ignore_S2, ignore_O2] = glm::m_glm(X=all_ones, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + [AIC_best, ignore_beta2, ignore_S2] = internal_glm(X=all_ones, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } #print ("Best AIC without any features: " + AIC_best); # First pass to examine single features AICs = matrix (AIC_best, rows = 1, cols = num_features); parfor (i in 1:num_features) { - [AIC_1, ignore_beta3, ignore_S3, ignore_O3] = glm::m_glm(X=X_orig[,i], Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + [AIC_1, ignore_beta3, ignore_S3] = internal_glm(X=X_orig[,i], Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); AICs[1,i] = AIC_1; } @@ -155,11 +136,11 @@ stepGLM = function ( #print ("AIC of an empty model is " + AIC_best + " and adding no feature achieves more than " + (thr * 100) + "% decrease in AIC!"); if (intercept_status == 0) { # Compute AIC of an empty model with no features and no intercept (all Ys are zero) - [AIC_best, ignore_beta4, ignore_S4, ignore_O4] = glm::m_glm(X=X_global, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + [AIC_best, ignore_beta4, ignore_S4] = internal_glm(X=X_global, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } else { # compute AIC of an empty model with only intercept (all Ys are constant) ###all_ones = matrix (1, rows = num_records, cols = 1); - [AIC_best, ignore_beta5, ignore_S5, ignore_O5] = glm::m_glm(X=all_ones, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + [AIC_best, ignore_beta5, ignore_S5] = internal_glm(X=all_ones, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } }; @@ -176,7 +157,7 @@ stepGLM = function ( # Construct the feature matrix X_loop = cbind (X_global, X_orig[,i]); - [AIC_2, ignore_beta6, ignore_S6, ignore_O6] = glm::m_glm(X=X_loop, Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + [AIC_2, ignore_beta6, ignore_S6] = internal_glm(X=X_loop, Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); AICs[1,i] = AIC_2; } } @@ -208,7 +189,84 @@ stepGLM = function ( # run GLM with selected set of features print ("Running GLM with selected features..."); - [AIC, B, S, O] = glm::m_glm(X=X_global, Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + [AIC, B, S] = internal_glm(X=X_global, Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + } + +compute_AIC = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] B, Int link, Int icpt) + return (Double aic) +{ + # Isolate unscaled parameters; accounts for m_glm returning 2 columns when icpt=2 + beta = B[, 1]; + + if (icpt > 0) { + ones = matrix(1, rows=nrow(X), cols=1); + X_design = cbind(X, ones); + } else { + X_design = X; + } + + eta = X_design %*% beta; + + # Map inverse link functions + if (link == 1) { + # log, see https://en.wikipedia.org/wiki/Generalized_linear_model#Link_function + mu = exp(eta); + } else if (link == 2) { + # logit, see https://en.wikipedia.org/wiki/Generalized_linear_model#Link_function + mu = 1 / (1 + exp(-eta)); + } else if (link == 3) { + # probit approximation, page 1487 in "Qualitative Response Models: A Survey (1981)" by Takeshi Amemiya + mu = 1 / (1 + exp(-1.6 * eta)); + } else if (link == 4) { + # cloglog, see https://search.r-project.org/CRAN/refmans/VGAM/html/clogloglink.html + mu = 1 - exp(-exp(eta)); + } else { + stop("Unsupported link function code: " + link); + mu = eta; } + # Constrain to prevent numerical discontinuity in log(0) + mu = max(mu, 1e-15); + mu = min(mu, 1 - 1e-15); + + Y_yes = Y[, 1]; + Y_neg = Y[, 2]; + + LL = sum(Y_yes * log(mu) + Y_neg * log(1 - mu)); + k = nrow(beta); + + aic = 2 * k - 2 * LL; +} + +internal_glm = function ( + Matrix[Double] X, + Matrix[Double] Y, + Int intercept_status, + Double num_features_orig, + Matrix[Double] Selected, + Int link, + Double disp, + Double tol, + Int moi, + Int mii +) return ( + Double AIC, + Matrix[Double] B, + Matrix[Double] S +) { + # bernoulli family parameters, see table in ./scripts/builtin/m_glm.dml for details. + new_link = link; + new_lpow = 1.0; + + if (link == 1) { + new_link = 1; + new_lpow = 0.0; + } + B = glm::m_glm(X=X, Y=Y, dfam=2, vpow=0.0, link=new_link, lpow=new_lpow, yneg=0.0, + icpt=intercept_status, disp=disp, reg=0.0, tol=tol, + moi=moi, mii=mii, verbose=FALSE); + AIC = compute_AIC(X, Y, B, link, intercept_status); + + S = Selected; +} From ca5429b3f3c99c1011118c728a031a7f79563470 Mon Sep 17 00:00:00 2001 From: bruno Date: Sun, 28 Jun 2026 05:52:08 +0200 Subject: [PATCH 009/132] edit test script --- scripts/algorithms/TestBuiltinStepGLM.dml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/algorithms/TestBuiltinStepGLM.dml b/scripts/algorithms/TestBuiltinStepGLM.dml index ffe1a1f64db..f22bb07d107 100644 --- a/scripts/algorithms/TestBuiltinStepGLM.dml +++ b/scripts/algorithms/TestBuiltinStepGLM.dml @@ -15,24 +15,27 @@ Y = (rand(rows=N, cols=1, min=0.0, max=1.0, seed=456) < P_y) * 1.0; [AIC, B, S] = stepGLM::stepGLM(X=X, Y=Y, link=2, yneg=0.0, icpt=0, tol=1e-6, disp=0.0, moi=200, mii=0, thr=0.01); +print("\n\n\n\n\n\n\nTest Results:"); print("Optimal AIC: " + AIC); print("Selected Feature Indices:\n" + toString(S)); print("Estimated Coefficients:\n" + toString(B)); -epsilon = 0.5; beta_est = matrix(0, rows=P, cols=1); for (i in 1:nrow(B)) { idx = as.scalar(S[1, i]); - if (nrow(S) > 1) { idx = as.scalar(S[i, 1]); } beta_est[idx, 1] = B[i, 1]; } +# Case 01 if (nrow(B) != 3 | sum(beta_est != 0 & beta_true == 0) > 0 | sum(beta_est == 0 & beta_true != 0) > 0) { stop("Test failed: Inexact feature support recovery."); } +print("passed test 1") +# Case 02 +epsilon = 0.5; if (max(abs(beta_est - beta_true)) > epsilon) { stop("Test failed: Parameter estimates exceed tolerance bound epsilon = " + epsilon + "."); } +print("passed test 2") -print("Automated convergence and estimation validation passed."); From 3321316255de955baf085d353a69ade2cee71026 Mon Sep 17 00:00:00 2001 From: bruno Date: Mon, 29 Jun 2026 15:45:21 +0200 Subject: [PATCH 010/132] add license to TestBuiltinStepGLM.dml --- scripts/algorithms/TestBuiltinStepGLM.dml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/scripts/algorithms/TestBuiltinStepGLM.dml b/scripts/algorithms/TestBuiltinStepGLM.dml index f22bb07d107..9924728b992 100644 --- a/scripts/algorithms/TestBuiltinStepGLM.dml +++ b/scripts/algorithms/TestBuiltinStepGLM.dml @@ -1,3 +1,25 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + source("scripts/builtin/stepGLM.dml") as stepGLM; N = 1000; From e13e83af0272959df105283b9f6cb3bd7cba4d6f Mon Sep 17 00:00:00 2001 From: bruno Date: Wed, 1 Jul 2026 15:31:40 +0200 Subject: [PATCH 011/132] rename stepGLM to m_stepGLM; register stepGLM in Builtins.java --- scripts/algorithms/TestBuiltinStepGLM.dml | 2 +- scripts/builtin/stepGLM.dml | 2 +- src/main/java/org/apache/sysds/common/Builtins.java | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/algorithms/TestBuiltinStepGLM.dml b/scripts/algorithms/TestBuiltinStepGLM.dml index 9924728b992..52d393bd29f 100644 --- a/scripts/algorithms/TestBuiltinStepGLM.dml +++ b/scripts/algorithms/TestBuiltinStepGLM.dml @@ -35,7 +35,7 @@ Z = X %*% beta_true; P_y = 1.0 / (1.0 + exp(-Z)); Y = (rand(rows=N, cols=1, min=0.0, max=1.0, seed=456) < P_y) * 1.0; -[AIC, B, S] = stepGLM::stepGLM(X=X, Y=Y, link=2, yneg=0.0, icpt=0, tol=1e-6, disp=0.0, moi=200, mii=0, thr=0.01); +[AIC, B, S] = stepGLM::m_stepGLM(X=X, Y=Y, link=2, yneg=0.0, icpt=0, tol=1e-6, disp=0.0, moi=200, mii=0, thr=0.01); print("\n\n\n\n\n\n\nTest Results:"); print("Optimal AIC: " + AIC); diff --git a/scripts/builtin/stepGLM.dml b/scripts/builtin/stepGLM.dml index d4f894a52ac..00230ca455c 100644 --- a/scripts/builtin/stepGLM.dml +++ b/scripts/builtin/stepGLM.dml @@ -58,7 +58,7 @@ source("./scripts/builtin/glm.dml") as glm; -stepGLM = function ( +m_stepGLM = function ( Matrix[Double] X, Matrix[Double] Y, Int link = 2, diff --git a/src/main/java/org/apache/sysds/common/Builtins.java b/src/main/java/org/apache/sysds/common/Builtins.java index e21c539d6d8..62145124d82 100644 --- a/src/main/java/org/apache/sysds/common/Builtins.java +++ b/src/main/java/org/apache/sysds/common/Builtins.java @@ -333,6 +333,7 @@ public enum Builtins { STATSNA("statsNA", true), STRATSTATS("stratstats", true), STEPLM("steplm",true, ReturnType.MULTI_RETURN), + STEPGLM("stepGLM",true, ReturnType.MULTI_RETURN), STFT("stft", false, ReturnType.MULTI_RETURN), SQRT("sqrt", false), SQRT_MATRIX("sqrtMatrix", true), From c3410638cf2748de20b429a34a58674f39fddd4e Mon Sep 17 00:00:00 2001 From: bruno Date: Wed, 1 Jul 2026 16:40:11 +0200 Subject: [PATCH 012/132] add BuiltinSTEPGlmTest.java --- .../builtin/part2/BuiltinSTEPGlmTest.java | 68 +++++++++++++++++++ .../scripts/functions/builtin/stepGLM.dml | 4 ++ 2 files changed, 72 insertions(+) create mode 100644 src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java rename scripts/algorithms/TestBuiltinStepGLM.dml => src/test/scripts/functions/builtin/stepGLM.dml (97%) diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java new file mode 100644 index 00000000000..7d95dc6ec28 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.builtin.part2; + +import org.junit.Test; +import org.apache.sysds.common.Types.ExecMode; +import org.apache.sysds.common.Types.ExecType; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; + +public class BuiltinSTEPGlmTest extends AutomatedTestBase +{ + private final static String TEST_NAME = "stepGLM"; + private final static String TEST_DIR = "functions/builtin/"; + private static final String TEST_CLASS_DIR = TEST_DIR + BuiltinSTEPGlmTest.class.getSimpleName() + "/"; + + @Override + public void setUp() { + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[]{})); + } + + @Test + public void testLmMatrixDenseCPlm() { + runSTEPGlmTest(ExecType.CP); + } + + @Test + public void testLmMatrixSparseSPlm() { + runSTEPGlmTest(ExecType.SPARK); + } + + private void runSTEPGlmTest(ExecType instType) { + ExecMode platformOld = setExecMode(instType); + + try { + loadTestConfiguration(getTestConfiguration(TEST_NAME)); + + String HOME = SCRIPT_DIR + TEST_DIR; + + // Pointing to the generated validation DML script + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + programArgs = new String[]{}; + + // runTest executes the script; fails if the DML script invokes stop() + runTest(true, false, null, -1); + } + finally { + rtplatform = platformOld; + } + } +} diff --git a/scripts/algorithms/TestBuiltinStepGLM.dml b/src/test/scripts/functions/builtin/stepGLM.dml similarity index 97% rename from scripts/algorithms/TestBuiltinStepGLM.dml rename to src/test/scripts/functions/builtin/stepGLM.dml index 52d393bd29f..95a4dfb9908 100644 --- a/scripts/algorithms/TestBuiltinStepGLM.dml +++ b/src/test/scripts/functions/builtin/stepGLM.dml @@ -61,3 +61,7 @@ if (max(abs(beta_est - beta_true)) > epsilon) { } print("passed test 2") + + +#stop("!!!Sucess!!!") # uncomment for letting the test fail + From 8887e5ae695990b6967aa464f6de3543b58462fb Mon Sep 17 00:00:00 2001 From: bruno Date: Wed, 1 Jul 2026 22:12:06 +0200 Subject: [PATCH 013/132] indent BuiltinSTEPGlmTest.java --- .../builtin/part2/BuiltinSTEPGlmTest.java | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java index 7d95dc6ec28..e84047d4b59 100644 --- a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java +++ b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java @@ -27,32 +27,32 @@ public class BuiltinSTEPGlmTest extends AutomatedTestBase { - private final static String TEST_NAME = "stepGLM"; - private final static String TEST_DIR = "functions/builtin/"; - private static final String TEST_CLASS_DIR = TEST_DIR + BuiltinSTEPGlmTest.class.getSimpleName() + "/"; + private final static String TEST_NAME = "stepGLM"; + private final static String TEST_DIR = "functions/builtin/"; + private static final String TEST_CLASS_DIR = TEST_DIR + BuiltinSTEPGlmTest.class.getSimpleName() + "/"; - @Override - public void setUp() { - addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[]{})); - } + @Override + public void setUp() { + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[]{})); + } - @Test - public void testLmMatrixDenseCPlm() { - runSTEPGlmTest(ExecType.CP); - } + @Test + public void testLmMatrixDenseCPlm() { + runSTEPGlmTest(ExecType.CP); + } - @Test - public void testLmMatrixSparseSPlm() { - runSTEPGlmTest(ExecType.SPARK); - } + @Test + public void testLmMatrixSparseSPlm() { + runSTEPGlmTest(ExecType.SPARK); + } - private void runSTEPGlmTest(ExecType instType) { - ExecMode platformOld = setExecMode(instType); + private void runSTEPGlmTest(ExecType instType) { + ExecMode platformOld = setExecMode(instType); - try { - loadTestConfiguration(getTestConfiguration(TEST_NAME)); + try { + loadTestConfiguration(getTestConfiguration(TEST_NAME)); - String HOME = SCRIPT_DIR + TEST_DIR; + String HOME = SCRIPT_DIR + TEST_DIR; // Pointing to the generated validation DML script fullDMLScriptName = HOME + TEST_NAME + ".dml"; @@ -60,9 +60,9 @@ private void runSTEPGlmTest(ExecType instType) { // runTest executes the script; fails if the DML script invokes stop() runTest(true, false, null, -1); - } - finally { - rtplatform = platformOld; - } - } + } + finally { + rtplatform = platformOld; + } + } } From 159fa87930cfe5be4a5a50ff0401e712799e1c59 Mon Sep 17 00:00:00 2001 From: bruno Date: Thu, 2 Jul 2026 07:02:58 +0200 Subject: [PATCH 014/132] replace spaces with tabs in BuiltinSTEPGlmTest.java --- .../builtin/part2/BuiltinSTEPGlmTest.java | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java index e84047d4b59..e6dc61ad38c 100644 --- a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java +++ b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java @@ -1,13 +1,13 @@ /* * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file + * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -33,36 +33,36 @@ public class BuiltinSTEPGlmTest extends AutomatedTestBase @Override public void setUp() { - addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[]{})); + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[]{})); } @Test public void testLmMatrixDenseCPlm() { - runSTEPGlmTest(ExecType.CP); + runSTEPGlmTest(ExecType.CP); } @Test public void testLmMatrixSparseSPlm() { - runSTEPGlmTest(ExecType.SPARK); + runSTEPGlmTest(ExecType.SPARK); } private void runSTEPGlmTest(ExecType instType) { - ExecMode platformOld = setExecMode(instType); + ExecMode platformOld = setExecMode(instType); - try { - loadTestConfiguration(getTestConfiguration(TEST_NAME)); + try { + loadTestConfiguration(getTestConfiguration(TEST_NAME)); - String HOME = SCRIPT_DIR + TEST_DIR; + String HOME = SCRIPT_DIR + TEST_DIR; - // Pointing to the generated validation DML script - fullDMLScriptName = HOME + TEST_NAME + ".dml"; - programArgs = new String[]{}; + // Pointing to the generated validation DML script + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + programArgs = new String[]{}; - // runTest executes the script; fails if the DML script invokes stop() - runTest(true, false, null, -1); - } - finally { - rtplatform = platformOld; - } + // runTest executes the script; fails if the DML script invokes stop() + runTest(true, false, null, -1); + } + finally { + rtplatform = platformOld; + } } } From b7e64b83cf0c88545f833b4ac1fe911aed34039b Mon Sep 17 00:00:00 2001 From: bruno Date: Thu, 2 Jul 2026 07:08:53 +0200 Subject: [PATCH 015/132] format BuiltinSTEPGlmTest.java via jdtls --- .../builtin/part2/BuiltinSTEPGlmTest.java | 60 +++++++++---------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java index e6dc61ad38c..e70aedcabe4 100644 --- a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java +++ b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java @@ -25,44 +25,42 @@ import org.apache.sysds.test.AutomatedTestBase; import org.apache.sysds.test.TestConfiguration; -public class BuiltinSTEPGlmTest extends AutomatedTestBase -{ - private final static String TEST_NAME = "stepGLM"; - private final static String TEST_DIR = "functions/builtin/"; - private static final String TEST_CLASS_DIR = TEST_DIR + BuiltinSTEPGlmTest.class.getSimpleName() + "/"; +public class BuiltinSTEPGlmTest extends AutomatedTestBase { + private final static String TEST_NAME = "stepGLM"; + private final static String TEST_DIR = "functions/builtin/"; + private static final String TEST_CLASS_DIR = TEST_DIR + BuiltinSTEPGlmTest.class.getSimpleName() + "/"; - @Override - public void setUp() { - addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[]{})); - } + @Override + public void setUp() { + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {})); + } - @Test - public void testLmMatrixDenseCPlm() { - runSTEPGlmTest(ExecType.CP); - } + @Test + public void testLmMatrixDenseCPlm() { + runSTEPGlmTest(ExecType.CP); + } - @Test - public void testLmMatrixSparseSPlm() { - runSTEPGlmTest(ExecType.SPARK); - } + @Test + public void testLmMatrixSparseSPlm() { + runSTEPGlmTest(ExecType.SPARK); + } - private void runSTEPGlmTest(ExecType instType) { - ExecMode platformOld = setExecMode(instType); + private void runSTEPGlmTest(ExecType instType) { + ExecMode platformOld = setExecMode(instType); - try { - loadTestConfiguration(getTestConfiguration(TEST_NAME)); + try { + loadTestConfiguration(getTestConfiguration(TEST_NAME)); - String HOME = SCRIPT_DIR + TEST_DIR; + String HOME = SCRIPT_DIR + TEST_DIR; - // Pointing to the generated validation DML script - fullDMLScriptName = HOME + TEST_NAME + ".dml"; - programArgs = new String[]{}; + // Pointing to the generated validation DML script + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + programArgs = new String[] {}; - // runTest executes the script; fails if the DML script invokes stop() - runTest(true, false, null, -1); - } - finally { - rtplatform = platformOld; + // runTest executes the script; fails if the DML script invokes stop() + runTest(true, false, null, -1); + } finally { + rtplatform = platformOld; + } } - } } From 59b8d883a0cf3ce8f0a7dd07da3fe9da34f1b15d Mon Sep 17 00:00:00 2001 From: bruno Date: Sun, 30 Aug 2026 23:12:49 +0200 Subject: [PATCH 016/132] remove print statments --- scripts/builtin/stepGLM.dml | 5 ----- src/test/scripts/functions/builtin/stepGLM.dml | 9 --------- 2 files changed, 14 deletions(-) diff --git a/scripts/builtin/stepGLM.dml b/scripts/builtin/stepGLM.dml index 00230ca455c..b0a664addb3 100644 --- a/scripts/builtin/stepGLM.dml +++ b/scripts/builtin/stepGLM.dml @@ -113,7 +113,6 @@ m_stepGLM = function ( all_ones = matrix (1, rows = num_records, cols = 1); [AIC_best, ignore_beta2, ignore_S2] = internal_glm(X=all_ones, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } - #print ("Best AIC without any features: " + AIC_best); # First pass to examine single features AICs = matrix (AIC_best, rows = 1, cols = num_features); @@ -133,7 +132,6 @@ m_stepGLM = function ( } if (column_best == 0) { - #print ("AIC of an empty model is " + AIC_best + " and adding no feature achieves more than " + (thr * 100) + "% decrease in AIC!"); if (intercept_status == 0) { # Compute AIC of an empty model with no features and no intercept (all Ys are zero) [AIC_best, ignore_beta4, ignore_S4] = internal_glm(X=X_global, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); @@ -144,7 +142,6 @@ m_stepGLM = function ( } }; - # print ("Best AIC " + AIC_best + " achieved with feature: " + column_best); columns_fixed[1,column_best] = 1; columns_fixed_ordered[1,1] = column_best; X_global = X_orig[,column_best]; @@ -173,7 +170,6 @@ m_stepGLM = function ( # cbind best found features (i.e., columns) to X_global if (as.scalar(columns_fixed[1,column_best]) == 0) { # new best feature found - #print ("Best AIC " + AIC_best + " achieved with feature: " + column_best); columns_fixed[1,column_best] = 1; columns_fixed_ordered = cbind (columns_fixed_ordered, as.matrix(column_best)); if (ncol(columns_fixed_ordered) == num_features) { # all features examined @@ -188,7 +184,6 @@ m_stepGLM = function ( } # run GLM with selected set of features - print ("Running GLM with selected features..."); [AIC, B, S] = internal_glm(X=X_global, Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } diff --git a/src/test/scripts/functions/builtin/stepGLM.dml b/src/test/scripts/functions/builtin/stepGLM.dml index 95a4dfb9908..e8945a7b3de 100644 --- a/src/test/scripts/functions/builtin/stepGLM.dml +++ b/src/test/scripts/functions/builtin/stepGLM.dml @@ -37,10 +37,6 @@ Y = (rand(rows=N, cols=1, min=0.0, max=1.0, seed=456) < P_y) * 1.0; [AIC, B, S] = stepGLM::m_stepGLM(X=X, Y=Y, link=2, yneg=0.0, icpt=0, tol=1e-6, disp=0.0, moi=200, mii=0, thr=0.01); -print("\n\n\n\n\n\n\nTest Results:"); -print("Optimal AIC: " + AIC); -print("Selected Feature Indices:\n" + toString(S)); -print("Estimated Coefficients:\n" + toString(B)); beta_est = matrix(0, rows=P, cols=1); for (i in 1:nrow(B)) { @@ -52,16 +48,11 @@ for (i in 1:nrow(B)) { if (nrow(B) != 3 | sum(beta_est != 0 & beta_true == 0) > 0 | sum(beta_est == 0 & beta_true != 0) > 0) { stop("Test failed: Inexact feature support recovery."); } -print("passed test 1") # Case 02 epsilon = 0.5; if (max(abs(beta_est - beta_true)) > epsilon) { stop("Test failed: Parameter estimates exceed tolerance bound epsilon = " + epsilon + "."); } -print("passed test 2") - -#stop("!!!Sucess!!!") # uncomment for letting the test fail - From bc4dfe26a156c81eda59c0b521443bcb05ccb994 Mon Sep 17 00:00:00 2001 From: bruno Date: Sun, 30 Aug 2026 23:15:25 +0200 Subject: [PATCH 017/132] remove comment --- scripts/builtin/stepGLM.dml | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/builtin/stepGLM.dml b/scripts/builtin/stepGLM.dml index b0a664addb3..42492c9b276 100644 --- a/scripts/builtin/stepGLM.dml +++ b/scripts/builtin/stepGLM.dml @@ -137,7 +137,6 @@ m_stepGLM = function ( [AIC_best, ignore_beta4, ignore_S4] = internal_glm(X=X_global, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } else { # compute AIC of an empty model with only intercept (all Ys are constant) - ###all_ones = matrix (1, rows = num_records, cols = 1); [AIC_best, ignore_beta5, ignore_S5] = internal_glm(X=all_ones, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); } }; From 69ccca95259f34d2e404dad89e16a0ebbcc6582a Mon Sep 17 00:00:00 2001 From: bruno Date: Mon, 31 Aug 2026 08:23:27 +0200 Subject: [PATCH 018/132] rm old stepGLM.dml script --- scripts/algorithms/StepGLM.dml | 1196 -------------------------------- 1 file changed, 1196 deletions(-) delete mode 100644 scripts/algorithms/StepGLM.dml diff --git a/scripts/algorithms/StepGLM.dml b/scripts/algorithms/StepGLM.dml deleted file mode 100644 index 213f373b1b9..00000000000 --- a/scripts/algorithms/StepGLM.dml +++ /dev/null @@ -1,1196 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - -# -# THIS SCRIPT CHOOSES A GLM REGRESSION MODEL IN A STEPWISE ALGIRITHM USING AIC -# EACH GLM REGRESSION IS SOLVED USING NEWTON/FISHER SCORING WITH TRUST REGIONS -# -# INPUT PARAMETERS: -# --------------------------------------------------------------------------------------------- -# NAME TYPE DEFAULT MEANING -# --------------------------------------------------------------------------------------------- -# X String --- Location to read the matrix X of feature vectors -# Y String --- Location to read response matrix Y with 1 column -# B String --- Location to store estimated regression parameters (the betas) -# S String --- Location to write the selected features ordered as computed by the algorithm -# O String " " Location to write the printed statistics; by default is standard output -# link Int 2 Link function code: 1 = log, 2 = Logit, 3 = Probit, 4 = Cloglog -# yneg Double 0.0 Response value for Bernoulli "No" label, usually 0.0 or -1.0 -# icpt Int 0 Intercept presence, X columns shifting and rescaling: -# 0 = no intercept, no shifting, no rescaling; -# 1 = add intercept, but neither shift nor rescale X; -# 2 = add intercept, shift & rescale X columns to mean = 0, variance = 1 -# tol Double 0.000001 Tolerance (epsilon) -# disp Double 0.0 (Over-)dispersion value, or 0.0 to estimate it from data -# moi Int 200 Maximum number of outer (Newton / Fisher Scoring) iterations -# mii Int 0 Maximum number of inner (Conjugate Gradient) iterations, 0 = no maximum -# thr Double 0.01 Threshold to stop the algorithm: if the decrease in the value of AIC falls below thr -# no further features are being checked and the algorithm stops -# fmt String "text" The betas matrix output format, such as "text" or "csv" -# --------------------------------------------------------------------------------------------- -# OUTPUT: Matrix beta, whose size depends on icpt: -# icpt=0: ncol(X) x 1; icpt=1: (ncol(X) + 1) x 1; icpt=2: (ncol(X) + 1) x 2 -# -# In addition, in the last run of GLM some statistics are provided in CSV format, one comma-separated name-value -# pair per each line, as follows: -# -# NAME MEANING -# ------------------------------------------------------------------------------------------- -# TERMINATION_CODE A positive integer indicating success/failure as follows: -# 1 = Converged successfully; 2 = Maximum number of iterations reached; -# 3 = Input (X, Y) out of range; 4 = Distribution/link is not supported -# BETA_MIN Smallest beta value (regression coefficient), excluding the intercept -# BETA_MIN_INDEX Column index for the smallest beta value -# BETA_MAX Largest beta value (regression coefficient), excluding the intercept -# BETA_MAX_INDEX Column index for the largest beta value -# INTERCEPT Intercept value, or NaN if there is no intercept (if icpt=0) -# DISPERSION Dispersion used to scale deviance, provided as "disp" input parameter -# or estimated (same as DISPERSION_EST) if the "disp" parameter is <= 0 -# DISPERSION_EST Dispersion estimated from the dataset -# DEVIANCE_UNSCALED Deviance from the saturated model, assuming dispersion == 1.0 -# DEVIANCE_SCALED Deviance from the saturated model, scaled by the DISPERSION value -# ------------------------------------------------------------------------------------------- -# -# HOW TO INVOKE THIS SCRIPT - EXAMPLE: -# hadoop jar SystemDS.jar -f StepGLM.dml -nvargs X=INPUT_DIR/X Y=INPUT_DIR/Y B=OUTPUT_DIR/betas -# S=OUTPUT_DIR_S/selected O=OUTPUT_DIR/stats link=2 yneg=-1.0 icpt=2 tol=0.00000001 -# disp=1.0 moi=100 mii=10 thr=0.01 fmt=csv -# -# THE StepGLM SCRIPT CURRENTLY SUPPORTS BERNOULLI DISTRIBUTION FAMILY AND THE FOLLOWING LINK FUNCTIONS ONLY! -# - LOG -# - LOGIT -# - PROBIT -# - CLOGLOG - -fileX = $X; -fileY = $Y; -fileB = $B; -intercept_status = ifdef ($icpt, 0); -thr = ifdef ($thr, 0.01); -bernoulli_No_label = ifdef ($yneg, 0.0); # $yneg = 0.0; -distribution_type = 2; - -bernoulli_No_label = as.double (bernoulli_No_label); - -# currently only the forward selection strategy in supported: start from one feature and iteratively add -# features until AIC improves -dir = "forward"; - -print("BEGIN STEPWISE GLM SCRIPT"); -print ("Reading X and Y..."); -X_orig = read (fileX); -Y = read (fileY); - -if (distribution_type == 2 & ncol(Y) == 1) { - is_Y_negative = (Y == bernoulli_No_label); - Y = cbind (1 - is_Y_negative, is_Y_negative); - count_Y_negative = sum (is_Y_negative); - if (count_Y_negative == 0) { - stop ("StepGLM Input Error: all Y-values encode Bernoulli YES-label, none encode NO-label"); - } - if (count_Y_negative == nrow(Y)) { - stop ("StepGLM Input Error: all Y-values encode Bernoulli NO-label, none encode YES-label"); - } -} - -num_records = nrow (X_orig); -num_features = ncol (X_orig); - -# BEGIN STEPWISE GENERALIZED LINEAR MODELS - -if (dir == "forward") { - - continue = TRUE; - columns_fixed = matrix (0, rows = 1, cols = num_features); - columns_fixed_ordered = matrix (0, rows = 1, cols = 1); - - # X_global stores the best model found at each step - X_global = matrix (0, rows = num_records, cols = 1); - - if (intercept_status == 0) { - # Compute AIC of an empty model with no features and no intercept (all Ys are zero) - [AIC_best] = glm_fit (X_global, Y, 0, num_features, columns_fixed_ordered, " "); - } else { - # compute AIC of an empty model with only intercept (all Ys are constant) - all_ones = matrix (1, rows = num_records, cols = 1); - [AIC_best] = glm_fit (all_ones, Y, 0, num_features, columns_fixed_ordered, " "); - } - print ("Best AIC without any features: " + AIC_best); - - # First pass to examine single features - AICs = matrix (AIC_best, rows = 1, cols = num_features); - parfor (i in 1:num_features) { - [AIC_1] = glm_fit (X_orig[,i], Y, intercept_status, num_features, columns_fixed_ordered, " "); - AICs[1,i] = AIC_1; - } - - # Determine the best AIC - column_best = 0; - for (k in 1:num_features) { - AIC_cur = as.scalar (AICs[1,k]); - if ( (AIC_cur < AIC_best) & ((AIC_best - AIC_cur) > abs (thr * AIC_best)) ) { - column_best = k; - AIC_best = as.scalar(AICs[1,k]); - } - } - - if (column_best == 0) { - print ("AIC of an empty model is " + AIC_best + " and adding no feature achieves more than " + (thr * 100) + "% decrease in AIC!"); - if (intercept_status == 0) { - # Compute AIC of an empty model with no features and no intercept (all Ys are zero) - [AIC_best] = glm_fit (X_global, Y, 0, num_features, columns_fixed_ordered, fileB); - } else { - # compute AIC of an empty model with only intercept (all Ys are constant) - ###all_ones = matrix (1, rows = num_records, cols = 1); - [AIC_best] = glm_fit (all_ones, Y, 0, num_features, columns_fixed_ordered, fileB); - } - }; - - print ("Best AIC " + AIC_best + " achieved with feature: " + column_best); - columns_fixed[1,column_best] = 1; - columns_fixed_ordered[1,1] = column_best; - X_global = X_orig[,column_best]; - - while (continue) { - # Subsequent passes over the features - parfor (i in 1:num_features) { - if (as.scalar(columns_fixed[1,i]) == 0) { - - # Construct the feature matrix - X = cbind (X_global, X_orig[,i]); - - [AIC_2] = glm_fit (X, Y, intercept_status, num_features, columns_fixed_ordered, " "); - AICs[1,i] = AIC_2; - } - } - - # Determine the best AIC - for (k in 1:num_features) { - AIC_cur = as.scalar (AICs[1,k]); - if ( (AIC_cur < AIC_best) & ((AIC_best - AIC_cur) > abs (thr * AIC_best)) & (as.scalar(columns_fixed[1,k]) == 0) ) { - column_best = k; - AIC_best = as.scalar(AICs[1,k]); - } - } - - # cbind best found features (i.e., columns) to X_global - if (as.scalar(columns_fixed[1,column_best]) == 0) { # new best feature found - print ("Best AIC " + AIC_best + " achieved with feature: " + column_best); - columns_fixed[1,column_best] = 1; - columns_fixed_ordered = cbind (columns_fixed_ordered, as.matrix(column_best)); - if (ncol(columns_fixed_ordered) == num_features) { # all features examined - X_global = cbind (X_global, X_orig[,column_best]); - continue = FALSE; - } else { - X_global = cbind (X_global, X_orig[,column_best]); - } - } else { - continue = FALSE; - } - } - - # run GLM with selected set of features - print ("Running GLM with selected features..."); - [AIC] = glm_fit (X_global, Y, intercept_status, num_features, columns_fixed_ordered, fileB); - -} else { - stop ("Currently only forward selection strategy is supported!"); -} - - -################### UDFS USED IN THIS SCRIPT ################## - -glm_fit = function (Matrix[Double] X, Matrix[Double] Y, Int intercept_status, Double num_features_orig, Matrix[Double] Selected, String fileB) return (Double AIC) { - - # distribution family code: 1 = Power, 2 = Bernoulli/Binomial; currently only Bernouli distribution family is supported! - distribution_type = 2; # $dfam = 2; - variance_as_power_of_the_mean = 0.0; # $vpow = 0.0; - # link function code: 0 = canonical (depends on distribution), 1 = Power, 2 = Logit, 3 = Probit, 4 = Cloglog, 5 = Cauchit; - # currently only log (link = 1), logit (link = 2), probit (link = 3), and cloglog (link = 4) are supported! - link_type = ifdef ($link, 2); # $link = 2; - link_as_power_of_the_mean = 0.0; # $lpow = 0.0; - - dispersion = ifdef ($disp, 0.0); # $disp = 0.0; - eps = ifdef ($tol, 0.000001); # $tol = 0.000001; - max_iteration_IRLS = ifdef ($moi, 200); # $moi = 200; - max_iteration_CG = ifdef ($mii, 0); # $mii = 0; - - variance_as_power_of_the_mean = as.double (variance_as_power_of_the_mean); - link_as_power_of_the_mean = as.double (link_as_power_of_the_mean); - - dispersion = as.double (dispersion); - eps = as.double (eps); - - # Default values for output statistics: - regularization = 0.0; - termination_code = 0.0; - min_beta = NaN; - i_min_beta = NaN; - max_beta = NaN; - i_max_beta = NaN; - intercept_value = NaN; - dispersion = NaN; - estimated_dispersion = NaN; - deviance_nodisp = NaN; - deviance = NaN; - - ##### INITIALIZE THE PARAMETERS ##### - - num_records = nrow (X); - num_features = ncol (X); - zeros_r = matrix (0, rows = num_records, cols = 1); - ones_r = 1 + zeros_r; - - # Introduce the intercept, shift and rescale the columns of X if needed - - if (intercept_status == 1 | intercept_status == 2) { # add the intercept column - X = cbind (X, ones_r); - num_features = ncol (X); - } - - scale_lambda = matrix (1, rows = num_features, cols = 1); - if (intercept_status == 1 | intercept_status == 2) { - scale_lambda [num_features, 1] = 0; - } - - if (intercept_status == 2) { # scale-&-shift X columns to mean 0, variance 1 - # Important assumption: X [, num_features] = ones_r - avg_X_cols = t(colSums(X)) / num_records; - var_X_cols = (t(colSums (X ^ 2)) - num_records * (avg_X_cols ^ 2)) / (num_records - 1); - is_unsafe = (var_X_cols <= 0); - scale_X = 1.0 / sqrt (var_X_cols * (1 - is_unsafe) + is_unsafe); - scale_X [num_features, 1] = 1; - shift_X = - avg_X_cols * scale_X; - shift_X [num_features, 1] = 0; - rowSums_X_sq = (X ^ 2) %*% (scale_X ^ 2) + X %*% (2 * scale_X * shift_X) + sum (shift_X ^ 2); - } else { - scale_X = matrix (1, rows = num_features, cols = 1); - shift_X = matrix (0, rows = num_features, cols = 1); - rowSums_X_sq = rowSums (X ^ 2); - } - - # Henceforth we replace "X" with "X %*% (SHIFT/SCALE TRANSFORM)" and rowSums(X ^ 2) - # with "rowSums_X_sq" in order to preserve the sparsity of X under shift and scale. - # The transform is then associatively applied to the other side of the expression, - # and is rewritten via "scale_X" and "shift_X" as follows: - # - # ssX_A = (SHIFT/SCALE TRANSFORM) %*% A --- is rewritten as: - # ssX_A = diag (scale_X) %*% A; - # ssX_A [num_features, ] = ssX_A [num_features, ] + t(shift_X) %*% A; - # - # tssX_A = t(SHIFT/SCALE TRANSFORM) %*% A --- is rewritten as: - # tssX_A = diag (scale_X) %*% A + shift_X %*% A [num_features, ]; - - # Initialize other input-dependent parameters - - lambda = scale_lambda * regularization; - if (max_iteration_CG == 0) { - max_iteration_CG = num_features; - } - - # Set up the canonical link, if requested [Then we have: Var(mu) * (d link / d mu) = const] - - if (link_type == 0) { - if (distribution_type == 1) { - link_type = 1; - link_as_power_of_the_mean = 1.0 - variance_as_power_of_the_mean; - } else { - if (distribution_type == 2) { - link_type = 2; - } - } - } - - # For power distributions and/or links, we use two constants, - # "variance as power of the mean" and "link_as_power_of_the_mean", - # to specify the variance and the link as arbitrary powers of the - # mean. However, the variance-powers of 1.0 (Poisson family) and - # 2.0 (Gamma family) have to be treated as special cases, because - # these values integrate into logarithms. The link-power of 0.0 - # is also special as it represents the logarithm link. - - num_response_columns = ncol (Y); - is_supported = 0; - if (num_response_columns == 2 & distribution_type == 2 & link_type >= 1 & link_type <= 4) { # BERNOULLI DISTRIBUTION - is_supported = 1; - } - if (num_response_columns == 1 & distribution_type == 2) { - print ("Error: Bernoulli response matrix has not been converted into two-column format."); - } - - if (is_supported == 1) { - - ##### INITIALIZE THE BETAS ##### - - [beta, saturated_log_l, isNaN] = - glm_initialize (X, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean, intercept_status, max_iteration_CG); - - # print(" --- saturated logLik " + saturated_log_l); - - if (isNaN == 0) { - - ##### START OF THE MAIN PART ##### - - sum_X_sq = sum (rowSums_X_sq); - trust_delta = 0.5 * sqrt (num_features) / max (sqrt (rowSums_X_sq)); - ### max_trust_delta = trust_delta * 10000.0; - log_l = 0.0; - deviance_nodisp = 0.0; - new_deviance_nodisp = 0.0; - isNaN_log_l = 2; - newbeta = beta; - g = matrix (0.0, rows = num_features, cols = 1); - g_norm = sqrt (sum ((g + lambda * beta) ^ 2)); - accept_new_beta = 1; - reached_trust_boundary = 0; - neg_log_l_change_predicted = 0.0; - i_IRLS = 0; - - # print ("BEGIN IRLS ITERATIONS..."); - - ssX_newbeta = diag (scale_X) %*% newbeta; - ssX_newbeta [num_features, ] = ssX_newbeta [num_features, ] + t(shift_X) %*% newbeta; - all_linear_terms = X %*% ssX_newbeta; - - [new_log_l, isNaN_new_log_l] = glm_log_likelihood_part - (all_linear_terms, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean); - - if (isNaN_new_log_l == 0) { - new_deviance_nodisp = 2.0 * (saturated_log_l - new_log_l); - new_log_l = new_log_l - 0.5 * sum (lambda * newbeta ^ 2); - } - - while (termination_code == 0) { - accept_new_beta = 1; - - if (i_IRLS > 0) { - if (isNaN_log_l == 0) { - accept_new_beta = 0; - } - - # Decide whether to accept a new iteration point and update the trust region - # See Alg. 4.1 on p. 69 of "Numerical Optimization" 2nd ed. by Nocedal and Wright - - rho = (- new_log_l + log_l) / neg_log_l_change_predicted; - if (rho < 0.25 | isNaN_new_log_l == 1) { - trust_delta = 0.25 * trust_delta; - } - if (rho > 0.75 & isNaN_new_log_l == 0 & reached_trust_boundary == 1) { - trust_delta = 2 * trust_delta; - - ### if (trust_delta > max_trust_delta) { - ### trust_delta = max_trust_delta; - ### } - } - if (rho > 0.1 & isNaN_new_log_l == 0) { - accept_new_beta = 1; - } - } - - if (accept_new_beta == 1) { - beta = newbeta; log_l = new_log_l; deviance_nodisp = new_deviance_nodisp; isNaN_log_l = isNaN_new_log_l; - - [g_Y, w] = glm_dist (all_linear_terms, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean); - - # We introduced these variables to avoid roundoff errors: - # g_Y = y_residual / (y_var * link_grad); - # w = 1.0 / (y_var * link_grad * link_grad); - - gXY = - t(X) %*% g_Y; - g = diag (scale_X) %*% gXY + shift_X %*% gXY [num_features, ]; - g_norm = sqrt (sum ((g + lambda * beta) ^ 2)); - } - - [z, neg_log_l_change_predicted, num_CG_iters, reached_trust_boundary] = - get_CG_Steihaug_point (X, scale_X, shift_X, w, g, beta, lambda, trust_delta, max_iteration_CG); - - newbeta = beta + z; - - ssX_newbeta = diag (scale_X) %*% newbeta; - ssX_newbeta [num_features, ] = ssX_newbeta [num_features, ] + t(shift_X) %*% newbeta; - all_linear_terms = X %*% ssX_newbeta; - - [new_log_l, isNaN_new_log_l] = glm_log_likelihood_part - (all_linear_terms, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean); - - if (isNaN_new_log_l == 0) { - new_deviance_nodisp = 2.0 * (saturated_log_l - new_log_l); - new_log_l = new_log_l - 0.5 * sum (lambda * newbeta ^ 2); - } - - log_l_change = new_log_l - log_l; # R's criterion for termination: |dev - devold|/(|dev| + 0.1) < eps - - if (reached_trust_boundary == 0 & isNaN_new_log_l == 0 & - (2.0 * abs (log_l_change) < eps * (deviance_nodisp + 0.1) | abs (log_l_change) < (abs (log_l) + abs (new_log_l)) * 0.00000000000001) ) { - termination_code = 1; - } - rho = - log_l_change / neg_log_l_change_predicted; - z_norm = sqrt (sum (z * z)); - - i_IRLS = i_IRLS + 1; - - if (i_IRLS == max_iteration_IRLS) { - termination_code = 2; - } - } - - beta = newbeta; - log_l = new_log_l; - deviance_nodisp = new_deviance_nodisp; - - #---------------------------- last part - - if (termination_code != 1) { - print ("One of the runs of GLM did not converged in " + i_IRLS + " steps!"); - } - - ##### COMPUTE AIC ##### - - if (distribution_type == 2 & link_type >= 1 & link_type <= 4) { - AIC = -2 * log_l; - if (sum (X) != 0) { - AIC = AIC + 2 * num_features; - } - } else { - stop ("Currently only the Bernoulli distribution family the following link functions are supported: log, logit, probit, and cloglog!"); - } - - if (fileB != " ") { - fileO = ifdef ($O, " "); - fileS = $S; - fmt = ifdef ($fmt, "text"); - - # Output which features give the best AIC and are being used for linear regression - write (Selected, fileS, format=fmt); - - ssX_beta = diag (scale_X) %*% beta; - ssX_beta [num_features, ] = ssX_beta [num_features, ] + t(shift_X) %*% beta; - if (intercept_status == 2) { - beta_out = cbind (ssX_beta, beta); - } else { - beta_out = ssX_beta; - } - - if (intercept_status == 0 & num_features == 1) { - p = sum (X == 1); - if (p == num_records) { - beta_out = beta_out[1,]; - } - } - - - if (intercept_status == 1 | intercept_status == 2) { - intercept_value = as.scalar (beta_out [num_features, 1]); - beta_noicept = beta_out [1 : (num_features - 1), 1]; - } else { - beta_noicept = beta_out [1 : num_features, 1]; - } - min_beta = min (beta_noicept); - max_beta = max (beta_noicept); - tmp_i_min_beta = rowIndexMin (t(beta_noicept)) - i_min_beta = as.scalar (tmp_i_min_beta [1, 1]); - tmp_i_max_beta = rowIndexMax (t(beta_noicept)) - i_max_beta = as.scalar (tmp_i_max_beta [1, 1]); - - ##### OVER-DISPERSION PART ##### - - all_linear_terms = X %*% ssX_beta; - [g_Y, w] = glm_dist (all_linear_terms, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean); - - pearson_residual_sq = g_Y ^ 2 / w; - pearson_residual_sq = replace (target = pearson_residual_sq, pattern = NaN, replacement = 0); - # pearson_residual_sq = (y_residual ^ 2) / y_var; - - if (num_records > num_features) { - estimated_dispersion = sum (pearson_residual_sq) / (num_records - num_features); - } - if (dispersion <= 0) { - dispersion = estimated_dispersion; - } - deviance = deviance_nodisp / dispersion; - - ##### END OF THE MAIN PART ##### - - str = "BETA_MIN," + min_beta; - str = append (str, "BETA_MIN_INDEX," + i_min_beta); - str = append (str, "BETA_MAX," + max_beta); - str = append (str, "BETA_MAX_INDEX," + i_max_beta); - str = append (str, "INTERCEPT," + intercept_value); - str = append (str, "DISPERSION," + dispersion); - str = append (str, "DISPERSION_EST," + estimated_dispersion); - str = append (str, "DEVIANCE_UNSCALED," + deviance_nodisp); - str = append (str, "DEVIANCE_SCALED," + deviance); - - if (fileO != " ") { - write (str, fileO); - } - else { - print (str); - } - - # Prepare the output matrix - print ("Writing the output matrix..."); - if (intercept_status == 0 & num_features == 1) { - if (p == num_records) { - beta_out_tmp = matrix (0, rows = num_features_orig + 1, cols = 1); - beta_out_tmp[num_features_orig + 1,] = beta_out; - beta_out = beta_out_tmp; - write (beta_out, fileB, format=fmt); - stop (""); - } else if (sum (X) == 0){ - beta_out = matrix (0, rows = num_features_orig, cols = 1); - write (beta_out, fileB, format=fmt); - stop (""); - } - } - - no_selected = ncol (Selected); - max_selected = max (Selected); - last = max_selected + 1; - - if (intercept_status != 0) { - - Selected_ext = cbind (Selected, as.matrix (last)); - P1 = table (seq (1, ncol (Selected_ext)), t(Selected_ext)); - - if (intercept_status == 2) { - - P1_ssX_beta = P1 * ssX_beta; - P2_ssX_beta = colSums (P1_ssX_beta); - P1_beta = P1 * beta; - P2_beta = colSums (P1_beta); - - if (max_selected < num_features_orig) { - - P2_ssX_beta = cbind (P2_ssX_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); - P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); - - P2_ssX_beta[1, num_features_orig+1] = P2_ssX_beta[1, max_selected + 1]; - P2_ssX_beta[1, max_selected + 1] = 0; - - P2_beta[1, num_features_orig+1] = P2_beta[1, max_selected + 1]; - P2_beta[1, max_selected + 1] = 0; - - } - beta_out = cbind (t(P2_ssX_beta), t(P2_beta)); - - } else { - - P1_beta = P1 * beta; - P2_beta = colSums (P1_beta); - - if (max_selected < num_features_orig) { - P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); - P2_beta[1, num_features_orig+1] = P2_beta[1, max_selected + 1] ; - P2_beta[1, max_selected + 1] = 0; - } - beta_out = t(P2_beta); - - } - } else { - - P1 = table (seq (1, no_selected), t(Selected)); - P1_beta = P1 * beta; - P2_beta = colSums (P1_beta); - - if (max_selected < num_features_orig) { - P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); - } - - beta_out = t(P2_beta); - } - - write ( beta_out, fileB, format=fmt ); - - } - - } else { - stop ("Input matrices X and/or Y are out of range!"); - } - } else { - stop ("Response matrix with " + num_response_columns + " columns, distribution family (" + distribution_type + ", " + variance_as_power_of_the_mean - + ") and link family (" + link_type + ", " + link_as_power_of_the_mean + ") are NOT supported together."); - } -} - -glm_initialize = function (Matrix[double] X, Matrix[double] Y, int dist_type, double var_power, int link_type, double link_power, int icept_status, int max_iter_CG) - return (Matrix[double] beta, double saturated_log_l, int isNaN) -{ - saturated_log_l = 0.0; - isNaN = 0; - y_corr = Y [, 1]; - if (dist_type == 2) { - n_corr = rowSums (Y); - is_n_zero = (n_corr == 0); - y_corr = Y [, 1] / (n_corr + is_n_zero) + (0.5 - Y [, 1]) * is_n_zero; - } - linear_terms = y_corr; - if (dist_type == 1 & link_type == 1) { # POWER DISTRIBUTION - if (link_power == 0) { - if (sum (y_corr < 0) == 0) { - is_zero_y_corr = (y_corr == 0); - linear_terms = log (y_corr + is_zero_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { isNaN = 1; } - } else { if (link_power == 1.0) { - linear_terms = y_corr; - } else { if (link_power == -1.0) { - linear_terms = 1.0 / y_corr; - } else { if (link_power == 0.5) { - if (sum (y_corr < 0) == 0) { - linear_terms = sqrt (y_corr); - } else { isNaN = 1; } - } else { if (link_power > 0) { - if (sum (y_corr < 0) == 0) { - is_zero_y_corr = (y_corr == 0); - linear_terms = (y_corr + is_zero_y_corr) ^ link_power - is_zero_y_corr; - } else { isNaN = 1; } - } else { - if (sum (y_corr <= 0) == 0) { - linear_terms = y_corr ^ link_power; - } else { isNaN = 1; } - }}}}} - } - if (dist_type == 2 & link_type >= 1 & link_type <= 5) - { # BINOMIAL/BERNOULLI DISTRIBUTION - if (link_type == 1 & link_power == 0) { # Binomial.log - if (sum (y_corr < 0) == 0) { - is_zero_y_corr = (y_corr == 0); - linear_terms = log (y_corr + is_zero_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { isNaN = 1; } - } else { if (link_type == 1 & link_power > 0) { # Binomial.power_nonlog pos - if (sum (y_corr < 0) == 0) { - is_zero_y_corr = (y_corr == 0); - linear_terms = (y_corr + is_zero_y_corr) ^ link_power - is_zero_y_corr; - } else { isNaN = 1; } - } else { if (link_type == 1) { # Binomial.power_nonlog neg - if (sum (y_corr <= 0) == 0) { - linear_terms = y_corr ^ link_power; - } else { isNaN = 1; } - } else { - is_zero_y_corr = (y_corr <= 0); - is_one_y_corr = (y_corr >= 1.0); - y_corr = y_corr * (1.0 - is_zero_y_corr) * (1.0 - is_one_y_corr) + 0.5 * (is_zero_y_corr + is_one_y_corr); - if (link_type == 2) { # Binomial.logit - linear_terms = log (y_corr / (1.0 - y_corr)) - + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { if (link_type == 3) { # Binomial.probit - y_below_half = y_corr + (1.0 - 2.0 * y_corr) * (y_corr > 0.5); - t = sqrt (- 2.0 * log (y_below_half)); - approx_inv_Gauss_CDF = - t + (2.515517 + t * (0.802853 + t * 0.010328)) / (1.0 + t * (1.432788 + t * (0.189269 + t * 0.001308))); - linear_terms = approx_inv_Gauss_CDF * (1.0 - 2.0 * (y_corr > 0.5)) - + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { if (link_type == 4) { # Binomial.cloglog - linear_terms = log (- log (1.0 - y_corr)) - - log (- log (0.5)) * (is_zero_y_corr + is_one_y_corr) - + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { if (link_type == 5) { # Binomial.cauchit - linear_terms = tan ((y_corr - 0.5) * pi) - + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - }} }}}}} - } - - if (isNaN == 0) { - [saturated_log_l, isNaN] = - glm_log_likelihood_part (linear_terms, Y, dist_type, var_power, link_type, link_power); - } - - if ((dist_type == 1 & link_type == 1 & link_power == 0) | - (dist_type == 2 & link_type >= 2)) - { - desired_eta = 0.0; - } else { if (link_type == 1 & link_power == 0) { - desired_eta = log (0.5); - } else { if (link_type == 1) { - desired_eta = 0.5 ^ link_power; - } else { - desired_eta = 0.5; - }}} - - beta = matrix (0.0, rows = ncol(X), cols = 1); - - if (desired_eta != 0) { - if (icept_status == 1 | icept_status == 2) { - beta [nrow(beta), 1] = desired_eta; - } else { - # We want: avg (X %*% ssX_transform %*% beta) = desired_eta - # Note that "ssX_transform" is trivial here, hence ignored - - beta = straightenX (X, 0.000001, max_iter_CG); - beta = beta * desired_eta; - } } } - - -glm_dist = function (Matrix[double] linear_terms, Matrix[double] Y, - int dist_type, double var_power, int link_type, double link_power) - return (Matrix[double] g_Y, Matrix[double] w) -# ORIGINALLY we returned more meaningful vectors, namely: -# Matrix[double] y_residual : y - y_mean, i.e. y observed - y predicted -# Matrix[double] link_gradient : derivative of the link function -# Matrix[double] var_function : variance without dispersion, i.e. the V(mu) function -# BUT, this caused roundoff errors, so we had to compute "directly useful" vectors -# and skip over the "meaningful intermediaries". Now we output these two variables: -# g_Y = y_residual / (var_function * link_gradient); -# w = 1.0 / (var_function * link_gradient ^ 2); -{ - num_records = nrow (linear_terms); - zeros_r = matrix (0.0, rows = num_records, cols = 1); - ones_r = 1 + zeros_r; - g_Y = zeros_r; - w = zeros_r; - - # Some constants - - one_over_sqrt_two_pi = 0.39894228040143267793994605993438; - ones_2 = matrix (1.0, rows = 1, cols = 2); - p_one_m_one = ones_2; - p_one_m_one [1, 2] = -1.0; - m_one_p_one = ones_2; - m_one_p_one [1, 1] = -1.0; - zero_one = ones_2; - zero_one [1, 1] = 0.0; - one_zero = ones_2; - one_zero [1, 2] = 0.0; - flip_pos = matrix (0, rows = 2, cols = 2); - flip_neg = flip_pos; - flip_pos [1, 2] = 1; - flip_pos [2, 1] = 1; - flip_neg [1, 2] = -1; - flip_neg [2, 1] = 1; - - if (dist_type == 1 & link_type == 1) { # POWER DISTRIBUTION - y_mean = zeros_r; - if (link_power == 0) { - y_mean = exp (linear_terms); - y_mean_pow = y_mean ^ (1 - var_power); - w = y_mean_pow * y_mean; - g_Y = y_mean_pow * (Y - y_mean); - } else { if (link_power == 1.0) { - y_mean = linear_terms; - w = y_mean ^ (- var_power); - g_Y = w * (Y - y_mean); - } else { - y_mean = linear_terms ^ (1.0 / link_power); - c1 = (1 - var_power) / link_power - 1; - c2 = (2 - var_power) / link_power - 2; - g_Y = (linear_terms ^ c1) * (Y - y_mean) / link_power; - w = (linear_terms ^ c2) / (link_power ^ 2); - } }} - if (dist_type == 2 & link_type >= 1 & link_type <= 5) - { # BINOMIAL/BERNOULLI DISTRIBUTION - if (link_type == 1) { # BINOMIAL.POWER LINKS - if (link_power == 0) { # Binomial.log - vec1 = 1 / (exp (- linear_terms) - 1); - g_Y = Y [, 1] - Y [, 2] * vec1; - w = rowSums (Y) * vec1; - } else { # Binomial.nonlog - vec1 = zeros_r; - if (link_power == 0.5) { - vec1 = 1 / (1 - linear_terms ^ 2); - } else { if (sum (linear_terms < 0) == 0) { - vec1 = linear_terms ^ (- 2 + 1 / link_power) / (1 - linear_terms ^ (1 / link_power)); - } else {isNaN = 1;}} - # We want a "zero-protected" version of - # vec2 = Y [, 1] / linear_terms; - is_y_0 = (Y [, 1] == 0); - vec2 = (Y [, 1] + is_y_0) / (linear_terms * (1 - is_y_0) + is_y_0) - is_y_0; - g_Y = (vec2 - Y [, 2] * vec1 * linear_terms) / link_power; - w = rowSums (Y) * vec1 / link_power ^ 2; - } - } else { - is_LT_pos_infinite = (linear_terms == Inf); - is_LT_neg_infinite = (linear_terms == -Inf); - is_LT_infinite = is_LT_pos_infinite %*% one_zero + is_LT_neg_infinite %*% zero_one; - finite_linear_terms = replace (target = linear_terms, pattern = Inf, replacement = 0); - finite_linear_terms = replace (target = finite_linear_terms, pattern = -Inf, replacement = 0); - if (link_type == 2) { # Binomial.logit - Y_prob = exp (finite_linear_terms) %*% one_zero + ones_r %*% zero_one; - Y_prob = Y_prob / (rowSums (Y_prob) %*% ones_2); - Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; - g_Y = rowSums (Y * (Y_prob %*% flip_neg)); ### = y_residual; - w = rowSums (Y * (Y_prob %*% flip_pos) * Y_prob); ### = y_variance; - } else { if (link_type == 3) { # Binomial.probit - is_lt_pos = (linear_terms >= 0); - t_gp = 1.0 / (1.0 + abs (finite_linear_terms) * 0.231641888); # 0.231641888 = 0.3275911 / sqrt (2.0) - pt_gp = t_gp * ( 0.254829592 - + t_gp * (-0.284496736 # "Handbook of Mathematical Functions", ed. by M. Abramowitz and I.A. Stegun, - + t_gp * ( 1.421413741 # U.S. Nat-l Bureau of Standards, 10th print (Dec 1972), Sec. 7.1.26, p. 299 - + t_gp * (-1.453152027 - + t_gp * 1.061405429)))); - the_gauss_exp = exp (- (linear_terms ^ 2) / 2.0); - vec1 = 0.25 * pt_gp * (2 - the_gauss_exp * pt_gp); - vec2 = Y [, 1] - rowSums (Y) * is_lt_pos + the_gauss_exp * pt_gp * rowSums (Y) * (is_lt_pos - 0.5); - w = the_gauss_exp * (one_over_sqrt_two_pi ^ 2) * rowSums (Y) / vec1; - g_Y = one_over_sqrt_two_pi * vec2 / vec1; - } else { if (link_type == 4) { # Binomial.cloglog - the_exp = exp (linear_terms) - the_exp_exp = exp (- the_exp); - is_too_small = ((10000000 + the_exp) == 10000000); - the_exp_ratio = (1 - is_too_small) * (1 - the_exp_exp) / (the_exp + is_too_small) + is_too_small * (1 - the_exp / 2); - g_Y = (rowSums (Y) * the_exp_exp - Y [, 2]) / the_exp_ratio; - w = the_exp_exp * the_exp * rowSums (Y) / the_exp_ratio; - } else { if (link_type == 5) { # Binomial.cauchit - Y_prob = 0.5 + (atan (finite_linear_terms) %*% p_one_m_one) / pi; - Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; - y_residual = Y [, 1] * Y_prob [, 2] - Y [, 2] * Y_prob [, 1]; - var_function = rowSums (Y) * Y_prob [, 1] * Y_prob [, 2]; - link_gradient_normalized = (1 + linear_terms ^ 2) * pi; - g_Y = rowSums (Y) * y_residual / (var_function * link_gradient_normalized); - w = (rowSums (Y) ^ 2) / (var_function * link_gradient_normalized ^ 2); - }}}} - } - } - } - - -glm_log_likelihood_part = function (Matrix[double] linear_terms, Matrix[double] Y, - int dist_type, double var_power, int link_type, double link_power) - return (double log_l, int isNaN) -{ - isNaN = 0; - log_l = 0.0; - num_records = nrow (Y); - zeros_r = matrix (0.0, rows = num_records, cols = 1); - - if (dist_type == 1 & link_type == 1) - { # POWER DISTRIBUTION - b_cumulant = zeros_r; - natural_parameters = zeros_r; - is_natural_parameter_log_zero = zeros_r; - if (var_power == 1.0 & link_power == 0) { # Poisson.log - b_cumulant = exp (linear_terms); - is_natural_parameter_log_zero = (linear_terms == -Inf); - natural_parameters = replace (target = linear_terms, pattern = -Inf, replacement = 0); - } else { if (var_power == 1.0 & link_power == 1.0) { # Poisson.id - if (sum (linear_terms < 0) == 0) { - b_cumulant = linear_terms; - is_natural_parameter_log_zero = (linear_terms == 0); - natural_parameters = log (linear_terms + is_natural_parameter_log_zero); - } else {isNaN = 1;} - } else { if (var_power == 1.0 & link_power == 0.5) { # Poisson.sqrt - if (sum (linear_terms < 0) == 0) { - b_cumulant = linear_terms ^ 2; - is_natural_parameter_log_zero = (linear_terms == 0); - natural_parameters = 2.0 * log (linear_terms + is_natural_parameter_log_zero); - } else {isNaN = 1;} - } else { if (var_power == 1.0 & link_power > 0) { # Poisson.power_nonlog, pos - if (sum (linear_terms < 0) == 0) { - is_natural_parameter_log_zero = (linear_terms == 0); - b_cumulant = (linear_terms + is_natural_parameter_log_zero) ^ (1.0 / link_power) - is_natural_parameter_log_zero; - natural_parameters = log (linear_terms + is_natural_parameter_log_zero) / link_power; - } else {isNaN = 1;} - } else { if (var_power == 1.0) { # Poisson.power_nonlog, neg - if (sum (linear_terms <= 0) == 0) { - b_cumulant = linear_terms ^ (1.0 / link_power); - natural_parameters = log (linear_terms) / link_power; - } else {isNaN = 1;} - } else { if (var_power == 2.0 & link_power == -1.0) { # Gamma.inverse - if (sum (linear_terms <= 0) == 0) { - b_cumulant = - log (linear_terms); - natural_parameters = - linear_terms; - } else {isNaN = 1;} - } else { if (var_power == 2.0 & link_power == 1.0) { # Gamma.id - if (sum (linear_terms <= 0) == 0) { - b_cumulant = log (linear_terms); - natural_parameters = - 1.0 / linear_terms; - } else {isNaN = 1;} - } else { if (var_power == 2.0 & link_power == 0) { # Gamma.log - b_cumulant = linear_terms; - natural_parameters = - exp (- linear_terms); - } else { if (var_power == 2.0) { # Gamma.power_nonlog - if (sum (linear_terms <= 0) == 0) { - b_cumulant = log (linear_terms) / link_power; - natural_parameters = - linear_terms ^ (- 1.0 / link_power); - } else {isNaN = 1;} - } else { if (link_power == 0) { # PowerDist.log - natural_parameters = exp (linear_terms * (1.0 - var_power)) / (1.0 - var_power); - b_cumulant = exp (linear_terms * (2.0 - var_power)) / (2.0 - var_power); - } else { # PowerDist.power_nonlog - if (-2 * link_power == 1.0 - var_power) { - natural_parameters = 1.0 / (linear_terms ^ 2) / (1.0 - var_power); - } else { if (-1 * link_power == 1.0 - var_power) { - natural_parameters = 1.0 / linear_terms / (1.0 - var_power); - } else { if ( link_power == 1.0 - var_power) { - natural_parameters = linear_terms / (1.0 - var_power); - } else { if ( 2 * link_power == 1.0 - var_power) { - natural_parameters = linear_terms ^ 2 / (1.0 - var_power); - } else { - if (sum (linear_terms <= 0) == 0) { - power = (1.0 - var_power) / link_power; - natural_parameters = (linear_terms ^ power) / (1.0 - var_power); - } else {isNaN = 1;} - }}}} - if (-2 * link_power == 2.0 - var_power) { - b_cumulant = 1.0 / (linear_terms ^ 2) / (2.0 - var_power); - } else { if (-1 * link_power == 2.0 - var_power) { - b_cumulant = 1.0 / linear_terms / (2.0 - var_power); - } else { if ( link_power == 2.0 - var_power) { - b_cumulant = linear_terms / (2.0 - var_power); - } else { if ( 2 * link_power == 2.0 - var_power) { - b_cumulant = linear_terms ^ 2 / (2.0 - var_power); - } else { - if (sum (linear_terms <= 0) == 0) { - power = (2.0 - var_power) / link_power; - b_cumulant = (linear_terms ^ power) / (2.0 - var_power); - } else {isNaN = 1;} - }}}} - }}}}} }}}}} - if (sum (is_natural_parameter_log_zero * abs (Y)) > 0) { - log_l = -Inf; - isNaN = 1; - } - if (isNaN == 0) - { - log_l = sum (Y * natural_parameters - b_cumulant); - if (log_l != log_l | (log_l == log_l + 1.0 & log_l == log_l * 2.0)) { - log_l = -Inf; - isNaN = 1; - } } } - - if (dist_type == 2 & link_type >= 1 & link_type <= 5) - { # BINOMIAL/BERNOULLI DISTRIBUTION - - [Y_prob, isNaN] = binomial_probability_two_column (linear_terms, link_type, link_power); - - if (isNaN == 0) { - does_prob_contradict = (Y_prob <= 0); - if (sum (does_prob_contradict * abs (Y)) == 0) { - log_l = sum (Y * log (Y_prob * (1 - does_prob_contradict) + does_prob_contradict)); - if (log_l != log_l | (log_l == log_l + 1.0 & log_l == log_l * 2.0)) { - isNaN = 1; - } - } else { - log_l = -Inf; - isNaN = 1; - } } } - - if (isNaN == 1) { - log_l = - Inf; - } - } - - - -binomial_probability_two_column = - function (Matrix[double] linear_terms, int link_type, double link_power) - return (Matrix[double] Y_prob, int isNaN) -{ - isNaN = 0; - num_records = nrow (linear_terms); - - # Define some auxiliary matrices - - ones_2 = matrix (1.0, rows = 1, cols = 2); - p_one_m_one = ones_2; - p_one_m_one [1, 2] = -1.0; - m_one_p_one = ones_2; - m_one_p_one [1, 1] = -1.0; - zero_one = ones_2; - zero_one [1, 1] = 0.0; - one_zero = ones_2; - one_zero [1, 2] = 0.0; - - zeros_r = matrix (0.0, rows = num_records, cols = 1); - ones_r = 1.0 + zeros_r; - - # Begin the function body - - Y_prob = zeros_r %*% ones_2; - if (link_type == 1) { # Binomial.power - if (link_power == 0) { # Binomial.log - Y_prob = exp (linear_terms) %*% p_one_m_one + ones_r %*% zero_one; - } else { if (link_power == 0.5) { # Binomial.sqrt - Y_prob = (linear_terms ^ 2) %*% p_one_m_one + ones_r %*% zero_one; - } else { # Binomial.power_nonlog - if (sum (linear_terms < 0) == 0) { - Y_prob = (linear_terms ^ (1.0 / link_power)) %*% p_one_m_one + ones_r %*% zero_one; - } else {isNaN = 1;} - }} - } else { # Binomial.non_power - is_LT_pos_infinite = (linear_terms == Inf); - is_LT_neg_infinite = (linear_terms == -Inf); - is_LT_infinite = is_LT_pos_infinite %*% one_zero + is_LT_neg_infinite %*% zero_one; - finite_linear_terms = replace (target = linear_terms, pattern = Inf, replacement = 0); - finite_linear_terms = replace (target = finite_linear_terms, pattern = -Inf, replacement = 0); - if (link_type == 2) { # Binomial.logit - Y_prob = exp (finite_linear_terms) %*% one_zero + ones_r %*% zero_one; - Y_prob = Y_prob / (rowSums (Y_prob) %*% ones_2); - } else { if (link_type == 3) { # Binomial.probit - lt_pos_neg = (finite_linear_terms >= 0) %*% p_one_m_one + ones_r %*% zero_one; - t_gp = 1.0 / (1.0 + abs (finite_linear_terms) * 0.231641888); # 0.231641888 = 0.3275911 / sqrt (2.0) - pt_gp = t_gp * ( 0.254829592 - + t_gp * (-0.284496736 # "Handbook of Mathematical Functions", ed. by M. Abramowitz and I.A. Stegun, - + t_gp * ( 1.421413741 # U.S. Nat-l Bureau of Standards, 10th print (Dec 1972), Sec. 7.1.26, p. 299 - + t_gp * (-1.453152027 - + t_gp * 1.061405429)))); - the_gauss_exp = exp (- (finite_linear_terms ^ 2) / 2.0); - Y_prob = lt_pos_neg + ((the_gauss_exp * pt_gp) %*% ones_2) * (0.5 - lt_pos_neg); - } else { if (link_type == 4) { # Binomial.cloglog - the_exp = exp (finite_linear_terms); - the_exp_exp = exp (- the_exp); - is_too_small = ((10000000 + the_exp) == 10000000); - Y_prob [, 1] = (1 - is_too_small) * (1 - the_exp_exp) + is_too_small * the_exp * (1 - the_exp / 2); - Y_prob [, 2] = the_exp_exp; - } else { if (link_type == 5) { # Binomial.cauchit - Y_prob = 0.5 + (atan (finite_linear_terms) %*% p_one_m_one) / pi; - } else { - isNaN = 1; - }}}} - Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; - } } - - -# THE CG-STEIHAUG PROCEDURE SCRIPT - -# Apply Conjugate Gradient - Steihaug algorithm in order to approximately minimize -# 0.5 z^T (X^T diag(w) X + diag (lambda)) z + (g + lambda * beta)^T z -# under constraint: ||z|| <= trust_delta. -# See Alg. 7.2 on p. 171 of "Numerical Optimization" 2nd ed. by Nocedal and Wright -# IN THE ABOVE, "X" IS UNDERSTOOD TO BE "X %*% (SHIFT/SCALE TRANSFORM)"; this transform -# is given separately because sparse "X" may become dense after applying the transform. -# -get_CG_Steihaug_point = - function (Matrix[double] X, Matrix[double] scale_X, Matrix[double] shift_X, Matrix[double] w, - Matrix[double] g, Matrix[double] beta, Matrix[double] lambda, double trust_delta, int max_iter_CG) - return (Matrix[double] z, double neg_log_l_change, int i_CG, int reached_trust_boundary) -{ - trust_delta_sq = trust_delta ^ 2; - size_CG = nrow (g); - z = matrix (0.0, rows = size_CG, cols = 1); - neg_log_l_change = 0.0; - reached_trust_boundary = 0; - g_reg = g + lambda * beta; - r_CG = g_reg; - p_CG = -r_CG; - rr_CG = sum(r_CG * r_CG); - eps_CG = rr_CG * min (0.25, sqrt (rr_CG)); - converged_CG = 0; - if (rr_CG < eps_CG) { - converged_CG = 1; - } - - max_iteration_CG = max_iter_CG; - if (max_iteration_CG <= 0) { - max_iteration_CG = size_CG; - } - i_CG = 0; - while (converged_CG == 0) - { - i_CG = i_CG + 1; - ssX_p_CG = diag (scale_X) %*% p_CG; - ssX_p_CG [size_CG, ] = ssX_p_CG [size_CG, ] + t(shift_X) %*% p_CG; - temp_CG = t(X) %*% (w * (X %*% ssX_p_CG)); - q_CG = (lambda * p_CG) + diag (scale_X) %*% temp_CG + shift_X %*% temp_CG [size_CG, ]; - pq_CG = sum (p_CG * q_CG); - if (pq_CG <= 0) { - pp_CG = sum (p_CG * p_CG); - if (pp_CG > 0) { - [z, neg_log_l_change] = - get_trust_boundary_point (g_reg, z, p_CG, q_CG, r_CG, pp_CG, pq_CG, trust_delta_sq); - reached_trust_boundary = 1; - } else { - neg_log_l_change = 0.5 * sum (z * (r_CG + g_reg)); - } - converged_CG = 1; - } - if (converged_CG == 0) { - alpha_CG = rr_CG / pq_CG; - new_z = z + alpha_CG * p_CG; - if (sum(new_z * new_z) >= trust_delta_sq) { - pp_CG = sum (p_CG * p_CG); - [z, neg_log_l_change] = - get_trust_boundary_point (g_reg, z, p_CG, q_CG, r_CG, pp_CG, pq_CG, trust_delta_sq); - reached_trust_boundary = 1; - converged_CG = 1; - } - if (converged_CG == 0) { - z = new_z; - old_rr_CG = rr_CG; - r_CG = r_CG + alpha_CG * q_CG; - rr_CG = sum(r_CG * r_CG); - if (i_CG == max_iteration_CG | rr_CG < eps_CG) { - neg_log_l_change = 0.5 * sum (z * (r_CG + g_reg)); - reached_trust_boundary = 0; - converged_CG = 1; - } - if (converged_CG == 0) { - p_CG = -r_CG + (rr_CG / old_rr_CG) * p_CG; - } } } } } - - -# An auxiliary function used twice inside the CG-STEIHAUG loop: -get_trust_boundary_point = - function (Matrix[double] g, Matrix[double] z, Matrix[double] p, - Matrix[double] q, Matrix[double] r, double pp, double pq, - double trust_delta_sq) - return (Matrix[double] new_z, double f_change) -{ - zz = sum (z * z); pz = sum (p * z); - sq_root_d = sqrt (pz * pz - pp * (zz - trust_delta_sq)); - tau_1 = (- pz + sq_root_d) / pp; - tau_2 = (- pz - sq_root_d) / pp; - zq = sum (z * q); gp = sum (g * p); - f_extra = 0.5 * sum (z * (r + g)); - f_change_1 = f_extra + (0.5 * tau_1 * pq + zq + gp) * tau_1; - f_change_2 = f_extra + (0.5 * tau_2 * pq + zq + gp) * tau_2; - ind1 = as.integer(f_change_1 < f_change_2); - ind2 = as.integer(f_change_1 >= f_change_2); - new_z = z + ((ind1 * tau_1 + ind2 * tau_2) * p); - f_change = ind1 * f_change_1 + ind2 * f_change_2; -} - - -# Computes vector w such that ||X %*% w - 1|| -> MIN given avg(X %*% w) = 1 -# We find z_LS such that ||X %*% z_LS - 1|| -> MIN unconditionally, then scale -# it to compute w = c * z_LS such that sum(X %*% w) = nrow(X). -straightenX = - function (Matrix[double] X, double eps, int max_iter_CG) - return (Matrix[double] w) -{ - w_X = t(colSums(X)); - lambda_LS = 0.000001 * sum(X ^ 2) / ncol(X); - eps_LS = eps * nrow(X); - - # BEGIN LEAST SQUARES - - r_LS = - w_X; - z_LS = matrix (0.0, rows = ncol(X), cols = 1); - p_LS = - r_LS; - norm_r2_LS = sum (r_LS ^ 2); - i_LS = 0; - while (i_LS < max_iter_CG & i_LS < ncol(X) & norm_r2_LS >= eps_LS) - { - q_LS = t(X) %*% X %*% p_LS; - q_LS = q_LS + lambda_LS * p_LS; - alpha_LS = norm_r2_LS / sum (p_LS * q_LS); - z_LS = z_LS + alpha_LS * p_LS; - old_norm_r2_LS = norm_r2_LS; - r_LS = r_LS + alpha_LS * q_LS; - norm_r2_LS = sum (r_LS ^ 2); - p_LS = -r_LS + (norm_r2_LS / old_norm_r2_LS) * p_LS; - i_LS = i_LS + 1; - } - - # END LEAST SQUARES - - w = (nrow(X) / sum (w_X * z_LS)) * z_LS; - } - - \ No newline at end of file From 8aa902b2c6fc4ac8d1d8b3a5fa3e888ffb450499 Mon Sep 17 00:00:00 2001 From: bruno Date: Mon, 31 Aug 2026 09:35:35 +0200 Subject: [PATCH 019/132] replace misleading comments --- src/test/scripts/functions/builtin/stepGLM.dml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/scripts/functions/builtin/stepGLM.dml b/src/test/scripts/functions/builtin/stepGLM.dml index e8945a7b3de..392c6b6d245 100644 --- a/src/test/scripts/functions/builtin/stepGLM.dml +++ b/src/test/scripts/functions/builtin/stepGLM.dml @@ -44,15 +44,15 @@ for (i in 1:nrow(B)) { beta_est[idx, 1] = B[i, 1]; } -# Case 01 +# if beta_est and beta_true have the same sparsity if (nrow(B) != 3 | sum(beta_est != 0 & beta_true == 0) > 0 | sum(beta_est == 0 & beta_true != 0) > 0) { - stop("Test failed: Inexact feature support recovery."); + stop("Test failed: Unexpected non-zero element in beta_est"); } -# Case 02 +# if maximal element divergence remains below epsilon bound epsilon = 0.5; if (max(abs(beta_est - beta_true)) > epsilon) { - stop("Test failed: Parameter estimates exceed tolerance bound epsilon = " + epsilon + "."); + stop("Test failed: Element divergence exceeds epsilon=" + epsilon +" tolerance."); } From 0d9e6eb66b0b95e09fd4a33b133575aa7df00c3d Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Tue, 26 May 2026 01:30:47 +0200 Subject: [PATCH 020/132] [SYSTEMDS-2651] Extend TCP port polling to federated monitoring backend Wire startLocalFedMonitoring through FederatedWorkerUtils.waitForWorker so the monitoring backend's port-bind is polled instead of slept on (fixes flaky FederatedCoordinatorIntegrationCRUDTest), migrate FederatedLogicalTest to the bulk startLocalFedWorkers(int[]) API, and drop the now-unused FED_WORKER_WAIT_S and FED_MONITOR_WAIT constants. --- .../apache/sysds/test/AutomatedTestBase.java | 62 ++++++++++++------- .../part4/FederatedLogicalTest.java | 22 +++---- 2 files changed, 49 insertions(+), 35 deletions(-) diff --git a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java index 85a37b7dbd4..150a358bdf0 100644 --- a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java @@ -20,7 +20,6 @@ package org.apache.sysds.test; import static java.lang.Math.ceil; -import static java.lang.Thread.sleep; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -118,15 +117,10 @@ public abstract class AutomatedTestBase { public static final double GPU_TOLERANCE = 1e-9; /** - * Default upper bound (ms) passed to federated worker readiness waits. The wait returns as soon - * as the worker's TCP port accepts a connection, so this value only affects the deadline used - * when a worker never becomes ready. {@link FederatedWorkerUtils} clamps caller values below its - * enforced floor up to that floor, so the effective ceiling is at least that floor regardless - * of this constant. + * Default deadline (ms) for federated worker/monitoring readiness waits and a few legacy + * {@code sleep()} calls. {@link FederatedWorkerUtils} enforces its own minimum floor. */ public static final int FED_WORKER_WAIT = 3000; - public static final int FED_MONITOR_WAIT = 10000; - public static final int FED_WORKER_WAIT_S = 50; // The timeout for a test to fail. all tests must execute in less than this time. @@ -1765,29 +1759,53 @@ private static Process spawnLocalFedWorker(int port, String[] addArgs) { } /** - * Start new JVM for a federated monitoring backend at the port. + * Start a new JVM for a federated monitoring backend at the port. * - * @param port Port to use for the JVM - * @return the process associated with the worker. + *

Returns once the backend's TCP port accepts connections (Netty's bind has completed), or + * throws a {@link RuntimeException} once the {@link FederatedWorkerUtils} readiness floor + * elapses. + * + * @param port Port to use for the JVM + * @param addArgs Extra CLI args to append, or null + * @return the process associated with the monitoring backend. */ protected Process startLocalFedMonitoring(int port, String[] addArgs) { - Process process = null; + return startLocalFedMonitoring(port, addArgs, FED_WORKER_WAIT); + } + + /** + * Start a new JVM for a federated monitoring backend at the port. + * + *

Returns once the backend's TCP port accepts connections, or throws a + * {@link RuntimeException} after {@code timeoutMs} elapses. The monitoring server opens the + * port after Netty's {@code bind().sync()} returns; a successful TCP connect therefore signals + * that the HTTP listener is ready to accept requests. + * + * @param port Port to use for the JVM + * @param addArgs Extra CLI args to append, or null + * @param timeoutMs Upper bound on the wait, in ms; raised to a minimum value enforced inside + * {@link FederatedWorkerUtils}. + * @return the process associated with the monitoring backend. + */ + protected Process startLocalFedMonitoring(int port, String[] addArgs, int timeoutMs) { + Process process = spawnLocalFedMonitoring(port, addArgs); + FederatedWorkerUtils.waitForWorker(port, timeoutMs, process::isAlive, "monitoring process"); + return process; + } + + /** Spawn a federated monitoring backend JVM and return without waiting for the port to bind. */ + private static Process spawnLocalFedMonitoring(int port, String[] addArgs) { String separator = System.getProperty("file.separator"); String classpath = System.getProperty("java.class.path"); String path = System.getProperty("java.home") + separator + "bin" + separator + "java"; - String[] args = ArrayUtils.addAll(new String[]{path, "-cp", classpath, DMLScript.class.getName(), - "-fedMonitoring", Integer.toString(port)}, addArgs); - ProcessBuilder processBuilder = new ProcessBuilder(args); - + String[] args = ArrayUtils.addAll(new String[] {path, "-cp", classpath, DMLScript.class.getName(), + "-fedMonitoring", Integer.toString(port)}, addArgs); try { - process = processBuilder.start(); - // Wait till process is started - sleep(FED_MONITOR_WAIT); + return new ProcessBuilder(args).start(); } - catch(IOException | InterruptedException e) { - throw new RuntimeException(e); + catch(IOException e) { + throw new RuntimeException("Failed to launch federated monitoring process on port " + port, e); } - return process; } /** diff --git a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java index ba1e7e0ea08..f8acdd07930 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java @@ -372,17 +372,15 @@ public void federatedLogicalTest(String testname, Type op_type, ExecMode execMod // empty script name because we don't execute any script, just start the worker fullDMLScriptName = ""; int port1 = getRandomAvailablePort(); - int port2 = (!single_fed_worker ? getRandomAvailablePort() : 0); - int port3 = (!single_fed_worker ? getRandomAvailablePort() : 0); - int port4 = (!single_fed_worker ? getRandomAvailablePort() : 0); - Process thread1 = startLocalFedWorker(port1, (!single_fed_worker ? FED_WORKER_WAIT_S : FED_WORKER_WAIT)); - Process thread2 = (!single_fed_worker ? startLocalFedWorker(port2, FED_WORKER_WAIT_S) : null); - Process thread3 = (!single_fed_worker ? startLocalFedWorker(port3, FED_WORKER_WAIT_S) : null); - Process thread4 = (!single_fed_worker ? startLocalFedWorker(port4) : null); - - + int port2 = single_fed_worker ? 0 : getRandomAvailablePort(); + int port3 = single_fed_worker ? 0 : getRandomAvailablePort(); + int port4 = single_fed_worker ? 0 : getRandomAvailablePort(); + Process[] workers = startLocalFedWorkers(single_fed_worker + ? new int[] {port1} + : new int[] {port1, port2, port3, port4}); + try { - if(!isAlive(thread1)) + if(!isAlive(workers)) throw new RuntimeException("Failed starting federated worker"); getAndLoadTestConfiguration(testname); @@ -449,9 +447,7 @@ public void federatedLogicalTest(String testname, Type op_type, ExecMode execMod } } finally { - TestUtils.shutdownThreads(thread1); - if(!single_fed_worker) - TestUtils.shutdownThreads(thread2, thread3, thread4); + TestUtils.shutdownThreads(workers); resetExecMode(platform_old); } From a8dabe0dfbbf4fdc38f2f6bd276f72b9585dc460 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Tue, 26 May 2026 16:46:13 +0200 Subject: [PATCH 021/132] [SYSTEMDS-2651] Poll for async compression in federated component tests (#2472) FedWorkerReadMatrixCompress.verifyRead failed roughly once per ten component-test CI runs because it called FederatedTestUtils.wait(1000) to give the worker time to finish its async compression (kicked off by CompressedMatrixBlockFactory.compressAsync), then asserted that the returned block was a CompressedMatrixBlock. On a contended runner the 1 s sleep was not enough, the subsequent read returned the still- uncompressed block, and the assertion failed. Surefire's rerunFailingTestsCount=2 hid this as a "Flake" rather than a job failure. Add FedWorkerBase.awaitCompressed(long id), which polls getMatrixBlock at 25 ms intervals for up to COMPRESS_TIMEOUT_MS (10 s) and returns as soon as the worker reports the compressed form, or returns the last- observed block on timeout so the caller's assertion still produces a meaningful failure. Convert the three call sites that used the fixed-sleep anti-pattern: - FedWorkerReadMatrixCompress.verifyRead (the actual CI flake) - FedWorkerMatrixCompress.verifySameOrAlsoCompressedAsLocalCompress (polls only when local compresses, so the "do not compress" parametrization stays fast) - FedWorkerMatrixMultiplyWorkload.verifySameOrAlsoCompressedAsLocalCompress Remove the now-unused FederatedTestUtils.wait helper so the anti-pattern is harder to reintroduce. --- .../component/federated/FedWorkerBase.java | 39 +++++++++++++++++++ .../federated/FedWorkerMatrixCompress.java | 12 +++--- .../FedWorkerMatrixMultiplyWorkload.java | 13 +++---- .../FedWorkerReadMatrixCompress.java | 7 ++-- .../federated/FederatedTestUtils.java | 9 ----- 5 files changed, 54 insertions(+), 26 deletions(-) diff --git a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java index 1bf5d330066..2c854b4a81b 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java @@ -26,12 +26,19 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.sysds.runtime.compress.CompressedMatrixBlock; import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.test.AutomatedTestBase; public abstract class FedWorkerBase { protected static final Log LOG = LogFactory.getLog(FedWorkerBase.class.getName()); + /** Upper bound (ms) for {@link #awaitCompressed(long)} polling against async worker-side compression. */ + protected static final int COMPRESS_TIMEOUT_MS = 10_000; + + /** Poll interval used by {@link #awaitCompressed(long)} between successive reads. */ + private static final int COMPRESS_POLL_INTERVAL_MS = 25; + private final InetSocketAddress addr; public final int port; @@ -70,6 +77,38 @@ public MatrixBlock getMatrixBlock(long id) { return FederatedTestUtils.getMatrixBlock(id, addr); } + /** + * Poll the federated worker until the matrix at {@code id} is observed as a + * {@link CompressedMatrixBlock}, or {@link #COMPRESS_TIMEOUT_MS} elapses. + * + *

Federated workers compress asynchronously after a PUT/READ_VAR (see + * {@code CompressedMatrixBlockFactory.compressAsync}), so a {@code getMatrixBlock} fired right + * after the operation can race against the in-flight compression and return the uncompressed + * block. Tests that need to observe the compressed form should poll instead of sleeping a fixed + * amount. + * + *

On timeout this returns the most recent (uncompressed) read so the caller can produce a + * meaningful assertion failure naming the variable. + * + * @param id federated variable id + * @return the matrix block, compressed if compression finished in time, otherwise the latest read + */ + public MatrixBlock awaitCompressed(long id) { + final long deadline = System.currentTimeMillis() + COMPRESS_TIMEOUT_MS; + MatrixBlock mb = getMatrixBlock(id); + while(!(mb instanceof CompressedMatrixBlock) && System.currentTimeMillis() < deadline) { + try { + Thread.sleep(COMPRESS_POLL_INTERVAL_MS); + } + catch(InterruptedException ie) { + Thread.currentThread().interrupt(); + fail("Interrupted while waiting for federated compression of id=" + id); + } + mb = getMatrixBlock(id); + } + return mb; + } + public long matrixMult(long idLeft, long idRight) { return FederatedTestUtils.exec_MM(idLeft, idRight, addr); } diff --git a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java index 29c6f94e7a3..2b5ff327ef3 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java @@ -65,14 +65,16 @@ public void verifySameOrAlsoCompressedAsLocalCompress() { // local final MatrixBlock mbcLocal = CompressedMatrixBlockFactory.compress(mb).getLeft(); - // federated + // federated. Compression on the worker is async; poll only when we expect compression to + // match the local result, otherwise a single read is enough. final long id = putMatrixBlock(mb); - // give the federated site time to compress async. - FederatedTestUtils.wait(1000); - final MatrixBlock mbr = getMatrixBlock(id); + final MatrixBlock mbr = (mbcLocal instanceof CompressedMatrixBlock) + ? awaitCompressed(id) + : getMatrixBlock(id); if(mbcLocal instanceof CompressedMatrixBlock && !(mbr instanceof CompressedMatrixBlock)) - fail("Invalid result, the federated site did not compress the matrix block"); + fail("Invalid result, the federated site did not compress the matrix block within " + + COMPRESS_TIMEOUT_MS + "ms"); TestUtils.compareMatricesBitAvgDistance(mbcLocal, mbr, 0, 0, "Not equivalent matrix block returned from federated site"); diff --git a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixMultiplyWorkload.java b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixMultiplyWorkload.java index 06a193368c1..59c9a093c40 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixMultiplyWorkload.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixMultiplyWorkload.java @@ -88,19 +88,16 @@ public void verifySameOrAlsoCompressedAsLocalCompress() { for(int i = 0; i < 9; i++) // chain left side compressed multiplications with idr. ide = matrixMult(ide, idr); - // give the federated site time to compress async (it should already be done, but just to be safe). - FederatedTestUtils.wait(1000); - - // Get back the matrix block stored behind mbr that should be compressed now. - final MatrixBlock mbr_compressed = getMatrixBlock(idr); + // Workload-driven compression runs async on the worker; poll instead of sleeping a fixed + // amount so a slow runner doesn't observe the still-uncompressed block. + final MatrixBlock mbr_compressed = awaitCompressed(idr); if(!(mbr_compressed instanceof CompressedMatrixBlock)) - fail("Invalid result, the federated site did not compress the matrix block based on workload"); + fail("Invalid result, the federated site did not compress the matrix block based on workload within " + + COMPRESS_TIMEOUT_MS + "ms"); TestUtils.compareMatricesBitAvgDistance(mbcLocal, mbr_compressed, 0, 0, "Not equivalent matrix block returned from federated site"); } - - } diff --git a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerReadMatrixCompress.java b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerReadMatrixCompress.java index ed47a87e1e8..d94cd367a1d 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerReadMatrixCompress.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerReadMatrixCompress.java @@ -65,15 +65,14 @@ public FedWorkerReadMatrixCompress(int port, String path) { public void verifyRead() { MatrixBlock expected = readCSV(); Long id = readMatrix(path); - // give the federated site time to compress async. - FederatedTestUtils.wait(1000); - MatrixBlock actual = getMatrixBlock(id); + // Compression happens async on the worker; poll instead of sleeping a fixed amount. + MatrixBlock actual = awaitCompressed(id); if(actual instanceof CompressedMatrixBlock){ TestUtils.compareMatricesBitAvgDistance(expected, actual, 0, 0, "Not equivalent matrix block read from federated site"); } else - fail("Did not compress the matrix input"); + fail("Did not compress the matrix input within " + COMPRESS_TIMEOUT_MS + "ms"); } protected MatrixBlock readCSV() { diff --git a/src/test/java/org/apache/sysds/test/component/federated/FederatedTestUtils.java b/src/test/java/org/apache/sysds/test/component/federated/FederatedTestUtils.java index 4d3796892aa..9b589c35f7d 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FederatedTestUtils.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FederatedTestUtils.java @@ -190,13 +190,4 @@ private static void exec(long id, String inst, InetSocketAddress addr, int timeo fail("Failed to get response from put Matrix Block"); } } - - protected static void wait(int ms) { - try { - Thread.sleep(ms); - } - catch(Exception e) { - fail("Failed to wait"); - } - } } From 4fccae46cfc3ee25b4849d4c63ef72dc28072123 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Thu, 28 May 2026 15:07:30 +0200 Subject: [PATCH 022/132] [MINOR] Upgrade Surefire to 3.5.2 and enforce fork exit timeout Fix intermittent ~26 minute hangs in the **.component.c**.** GitHub Actions job. The forks were finishing their test classes but failing to exit cleanly (leaked non-daemon threads from test executors), and Surefire 3.0.0 did not reliably enforce its fork shutdown timeout. --- pom.xml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c4206532a62..dc9783f8571 100644 --- a/pom.xml +++ b/pom.xml @@ -53,7 +53,7 @@ 2.0.11 2.22.1 3.2.0 - 3.0.0 + 3.5.2 3.3.1 3.0.0 3.5.0 @@ -410,6 +410,10 @@ false ${test-forkedProcessTimeout} + + 30 + + native true ${rerun.failing.tests.count} From 98760575f1180ade6848558252b37d7a869032cd Mon Sep 17 00:00:00 2001 From: Matthias Boehm Date: Tue, 26 May 2026 19:57:00 +0200 Subject: [PATCH 023/132] [MINOR] Fix incorrect formatting of various tests --- .../functions/indexing/LeftIndexingTest.java | 75 ++-- .../io/parquet/FrameParquetSchemaTest.java | 354 +++++++++--------- .../functions/jmlc/JMLConnectionTest.java | 62 +-- .../misc/NrowNcolUnknownCSVReadTest.java | 4 +- .../functions/reorg/MatrixReshapeTest.java | 2 +- .../functions/reorg/VectorReshapeTest.java | 2 +- ...writeQuantizationFusedCompressionTest.java | 222 +++++------ .../TransformFrameEncodeBagOfWords.java | 10 +- .../vect/LeftIndexingChainUpdateTest.java | 10 +- 9 files changed, 370 insertions(+), 371 deletions(-) diff --git a/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java b/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java index ced7efb71a9..dbbf199f8fa 100644 --- a/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java +++ b/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java @@ -72,34 +72,33 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod } if(et == ExecType.SPARK) { - rtplatform = ExecMode.SPARK; - } + rtplatform = ExecMode.SPARK; + } else { // rtplatform = (et==ExecType.MR)? ExecMode.HADOOP : ExecMode.SINGLE_NODE; - rtplatform = ExecMode.HYBRID; + rtplatform = ExecMode.HYBRID; } if( rtplatform == ExecMode.SPARK ) DMLScript.USE_LOCAL_SPARK_CONFIG = true; - - config.addVariable("rows", rows); - config.addVariable("cols", cols); - - long rowstart=816, rowend=1229, colstart=967, colend=1009; - // long rowstart=2, rowend=4, colstart=9, colend=10; - /* - Random rand=new Random(System.currentTimeMillis()); - rowstart=(long)(rand.nextDouble()*((double)rows))+1; - rowend=(long)(rand.nextDouble()*((double)(rows-rowstart+1)))+rowstart; - colstart=(long)(rand.nextDouble()*((double)cols))+1; - colend=(long)(rand.nextDouble()*((double)(cols-colstart+1)))+colstart; - */ - config.addVariable("rowstart", rowstart); - config.addVariable("rowend", rowend); - config.addVariable("colstart", colstart); - config.addVariable("colend", colend); + config.addVariable("rows", rows); + config.addVariable("cols", cols); + + long rowstart=816, rowend=1229, colstart=967, colend=1009; + // long rowstart=2, rowend=4, colstart=9, colend=10; + /* + Random rand=new Random(System.currentTimeMillis()); + rowstart=(long)(rand.nextDouble()*((double)rows))+1; + rowend=(long)(rand.nextDouble()*((double)(rows-rowstart+1)))+rowstart; + colstart=(long)(rand.nextDouble()*((double)cols))+1; + colend=(long)(rand.nextDouble()*((double)(cols-colstart+1)))+colstart; + */ + config.addVariable("rowstart", rowstart); + config.addVariable("rowend", rowend); + config.addVariable("colstart", colstart); + config.addVariable("colend", colend); loadTestConfiguration(config); - + /* This is for running the junit test the new way, i.e., construct the arguments directly */ String LI_HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = LI_HOME + "LeftIndexingTest" + ".dml"; @@ -118,31 +117,31 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod inputDir() + " " + rowstart + " " + rowend + " " + colstart + " " + colend + " " + expectedDir(); double sparsity=1.0;//rand.nextDouble(); - double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, System.currentTimeMillis()); - writeInputMatrix("A", A, true); - - sparsity=0.1;//rand.nextDouble(); - double[][] B = getRandomMatrix((int)(rowend-rowstart+1), (int)(colend-colstart+1), min, max, sparsity, System.currentTimeMillis()); - writeInputMatrix("B", B, true); - - sparsity=0.5;//rand.nextDouble(); - double[][] C = getRandomMatrix((int)(rowend), (int)(cols-colstart+1), min, max, sparsity, System.currentTimeMillis()); - writeInputMatrix("C", C, true); - - sparsity=0.01;//rand.nextDouble(); - double[][] D = getRandomMatrix(rows, (int)(colend-colstart+1), min, max, sparsity, System.currentTimeMillis()); - writeInputMatrix("D", D, true); - + double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, System.currentTimeMillis()); + writeInputMatrix("A", A, true); + + sparsity=0.1;//rand.nextDouble(); + double[][] B = getRandomMatrix((int)(rowend-rowstart+1), (int)(colend-colstart+1), min, max, sparsity, System.currentTimeMillis()); + writeInputMatrix("B", B, true); + + sparsity=0.5;//rand.nextDouble(); + double[][] C = getRandomMatrix((int)(rowend), (int)(cols-colstart+1), min, max, sparsity, System.currentTimeMillis()); + writeInputMatrix("C", C, true); + + sparsity=0.01;//rand.nextDouble(); + double[][] D = getRandomMatrix(rows, (int)(colend-colstart+1), min, max, sparsity, System.currentTimeMillis()); + writeInputMatrix("D", D, true); + /* * Expected number of jobs: * Reblock - 1 job * While loop iteration - 10 jobs * Final output write - 1 job */ - //boolean exceptionExpected = false; + //boolean exceptionExpected = false; //int expectedNumberOfJobs = 12; //runTest(exceptionExpected, null, expectedNumberOfJobs); - boolean exceptionExpected = false; + boolean exceptionExpected = false; int expectedNumberOfJobs = -1; runTest(true, exceptionExpected, null, expectedNumberOfJobs); } diff --git a/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java index dc776c8eab2..1e4334891ed 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java @@ -43,181 +43,181 @@ */ public class FrameParquetSchemaTest extends AutomatedTestBase { - private final static String TEST_NAME = "FrameParquetSchemaTest"; - private final static String TEST_DIR = "functions/io/parquet"; - private final static String TEST_CLASS_DIR = TEST_DIR + FrameParquetSchemaTest.class.getSimpleName() + "/"; - - @Override - public void setUp() { - addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[]{"Rout"})); - } - - - /** - * Test for sequential writer and reader - * - */ - @Test - public void testParquetWriteReadAllSchemaTypes() { - String fname = output("Rout"); - - // Define a schema with one column per type - ValueType[] schema = new ValueType[] { - ValueType.FP64, - ValueType.FP32, - ValueType.INT32, - ValueType.INT64, - ValueType.BOOLEAN, - ValueType.STRING - }; - - // Create an empty frame block with the above schema - FrameBlock fb = new FrameBlock(schema); - - // Populate frame block - Object[][] rows = new Object[][] { - { 1.0, 1.1f, 10, 100L, true, "A" }, - { 2.0, 2.1f, 20, 200L, false, "B" }, - { 3.0, 3.1f, 30, 300L, true, "C" }, - { 4.0, 4.1f, 40, 400L, false, "D" }, - { 5.0, 5.1f, 50, 500L, true, "E" } - }; - - for (Object[] row : rows) { - fb.appendRow(row); - } - - System.out.println(fb); - - int numRows = fb.getNumRows(); - int numCols = fb.getNumColumns(); - - // Write the FrameBlock to a Parquet file using the sequential writer - try { - FrameWriter writer = new FrameWriterParquet(); - writer.writeFrameToHDFS(fb, fname, numRows, numCols); - } - catch (IOException e) { - e.printStackTrace(); - Assert.fail("Failed to write frame block to Parquet: " + e.getMessage()); - } - - // Read the Parquet file back into a new FrameBlock - FrameBlock fbRead = null; - try { - FrameReader reader = new FrameReaderParquet(); - String[] colNames = fb.getColumnNames(); - fbRead = reader.readFrameFromHDFS(fname, schema, colNames, numRows, numCols); - } - catch (IOException e) { - e.printStackTrace(); - Assert.fail("Failed to read frame block from Parquet: " + e.getMessage()); - } - - // Compare the original and the read frame blocks - compareFrameBlocks(fb, fbRead, 1e-6); - } - - /** - * Test for multithreaded writer and reader - * - */ - @Test - public void testParquetWriteReadAllSchemaTypesParallel() { - String fname = output("Rout_parallel"); - - ValueType[] schema = new ValueType[] { - ValueType.FP64, - ValueType.FP32, - ValueType.INT32, - ValueType.INT64, - ValueType.BOOLEAN, - ValueType.STRING - }; - - FrameBlock fb = new FrameBlock(schema); - - Object[][] rows = new Object[][] { - { 1.0, 1.1f, 10, 100L, true, "A" }, - { 2.0, 2.1f, 20, 200L, false, "B" }, - { 3.0, 3.1f, 30, 300L, true, "C" }, - { 4.0, 4.1f, 40, 400L, false, "D" }, - { 5.0, 5.1f, 50, 500L, true, "E" } - }; - - for (Object[] row : rows) { - fb.appendRow(row); - } - - int numRows = fb.getNumRows(); - int numCols = fb.getNumColumns(); - - try { - FrameWriter writer = new FrameWriterParquetParallel(); - writer.writeFrameToHDFS(fb, fname, numRows, numCols); - } - catch (IOException e) { - e.printStackTrace(); - Assert.fail("Failed to write frame block to Parquet (parallel): " + e.getMessage()); - } - - FrameBlock fbRead = null; - try { - FrameReader reader = new FrameReaderParquetParallel(); - String[] colNames = fb.getColumnNames(); - fbRead = reader.readFrameFromHDFS(fname, schema, colNames, numRows, numCols); - } - catch (IOException e) { - e.printStackTrace(); - Assert.fail("Failed to read frame block from Parquet (parallel): " + e.getMessage()); - } - - compareFrameBlocks(fb, fbRead, 1e-6); - } - - private void compareFrameBlocks(FrameBlock expected, FrameBlock actual, double eps) { - Assert.assertEquals("Number of rows mismatch", expected.getNumRows(), actual.getNumRows()); - Assert.assertEquals("Number of columns mismatch", expected.getNumColumns(), actual.getNumColumns()); - - int rows = expected.getNumRows(); - int cols = expected.getNumColumns(); - - for (int i = 0; i < rows; i++) { - for (int j = 0; j < cols; j++) { - Object expVal = expected.get(i, j); - Object actVal = actual.get(i, j); - ValueType vt = expected.getSchema()[j]; - - // Handle nulls first - if(expVal == null || actVal == null) { - Assert.assertEquals("Mismatch at (" + i + "," + j + ")", expVal, actVal); - } else { - switch(vt) { - case FP64: - case FP32: - double dExp = ((Number) expVal).doubleValue(); - double dAct = ((Number) actVal).doubleValue(); - Assert.assertEquals("Mismatch at (" + i + "," + j + ")", dExp, dAct, eps); - break; - case INT32: - case INT64: - long lExp = ((Number) expVal).longValue(); - long lAct = ((Number) actVal).longValue(); - Assert.assertEquals("Mismatch at (" + i + "," + j + ")", lExp, lAct); - break; - case BOOLEAN: - boolean bExp = (Boolean) expVal; - boolean bAct = (Boolean) actVal; - Assert.assertEquals("Mismatch at (" + i + "," + j + ")", bExp, bAct); - break; - case STRING: - Assert.assertEquals("Mismatch at (" + i + "," + j + ")", expVal.toString(), actVal.toString()); - break; - default: - Assert.fail("Unsupported type in comparison: " + vt); - } - } - } - } - } + private final static String TEST_NAME = "FrameParquetSchemaTest"; + private final static String TEST_DIR = "functions/io/parquet"; + private final static String TEST_CLASS_DIR = TEST_DIR + FrameParquetSchemaTest.class.getSimpleName() + "/"; + + @Override + public void setUp() { + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[]{"Rout"})); + } + + + /** + * Test for sequential writer and reader + * + */ + @Test + public void testParquetWriteReadAllSchemaTypes() { + String fname = output("Rout"); + + // Define a schema with one column per type + ValueType[] schema = new ValueType[] { + ValueType.FP64, + ValueType.FP32, + ValueType.INT32, + ValueType.INT64, + ValueType.BOOLEAN, + ValueType.STRING + }; + + // Create an empty frame block with the above schema + FrameBlock fb = new FrameBlock(schema); + + // Populate frame block + Object[][] rows = new Object[][] { + { 1.0, 1.1f, 10, 100L, true, "A" }, + { 2.0, 2.1f, 20, 200L, false, "B" }, + { 3.0, 3.1f, 30, 300L, true, "C" }, + { 4.0, 4.1f, 40, 400L, false, "D" }, + { 5.0, 5.1f, 50, 500L, true, "E" } + }; + + for (Object[] row : rows) { + fb.appendRow(row); + } + + System.out.println(fb); + + int numRows = fb.getNumRows(); + int numCols = fb.getNumColumns(); + + // Write the FrameBlock to a Parquet file using the sequential writer + try { + FrameWriter writer = new FrameWriterParquet(); + writer.writeFrameToHDFS(fb, fname, numRows, numCols); + } + catch (IOException e) { + e.printStackTrace(); + Assert.fail("Failed to write frame block to Parquet: " + e.getMessage()); + } + + // Read the Parquet file back into a new FrameBlock + FrameBlock fbRead = null; + try { + FrameReader reader = new FrameReaderParquet(); + String[] colNames = fb.getColumnNames(); + fbRead = reader.readFrameFromHDFS(fname, schema, colNames, numRows, numCols); + } + catch (IOException e) { + e.printStackTrace(); + Assert.fail("Failed to read frame block from Parquet: " + e.getMessage()); + } + + // Compare the original and the read frame blocks + compareFrameBlocks(fb, fbRead, 1e-6); + } + + /** + * Test for multithreaded writer and reader + * + */ + @Test + public void testParquetWriteReadAllSchemaTypesParallel() { + String fname = output("Rout_parallel"); + + ValueType[] schema = new ValueType[] { + ValueType.FP64, + ValueType.FP32, + ValueType.INT32, + ValueType.INT64, + ValueType.BOOLEAN, + ValueType.STRING + }; + + FrameBlock fb = new FrameBlock(schema); + + Object[][] rows = new Object[][] { + { 1.0, 1.1f, 10, 100L, true, "A" }, + { 2.0, 2.1f, 20, 200L, false, "B" }, + { 3.0, 3.1f, 30, 300L, true, "C" }, + { 4.0, 4.1f, 40, 400L, false, "D" }, + { 5.0, 5.1f, 50, 500L, true, "E" } + }; + + for (Object[] row : rows) { + fb.appendRow(row); + } + + int numRows = fb.getNumRows(); + int numCols = fb.getNumColumns(); + + try { + FrameWriter writer = new FrameWriterParquetParallel(); + writer.writeFrameToHDFS(fb, fname, numRows, numCols); + } + catch (IOException e) { + e.printStackTrace(); + Assert.fail("Failed to write frame block to Parquet (parallel): " + e.getMessage()); + } + + FrameBlock fbRead = null; + try { + FrameReader reader = new FrameReaderParquetParallel(); + String[] colNames = fb.getColumnNames(); + fbRead = reader.readFrameFromHDFS(fname, schema, colNames, numRows, numCols); + } + catch (IOException e) { + e.printStackTrace(); + Assert.fail("Failed to read frame block from Parquet (parallel): " + e.getMessage()); + } + + compareFrameBlocks(fb, fbRead, 1e-6); + } + + private void compareFrameBlocks(FrameBlock expected, FrameBlock actual, double eps) { + Assert.assertEquals("Number of rows mismatch", expected.getNumRows(), actual.getNumRows()); + Assert.assertEquals("Number of columns mismatch", expected.getNumColumns(), actual.getNumColumns()); + + int rows = expected.getNumRows(); + int cols = expected.getNumColumns(); + + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + Object expVal = expected.get(i, j); + Object actVal = actual.get(i, j); + ValueType vt = expected.getSchema()[j]; + + // Handle nulls first + if(expVal == null || actVal == null) { + Assert.assertEquals("Mismatch at (" + i + "," + j + ")", expVal, actVal); + } else { + switch(vt) { + case FP64: + case FP32: + double dExp = ((Number) expVal).doubleValue(); + double dAct = ((Number) actVal).doubleValue(); + Assert.assertEquals("Mismatch at (" + i + "," + j + ")", dExp, dAct, eps); + break; + case INT32: + case INT64: + long lExp = ((Number) expVal).longValue(); + long lAct = ((Number) actVal).longValue(); + Assert.assertEquals("Mismatch at (" + i + "," + j + ")", lExp, lAct); + break; + case BOOLEAN: + boolean bExp = (Boolean) expVal; + boolean bAct = (Boolean) actVal; + Assert.assertEquals("Mismatch at (" + i + "," + j + ")", bExp, bAct); + break; + case STRING: + Assert.assertEquals("Mismatch at (" + i + "," + j + ")", expVal.toString(), actVal.toString()); + break; + default: + Assert.fail("Unsupported type in comparison: " + vt); + } + } + } + } + } } diff --git a/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java b/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java index ac77371dfc3..dfb3d8a19de 100644 --- a/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java @@ -43,11 +43,11 @@ @net.jcip.annotations.NotThreadSafe public class JMLConnectionTest extends AutomatedTestBase { public static final String META = "{\"data_type\": \"matrix\",\n" + - " \"value_type\": \"double\", \n" + - " \"rows\": 1,\n" + - " \"cols\": 1,\n" + - " \"nnz\": 1,\n" + - " \"format\": \"csv\"}"; + " \"value_type\": \"double\", \n" + + " \"rows\": 1,\n" + + " \"cols\": 1,\n" + + " \"nnz\": 1,\n" + + " \"format\": \"csv\"}"; private final static String TEST_NAME = "JMLConnectionTest"; private final static String TEST_DIR = "functions/jmlc/"; @@ -99,36 +99,36 @@ public void testConnectionInvalidInName() throws DMLException { conn.gatherMemStats(false); Assert.assertFalse(DMLScript.STATISTICS); - try (conn) { - conn.prepareScript("printx('hello')", new String[]{"$inScalar1", null}, new String[]{null}); - throw new AssertionError("Test should have thrown a LanguageException"); - } catch (LanguageException e) { - Assert.assertTrue(e.getMessage().startsWith("Invalid variable names")); - } finally { - DMLScript.STATISTICS = oldStat; - DMLScript.JMLC_MEM_STATISTICS = oldJMLCStat; - } + try (conn) { + conn.prepareScript("printx('hello')", new String[]{"$inScalar1", null}, new String[]{null}); + throw new AssertionError("Test should have thrown a LanguageException"); + } catch (LanguageException e) { + Assert.assertTrue(e.getMessage().startsWith("Invalid variable names")); + } finally { + DMLScript.STATISTICS = oldStat; + DMLScript.JMLC_MEM_STATISTICS = oldJMLCStat; + } } @Test public void testConnectionParseLanguageException() { - try (Connection conn = new Connection()) { - conn.prepareScript("printx('hello')", new String[]{}, new String[]{}); - throw new AssertionError("Test should have thrown a DMLException"); - } catch (DMLException e) { - Throwable cause = e.getCause(); - Assert.assertTrue(cause.getMessage().startsWith("ERROR: [line 1:0] -> printx('hello') -- function printx is undefined in namespace .builtinNS")); - } + try (Connection conn = new Connection()) { + conn.prepareScript("printx('hello')", new String[]{}, new String[]{}); + throw new AssertionError("Test should have thrown a DMLException"); + } catch (DMLException e) { + Throwable cause = e.getCause(); + Assert.assertTrue(cause.getMessage().startsWith("ERROR: [line 1:0] -> printx('hello') -- function printx is undefined in namespace .builtinNS")); + } } @Test public void testConnectionParseException() { - try (Connection conn = new Connection()) { - conn.prepareScript("print('hello'", new String[]{}, new String[]{}); - throw new AssertionError("Test should have thrown a ParseException"); - } catch (Exception e) { - Assert.assertEquals("ParseException", e.getClass().getSimpleName()); - } + try (Connection conn = new Connection()) { + conn.prepareScript("print('hello'", new String[]{}, new String[]{}); + throw new AssertionError("Test should have thrown a ParseException"); + } catch (Exception e) { + Assert.assertEquals("ParseException", e.getClass().getSimpleName()); + } } @Test @@ -144,11 +144,11 @@ public void testConnectionClose() { @Test public void testReadScriptHDFS() { - try (Connection conn = new Connection()) { - conn.readScript("hdfs://localhost:9000/Test"); - } catch (IOException e) { + try (Connection conn = new Connection()) { + conn.readScript("hdfs://localhost:9000/Test"); + } catch (IOException e) { Assert.assertEquals("ConnectException",e.getClass().getSimpleName()); - } + } } @Test diff --git a/src/test/java/org/apache/sysds/test/functions/misc/NrowNcolUnknownCSVReadTest.java b/src/test/java/org/apache/sysds/test/functions/misc/NrowNcolUnknownCSVReadTest.java index 1fe3913ad6f..65d5a5092ee 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/NrowNcolUnknownCSVReadTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/NrowNcolUnknownCSVReadTest.java @@ -93,10 +93,10 @@ private void runNxxUnkownCSVTest( String testName ) MatrixBlock mb = DataConverter.convertToMatrixBlock(A); DataConverter.writeMatrixToHDFS(mb, input("A"), FileFormat.CSV, new MatrixCharacteristics(rows,cols,-1,-1)); - HDFSTool.deleteFileIfExistOnHDFS(input("A.mtd")); + HDFSTool.deleteFileIfExistOnHDFS(input("A.mtd")); //run tests - runTest(true, false, null, -1); + runTest(true, false, null, -1); } catch(Exception ex) { diff --git a/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java index 1c72d026232..5f42db7d733 100644 --- a/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java @@ -336,7 +336,7 @@ private void runTestMatrixReshape( ReshapeType type, boolean rowwise, boolean sp fullRScriptName = HOME + TEST_NAME + ".R"; rCmd = "Rscript" + " " + fullRScriptName + " " + - inputDir() + " " + trows + " " + tcols + " " + expectedDir(); + inputDir() + " " + trows + " " + tcols + " " + expectedDir(); double[][] X = getRandomMatrix(rows, cols, 0, 1, sparsity, 7); writeInputMatrix("X", X, true); diff --git a/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java index 72622ab0154..dcdafddcd47 100644 --- a/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java @@ -95,7 +95,7 @@ private void runVectorReshape(boolean sparse, ExecType et) fullRScriptName = HOME + TEST_NAME + ".R"; rCmd = "Rscript" + " " + fullRScriptName + " " + - inputDir() + " " + rows2 + " " + cols2 + " " + expectedDir(); + inputDir() + " " + rows2 + " " + cols2 + " " + expectedDir(); double sparsity = sparse ? sparsitySparse : sparsityDense; double[][] X = getRandomMatrix(rows1, cols1, 0, 1, sparsity, 7); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java index 3a9dfa48dda..e8e885f905f 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java @@ -35,115 +35,115 @@ * */ public class RewriteQuantizationFusedCompressionTest extends AutomatedTestBase { - private static final String TEST_NAME1 = "RewriteQuantizationFusedCompressionScalar"; - private static final String TEST_NAME2 = "RewriteQuantizationFusedCompressionMatrix"; - private static final String TEST_DIR = "functions/rewrite/"; - private static final String TEST_CLASS_DIR = TEST_DIR - + RewriteQuantizationFusedCompressionTest.class.getSimpleName() + "/"; - - private static final int rows = 500; - private static final int cols = 500; - private static final double sfValue = 0.5; // Value used to fill the scale factor matrix or as a standalone scalar - - @Override - public void setUp() { - TestUtils.clearAssertionInformation(); - addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"})); - addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"})); - } - - @Test - public void testRewriteQuantizationFusedCompressionScalar() { - testRewriteQuantizationFusedCompression(TEST_NAME1, true, true); - } - - @Test - public void testRewriteQuantizationFusedCompressionNoRewriteScalar() { - testRewriteQuantizationFusedCompression(TEST_NAME1, false, true); - } - - @Test - public void testRewriteQuantizationFusedCompression() { - testRewriteQuantizationFusedCompression(TEST_NAME2, true, false); - } - - @Test - public void testRewriteQuantizationFusedCompressionNoRewrite() { - testRewriteQuantizationFusedCompression(TEST_NAME2, false, false); - } - - /** - * Unified method to test both scalar and matrix scale factors. - * - * @param testname Test name - * @param rewrites Whether to enable fusion rewrites - * @param isScalar Whether the scale factor is a scalar or a matrix - */ - private void testRewriteQuantizationFusedCompression(String testname, boolean rewrites, boolean isScalar) { - boolean oldRewriteFlag = OptimizerUtils.ALLOW_QUANTIZE_COMPRESS_REWRITE; - OptimizerUtils.ALLOW_QUANTIZE_COMPRESS_REWRITE = rewrites; - - try { - TestConfiguration config = getTestConfiguration(testname); - loadTestConfiguration(config); - - String HOME = SCRIPT_DIR + TEST_DIR; - fullDMLScriptName = HOME + testname + ".dml"; - - double[][] A = getRandomMatrix(rows, cols, -1, 1, 0.70d, 7); - - String[] programArgs; - if(isScalar) { - // Scalar case: pass sfValue as a string - String s = Double.toString(sfValue); - programArgs = new String[] {"-stats", "-args", input("A"), s, output("R")}; - writeInputMatrixWithMTD("A", A, 174522, false); - } - else { - // Matrix case: pass S as a separate matrix - double[][] S = new double[rows][1]; - for(int i = 0; i < rows; i++) { - S[i][0] = sfValue; - } - programArgs = new String[] {"-stats", "-args", input("A"), input("S"), output("R")}; - writeInputMatrixWithMTD("A", A, 174522, false); - writeInputMatrixWithMTD("S", S, 500, false); - } - - this.programArgs = programArgs; - runTest(true, false, null, -1); - - // Simple check if quantization indeed occured by computing expected sum - // Even if compression is aborted, the quantization step should still take effect - double expectedR = Arrays.stream(A).flatMapToDouble(Arrays::stream).map(x -> Math.floor(x * sfValue)).sum(); - double actualR = TestUtils.readDMLScalar(output("R")); - - Assert.assertEquals("Mismatch in expected sum after quantization and compression", expectedR, actualR, 0.0); - - // Check if fusion occurred - if(rewrites) { - Assert.assertEquals("Expected fused operation count mismatch", 1, - Statistics.getCPHeavyHitterCount(Opcodes.QUANTIZE_COMPRESS.toString())); - Assert.assertEquals("Expected no separate floor op", 0, - Statistics.getCPHeavyHitterCount(Opcodes.FLOOR.toString())); - Assert.assertEquals("Expected no separate compress op", 0, - Statistics.getCPHeavyHitterCount(Opcodes.COMPRESS.toString())); - Assert.assertEquals("Expected no separate multiplication op", 0, - Statistics.getCPHeavyHitterCount(Opcodes.MULT.toString())); - } - else { - Assert.assertEquals("Expected no fused op", 0, - Statistics.getCPHeavyHitterCount(Opcodes.QUANTIZE_COMPRESS.toString())); - Assert.assertEquals("Expected separate floor op", 1, - Statistics.getCPHeavyHitterCount(Opcodes.FLOOR.toString())); - Assert.assertEquals("Expected separate compress op", 1, - Statistics.getCPHeavyHitterCount(Opcodes.COMPRESS.toString())); - Assert.assertEquals("Expected separate multiplication op", 1, - Statistics.getCPHeavyHitterCount(Opcodes.MULT.toString())); - } - } - finally { - OptimizerUtils.ALLOW_QUANTIZE_COMPRESS_REWRITE = oldRewriteFlag; - } - } + private static final String TEST_NAME1 = "RewriteQuantizationFusedCompressionScalar"; + private static final String TEST_NAME2 = "RewriteQuantizationFusedCompressionMatrix"; + private static final String TEST_DIR = "functions/rewrite/"; + private static final String TEST_CLASS_DIR = TEST_DIR + + RewriteQuantizationFusedCompressionTest.class.getSimpleName() + "/"; + + private static final int rows = 500; + private static final int cols = 500; + private static final double sfValue = 0.5; // Value used to fill the scale factor matrix or as a standalone scalar + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"})); + addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"})); + } + + @Test + public void testRewriteQuantizationFusedCompressionScalar() { + testRewriteQuantizationFusedCompression(TEST_NAME1, true, true); + } + + @Test + public void testRewriteQuantizationFusedCompressionNoRewriteScalar() { + testRewriteQuantizationFusedCompression(TEST_NAME1, false, true); + } + + @Test + public void testRewriteQuantizationFusedCompression() { + testRewriteQuantizationFusedCompression(TEST_NAME2, true, false); + } + + @Test + public void testRewriteQuantizationFusedCompressionNoRewrite() { + testRewriteQuantizationFusedCompression(TEST_NAME2, false, false); + } + + /** + * Unified method to test both scalar and matrix scale factors. + * + * @param testname Test name + * @param rewrites Whether to enable fusion rewrites + * @param isScalar Whether the scale factor is a scalar or a matrix + */ + private void testRewriteQuantizationFusedCompression(String testname, boolean rewrites, boolean isScalar) { + boolean oldRewriteFlag = OptimizerUtils.ALLOW_QUANTIZE_COMPRESS_REWRITE; + OptimizerUtils.ALLOW_QUANTIZE_COMPRESS_REWRITE = rewrites; + + try { + TestConfiguration config = getTestConfiguration(testname); + loadTestConfiguration(config); + + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + testname + ".dml"; + + double[][] A = getRandomMatrix(rows, cols, -1, 1, 0.70d, 7); + + String[] programArgs; + if(isScalar) { + // Scalar case: pass sfValue as a string + String s = Double.toString(sfValue); + programArgs = new String[] {"-stats", "-args", input("A"), s, output("R")}; + writeInputMatrixWithMTD("A", A, 174522, false); + } + else { + // Matrix case: pass S as a separate matrix + double[][] S = new double[rows][1]; + for(int i = 0; i < rows; i++) { + S[i][0] = sfValue; + } + programArgs = new String[] {"-stats", "-args", input("A"), input("S"), output("R")}; + writeInputMatrixWithMTD("A", A, 174522, false); + writeInputMatrixWithMTD("S", S, 500, false); + } + + this.programArgs = programArgs; + runTest(true, false, null, -1); + + // Simple check if quantization indeed occured by computing expected sum + // Even if compression is aborted, the quantization step should still take effect + double expectedR = Arrays.stream(A).flatMapToDouble(Arrays::stream).map(x -> Math.floor(x * sfValue)).sum(); + double actualR = TestUtils.readDMLScalar(output("R")); + + Assert.assertEquals("Mismatch in expected sum after quantization and compression", expectedR, actualR, 0.0); + + // Check if fusion occurred + if(rewrites) { + Assert.assertEquals("Expected fused operation count mismatch", 1, + Statistics.getCPHeavyHitterCount(Opcodes.QUANTIZE_COMPRESS.toString())); + Assert.assertEquals("Expected no separate floor op", 0, + Statistics.getCPHeavyHitterCount(Opcodes.FLOOR.toString())); + Assert.assertEquals("Expected no separate compress op", 0, + Statistics.getCPHeavyHitterCount(Opcodes.COMPRESS.toString())); + Assert.assertEquals("Expected no separate multiplication op", 0, + Statistics.getCPHeavyHitterCount(Opcodes.MULT.toString())); + } + else { + Assert.assertEquals("Expected no fused op", 0, + Statistics.getCPHeavyHitterCount(Opcodes.QUANTIZE_COMPRESS.toString())); + Assert.assertEquals("Expected separate floor op", 1, + Statistics.getCPHeavyHitterCount(Opcodes.FLOOR.toString())); + Assert.assertEquals("Expected separate compress op", 1, + Statistics.getCPHeavyHitterCount(Opcodes.COMPRESS.toString())); + Assert.assertEquals("Expected separate multiplication op", 1, + Statistics.getCPHeavyHitterCount(Opcodes.MULT.toString())); + } + } + finally { + OptimizerUtils.ALLOW_QUANTIZE_COMPRESS_REWRITE = oldRewriteFlag; + } + } } diff --git a/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java b/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java index f1cdd4b0f44..8c4ba6ae8ad 100644 --- a/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java +++ b/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java @@ -275,17 +275,17 @@ private void runTransformTest(String testname, ExecMode rt, boolean recode, bool } private String[][] readTwoColumnStringCSV(String s) { - try { - FrameBlock in = readDMLFrameFromHDFS(s, Types.FileFormat.CSV, false); + try { + FrameBlock in = readDMLFrameFromHDFS(s, Types.FileFormat.CSV, false); String[][] out = new String[2][in.getNumRows()]; for (int i = 0; i < in.getNumRows(); i++) { out[0][i] = in.getString(i, 0); out[1][i] = in.getString(i, 1); } return out; - } catch (IOException e) { - throw new RuntimeException(e); - } + } catch (IOException e) { + throw new RuntimeException(e); + } } @SuppressWarnings("unchecked") diff --git a/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java b/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java index dfe1d3538b1..d3d71d820d6 100644 --- a/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java +++ b/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java @@ -89,11 +89,11 @@ private void runVectorizationTest( String testName, boolean rewrites ) rCmd = getRCmd(inputDir(), expectedDir()); //run tests - runTest(true, false, null, -1); - runRScript(true); - - //compare results - HashMap dmlfile = readDMLMatrixFromOutputDir("R"); + runTest(true, false, null, -1); + runRScript(true); + + //compare results + HashMap dmlfile = readDMLMatrixFromOutputDir("R"); HashMap rfile = readRMatrixFromExpectedDir("R"); TestUtils.compareMatrices(dmlfile, rfile, 1e-14, "DML", "R"); } From a221629d75066adde287826af1a610d0c584bdce Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Tue, 2 Jun 2026 11:36:06 +0200 Subject: [PATCH 024/132] [SYSTEMDS-3887] Add video modality to Scuro optimizer pipeline In this patch the video modality is included in the unimodal optimizer pipeline. Since the video modality is sometimes loaded using the chunked data loader support for the chunked representation is added to the optimizer. --- .../systemds/scuro/dataloader/video_loader.py | 15 +- .../scuro/drsearch/modality_shared_memory.py | 23 + .../systemds/scuro/drsearch/node_executor.py | 399 ++++++++++++------ .../systemds/scuro/drsearch/node_scheduler.py | 28 +- .../scuro/drsearch/representation_dag.py | 54 ++- .../scuro/drsearch/unimodal_optimizer.py | 134 +++--- .../scuro/modality/unimodal_modality.py | 40 +- .../systemds/scuro/representations/clip.py | 153 ++++++- .../representations/covarep_audio_features.py | 120 +++++- .../scuro/representations/mel_spectrogram.py | 57 ++- .../systemds/scuro/representations/mfcc.py | 67 ++- .../systemds/scuro/representations/resnet.py | 38 +- .../representations/swin_video_transformer.py | 50 ++- .../systemds/scuro/representations/vgg.py | 25 +- .../systemds/scuro/representations/wav2vec.py | 36 +- .../systemds/scuro/representations/x3d.py | 62 ++- src/main/python/tests/scuro/data_generator.py | 1 + .../tests/scuro/test_multimodal_join.py | 4 +- .../tests/scuro/test_unimodal_optimizer.py | 28 +- .../scuro/test_unimodal_representations.py | 92 +++- .../tests/scuro/test_window_operations.py | 2 +- 21 files changed, 1124 insertions(+), 304 deletions(-) diff --git a/src/main/python/systemds/scuro/dataloader/video_loader.py b/src/main/python/systemds/scuro/dataloader/video_loader.py index a60b7acc60b..b35a22a8b66 100644 --- a/src/main/python/systemds/scuro/dataloader/video_loader.py +++ b/src/main/python/systemds/scuro/dataloader/video_loader.py @@ -37,6 +37,7 @@ class VideoStats: max_height: int max_channels: int num_instances: int + num_total_instances: int @property def output_shape(self): @@ -132,8 +133,20 @@ def get_stats(self, source_path: str): max_height = max(max_height, height) max_num_channels = max(max_num_channels, num_channels) num_instances += 1 + num_total_instances = num_instances + num_instances = ( + min(num_instances, self.chunk_size) + if self.chunk_size is not None + else num_instances + ) return VideoStats( - fps, max_length, max_width, max_height, max_num_channels, num_instances + fps, + max_length, + max_width, + max_height, + max_num_channels, + num_instances, + num_total_instances, ) def estimate_peak_memory_bytes(self) -> dict: diff --git a/src/main/python/systemds/scuro/drsearch/modality_shared_memory.py b/src/main/python/systemds/scuro/drsearch/modality_shared_memory.py index a98592b6204..d4092b90cfc 100644 --- a/src/main/python/systemds/scuro/drsearch/modality_shared_memory.py +++ b/src/main/python/systemds/scuro/drsearch/modality_shared_memory.py @@ -401,3 +401,26 @@ def add_shared_memory_candidate(data: Any, resident_bytes: int = 0) -> bool: return data, shm.name, data_nbytes, resident_bytes return None, None, 0, resident_bytes + + +_SHARED_MEMORY_WRAPPER_TYPES = ( + SharedStringList, + SharedGroupedArrayList, + SharedArrayList, + SharedNDArray, +) + + +def collect_shm_names_from_payload(data: Any) -> List[str]: + if data is None: + return [] + if isinstance(data, _SHARED_MEMORY_WRAPPER_TYPES): + return [data.shm_name] + if hasattr(data, "data"): + return collect_shm_names_from_payload(data.data) + if isinstance(data, (list, tuple)): + names: List[str] = [] + for item in data: + names.extend(collect_shm_names_from_payload(item)) + return names + return [] diff --git a/src/main/python/systemds/scuro/drsearch/node_executor.py b/src/main/python/systemds/scuro/drsearch/node_executor.py index 418b11fe700..4b9b2acc658 100644 --- a/src/main/python/systemds/scuro/drsearch/node_executor.py +++ b/src/main/python/systemds/scuro/drsearch/node_executor.py @@ -23,13 +23,17 @@ import os from multiprocessing import shared_memory from systemds.scuro import Modality -from systemds.scuro.drsearch.modality_shared_memory import add_shared_memory_candidate +from systemds.scuro.drsearch.modality_shared_memory import ( + add_shared_memory_candidate, + collect_shm_names_from_payload, +) from systemds.scuro.drsearch.node_scheduler import MemoryAwareNodeScheduler from systemds.scuro.drsearch.representation_dag import ( RepresentationDag, RepresentationNode, ) +import threading import numpy as np from typing import Any, Dict, List, Optional import multiprocessing as mp @@ -85,6 +89,7 @@ def __init__(self): self.ref_count = {} self.memory_usage_per_node = {} self.shared_memory_names = {} + self._shm_retain_count: Dict[str, int] = {} def get(self, node_id: str) -> Any: return self.cache[node_id] @@ -125,21 +130,44 @@ def inc_ref(self, node_id: str): self.ref_count[node_id] += 1 def dec_ref(self, node_id: str): + if node_id not in self.ref_count: + return self.ref_count[node_id] -= 1 - if self.ref_count[node_id] == 0: - del self.cache[node_id] - del self.ref_count[node_id] - del self.memory_usage_per_node[node_id] - self._cleanup_shared_memory(node_id) + if self.ref_count[node_id] <= 0: + self.ref_count[node_id] = 0 + self._try_cleanup_node(node_id) def clear(self, node_id: str): - if node_id in self.cache: - del self.cache[node_id] if node_id in self.ref_count: - del self.ref_count[node_id] - if node_id in self.memory_usage_per_node: - del self.memory_usage_per_node[node_id] - self._cleanup_shared_memory(node_id) + self.ref_count[node_id] = 0 + self._try_cleanup_node(node_id) + + def retain_shm_names(self, shm_names: List[str]) -> List[str]: + retained: List[str] = [] + for shm_name in shm_names: + if not shm_name: + continue + self._shm_retain_count[shm_name] = ( + self._shm_retain_count.get(shm_name, 0) + 1 + ) + retained.append(shm_name) + return retained + + def release_shm_names(self, shm_names: List[str]) -> None: + nodes_to_recheck: List[str] = [] + for shm_name in shm_names: + if not shm_name: + continue + count = self._shm_retain_count.get(shm_name, 0) - 1 + if count <= 0: + self._shm_retain_count.pop(shm_name, None) + else: + self._shm_retain_count[shm_name] = count + for node_id, node_names in self.shared_memory_names.items(): + if shm_name in node_names and node_id not in nodes_to_recheck: + nodes_to_recheck.append(node_id) + for node_id in nodes_to_recheck: + self._try_cleanup_node(node_id) def __len__(self): return len(self.cache) @@ -147,6 +175,20 @@ def __len__(self): def get_memory_total_memory_usage(self): return sum(self.memory_usage_per_node.values()) + def _shm_names_in_use(self, shm_names: List[str]) -> bool: + return any(self._shm_retain_count.get(name, 0) > 0 for name in shm_names) + + def _try_cleanup_node(self, node_id: str) -> None: + if self.ref_count.get(node_id, 0) > 0: + return + shm_names = self.shared_memory_names.get(node_id, []) + if shm_names and self._shm_names_in_use(shm_names): + return + self.cache.pop(node_id, None) + self.ref_count.pop(node_id, None) + self.memory_usage_per_node.pop(node_id, None) + self._cleanup_shared_memory(node_id) + def _cleanup_shared_memory(self, node_id: str): names = self.shared_memory_names.pop(node_id, []) for shm_name in names: @@ -160,10 +202,37 @@ def _cleanup_shared_memory(self, node_id: str): pass def cleanup_all(self): + self._shm_retain_count.clear() for node_id in list(self.shared_memory_names.keys()): + self.ref_count.pop(node_id, None) + self.cache.pop(node_id, None) + self.memory_usage_per_node.pop(node_id, None) self._cleanup_shared_memory(node_id) +def _execute_multiple_reps_for_leaf_dependencies( + nodes: List[RepresentationNode], + modalities: List[Modality], + gpu_id: Optional[int], +): + representations = [] + node_id_by_representation = {} + for node in nodes: + operation = node.operation(params=node.parameters) + if hasattr(operation, "gpu_id"): + operation.gpu_id = gpu_id + representations.append(operation) + node_id_by_representation[operation.name] = node.node_id + + modality_results = modalities[0].apply_representations( + representations, parallel=True + ) + return { + "results": modality_results, + "node_id_by_representation": node_id_by_representation, + } + + def _execute_node_worker(node, input_mods, task, rep_cache, gpu_id): if gpu_id is not None: device = torch.device(f"cuda:{gpu_id}") @@ -205,14 +274,19 @@ def _run_node_op(): ) return input_mods[0].combine(input_mods[1:], fusion_op) - result, peak_delta_bytes, peak_abs_rss = measure_peak_rss_during( - _run_node_op, - sample_s=0.01, - ) - - gpu_peak_bytes = ( - torch.cuda.max_memory_allocated(device) if gpu_id is not None else 0 - ) + gpu_peak_bytes = -1 + peak_delta_bytes = -1 + peak_abs_rss = -1 + if DEBUG: + result, peak_delta_bytes, peak_abs_rss = measure_peak_rss_during( + _run_node_op, + sample_s=0.01, + ) + gpu_peak_bytes = ( + torch.cuda.max_memory_allocated(device) if gpu_id is not None else 0 + ) + else: + result = _run_node_op() return { "result": result, @@ -224,7 +298,10 @@ def _run_node_op(): def _execute_task_worker( - task_node_id: str, task: Any, data: Any, gpu_id: Optional[int] + task_node_id: str, + task: Any, + data: Any, + gpu_id: Optional[int], ) -> Dict[str, Any]: if DEBUG: @@ -243,17 +320,23 @@ def _run_task(): end = time.perf_counter() return scores, end - start - gpu_peak_bytes = ( - torch.cuda.max_memory_allocated(device) if gpu_id is not None else 0 - ) - result, peak_delta_bytes, peak_abs_rss = measure_peak_rss_during( - _run_task, - sample_s=0.01, - ) + gpu_peak_bytes = -1 + peak_delta_bytes = -1 if DEBUG: + gpu_peak_bytes = ( + torch.cuda.max_memory_allocated(device) if gpu_id is not None else 0 + ) + result, peak_delta_bytes, peak_abs_rss = measure_peak_rss_during( + _run_task, + sample_s=0.01, + ) + print( f"Task {task_node_id} has a CPU peak memory usage of {peak_delta_bytes/1024**3:.2f} GB, and a GPU peak memory usage of {gpu_peak_bytes/1024**3:.2f} GB" ) + else: + result = _run_task() + return { "scores": result[0], "task_time": result[1], @@ -303,6 +386,31 @@ def __init__( resume=False, ) + def _shm_names_for_submit( + self, parent_node_ids: List[str], payload_data: Any + ) -> List[str]: + names: List[str] = [] + for parent_id in parent_node_ids or []: + names.extend(self.result_cache.shared_memory_names.get(parent_id, [])) + if parent_node_ids: + names.extend(collect_shm_names_from_payload(payload_data)) + else: + names.extend(getattr(self, "_leaf_shm_names", [])) + names.extend(collect_shm_names_from_payload(payload_data)) + # preserve order, drop duplicates + return list(dict.fromkeys(names)) + + def _retain_for_submit( + self, parent_node_ids: List[str], payload_data: Any + ) -> List[str]: + return self.result_cache.retain_shm_names( + self._shm_names_for_submit(parent_node_ids, payload_data) + ) + + def _release_for_future(self, retained_shm_names: List[str]) -> None: + if retained_shm_names: + self.result_cache.release_shm_names(retained_shm_names) + def run(self) -> None: task_results = {} memory_usage_data = {} @@ -314,6 +422,22 @@ def run(self) -> None: max_workers=self.max_num_workers, mp_context=ctx ) as executor: future_to_node_id = {} + future_to_retained_shm: Dict[Any, List[str]] = {} + + def submit_nodes_with_leaf_dependencies(node_ids: List[str]): + nodes = [self.scheduler.mapping[node_id] for node_id in node_ids] + gpu_id = nodes[0].gpu_id + self.scheduler.move_to_running(node_ids) + + retained = self._retain_for_submit([], self.modalities[0].data) + future = executor.submit( + _execute_multiple_reps_for_leaf_dependencies, + nodes, + self.modalities, + gpu_id, + ) + future_to_node_id[future] = node_ids + future_to_retained_shm[future] = retained def submit_node(node_id: str): node = self.scheduler.mapping[node_id] @@ -325,6 +449,7 @@ def submit_node(node_id: str): self.result_cache.get(parent_node_id) for parent_node_id in parent_node_ids ] + if self._is_task_node(node): task_result = ResultEntry( dag=self._get_dag_from_node_ids(node_id), @@ -332,31 +457,42 @@ def submit_node(node_id: str): ) task_results[node_id] = task_result task_idx = int(node.parameters.get("_task_idx", 0)) + payload_data = ( + self.modalities[0].data + if parent_results is None + else parent_results[0].data + ) + retained = self._retain_for_submit(parent_node_ids, payload_data) future = executor.submit( _execute_task_worker, node_id, self.tasks[task_idx], - ( - self.modalities[0].data - if parent_results is None - else parent_results[0].data - ), + payload_data, gpu_id, ) else: + payload_data = ( + self.modalities if parent_results is None else parent_results + ) + retained = self._retain_for_submit(parent_node_ids, payload_data) future = executor.submit( _execute_node_worker, node, - self.modalities if parent_results is None else parent_results, + payload_data, None, None, gpu_id, ) self.scheduler.move_to_running(node_id) future_to_node_id[future] = node_id + future_to_retained_shm[future] = retained def submit_new_ready_nodes(): - for node_id in self.scheduler.get_runnable().copy(): + ready_nodes = self.scheduler.get_runnable().copy() + for node_id in ready_nodes: + if isinstance(node_id, list): + submit_nodes_with_leaf_dependencies(node_id) + continue submit_node(node_id) submit_new_ready_nodes() @@ -372,94 +508,127 @@ def submit_new_ready_nodes(): for future in done: node_id = future_to_node_id.pop(future) + retained_shm = future_to_retained_shm.pop(future, []) try: result = future.result() - except Exception as e: - err_cls = type(e) - err_mod = err_cls.__module__ - if err_mod.startswith("torch"): - torch.cuda.empty_cache() - print(f"Error executing node {node_id}: {e}") - self.scheduler.add_failed_node(node_id) - continue - - peak_bytes = result["peak_bytes"] - gpu_peak_bytes = result["gpu_peak_bytes"] - - node = self.scheduler.mapping[node_id] - if self._is_task_node(node): - task_results[node_id].task_time = result["task_time"] - task_results[node_id].train_score = result["scores"][ - 0 - ].average_scores - task_results[node_id].val_score = result["scores"][ - 1 - ].average_scores - task_results[node_id].test_score = result["scores"][ - 2 - ].average_scores - if self.enable_checkpointing: - self.checkpoint_manager.increment(node_id) - self.checkpoint_manager.checkpoint_if_due(task_results) - self._checkpoint_memory_usage( - node_id, - peak_bytes, - gpu_peak_bytes, - "task", - memory_usage_data, - None, - ) - parent_node_ids = self.scheduler.get_valid_parents(node_id) - if len(parent_node_ids) > 0: + if isinstance(node_id, list): + results = result["results"] + node_id_by_representation = result[ + "node_id_by_representation" + ] + for ( + representation, + transformed_modality, + ) in results.items(): + batch_node_id = node_id_by_representation[ + representation + ] + self._handle_modality_result( + transformed_modality, + batch_node_id, + None, + None, + memory_usage_data, + representation, + ) + submit_new_ready_nodes() + continue + + peak_bytes = result["peak_bytes"] + gpu_peak_bytes = result["gpu_peak_bytes"] + node = self.scheduler.mapping[node_id] + if self._is_task_node(node): + task_results[node_id].task_time = result["task_time"] + task_results[node_id].train_score = result["scores"][ + 0 + ].average_scores + task_results[node_id].val_score = result["scores"][ + 1 + ].average_scores + task_results[node_id].test_score = result["scores"][ + 2 + ].average_scores + if self.enable_checkpointing: + self.checkpoint_manager.increment(node_id) + self.checkpoint_manager.checkpoint_if_due(task_results) + self._checkpoint_memory_usage( + node_id, + peak_bytes, + gpu_peak_bytes, + "task", + memory_usage_data, + None, + ) + + parent_node_ids = self.scheduler.get_valid_parents(node_id) for parent_node_id in parent_node_ids: self.result_cache.dec_ref(parent_node_id) - if ( - parent_node_id in self.result_cache.ref_count - and self.result_cache.ref_count[parent_node_id] == 0 - ): - self.result_cache.clear(parent_node_id) - self.scheduler.complete_node(node_id) - - else: - transformed_modality = result["result"] - actual_stats = self._infer_actual_output_stats( - transformed_modality - ) - estimated_stats = self.scheduler.node_stats.get(node_id) - - if actual_stats is not None and ( - estimated_stats is None - or not getattr( - estimated_stats, "output_shape_is_known", True - ) - ): - self.scheduler.update_node_stats_and_reestimate_descendants( - node_id, actual_stats - ) - if self.enable_checkpointing: - self._checkpoint_memory_usage( + self.scheduler.complete_node(node_id) + else: + transformed_modality = result["result"] + self._handle_modality_result( + transformed_modality, node_id, peak_bytes, gpu_peak_bytes, - result["operation_name"], memory_usage_data, - transformed_modality.data, + result["operation_name"], ) - before_bytes = self.result_cache.get_memory_total_memory_usage() - self._manage_result_cache(node_id, transformed_modality) - after_bytes = self.result_cache.get_memory_total_memory_usage() - self.scheduler.update_cpu_memory_in_use( - after_bytes - before_bytes - ) - self.scheduler.complete_node(node_id) - submit_new_ready_nodes() - assert len(self.result_cache.ref_count.keys()) == 0 + submit_new_ready_nodes() + except Exception: + parent_node_ids = [] + if not isinstance(node_id, list): + parent_node_ids = self.scheduler.get_valid_parents(node_id) + for parent_node_id in parent_node_ids: + self.result_cache.dec_ref(parent_node_id) + if not isinstance(node_id, list): + self.scheduler.add_failed_node(node_id) + raise + finally: + self._release_for_future(retained_shm) + + assert not self.result_cache.ref_count + assert not self.result_cache._shm_retain_count self.result_cache.cleanup_all() self._cleanup_leaf_shared_memory() - return list(task_results.values()) + return {"task_results": list(task_results.values())} + + def _handle_modality_result( + self, + transformed_modality: Any, + node_id: str, + peak_bytes: int, + gpu_peak_bytes: int, + memory_usage_data, + operation_name: str, + ): + actual_stats = self._infer_actual_output_stats(transformed_modality) + estimated_stats = self.scheduler.node_stats.get(node_id) + + if actual_stats is not None and ( + estimated_stats is None + or not getattr(estimated_stats, "output_shape_is_known", True) + ): + self.scheduler.update_node_stats_and_reestimate_descendants( + node_id, actual_stats + ) + if self.enable_checkpointing: + self._checkpoint_memory_usage( + node_id, + peak_bytes, + gpu_peak_bytes, + operation_name, + memory_usage_data, + transformed_modality.data, + ) + before_bytes = self.result_cache.get_memory_total_memory_usage() + self._manage_result_cache(node_id, transformed_modality) + after_bytes = self.result_cache.get_memory_total_memory_usage() + self.scheduler.update_cpu_memory_in_use(after_bytes - before_bytes) + self.scheduler.complete_node(node_id) def _materialize_leaf_modalities_in_shared_memory(self): self._leaf_shm_names = [] @@ -587,22 +756,14 @@ def _infer_actual_output_stats( def _manage_result_cache(self, node_id: str, result: Any): parent_node_ids = self.scheduler.get_valid_parents(node_id) - if len(parent_node_ids) > 0: - for parent_node_id in parent_node_ids: - self.result_cache.dec_ref(parent_node_id) + for parent_node_id in parent_node_ids: + self.result_cache.dec_ref(parent_node_id) if self.scheduler.get_children(node_id): for _ in self.scheduler.get_children(node_id): self.result_cache.inc_ref(node_id) self.result_cache.add_result(node_id, result) - for parent_node_id in parent_node_ids: - if ( - parent_node_id in self.result_cache.ref_count - and self.result_cache.ref_count[parent_node_id] == 0 - ): - self.result_cache.clear(parent_node_id) - def _get_nodes_by_ids(self, nodes_ids: List[str]) -> List[RepresentationNode]: return [self.scheduler.mapping[node_id] for node_id in nodes_ids] diff --git a/src/main/python/systemds/scuro/drsearch/node_scheduler.py b/src/main/python/systemds/scuro/drsearch/node_scheduler.py index ef3ccc844e5..209f4503860 100644 --- a/src/main/python/systemds/scuro/drsearch/node_scheduler.py +++ b/src/main/python/systemds/scuro/drsearch/node_scheduler.py @@ -19,7 +19,7 @@ # # ------------------------------------------------------------- from __future__ import annotations - +import re from typing import List, Dict, Optional, Any from collections import defaultdict, deque from collections import deque @@ -93,8 +93,25 @@ def get_runnable(self) -> List[RepresentationNode]: ok, gpu_id = self._check_memory_constraints(node) if ok: self.mapping[node].gpu_id = gpu_id - self.ready_nodes.append(node) self._reserve_memory(node, gpu_id) + self.ready_nodes.append(node) + contains_leaf = [] + for node in self.ready_nodes: + if any(re.fullmatch(r"leaf_\d+", i) for i in self.mapping[node].inputs): + for mod in self.modalities: + if ( + mod.modality_id + == self.mapping[self.mapping[node].inputs[0]].modality_id + ): + if mod.data_loader.chunk_size is not None: + contains_leaf.append(node) + break + + for node in contains_leaf: + self.ready_nodes.remove(node) + + if len(contains_leaf) > 0: + self.ready_nodes.append(contains_leaf) return self.ready_nodes def _get_runnable_nodes(self) -> List[str]: @@ -128,9 +145,12 @@ def add_failed_node(self, node_id: str): self._release_memory(node_id, self.mapping[node_id].gpu_id) - def move_to_running(self, node_id: str): + def move_to_running(self, node_id: str | list): self.ready_nodes.remove(node_id) - self.running_nodes.append(node_id) + if isinstance(node_id, list): + self.running_nodes.extend(node_id) + else: + self.running_nodes.append(node_id) def complete_node(self, node_id: str): self.running_nodes.remove(node_id) diff --git a/src/main/python/systemds/scuro/drsearch/representation_dag.py b/src/main/python/systemds/scuro/drsearch/representation_dag.py index 099732e46df..b1d5835ad3f 100644 --- a/src/main/python/systemds/scuro/drsearch/representation_dag.py +++ b/src/main/python/systemds/scuro/drsearch/representation_dag.py @@ -37,6 +37,17 @@ from collections import OrderedDict, defaultdict, deque +def pushdown_aggregation_for_node( + node_parameters: Optional[Dict[str, Any]], +) -> Optional[AggregatedRepresentation]: + if not node_parameters: + return None + pushdown_config = node_parameters.get("_pushdown_aggregation") + if pushdown_config is None: + return None + return AggregatedRepresentation(params=pushdown_config) + + class LRUCache: def __init__(self, max_size: int = 256): self.max_size = max_size @@ -231,6 +242,23 @@ def _compute_node_signature(self, node, input_sig_tuple) -> Hashable: params_items = tuple(sorted((node.parameters or {}).items())) return ("op", op_cls, params_items, input_sig_tuple) + def get_represntation_names(self) -> str: + representation_names = [] + visited = set() + + def visit_node(node_id): + if node_id in visited: + return + node = self.get_node_by_id(node_id) + for input_id in node.inputs: + visit_node(input_id) + visited.add(node_id) + if node.operation is not None: + representation_names.append(node.operation().name) + + visit_node(self.root_node_id) + return " -> ".join(representation_names) + def execute( self, modalities: List[Modality], @@ -282,9 +310,9 @@ def execute_node(node_id: str, task) -> TransformedModality: if rep_cache is not None: result = rep_cache[node_operation.name] else: - # Compute the representation + agg = pushdown_aggregation_for_node(node.parameters) result = input_mods[0].apply_representation( - node_operation + node_operation, aggregation=agg ) else: # It's a fusion operation @@ -314,8 +342,10 @@ def execute_node(node_id: str, task) -> TransformedModality: if rep_cache is not None: result = rep_cache[node_operation.name] else: - # Compute the representation - result = input_mods[0].apply_representation(node_operation) + agg = pushdown_aggregation_for_node(node.parameters) + result = input_mods[0].apply_representation( + node_operation, aggregation=agg + ) else: # It's a fusion operation fusion_op = node_operation @@ -387,13 +417,17 @@ def get_modality_by_id_and_instance_id( modalities: List[Modality], modality_id: int, instance_id: int ): counter = 0 + modality_per_id = {} for modality in modalities: - if modality.modality_id == modality_id: - if counter == instance_id or instance_id == -1: - return modality - else: - counter += 1 - return None + if modality.modality_id not in modality_per_id: + modality_per_id[modality.modality_id] = [] + modality_per_id[modality.modality_id].append(modality) + if modality_id not in modality_per_id: + return None + if instance_id == -1 or len(modality_per_id[modality_id]) == 1: + return modality_per_id[modality_id][0] + else: + return modality_per_id[modality_id][instance_id] class RepresentationDAGBuilder: diff --git a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py index 215f27929f3..8dc2e1a082f 100644 --- a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py +++ b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py @@ -313,6 +313,37 @@ def _process_modality(self, modality, skip_remaining: int = 0, scheduler=None): metric_name=self.metric_name, ) + dags, dags_with_pushdown, expanded_dags_with_task_roots = ( + self._build_execution_dags_for_modality(modality, skip_remaining) + ) + + node_executor = NodeExecutor( + expanded_dags_with_task_roots, + [modality], + self.tasks, + self._checkpoint_manager, + self.max_num_workers, + self.result_path, + enable_checkpointing=self.enable_checkpointing, + ) + + exec_out = node_executor.run() + task_results = exec_out["task_results"] + + for task_result in task_results: + local_results.add_task_result(task_result, dags) + + if self.save_all_results: + timestr = time.strftime("%Y%m%d-%H%M%S") + file_name = f"{modality.modality_id}_unimodal_results_{timestr}.pkl" + with open(file_name, "wb") as f: + pickle.dump(local_results.results, f) + + return local_results + + def _build_execution_dags_for_modality( + self, modality: Modality, skip_remaining: int = 0 + ) -> tuple: modality_specific_operators = self._get_modality_operators( modality.modality_type ) @@ -341,28 +372,7 @@ def _process_modality(self, modality, skip_remaining: int = 0, scheduler=None): expanded_dags_with_task_roots = self._expand_dags_with_task_roots( dags_with_pushdown ) - - node_executor = NodeExecutor( - expanded_dags_with_task_roots, - [modality], - self.tasks, - self._checkpoint_manager, - self.max_num_workers, - self.result_path, - enable_checkpointing=self.enable_checkpointing, - ) - task_results = node_executor.run() - - for task_result in task_results: - local_results.add_task_result(task_result, dags) - - if self.save_all_results: - timestr = time.strftime("%Y%m%d-%H%M%S") - file_name = f"{modality.modality_id}_unimodal_results_{timestr}.pkl" - with open(file_name, "wb") as f: - pickle.dump(local_results.results, f) - - return local_results + return dags, dags_with_pushdown, expanded_dags_with_task_roots def _merge_results(self, local_results): for modality_id in local_results.results: @@ -498,15 +508,22 @@ def _build_modality_dag( return dags def _aggregation_needed(self, dag: RepresentationDag) -> bool: + input_stats = {} + # TODO: adapt this to the fusion of multiple modalities, list of input stats needed for modality in self.modalities: if modality.modality_id == dag.nodes[0].modality_id: - last_stats = modality.stats + input_stats[dag.nodes[0].node_id] = modality.stats break for node in dag.nodes[1:]: - last_stats = node.operation(params=node.parameters).get_output_stats( - last_stats + previous_stats = [ + input_stats.get(input_node_id, None) for input_node_id in node.inputs + ] + current_stats = node.operation(params=node.parameters).get_output_stats( + previous_stats if len(previous_stats) > 1 else previous_stats[0] ) - return len(last_stats.output_shape) > 1 + input_stats[node.node_id] = current_stats + + return len(input_stats.get(dag.root_node_id, None).output_shape) > 1 def add_aggregation_operator(self, builder, dags): new_dags = [] @@ -675,7 +692,12 @@ def print_results(self): print(f"{modality}_{task_name}: {entry}") def get_k_best_results( - self, modality, task, performance_metric_name, prune_cache=False + self, + modality, + task, + performance_metric_name, + prune_cache=False, + cache_needed=True, ): """ Get the k best results for the given modality @@ -693,36 +715,38 @@ def get_k_best_results( results = results[: self.k] sorted_indices = sorted_indices[: self.k] - task_cache = self.cache.get(modality.modality_id, {}).get(task.model.name, None) - if not task_cache: - cache = [] - for result in results: - if result.dag.nodes[-1].parameters.get("_node_kind", False) == "task": - dag = copy.deepcopy(result.dag) - dag.nodes = dag.nodes[:-1] - dag.root_node_id = dag.nodes[-1].node_id - cache.append(dag.execute([modality])) - - elif isinstance(task_cache, list): - cache = task_cache - else: - cache_items = list(task_cache.items()) if task_cache else [] - cache = [cache_items[i][1] for i in sorted_indices if i < len(cache_items)] - - if prune_cache: - # Note: in case the unimodal results are loaded from a file, we need to initialize the cache for the modality and task - if modality.modality_id not in self.operator_performance.cache: - self.operator_performance.cache[modality.modality_id] = {} - if ( - task.model.name - not in self.operator_performance.cache[modality.modality_id] - ): + cache = [] + if cache_needed: + task_cache = self.cache.get(modality.modality_id, {}).get( + task.model.name, None + ) + if not task_cache: + cache = [] + for result in results: + cache.append(result.dag.execute([modality])) + + elif isinstance(task_cache, list): + cache = task_cache + else: + cache_items = list(task_cache.items()) if task_cache else [] + cache = [ + cache_items[i][1] for i in sorted_indices if i < len(cache_items) + ] + + if prune_cache: + # Note: in case the unimodal results are loaded from a file, we need to initialize the cache for the modality and task + if modality.modality_id not in self.operator_performance.cache: + self.operator_performance.cache[modality.modality_id] = {} + if ( + task.model.name + not in self.operator_performance.cache[modality.modality_id] + ): + self.operator_performance.cache[modality.modality_id][ + task.model.name + ] = {} self.operator_performance.cache[modality.modality_id][ task.model.name - ] = {} - self.operator_performance.cache[modality.modality_id][ - task.model.name - ] = cache + ] = cache return results, cache diff --git a/src/main/python/systemds/scuro/modality/unimodal_modality.py b/src/main/python/systemds/scuro/modality/unimodal_modality.py index 84204ac570d..0535c64bcee 100644 --- a/src/main/python/systemds/scuro/modality/unimodal_modality.py +++ b/src/main/python/systemds/scuro/modality/unimodal_modality.py @@ -18,6 +18,7 @@ # under the License. # # ------------------------------------------------------------- +from concurrent.futures import ThreadPoolExecutor, as_completed import gc import time import numpy as np @@ -135,7 +136,7 @@ def aggregate(self, aggregation_function): if self.data is None: raise Exception("Data is None") - def apply_representations(self, representations, aggregation=None): + def apply_representations(self, representations, aggregation=None, parallel=False): """ Applies a list of representations to the modality. Specifically, it applies the representations to the modality in a chunked manner. :param representations: List of representations to apply @@ -158,20 +159,29 @@ def apply_representations(self, representations, aggregation=None): time.time() ) # TODO: should be repalced in unimodal_representation.transform if self.data_loader.chunk_size: - for _ in self.iter_raw_data_chunks(reset=True): - for representation in representations: - transformed_chunk = representation.transform(self) - transformed_modalities_per_representation[ - representation.name - ].data.extend(transformed_chunk.data) - transformed_modalities_per_representation[ - representation.name - ].metadata.extend(transformed_chunk.metadata) - for d in transformed_chunk.data: - original_lengths_per_representation[representation.name].append( - d.shape[0] - ) - + with ThreadPoolExecutor( + max_workers=len(representations) if parallel else 1 + ) as executor: + time_s = time.time() + for _ in self.iter_raw_data_chunks(reset=True): + representations_futures = {} + for representation in representations: + future = executor.submit(representation.transform, self) + representations_futures[future] = representation.name + for future in as_completed(representations_futures.keys()): + representation_name = representations_futures.get(future) + transformed_chunk = future.result() + transformed_modalities_per_representation[ + representation_name + ].data.extend(transformed_chunk.data) + transformed_modalities_per_representation[ + representation_name + ].metadata.extend(transformed_chunk.metadata) + for d in transformed_chunk.data: + original_lengths_per_representation[ + representation_name + ].append(d.shape[0]) + print(f"Time for transforming data chunks: {time.time() - time_s}") else: if not self.has_data(): self.extract_raw_data() diff --git a/src/main/python/systemds/scuro/representations/clip.py b/src/main/python/systemds/scuro/representations/clip.py index 518cc1eb5dc..2c880686b11 100644 --- a/src/main/python/systemds/scuro/representations/clip.py +++ b/src/main/python/systemds/scuro/representations/clip.py @@ -21,6 +21,7 @@ import numpy as np from torchvision import transforms +from systemds.scuro.dataloader.video_loader import VideoStats from systemds.scuro.modality.transformed import TransformedModality from systemds.scuro.representations.representation import RepresentationStats from systemds.scuro.representations.unimodal import UnimodalRepresentation @@ -48,14 +49,20 @@ @register_representation([ModalityType.VIDEO, ModalityType.IMAGE]) class CLIPVisual(UnimodalRepresentation): - def __init__(self, output_file=None, batch_size=32, params=None): - parameters = {} + def __init__(self, output_file=None, batch_size=32, layer_name="", params=None): + parameters = self._get_parameters() super().__init__("CLIPVisual", ModalityType.EMBEDDING, parameters) self.model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32") self.processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") + if params is not None: + self.batch_size = int(params.get("batch_size", batch_size)) + self.layer_name = params.get("layer_name", layer_name) + else: + self.batch_size = batch_size + self.layer_name = layer_name self.output_file = output_file + self.data_type = torch.float32 - self.batch_size = batch_size self.gpu_id = None self.device = get_device() @@ -68,11 +75,42 @@ def gpu_id(self, gpu_id): self._gpu_id = gpu_id self.device = get_device(gpu_id) + def _get_parameters(self): + parameters = { + "batch_size": [1, 2, 4, 8, 16, 32, 64, 128], + "layer_name": [ + "", + "encoder.layers.0.layer_norm2", + "encoder.layers.1.layer_norm2", + "encoder.layers.2.layer_norm2", + "encoder.layers.3.layer_norm2", + "encoder.layers.4.layer_norm2", + "encoder.layers.5.layer_norm2", + "encoder.layers.6.layer_norm2", + "encoder.layers.7.layer_norm2", + "encoder.layers.8.layer_norm2", + "encoder.layers.9.layer_norm2", + "encoder.layers.10.layer_norm2", + "encoder.layers.11.layer_norm2", + "post_layernorm", + ], + } + + return parameters + def estimate_output_memory_bytes(self, input_stats) -> int: return input_stats.num_instances * 512 * self.data_type.itemsize def get_output_stats(self, input_stats) -> RepresentationStats: - if not isinstance(input_stats, RepresentationStats): + if isinstance(input_stats, VideoStats): + return RepresentationStats( + input_stats.num_instances, + ( + input_stats.max_length, + 512, + ), + ) + elif not isinstance(input_stats, RepresentationStats): return RepresentationStats(input_stats.num_instances, (512,)) else: return RepresentationStats( @@ -177,6 +215,21 @@ def transform(self, modality, aggregation=None): self.model = self.model.to(self.data_type) self.model = self.model.to(self.device) + self.clip_output = None + + def get_activation(name): + def hook(model, input, output): + self.clip_output = ( + output[0].detach() if isinstance(output, tuple) else output.detach() + ) + + return hook + + if self.layer_name != "": + for name, layer in self.model.vision_model.named_modules(): + if name == self.layer_name: + layer.register_forward_hook(get_activation(name)) + break embeddings = self.create_visual_embeddings(modality) @@ -212,9 +265,14 @@ def create_visual_embeddings(self, modality): inputs.to(self.device) with torch.no_grad(): - output = self.model.get_image_features(**inputs) - if len(output.shape) > 2: - output = torch.nn.functional.adaptive_avg_pool2d(output, (1, 1)) + if self.layer_name != "": + _ = self.model.vision_model(**inputs) + output = self.clip_output + else: + output = self.model.get_image_features(**inputs) + + output = self._pool_visual_output(output) + embeddings.extend( torch.flatten(output, 1) .detach() @@ -241,13 +299,13 @@ def create_visual_embeddings(self, modality): ) inputs.to(self.device) with torch.no_grad(): - output = self.model.get_image_features(**inputs) + if self.layer_name != "": + _ = self.model.vision_model(**inputs) + output = self.clip_output + else: + output = self.model.get_image_features(**inputs) - if hasattr(output, "pooler_output"): - output = output.pooler_output - - if len(output.shape) > 2: - output = torch.nn.functional.adaptive_avg_pool2d(output, (1, 1)) + output = self._pool_visual_output(output) embeddings[id].extend( torch.flatten(output, 1) @@ -261,13 +319,28 @@ def create_visual_embeddings(self, modality): embeddings[id] = np.array(embeddings[id]) return list(embeddings.values()) + def _pool_visual_output(self, output: torch.Tensor) -> torch.Tensor: + if output.ndim == 4: + output = torch.nn.functional.adaptive_avg_pool2d(output, (1, 1)) + return torch.flatten(output, 1) + if output.ndim == 3: + return output.mean(dim=1) + if output.ndim == 2: + return output + raise ValueError(f"Unexpected CLIP visual output shape: {tuple(output.shape)}") + @register_representation(ModalityType.TEXT) class CLIPText(UnimodalRepresentation): - def __init__(self, output_file=None, batch_size=32, params=None): - self.batch_size = batch_size + def __init__(self, output_file=None, batch_size=32, layer_name="", params=None): + if params is not None: + self.batch_size = int(params.get("batch_size", batch_size)) + self.layer_name = params.get("layer_name", layer_name) + else: + self.batch_size = batch_size + self.layer_name = layer_name self.max_seq_length = 77 - parameters = {"batch_size": [1, 2, 4, 8, 16, 32, 64, 128]} + parameters = self._get_parameters() super().__init__("CLIPText", ModalityType.EMBEDDING, parameters) self.model = None @@ -295,6 +368,29 @@ def estimate_output_memory_bytes(self, input_stats) -> int: input_stats.num_instances * np.prod(output_stats) * self.data_type.itemsize ) + def _get_parameters(self): + parameters = { + "batch_size": [1, 2, 4, 8, 16, 32, 64, 128], + "layer_name": [ + "", + "encoder.layers.0.layer_norm2", + "encoder.layers.1.layer_norm2", + "encoder.layers.2.layer_norm2", + "encoder.layers.3.layer_norm2", + "encoder.layers.4.layer_norm2", + "encoder.layers.5.layer_norm2", + "encoder.layers.6.layer_norm2", + "encoder.layers.7.layer_norm2", + "encoder.layers.8.layer_norm2", + "encoder.layers.9.layer_norm2", + "encoder.layers.10.layer_norm2", + "encoder.layers.11.layer_norm2", + "final_layer_norm", + ], + } + + return parameters + def get_output_stats(self, input_stats) -> RepresentationStats: if not isinstance(input_stats, RepresentationStats): self.stats = RepresentationStats( @@ -379,6 +475,21 @@ def transform(self, modality, aggregation=None): self.processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") self.model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32") self.model = self.model.to(self.device) + self.clip_output = None + + def get_activation(name): + def hook(model, input, output): + self.clip_output = ( + output[0].detach() if isinstance(output, tuple) else output.detach() + ) + + return hook + + if self.layer_name != "": + for name, layer in self.model.text_model.named_modules(): + if name == self.layer_name: + layer.register_forward_hook(get_activation(name)) + break if ModalityType.TEXT.has_field(modality.metadata, "text_spans"): dataset = TextSpanDataset(modality.data, modality.metadata) @@ -415,9 +526,15 @@ def create_text_embeddings(self, data, model, aggregation=None): ) inputs.to(self.device) with torch.no_grad(): - text_features = model.get_text_features(**inputs) + if self.layer_name != "": + _ = model.text_model(**inputs) + + batch_np = self.clip_output.cpu().float().numpy() + if batch_np.ndim == 3: + batch_np = batch_np.mean(axis=1) + else: + batch_np = model.get_text_features(**inputs).cpu().float().numpy() - batch_np = text_features.detach().cpu().float().numpy() if aggregation is not None: batch_np = aggregation.execute(batch_np) diff --git a/src/main/python/systemds/scuro/representations/covarep_audio_features.py b/src/main/python/systemds/scuro/representations/covarep_audio_features.py index 01098ef4ee1..973b7a99c2f 100644 --- a/src/main/python/systemds/scuro/representations/covarep_audio_features.py +++ b/src/main/python/systemds/scuro/representations/covarep_audio_features.py @@ -30,6 +30,11 @@ register_representation, register_context_representation_operator, ) +from systemds.scuro.utils.static_variables import ( + NP_ARRAY_HEADER_BYTES, + PY_LIST_HEADER_BYTES, + PY_LIST_SLOT_BYTES, +) @register_representation(ModalityType.AUDIO) @@ -97,11 +102,32 @@ def get_output_stats(self, input_stats) -> RepresentationStats: return RepresentationStats(num_instances, (num_frames, 4)) def estimate_peak_memory_bytes(self, input_stats) -> dict: - # TODO - return { - "cpu_peak_bytes": 0, - "gpu_peak_bytes": 0, - } + num_frames = 1 + max((input_stats.max_length - 1) // int(self.hop_length), 0) + num_frames = max(int(num_frames), 1) + + out_elem = np.dtype(np.float32).itemsize + output_payload_per_instance = num_frames * 4 * out_elem + retained_output_bytes = PY_LIST_HEADER_BYTES + input_stats.num_instances * ( + output_payload_per_instance + NP_ARRAY_HEADER_BYTES + PY_LIST_SLOT_BYTES + ) + + num_freq_bins = 1 + 2048 // 2 + stft_bytes = num_frames * num_freq_bins * np.dtype(np.complex64).itemsize + magnitude_bytes = num_frames * num_freq_bins * np.dtype(np.float32).itemsize + per_feature_bytes = num_frames * out_elem + stacked_bytes = 4 * num_frames * out_elem + fft_workspace_bytes = max(2 * 1024 * 1024, stft_bytes // 2) + transient_one_instance = ( + 4 * per_feature_bytes + + stacked_bytes + + stft_bytes + + magnitude_bytes + + fft_workspace_bytes + ) + cpu_peak = int( + (retained_output_bytes + transient_one_instance) * 1.15 + 16 * 1024 * 1024 + ) + return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} @register_representation(ModalityType.AUDIO) @@ -145,14 +171,26 @@ def get_output_stats(self, input_stats) -> RepresentationStats: num_frames = 1 + max(int((signal_length - 1) // self.hop_length), 0) num_frames = max(int(num_frames), 1) - return RepresentationStats(num_instances, (num_frames, 1)) + return RepresentationStats(num_instances, (1, num_frames)) def estimate_peak_memory_bytes(self, input_stats) -> dict: - # TODO - return { - "cpu_peak_bytes": 0, - "gpu_peak_bytes": 0, - } + num_frames = 1 + max((input_stats.max_length - 1) // int(self.hop_length), 0) + + out_elem = np.dtype(np.float32).itemsize + output_payload_per_instance = num_frames * out_elem + retained_output_bytes = PY_LIST_HEADER_BYTES + input_stats.num_instances * ( + output_payload_per_instance + NP_ARRAY_HEADER_BYTES + PY_LIST_SLOT_BYTES + ) + framed_bytes = 2048 * num_frames * np.dtype(np.float32).itemsize + crossings_mask_bytes = 2048 * num_frames + output_instance_bytes = output_payload_per_instance + transient_one_instance = ( + framed_bytes + crossings_mask_bytes + output_instance_bytes + ) + cpu_peak = int( + (retained_output_bytes + transient_one_instance) * 1.15 + 8 * 1024 * 1024 + ) + return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} @register_representation(ModalityType.AUDIO) @@ -197,14 +235,28 @@ def get_output_stats(self, input_stats) -> RepresentationStats: num_frames = 1 + max(int((signal_length - 1) // self.hop_length), 0) num_frames = max(int(num_frames), 1) - return RepresentationStats(num_instances, (num_frames, 1)) + return RepresentationStats(num_instances, (1, num_frames)) def estimate_peak_memory_bytes(self, input_stats) -> dict: - # TODO - return { - "cpu_peak_bytes": 0, - "gpu_peak_bytes": 0, - } + num_frames = 1 + max((input_stats.max_length - 1) // int(self.hop_length), 0) + num_frames = max(int(num_frames), 1) + out_elem = np.dtype(np.float32).itemsize + output_payload_per_instance = num_frames * out_elem + retained_output_bytes = PY_LIST_HEADER_BYTES + input_stats.num_instances * ( + output_payload_per_instance + NP_ARRAY_HEADER_BYTES + PY_LIST_SLOT_BYTES + ) + frame_len = int(self.frame_length) + framed_bytes = frame_len * num_frames * np.dtype(np.float32).itemsize + squared_bytes = framed_bytes + mean_bytes = num_frames * np.dtype(np.float32).itemsize + output_instance_bytes = output_payload_per_instance + transient_one_instance = ( + framed_bytes + squared_bytes + mean_bytes + output_instance_bytes + ) + cpu_peak = int( + (retained_output_bytes + transient_one_instance) * 1.15 + 8 * 1024 * 1024 + ) + return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} @register_representation(ModalityType.AUDIO) @@ -251,11 +303,33 @@ def get_output_stats(self, input_stats) -> RepresentationStats: num_frames = 1 + max(int((signal_length - 1) // self.hop_length), 0) num_frames = max(int(num_frames), 1) - return RepresentationStats(num_instances, (num_frames, 1)) + return RepresentationStats(num_instances, (1, num_frames)) def estimate_peak_memory_bytes(self, input_stats) -> dict: - # TODO - return { - "cpu_peak_bytes": 0, - "gpu_peak_bytes": 0, - } + num_frames = 1 + max((input_stats.max_length - 1) // int(self.hop_length), 0) + num_frames = max(int(num_frames), 1) + out_elem = np.dtype(np.float32).itemsize + output_payload_per_instance = num_frames * out_elem + retained_output_bytes = PY_LIST_HEADER_BYTES + input_stats.num_instances * ( + output_payload_per_instance + NP_ARRAY_HEADER_BYTES + PY_LIST_SLOT_BYTES + ) + n_fft = 2048 + num_freq_bins = 1 + n_fft // 2 + stft_bytes = num_frames * num_freq_bins * np.dtype(np.complex64).itemsize + pitches_bytes = num_frames * num_freq_bins * np.dtype(np.float32).itemsize + magnitudes_bytes = pitches_bytes + argmax_idx_bytes = num_frames * np.dtype(np.int64).itemsize + gathered_pitch_bytes = output_payload_per_instance + fft_workspace_bytes = max(2 * 1024 * 1024, stft_bytes // 2) + transient_one_instance = ( + stft_bytes + + pitches_bytes + + magnitudes_bytes + + argmax_idx_bytes + + gathered_pitch_bytes + + fft_workspace_bytes + ) + cpu_peak = int( + (retained_output_bytes + transient_one_instance) * 1.15 + 16 * 1024 * 1024 + ) + return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} diff --git a/src/main/python/systemds/scuro/representations/mel_spectrogram.py b/src/main/python/systemds/scuro/representations/mel_spectrogram.py index 46e5045b2eb..6d378806475 100644 --- a/src/main/python/systemds/scuro/representations/mel_spectrogram.py +++ b/src/main/python/systemds/scuro/representations/mel_spectrogram.py @@ -30,6 +30,11 @@ register_representation, register_context_representation_operator, ) +from systemds.scuro.utils.static_variables import ( + NP_ARRAY_HEADER_BYTES, + PY_LIST_HEADER_BYTES, + PY_LIST_SLOT_BYTES, +) @register_representation(ModalityType.AUDIO) @@ -76,7 +81,10 @@ def compute_feature(self, instance, sr=None): hop_length=self.hop_length, n_fft=self.n_fft, ) - return S.T + if instance.ndim == 1: + return S.T + + return S.transpose(0, 2, 1) def get_output_stats(self, input_stats) -> RepresentationStats: num_instances = getattr(input_stats, "num_instances", 0) @@ -94,7 +102,52 @@ def get_output_stats(self, input_stats) -> RepresentationStats: if signal_length < self.n_fft: num_frames = 1 else: - num_frames = 1 + (signal_length - self.n_fft) // self.hop_length + num_frames = 1 + signal_length // self.hop_length num_frames = max(int(num_frames), 1) return RepresentationStats(num_instances, (num_frames, self.n_mels)) + + def estimate_peak_memory_bytes(self, input_stats) -> dict: + n = int(getattr(input_stats, "num_instances", 0)) + if hasattr(input_stats, "max_length"): + signal_length = int(getattr(input_stats, "max_length", 0)) + elif hasattr(input_stats, "output_shape") and input_stats.output_shape: + signal_length = int(input_stats.output_shape[0]) + else: + signal_length = 0 + + if signal_length <= 0: + num_frames = 1 + elif signal_length < self.n_fft: + num_frames = 1 + else: + num_frames = 1 + (signal_length) // self.hop_length + num_frames = max(int(num_frames), 1) + + out_elem = np.dtype(np.float32).itemsize + num_freq_bins = 1 + self.n_fft // 2 + output_payload_per_instance = num_frames * self.n_mels * out_elem + retained_output_bytes = PY_LIST_HEADER_BYTES + n * ( + output_payload_per_instance + NP_ARRAY_HEADER_BYTES + PY_LIST_SLOT_BYTES + ) + + input_copy_bytes = max(signal_length, 1) * out_elem + stft_bytes = num_frames * num_freq_bins * np.dtype(np.complex64).itemsize + power_spec_bytes = num_frames * num_freq_bins * out_elem + mel_output_bytes = output_payload_per_instance + fft_workspace_bytes = max(2 * 1024 * 1024, stft_bytes // 2) + + transient_one_instance = ( + input_copy_bytes + + stft_bytes + + power_spec_bytes + + mel_output_bytes + + fft_workspace_bytes + ) + cpu_peak = int( + (retained_output_bytes + transient_one_instance) * 2 + 12 * 1024 * 1024 + ) + return { + "cpu_peak_bytes": cpu_peak, + "gpu_peak_bytes": 0, + } diff --git a/src/main/python/systemds/scuro/representations/mfcc.py b/src/main/python/systemds/scuro/representations/mfcc.py index 737a3dffe95..406fc6616c1 100644 --- a/src/main/python/systemds/scuro/representations/mfcc.py +++ b/src/main/python/systemds/scuro/representations/mfcc.py @@ -30,6 +30,11 @@ register_representation, register_context_representation_operator, ) +from systemds.scuro.utils.static_variables import ( + NP_ARRAY_HEADER_BYTES, + PY_LIST_HEADER_BYTES, + PY_LIST_SLOT_BYTES, +) @register_representation(ModalityType.AUDIO) @@ -81,8 +86,18 @@ def compute_feature(self, instance, sr=None): hop_length=self.hop_length, n_mels=self.n_mels, ) - mfcc = (mfcc - np.mean(mfcc)) / np.std(mfcc) - return mfcc.T + if mfcc.ndim == 2: + mean = np.mean(mfcc, keepdims=True) + std = np.std(mfcc, keepdims=True) + else: + mean = np.mean(mfcc, axis=(1, 2), keepdims=True) + std = np.std(mfcc, axis=(1, 2), keepdims=True) + mfcc = (mfcc - mean) / np.maximum(std, 1e-8) + + if instance.ndim == 1: + return mfcc.T + + return mfcc.transpose(0, 2, 1) def get_output_stats(self, input_stats) -> RepresentationStats: num_instances = getattr(input_stats, "num_instances", 0) @@ -101,3 +116,51 @@ def get_output_stats(self, input_stats) -> RepresentationStats: num_frames = max(int(num_frames), 1) return RepresentationStats(num_instances, (num_frames, self.n_mfcc)) + + def estimate_peak_memory_bytes(self, input_stats) -> dict: + n = int(getattr(input_stats, "num_instances", 0)) + if hasattr(input_stats, "max_length"): + signal_length = int(getattr(input_stats, "max_length", 0)) + elif hasattr(input_stats, "output_shape") and input_stats.output_shape: + signal_length = int(input_stats.output_shape[0]) + else: + signal_length = 0 + + if signal_length <= 0: + num_frames = 1 + else: + num_frames = 1 + max(int((signal_length - 1) // self.hop_length), 0) + num_frames = max(int(num_frames), 1) + + out_elem = np.dtype(np.float32).itemsize + n_fft = 2048 + num_freq_bins = 1 + n_fft // 2 + output_payload_per_instance = num_frames * self.n_mfcc * out_elem + retained_output_bytes = PY_LIST_HEADER_BYTES + n * ( + output_payload_per_instance + NP_ARRAY_HEADER_BYTES + PY_LIST_SLOT_BYTES + ) + + input_copy_bytes = max(signal_length, 1) * out_elem + stft_bytes = num_frames * num_freq_bins * np.dtype(np.complex64).itemsize + magnitude_bytes = num_frames * num_freq_bins * out_elem + mel_projection_bytes = num_frames * self.n_mels * out_elem + dct_output_bytes = output_payload_per_instance + norm_workspace_bytes = max(dct_output_bytes, 256 * 1024) + fft_workspace_bytes = max(2 * 1024 * 1024, stft_bytes // 2) + + transient_one_instance = ( + input_copy_bytes + + stft_bytes + + magnitude_bytes + + mel_projection_bytes + + dct_output_bytes + + norm_workspace_bytes + + fft_workspace_bytes + ) + cpu_peak = int( + (retained_output_bytes + transient_one_instance) * 1.15 + 16 * 1024 * 1024 + ) + return { + "cpu_peak_bytes": cpu_peak, + "gpu_peak_bytes": 0, + } diff --git a/src/main/python/systemds/scuro/representations/resnet.py b/src/main/python/systemds/scuro/representations/resnet.py index 299ccad3683..1202748aa63 100644 --- a/src/main/python/systemds/scuro/representations/resnet.py +++ b/src/main/python/systemds/scuro/representations/resnet.py @@ -19,6 +19,7 @@ # # ------------------------------------------------------------- from systemds.scuro.dataloader.image_loader import ImageStats +from systemds.scuro.dataloader.video_loader import VideoStats from systemds.scuro.representations.representation import RepresentationStats from systemds.scuro.utils.torch_dataset import CustomDataset from systemds.scuro.modality.transformed import TransformedModality @@ -33,12 +34,17 @@ from systemds.scuro.utils.static_variables import get_device +class Identity(torch.nn.Module): + def forward(self, input_: torch.Tensor) -> torch.Tensor: + return input_ + + @register_representation([ModalityType.IMAGE, ModalityType.VIDEO]) class ResNet(UnimodalRepresentation): def __init__( self, model_name="ResNet18", - layer="avgpool", + layer_name="avgpool", output_file=None, batch_size=32, params=None, @@ -47,21 +53,21 @@ def __init__( self.model = None self.gpu_id = None self.device = get_device() + if params is not None: + self.batch_size = int(params.get("batch_size", batch_size)) + self.layer_name = params.get("layer_name", layer_name) + else: + self.batch_size = batch_size + self.layer_name = layer_name self.model_name = model_name - self.batch_size = batch_size parameters = self._get_parameters() super().__init__("ResNet", ModalityType.EMBEDDING, parameters) self.output_file = output_file - self.layer_name = layer self.model.eval() for param in self.model.parameters(): param.requires_grad = False - class Identity(torch.nn.Module): - def forward(self, input_: torch.Tensor) -> torch.Tensor: - return input_ - self.model.fc = Identity() @property @@ -109,9 +115,24 @@ def model_name(self, model_name): raise NotImplementedError def estimate_output_memory_bytes(self, input_stats: ImageStats) -> int: + if isinstance(input_stats, VideoStats): + return ( + input_stats.num_instances + * input_stats.max_length + * 512 + * self.data_type.itemsize + ) return input_stats.num_instances * 512 * self.data_type.itemsize def get_output_stats(self, input_stats) -> RepresentationStats: + if isinstance(input_stats, VideoStats): + return RepresentationStats( + input_stats.num_instances, + ( + input_stats.max_length, + 512, + ), + ) return RepresentationStats(input_stats.num_instances, (512,)) def estimate_peak_memory_bytes(self, input_stats: ImageStats) -> dict: @@ -122,6 +143,9 @@ def estimate_peak_memory_bytes(self, input_stats: ImageStats) -> dict: * input_stats.max_channels * self.data_type.itemsize ) + if isinstance(input_stats, VideoStats): + input_bytes = input_bytes * input_stats.max_length + output_bytes = self.estimate_output_memory_bytes(input_stats) output_bytes_batch = output_bytes / input_stats.num_instances * self.batch_size diff --git a/src/main/python/systemds/scuro/representations/swin_video_transformer.py b/src/main/python/systemds/scuro/representations/swin_video_transformer.py index c46d12bcb56..39191f2f252 100644 --- a/src/main/python/systemds/scuro/representations/swin_video_transformer.py +++ b/src/main/python/systemds/scuro/representations/swin_video_transformer.py @@ -30,6 +30,7 @@ import numpy as np from systemds.scuro.modality.type import ModalityType from systemds.scuro.drsearch.operator_registry import register_representation +from systemds.scuro.dataloader.video_loader import VideoStats from systemds.scuro.utils.torch_dataset import CustomDataset from systemds.scuro.utils.static_variables import ( @@ -41,6 +42,8 @@ @register_representation([ModalityType.VIDEO]) class SwinVideoTransformer(UnimodalRepresentation): + _EMBED_DIM = 768 + def __init__(self, layer_name="avgpool", params=None): parameters = { "layer_name": [ @@ -54,7 +57,7 @@ def __init__(self, layer_name="avgpool", params=None): "avgpool", ], } - self.data_type = torch.float + self.data_type = torch.float32 super().__init__("SwinVideoTransformer", ModalityType.EMBEDDING, parameters) self.layer_name = layer_name self.model = swin3d_t(weights=models.video.Swin3D_T_Weights.KINETICS400_V1) @@ -65,7 +68,50 @@ def __init__(self, layer_name="avgpool", params=None): param.requires_grad = False def get_output_stats(self, input_stats) -> RepresentationStats: - return RepresentationStats(input_stats.num_instances, (768,)) + num_instances = getattr(input_stats, "num_instances", 0) + return RepresentationStats(num_instances, (self._EMBED_DIM,)) + + def estimate_output_memory_bytes(self, input_stats: VideoStats) -> int: + dt = int(torch.tensor([], dtype=self.data_type).element_size()) + return input_stats.num_instances * self._EMBED_DIM * dt + + def estimate_peak_memory_bytes(self, input_stats: VideoStats) -> dict: + dt = int(torch.tensor([], dtype=self.data_type).element_size()) + temporal = max(input_stats.max_length, 1) + input_bytes = ( + dt + * input_stats.max_channels + * temporal + * input_stats.max_height + * input_stats.max_width + ) + output_bytes = self.estimate_output_memory_bytes(input_stats) + n = max(input_stats.num_instances, 1) + output_bytes_batch = output_bytes / n + + batch_peak_bytes = (input_bytes + self._EMBED_DIM * dt) * 2 + + safety_margin_bytes = 100 * 1024 * 1024 + + param_size = 0 + for param in self.model.parameters(): + param_size += param.nelement() * param.element_size() + + buffer_size = 0 + for buffer in self.model.buffers(): + buffer_size += buffer.nelement() * buffer.element_size() + + size_all_bytes = param_size + buffer_size + + cpu_peak = ( + size_all_bytes * 2 * dt + + output_bytes_batch + + output_bytes + + input_bytes + + safety_margin_bytes + ) + gpu_peak = (size_all_bytes * dt + batch_peak_bytes) * 6 + return {"cpu_peak_bytes": int(cpu_peak), "gpu_peak_bytes": int(gpu_peak)} def transform(self, modality, aggregation=None): embeddings = {} diff --git a/src/main/python/systemds/scuro/representations/vgg.py b/src/main/python/systemds/scuro/representations/vgg.py index fa8f121438b..35bc07d8a29 100644 --- a/src/main/python/systemds/scuro/representations/vgg.py +++ b/src/main/python/systemds/scuro/representations/vgg.py @@ -21,6 +21,7 @@ from systemds.scuro.utils.converter import numpy_dtype_to_torch_dtype from systemds.scuro.utils.torch_dataset import CustomDataset from systemds.scuro.modality.transformed import TransformedModality +from systemds.scuro.dataloader.video_loader import VideoStats from systemds.scuro.representations.unimodal import UnimodalRepresentation from typing import Tuple, Any from systemds.scuro.drsearch.operator_registry import register_representation @@ -37,6 +38,11 @@ from systemds.scuro.representations.representation import RepresentationStats +class Identity(torch.nn.Module): + def forward(self, input_: torch.Tensor) -> torch.Tensor: + return input_ + + @register_representation([ModalityType.IMAGE, ModalityType.VIDEO]) class VGG19(UnimodalRepresentation): def __init__( @@ -58,10 +64,6 @@ def __init__( for param in self.model.parameters(): param.requires_grad = False - class Identity(torch.nn.Module): - def forward(self, input_: torch.Tensor) -> torch.Tensor: - return input_ - self.model.fc = Identity() @property @@ -88,9 +90,24 @@ def _get_parameters(self): return parameters def estimate_output_memory_bytes(self, input_stats: ImageStats) -> int: + if isinstance(input_stats, VideoStats): + return ( + input_stats.num_instances + * input_stats.max_length + * 4096 + * np.dtype(np.float32).itemsize + ) return input_stats.num_instances * 4096 * np.dtype(np.float32).itemsize def get_output_stats(self, input_stats) -> RepresentationStats: + if isinstance(input_stats, VideoStats): + return RepresentationStats( + input_stats.num_instances, + ( + input_stats.max_length, + 4096, + ), + ) return RepresentationStats(input_stats.num_instances, (4096,)) def estimate_peak_memory_bytes(self, input_stats: ImageStats) -> dict: diff --git a/src/main/python/systemds/scuro/representations/wav2vec.py b/src/main/python/systemds/scuro/representations/wav2vec.py index 38dcb848436..ece034e099b 100644 --- a/src/main/python/systemds/scuro/representations/wav2vec.py +++ b/src/main/python/systemds/scuro/representations/wav2vec.py @@ -65,12 +65,42 @@ def transform(self, modality, aggregation=None): outputs = self.model(**input) features = outputs.extract_features # TODO: check how to get intermediate representations - result.append(torch.flatten(features.mean(dim=1), 1).detach().cpu().numpy()) + result.append(torch.flatten(features.mean(dim=1)).detach().cpu().numpy()) - transformed_modality.data = result + transformed_modality.data = np.array(result) return transformed_modality def get_output_stats(self, input_stats) -> RepresentationStats: num_instances = getattr(input_stats, "num_instances", 0) - embedding_dim = 768 + embedding_dim = 512 return RepresentationStats(num_instances, (embedding_dim,)) + + def estimate_peak_memory_bytes(self, input_stats) -> dict: + n = int(getattr(input_stats, "num_instances", 1)) + + if hasattr(input_stats, "max_length"): + signal_len = int(getattr(input_stats, "max_length", 16000)) + elif hasattr(input_stats, "output_shape") and input_stats.output_shape: + signal_len = int(input_stats.output_shape[0]) + else: + signal_len = 16000 + signal_len = max(signal_len, 1) + + hidden = 768 + stride = 320 # conv frontend effective stride + frames = max(1, int(np.ceil(signal_len / stride))) + + model_resident = 420 * 1024 * 1024 # ~420 MB + activation_bytes = int(frames * hidden * 4 * 24) + io_temp = int(signal_len * 4 * 4) + 16 * 1024 * 1024 + + output_bytes = n * 512 * 4 + + cpu_peak = int( + (model_resident + activation_bytes + io_temp + output_bytes) * 1.25 + ) + + gpu_peak = 0 + cpu_peak = max(cpu_peak, 600 * 1024 * 1024) + + return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": gpu_peak} diff --git a/src/main/python/systemds/scuro/representations/x3d.py b/src/main/python/systemds/scuro/representations/x3d.py index f7a70921532..ace4cf4b8ca 100644 --- a/src/main/python/systemds/scuro/representations/x3d.py +++ b/src/main/python/systemds/scuro/representations/x3d.py @@ -27,7 +27,7 @@ from systemds.scuro.modality.transformed import TransformedModality from systemds.scuro.representations.unimodal import UnimodalRepresentation from systemds.scuro.representations.representation import RepresentationStats -from typing import Tuple, Any +from typing import Tuple, Any, Union import torch.utils.data import torch from torchvision.models.video import r3d_18, s3d @@ -35,6 +35,13 @@ import numpy as np from systemds.scuro.modality.type import ModalityType from systemds.scuro.drsearch.operator_registry import register_representation +from systemds.scuro.dataloader.video_loader import VideoStats +import math + + +class Identity(torch.nn.Module): + def forward(self, input_: torch.Tensor) -> torch.Tensor: + return input_ @register_representation([ModalityType.VIDEO]) @@ -53,14 +60,52 @@ def __init__( for param in self.model.parameters(): param.requires_grad = False - class Identity(torch.nn.Module): - def forward(self, input_: torch.Tensor) -> torch.Tensor: - return input_ - self.model.fc = Identity() def get_output_stats(self, input_stats) -> RepresentationStats: - return RepresentationStats(input_stats.num_instances, (512,)) + embedding_dim = 400 * math.floor((max(input_stats.max_length, 14) - 5) / 8) + return RepresentationStats(input_stats.num_instances, (embedding_dim,)) + + def estimate_output_memory_bytes(self, input_stats: VideoStats) -> int: + embedding_dim = 400 * math.floor((max(input_stats.max_length, 14) - 5) / 8) + return input_stats.num_instances * embedding_dim * self.data_type.itemsize + + def estimate_peak_memory_bytes(self, input_stats: VideoStats) -> dict: + temporal = max(input_stats.max_length, 14) + input_bytes = ( + self.data_type.itemsize + * input_stats.max_channels + * temporal + * input_stats.max_height + * input_stats.max_width + ) + output_bytes = self.estimate_output_memory_bytes(input_stats) + n = max(input_stats.num_instances, 1) + output_bytes_batch = output_bytes / n + + batch_peak_bytes = (input_bytes + 512 * self.data_type.itemsize) * 2 + + safety_margin_bytes = 100 * 1024 * 1024 + + param_size = 0 + for param in self.model.parameters(): + param_size += param.nelement() * param.element_size() + + buffer_size = 0 + for buffer in self.model.buffers(): + buffer_size += buffer.nelement() * buffer.element_size() + + size_all_bytes = param_size + buffer_size + + cpu_peak = ( + size_all_bytes * 2 * self.data_type.itemsize + + output_bytes_batch + + output_bytes + + input_bytes + + safety_margin_bytes + ) + gpu_peak = (size_all_bytes * self.data_type.itemsize + batch_peak_bytes) * 6 + return {"cpu_peak_bytes": int(cpu_peak), "gpu_peak_bytes": int(gpu_peak)} @property def model_name(self): @@ -85,6 +130,7 @@ def _get_parameters(self, high_level=True): for m in ["c3d", "s3d"]: parameters["model_name"].append(m) + # TODO: add embedding dimensions for each layer if high_level: parameters["layer_name"] = [ "features.1", @@ -155,12 +201,10 @@ def hook( values = activation pooled = torch.nn.functional.adaptive_avg_pool2d(values, (1, 1)) - embeddings[video_id].extend( + embeddings[video_id] = ( torch.flatten(pooled, 1).detach().cpu().numpy().flatten() ) - embeddings[video_id] = np.array(embeddings[video_id]) - transformed_modality = TransformedModality( modality, self, self.output_modality_type ) diff --git a/src/main/python/tests/scuro/data_generator.py b/src/main/python/tests/scuro/data_generator.py index 14a373d98cf..a51ea510ea3 100644 --- a/src/main/python/tests/scuro/data_generator.py +++ b/src/main/python/tests/scuro/data_generator.py @@ -78,6 +78,7 @@ def __init__(self, indices, chunk_size, modality_type, data, data_type, metadata max(d.shape[1] for d in data), max(d.shape[2] for d in data), max(d.shape[3] for d in data), + chunk_size if chunk_size is not None else len(data), len(data), ) elif modality_type == ModalityType.TIMESERIES: diff --git a/src/main/python/tests/scuro/test_multimodal_join.py b/src/main/python/tests/scuro/test_multimodal_join.py index 5fd22dc8d98..14ce9376be1 100644 --- a/src/main/python/tests/scuro/test_multimodal_join.py +++ b/src/main/python/tests/scuro/test_multimodal_join.py @@ -118,7 +118,9 @@ def _join(self, left_modality, right_modality, window_size): left_modality.join( right_modality, JoinCondition("timestamp", "timestamp", "<") ) - .apply_representation(ResNet(layer="layer1.0.conv2", model_name="ResNet18")) + .apply_representation( + ResNet(layer_name="layer1.0.conv2", model_name="ResNet18") + ) .window_aggregation(window_size, "mean") .combine("concat") ) diff --git a/src/main/python/tests/scuro/test_unimodal_optimizer.py b/src/main/python/tests/scuro/test_unimodal_optimizer.py index 3c5ce2a67f7..ad824b0335f 100644 --- a/src/main/python/tests/scuro/test_unimodal_optimizer.py +++ b/src/main/python/tests/scuro/test_unimodal_optimizer.py @@ -27,7 +27,8 @@ from systemds.scuro.representations.color_histogram import ColorHistogram from systemds.scuro.drsearch.operator_registry import Registry from systemds.scuro.drsearch.unimodal_optimizer import UnimodalOptimizer - +from systemds.scuro.representations.mfcc import MFCC +from systemds.scuro.representations.mel_spectrogram import MelSpectrogram from systemds.scuro.representations.word2vec import W2V from systemds.scuro.representations.bow import BoW from systemds.scuro.representations.bert import Bert @@ -48,7 +49,6 @@ from systemds.scuro.representations.aggregated_representation import ( AggregatedRepresentation, ) -from systemds.scuro.representations.bert import Bert from systemds.scuro.modality.type import ModalityType from unittest.mock import patch @@ -67,7 +67,6 @@ def setUpClass(cls): cls.tasks = [ TestTask("UnimodalRepresentationTask1", "Test1", cls.num_instances), - TestTask("UnimodalRepresentationTask2", "Test2", cls.num_instances), ] def test_unimodal_optimizer_for_text_modality(self): @@ -111,6 +110,18 @@ def test_unimodal_optimizer_for_multiple_modalities(self): ) self.optimize_unimodal_representation_for_modality([text, image]) + def test_unimodal_optimizer_for_audio_modality(self): + audio_data, audio_md = ModalityRandomDataGenerator().create_audio_data( + self.num_instances, 3000 + ) + audio = UnimodalModality( + TestDataLoader( + self.indices, None, ModalityType.AUDIO, audio_data, np.float32, audio_md + ) + ) + + self.optimize_unimodal_representation_for_modality([audio]) + def test_unimodal_optimizer_for_video_modality(self): video_data, video_md = ModalityRandomDataGenerator().create_visual_modality( self.num_instances, 10, 10 @@ -194,7 +205,14 @@ def optimize_unimodal_representation_for_modality(self, modalities): Bert, CLIPText, ], - ModalityType.VIDEO: [ResNet], + ModalityType.AUDIO: [ + MFCC, + MelSpectrogram, + ], + ModalityType.VIDEO: [ + ResNet, + CLIPVisual, + ], ModalityType.IMAGE: [ColorHistogram, CLIPVisual], ModalityType.EMBEDDING: [], }, @@ -216,7 +234,7 @@ def optimize_unimodal_representation_for_modality(self, modalities): in unimodal_optimizer.operator_performance.modality_ids ) - assert len(unimodal_optimizer.operator_performance.task_names) == 2 + assert len(unimodal_optimizer.operator_performance.task_names) == 1 result, cached = unimodal_optimizer.operator_performance.get_k_best_results( modalities[0], self.tasks[0], "accuracy" ) diff --git a/src/main/python/tests/scuro/test_unimodal_representations.py b/src/main/python/tests/scuro/test_unimodal_representations.py index bdc7af50b4c..a4e18743090 100644 --- a/src/main/python/tests/scuro/test_unimodal_representations.py +++ b/src/main/python/tests/scuro/test_unimodal_representations.py @@ -19,6 +19,7 @@ # # ------------------------------------------------------------- +import time import unittest import copy import numpy as np @@ -40,6 +41,7 @@ from systemds.scuro.representations.glove import GloVe from systemds.scuro.representations.wav2vec import Wav2Vec from systemds.scuro.representations.spectrogram import Spectrogram +from systemds.scuro.representations.window_aggregation import WindowAggregation from systemds.scuro.representations.word2vec import W2V from systemds.scuro.representations.tfidf import TfIdf from systemds.scuro.representations.x3d import X3D @@ -84,9 +86,54 @@ class TestUnimodalRepresentations(unittest.TestCase): @classmethod def setUpClass(cls): - cls.num_instances = 2 + cls.num_instances = 100 cls.indices = np.array(range(cls.num_instances)) + def _create_audio_modality(self, signal_length=1000): + audio_data, audio_md = ModalityRandomDataGenerator().create_audio_data( + self.num_instances, signal_length + ) + + audio = UnimodalModality( + TestDataLoader( + self.indices, None, ModalityType.AUDIO, audio_data, np.float32, audio_md + ) + ) + audio.extract_raw_data() + return audio + + def test_audio_representation_transform_output_shapes(self): + audio = self._create_audio_modality() + audio_representations = [ + (MFCC(), (2, 12)), + (MelSpectrogram(), (2, 128)), + (Spectrogram(), (2, 1025)), + (Wav2Vec(), (1, None)), + (Spectral(), (2, 4)), + (ZeroCrossing(), (2, None)), + (RMSE(), (2, None)), + (Pitch(), (2, None)), + ] + + for representation, expected_shape_signature in audio_representations: + with self.subTest(representation=representation.name): + transformed_modality = representation.transform(audio) + print(representation.name) + self.assertIsNotNone(transformed_modality.data) + self.assertEqual(len(transformed_modality.data), self.num_instances) + + for transformed_instance in transformed_modality.data: + self.assertEqual( + transformed_instance.ndim, + expected_shape_signature[0], + ) + if expected_shape_signature[1] is not None: + self.assertEqual( + transformed_instance.shape[1], + expected_shape_signature[1], + ) + self.assertGreater(transformed_instance.shape[0], 0) + def test_audio_representations(self): audio_representations = [ MFCC(), @@ -117,7 +164,6 @@ def test_audio_representations(self): assert len(r.data) == self.num_instances for i in range(self.num_instances): assert (audio.data[i] == original_data[i]).all() - assert r.data[0].ndim == 2 def test_timeseries_representations(self): ts_representations = [ @@ -173,27 +219,27 @@ def test_image_representations(self): assert r.data is not None assert len(r.data) == self.num_instances - def test_video_representations(self): - video_representations = [ - CLIPVisual(), - I3D(), - X3D(), - VGG19(), - ResNet(), - SwinVideoTransformer(), - ] - video_data, video_md = ModalityRandomDataGenerator().create_visual_modality( - self.num_instances, 25 - ) - video = UnimodalModality( - TestDataLoader( - self.indices, None, ModalityType.VIDEO, video_data, np.float32, video_md - ) - ) - for representation in video_representations: - r = video.apply_representation(representation) - assert r.data is not None - assert len(r.data) == self.num_instances + # def test_video_representations(self): + # video_representations = [ + # CLIPVisual(layer_name="post_layernorm"), + # I3D(), + # X3D(), + # VGG19(), + # ResNet(), + # SwinVideoTransformer(), + # ] + # video_data, video_md = ModalityRandomDataGenerator().create_visual_modality( + # self.num_instances, 25 + # ) + # video = UnimodalModality( + # TestDataLoader( + # self.indices, None, ModalityType.VIDEO, video_data, np.float32, video_md + # ) + # ) + # for representation in video_representations: + # r = video.apply_representation(representation) + # assert r.data is not None + # assert len(r.data) == self.num_instances def test_text_representations(self): test_representations = [ diff --git a/src/main/python/tests/scuro/test_window_operations.py b/src/main/python/tests/scuro/test_window_operations.py index 1f954cfeb04..2eaf5985db1 100644 --- a/src/main/python/tests/scuro/test_window_operations.py +++ b/src/main/python/tests/scuro/test_window_operations.py @@ -93,7 +93,7 @@ def test_window_operations_on_text_representations(self): self.run_window_aggregation_for_modality(ModalityType.TEXT, window_size) def run_window_aggregation_for_modality(self, modality_type, window_size): - r = self.data_generator.create1DModality(40, 100, modality_type) + r = self.data_generator.create1DModality(40, 5000, modality_type) for aggregation in self.aggregations: windowed_modality = r.window_aggregation(window_size, aggregation) From a9fd90d415e23ecd7b194a8703b711769ac8b05b Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Tue, 2 Jun 2026 12:57:35 +0200 Subject: [PATCH 025/132] [SYSTEMDS-3947] Use optuna for Scuro hyperparameter tuning This patch includes the optuna hyperparameter tuning library in the Scuro hyperparameter tuner. It adds support for wandb to observe the progress and parameter configurations. --- .github/workflows/python.yml | 3 +- .../scuro/drsearch/hyperparameter_tuner.py | 499 ++++++++++++------ .../scuro/representations/aggregate.py | 2 +- .../systemds/scuro/representations/bert.py | 45 +- .../representations/window_aggregation.py | 93 +++- 5 files changed, 463 insertions(+), 179 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 73d53ea679d..035965cf550 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -174,7 +174,8 @@ jobs: nltk \ fvcore \ scikit-optimize \ - flair + flair \ + optuna kill $KA cd src/main/python python -m unittest discover -s tests/scuro -p 'test_*.py' -v diff --git a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py index 0737f18d62d..0305f613b63 100644 --- a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py +++ b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py @@ -19,6 +19,7 @@ # # ------------------------------------------------------------- from typing import Dict, List, Tuple, Any, Optional +import inspect import os import numpy as np import logging @@ -31,7 +32,6 @@ import random from systemds.scuro.drsearch.representation_dag import ( RepresentationDAGBuilder, - RepresentationDag, RepresentationNode, ) from systemds.scuro.modality.modality import Modality @@ -40,12 +40,173 @@ from systemds.scuro.utils.checkpointing import CheckpointManager -def get_params_for_node(node_id, params): +def _get_params_for_node(node_id, params): return { k.split("-")[-1]: v for k, v in params.items() if k.startswith(node_id + "-") } +def _param_values_to_spec( + full_name: str, param_values: Any +) -> Optional[Dict[str, Any]]: + if isinstance(param_values, list): + return {"name": full_name, "type": "categorical", "domain": list(param_values)} + if isinstance(param_values, tuple) and len(param_values) == 2: + lo, hi = param_values + if isinstance(lo, int) and isinstance(hi, int): + return {"name": full_name, "type": "integer", "domain": (lo, hi)} + return {"name": full_name, "type": "real", "domain": (float(lo), float(hi))} + if isinstance(param_values, (str, int, float, bool)): + return {"name": full_name, "type": "categorical", "domain": [param_values]} + if hasattr(param_values, "__iter__") and not isinstance( + param_values, (str, bytes, dict) + ): + try: + domain = list(param_values) + except TypeError: + return None + if domain: + return {"name": full_name, "type": "categorical", "domain": domain} + return None + + +def _expand_aggregation_param_specs(op_id: str, agg_cls: Any) -> List[Dict[str, Any]]: + if not inspect.isclass(agg_cls): + return [] + + from systemds.scuro.representations.window_aggregation import ( + nested_aggregation_param_names, + ) + + nested_names = nested_aggregation_param_names(agg_cls) + if not nested_names: + return [] + + try: + instance = agg_cls() + except Exception: + return [] + + search_template = getattr(instance, "parameters", None) or {} + specs = [] + for nested_name in nested_names: + nested_values = search_template.get(nested_name) + if nested_values is None: + continue + full_name = f"{op_id}-aggregation_function_{nested_name}" + spec = _param_values_to_spec(full_name, nested_values) + if spec is not None: + specs.append(spec) + return specs + + +def _is_window_operation(op: Any) -> bool: + if not inspect.isclass(op): + return False + try: + from systemds.scuro.representations.window_aggregation import Window + + return issubclass(op, Window) + except ImportError: + return False + + +def _materialize_node_params( + node: RepresentationNode, flat_params: Dict[str, Any] +) -> Dict[str, Any]: + if not flat_params or node.operation is None: + return flat_params + if not _is_window_operation(node.operation): + return flat_params + + from systemds.scuro.representations.window_aggregation import ( + instantiate_nested_aggregation, + ) + + template = node.parameters or {} + agg_cls = template.get("aggregation_function") + + out: Dict[str, Any] = {} + agg_sub: Dict[str, Any] = {} + prefix = "aggregation_function_" + for key, value in flat_params.items(): + if key.startswith(prefix): + agg_sub[key[len(prefix) :]] = value + else: + out[key] = value + + if inspect.isclass(agg_cls): + out["aggregation_function"] = instantiate_nested_aggregation(agg_cls, agg_sub) + + return out + + +def _is_aggregated_representation_operation(op: Any) -> bool: + if not inspect.isclass(op): + return False + try: + from systemds.scuro.representations.aggregated_representation import ( + AggregatedRepresentation, + ) + + return issubclass(op, AggregatedRepresentation) + except ImportError: + return False + + +def _has_pushdown_aggregation(node_parameters: Optional[Dict[str, Any]]) -> bool: + return bool(node_parameters and "_pushdown_aggregation" in node_parameters) + + +def _apply_pushdown_trial_params( + base_params: Dict[str, Any], trial_params: Dict[str, Any] +) -> Dict[str, Any]: + """Merge trial values into node.parameters['_pushdown_aggregation'].""" + pushdown = copy.deepcopy(base_params.get("_pushdown_aggregation", {})) + prefix = "aggregation_function_" + top_level: Dict[str, Any] = {} + + for key, value in trial_params.items(): + if key.startswith(prefix): + pushdown[key] = value + elif key == "aggregation": + pushdown["aggregation_function_aggregation_function"] = value + elif key != "_pushdown_aggregation": + top_level[key] = value + + result = {**base_params, **top_level} + result["_pushdown_aggregation"] = pushdown + + for key in list(result.keys()): + if key.startswith(prefix): + result.pop(key, None) + return result + + +def _apply_trial_params_to_node( + node: RepresentationNode, global_params: Dict[str, Any] +) -> Dict[str, Any]: + base_params = copy.deepcopy(node.parameters) if node.parameters else {} + flat_params = _get_params_for_node(node.node_id, global_params) + if not flat_params: + return base_params + + trial_params = _materialize_node_params(node, flat_params) + + if _has_pushdown_aggregation(base_params): + return _apply_pushdown_trial_params(base_params, trial_params) + + if _is_aggregated_representation_operation(node.operation): + if "aggregation" in trial_params: + trial_params["aggregation_function_aggregation_function"] = trial_params[ + "aggregation" + ] + base_params.pop("aggregation_function_aggregation_function", None) + base_params.pop("aggregation_function_pad_modality", None) + + return {**base_params, **trial_params} + + @dataclass class HyperparamResult: representation_name: str @@ -91,7 +252,7 @@ def get_k_best_results(self, modality, task, performance_metric_name): prev_node_id = None for node in result.dag.nodes: if node.operation is not None and node.parameters: - params = get_params_for_node(node.node_id, result.best_params) + params = _apply_trial_params_to_node(node, result.best_params) prev_node_id = dag_with_best_params.create_operation_node( node.operation, [prev_node_id], params ) @@ -123,6 +284,12 @@ def __init__( random_state: int = 42, exhaustive_threshold: int = 256, local_search_patience: int = 3, + optuna_sampler: str = "tpe", # "tpe" | "random" | "bayes" + use_wandb: bool = False, + wandb_project: Optional[str] = None, + wandb_entity: Optional[str] = None, + wandb_group: Optional[str] = None, + wandb_tags: Optional[List[str]] = None, ): self.tasks = tasks self.unimodal_optimization_results = optimization_results @@ -137,7 +304,7 @@ def __init__( self.k_best_cache = None self.k_best_cache_by_modality = None self.k_best_representations = None - self.extract_k_best_modalities_per_task() + self.extract_k_best_modalities_per_task() # TODO: cache needed for multimodal optimization self.debug = debug self.logger = logging.getLogger(__name__) self.checkpoint_every = checkpoint_every @@ -152,10 +319,13 @@ def __init__( checkpoint_every=self.checkpoint_every, resume=self.resume, ) - if debug: - logging.basicConfig( - level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" - ) + self.optuna_sampler = optuna_sampler + self.use_wandb = use_wandb + self.wandb_project = wandb_project or "scuro-hyperparam" + self.wandb_entity = wandb_entity + self.wandb_group = wandb_group + self.wandb_tags = wandb_tags or [] + self._wandb_run = None def get_modalities_by_id(self, modality_ids: List[int]) -> Modality: modalities = [] @@ -187,7 +357,7 @@ def extract_k_best_modalities_per_task(self): for modality in self.modalities: k_best_results, cached_data = ( self.unimodal_optimization_results.get_k_best_results( - modality, task, self.scoring_metric + modality, task, self.scoring_metric, cache_needed=False ) ) representations[task.model.name][modality.modality_id] = k_best_results @@ -216,7 +386,7 @@ def resume_from_checkpoint(self): self.optimization_results.results = results def tune_unimodal_representations(self, max_eval_per_rep: Optional[int] = None): - self.resume_from_checkpoint() + # self.resume_from_checkpoint() for task in self.tasks: reps = self.k_best_representations[task.model.name] skip_remaining = 0 @@ -240,11 +410,11 @@ def tune_unimodal_representations(self, max_eval_per_rep: Optional[int] = None): self.optimization_results.add_result(results) self._checkpoint_manager.increment(task.model.name, len(results)) self._checkpoint_manager.checkpoint_if_due( - self.optimization_results.results, "eval_count_by_task" + self.optimization_results.results, ) except Exception: self._checkpoint_manager.save_checkpoint( - self.optimization_results.results, "eval_count_by_task", {} + self.optimization_results.results, {} ) raise @@ -269,7 +439,7 @@ def visit_node(node_id): visit_node(input_id) visited.add(node_id) if node.operation is not None: - params = self._get_params_for_node(node) + params = self.__get_params_for_node(node) if params: hyperparams[node_id] = params reps.append(node.operation) @@ -296,9 +466,9 @@ def visit_node(node_id): ) all_results = [baseline] else: - n_calls = max_evals if max_evals else 50 param_specs = self._build_param_specs(hyperparams) - default_config = {} + discrete_size = self._estimate_discrete_search_size(param_specs) + n_calls = min(discrete_size, max_evals) if max_evals else discrete_size all_results = self._search_best_configs( dag=dag, task=task, @@ -308,6 +478,7 @@ def visit_node(node_id): param_specs=param_specs, budget=n_calls, initial_config=None, + rep_name=rep_name, ) if not all_results: @@ -317,6 +488,8 @@ def get_score(result): score = result[1] if isinstance(score, PerformanceMeasure): return score.average_scores[self.scoring_metric] + elif isinstance(score, list): + return score[1] return score if self.maximize_metric: @@ -325,21 +498,7 @@ def get_score(result): best_params, best_score = min(all_results, key=get_score) tuning_time = time.time() - start_time - # results = self.unimodal_optimization_results.results[self.modalities[0].modality_id][task.model.name] - - # default_result = sorted( - # results, - # key=lambda r: r.val_score[self.scoring_metric], - # reverse=True, - # )[0] - # pm = PerformanceMeasure(name=self.scoring_metric, metrics=self.scoring_metric, higher_is_better=self.maximize_metric) - # pm.add_scores({self.scoring_metric: default_result.val_score[self.scoring_metric]}) - # default_params = self._get_default_params(dag) - # def_par ={} - # for k, v in default_params.items(): - # for k_v, v_v in v.items(): - # def_par[k+"-"+k_v] = v_v - # all_results.append((def_par, pm)) + best_result = HyperparamResult( representation_name=rep_name, best_params=best_params, @@ -354,11 +513,31 @@ def get_score(result): return best_result - def _get_params_for_node(self, node: RepresentationNode) -> Dict[str, Any]: - if not node.operation().parameters: + def __get_params_for_node(self, node: RepresentationNode) -> Dict[str, Any]: + try: + if node.parameters: + op = node.operation(params=node.parameters) + else: + op = node.operation() + except (TypeError, ValueError): + op = node.operation() + + if not op.parameters: return None - params = copy.deepcopy(node.operation().parameters) + params = copy.deepcopy(op.parameters) + if node.parameters: + if inspect.isclass(node.parameters.get("aggregation_function")): + params["aggregation_function"] = node.parameters["aggregation_function"] + for fixed_key in ("target_dimensions", "self_contained"): + if fixed_key in node.parameters: + params[fixed_key] = node.parameters[fixed_key] + + if _has_pushdown_aggregation(node.parameters): + from systemds.scuro.representations.aggregate import Aggregation + + params["aggregation_function"] = Aggregation + return params def _build_param_specs( @@ -367,23 +546,15 @@ def _build_param_specs( param_specs = [] for op_id, op_params in hyperparams.items(): for param_name, param_values in op_params.items(): + if param_name == "aggregation_function": + expanded = _expand_aggregation_param_specs(op_id, param_values) + if expanded: + param_specs.extend(expanded) + continue full_name = op_id + "-" + param_name - if isinstance(param_values, list): - param_type = "categorical" - domain = list(param_values) - elif isinstance(param_values, tuple) and len(param_values) == 2: - lo, hi = param_values - if isinstance(lo, int) and isinstance(hi, int): - param_type = "integer" - else: - param_type = "real" - domain = (lo, hi) - else: - param_type = "categorical" - domain = [param_values] - param_specs.append( - {"name": full_name, "type": param_type, "domain": domain} - ) + spec = _param_values_to_spec(full_name, param_values) + if spec is not None: + param_specs.append(spec) return param_specs def _config_key(self, params: Dict[str, Any]) -> Tuple[Tuple[str, Any], ...]: @@ -536,6 +707,24 @@ def _evaluate_configs( seen_configs[key] for key in unique_keys_in_order if key in seen_configs ] + def _suggest_config_from_specs( + self, trial, param_specs: List[Dict[str, Any]] + ) -> Dict[str, Any]: + + config = {} + for spec in param_specs: + name = spec["name"] + domain = spec["domain"] + if spec["type"] == "categorical": + config[name] = trial.suggest_categorical(name, list(domain)) + elif spec["type"] == "integer": + lo, hi = int(domain[0]), int(domain[1]) + config[name] = trial.suggest_int(name, lo, hi) + else: + lo, hi = float(domain[0]), float(domain[1]) + config[name] = trial.suggest_float(name, lo, hi) + return config + def _search_best_configs( self, dag, @@ -545,126 +734,137 @@ def _search_best_configs( modalities_override, param_specs: List[Dict[str, Any]], budget: int, - initial_config: Dict[str, Any], + initial_config: Optional[Dict[str, Any]], + rep_name: str = "", ) -> List[Tuple[Dict[str, Any], Any]]: + import optuna + from optuna.trial import TrialState + + optuna.logging.set_verbosity( + optuna.logging.INFO if self.debug else optuna.logging.WARNING + ) + budget = max(1, budget) - seen_configs: Dict[Tuple[Tuple[str, Any], ...], Tuple[Dict[str, Any], Any]] = {} all_results: List[Tuple[Dict[str, Any], Any]] = [] - best_score = np.nan - best_config = None - if initial_config is not None and budget > 0: - initial_results = self._evaluate_configs( + seen: Dict[Tuple[Tuple[str, Any], ...], Tuple[Dict[str, Any], Any]] = {} + + if initial_config is not None: + batch = self._evaluate_configs( dag, task, node_order, modality_ids, modalities_override, [initial_config], - seen_configs, + seen, ) - all_results.extend(initial_results) - if initial_results: - p, s = initial_results[0] - best_config = p - best_score = self._score_value(s) - budget -= 1 - - discrete_size = self._estimate_discrete_search_size(param_specs) - if discrete_size is not None and discrete_size <= min( - self.exhaustive_threshold, budget - ): - candidates = self._enumerate_configs(param_specs) - self._rng.shuffle(candidates) - candidates = candidates[:budget] - batch_results = self._evaluate_configs( - dag, - task, - node_order, - modality_ids, - modalities_override, - candidates, - seen_configs, - ) - all_results.extend(batch_results) + all_results.extend(batch) + budget = max(0, budget - len(batch)) + + if budget <= 0: return all_results - initial_budget = min(budget, max(8, len(param_specs) * 4)) - initial_candidates = [ - self._sample_random_config(param_specs) for _ in range(initial_budget) - ] - initial_results = self._evaluate_configs( - dag, - task, - node_order, - modality_ids, - modalities_override, - initial_candidates, - seen_configs, + direction = "maximize" if self.maximize_metric else "minimize" + sampler = ( + optuna.samplers.TPESampler(seed=self.random_state) + if self.optuna_sampler == "tpe" + else optuna.samplers.RandomSampler(seed=self.random_state) ) - all_results.extend(initial_results) - - for params, score in initial_results: - numeric_score = self._score_value(score) - if self._is_better(numeric_score, best_score): - best_score = numeric_score - best_config = params - eval_count = len(seen_configs) - no_improvement_rounds = 0 - step_scale = 0.5 + study = optuna.create_study( + direction=direction, + sampler=sampler, + study_name=f"{task.model.name}-{rep_name}"[:64], + ) - while eval_count < budget: - if best_config is None: - candidate_batch = [self._sample_random_config(param_specs)] - else: - candidate_batch = [] - batch_size = min( - max(2, abs(self.n_jobs) if self.n_jobs != 0 else 1), - budget - eval_count, + wandb_kwargs = {} + if self.use_wandb: + try: + import wandb + from optuna.integration.wandb import WeightsAndBiasesCallback + + wandb_kwargs["wandb_kwargs"] = { + "project": self.wandb_project, + "entity": self.wandb_entity, + "group": self.wandb_group or task.model.name, + "tags": self.wandb_tags + [rep_name, task.model.name], + "name": f"{task.model.name}-{rep_name}-{int(time.time())}", + "config": { + "task": task.model.name, + "representation": rep_name, + "scoring_metric": self.scoring_metric, + "budget": budget, + }, + } + wandb_cb = WeightsAndBiasesCallback( + metric_name=self.scoring_metric, + wandb_kwargs=wandb_kwargs["wandb_kwargs"], ) - for _ in range(batch_size): - candidate_batch.append( - self._generate_neighbor_config( - best_config, param_specs, step_scale - ) - ) + except ImportError: + self.logger.warning( + "wandb/optuna-integration not installed; disabling W&B" + ) + wandb_cb = None + else: + wandb_cb = None + + trial_results: List[Tuple[Dict[str, Any], Any]] = [] - if budget - eval_count > 3: - candidate_batch.append(self._sample_random_config(param_specs)) + def objective(trial: optuna.Trial) -> float: + config = self._suggest_config_from_specs(trial, param_specs) + key = self._config_key(config) + if key in seen: + trial.set_user_attr("duplicate", True) + raise optuna.TrialPruned() - batch_results = self._evaluate_configs( + params, scores = self.evaluate_dag_config( dag, - task, + config, node_order, modality_ids, - modalities_override, - candidate_batch, - seen_configs, + task, + modalities_override=modalities_override, ) - if not batch_results: - step_scale = max(0.05, step_scale * 0.5) - if step_scale <= 0.05: - break - continue + train_score = self._score_value(scores[0]) + val_score = self._score_value(scores[1]) + test_score = self._score_value(scores[2]) + if np.isnan(val_score): + raise optuna.TrialPruned() + + seen[self._config_key(params)] = ( + params, + [train_score, val_score, test_score], + ) + + trial_results.append((params, [train_score, val_score, test_score])) + return val_score + + callbacks = [c for c in [wandb_cb] if c is not None] + n_jobs = 1 if self.n_jobs == 0 else max(1, abs(self.n_jobs)) + try: + study.optimize( + objective, + n_trials=budget, + n_jobs=n_jobs, + callbacks=callbacks, + show_progress_bar=self.debug, + catch=(Exception,), + ) + finally: + if self.use_wandb and wandb.run is not None: + wandb.run.finish() - improved = False - for params, score in batch_results: - numeric_score = self._score_value(score) - if self._is_better(numeric_score, best_score): - best_score = numeric_score - best_config = params - improved = True - all_results.extend(batch_results) - eval_count = len(seen_configs) - - if improved: - no_improvement_rounds = 0 - step_scale = min(0.5, step_scale * 1.1) + all_results.extend(trial_results) + + for trial in study.trials: + if trial.state != TrialState.COMPLETE: + continue + config = trial.params + key = self._config_key(config) + if key in seen: + all_results.append(seen[key]) else: - no_improvement_rounds += 1 - step_scale = max(0.05, step_scale * 0.7) - if no_improvement_rounds >= self.local_search_patience: - break + pass return all_results @@ -684,11 +884,10 @@ def evaluate_dag_config( ): try: dag_copy = copy.deepcopy(dag) - for node_id in node_order: node = dag_copy.get_node_by_id(node_id) - if node.operation is not None and node.parameters: - node.parameters = get_params_for_node(node_id, params) + if node.operation is not None: + node.parameters = _apply_trial_params_to_node(node, params) modalities = ( modalities_override @@ -696,7 +895,7 @@ def evaluate_dag_config( else self.get_modalities_by_id(modality_ids) ) modified_modality = dag_copy.execute(modalities, task) - score = task.run(modified_modality.data)[1] + score = task.run(modified_modality.data) return params, score except Exception as e: @@ -704,7 +903,7 @@ def evaluate_dag_config( traceback.print_exc() self.logger.error(f"Error evaluating DAG with params {params}: {e}") - return params, np.nan + return params, [np.nan, np.nan, np.nan] def tune_multimodal_representations( self, diff --git a/src/main/python/systemds/scuro/representations/aggregate.py b/src/main/python/systemds/scuro/representations/aggregate.py index 8389fadbd0d..cf2c371676f 100644 --- a/src/main/python/systemds/scuro/representations/aggregate.py +++ b/src/main/python/systemds/scuro/representations/aggregate.py @@ -51,7 +51,7 @@ def _sum_agg(data, aggregate_dim=0): def __init__(self, aggregation_function="mean", pad_modality=True, params=None): if params is not None: aggregation_function = params["aggregation_function"] - pad_modality = params["pad_modality"] + pad_modality = params.get("pad_modality", True) if aggregation_function not in list(self._aggregation_function.keys()): raise ValueError("Invalid aggregation function") diff --git a/src/main/python/systemds/scuro/representations/bert.py b/src/main/python/systemds/scuro/representations/bert.py index fcaed8d4935..245466afb43 100644 --- a/src/main/python/systemds/scuro/representations/bert.py +++ b/src/main/python/systemds/scuro/representations/bert.py @@ -51,11 +51,13 @@ def __init__( aggregation=None, params=None, ): - parameters = {"batch_size": [1, 2, 4, 8, 16, 32, 64, 128]} + parameters = { + **(parameters or {}), + "batch_size": [1, 2, 4, 8, 16, 32, 64, 128], + } self.model_name = model_name super().__init__(representation_name, ModalityType.EMBEDDING, parameters) - - self.layer_name = layer + self.layer = layer self.output_file = output_file self.max_seq_length = max_seq_length self.needs_context = True @@ -67,6 +69,9 @@ def __init__( self.data_type = torch.float32 self.aggregation = aggregation self.params = params + if params is not None: + self.layer = params.get("layer", self.layer) + self.batch_size = int(params.get("batch_size", self.batch_size)) @property def gpu_id(self): @@ -83,12 +88,12 @@ def set_parameters( if params is not None: self.max_seq_length = int(params.get("max_seq_length", max_seq_length)) self.batch_size = int(params.get("batch_size", batch_size)) - self.layer_name = params.get("layer_name", layer) + self.layer = params.get("layer", layer) self.output_file = params.get("output_file", output_file) else: self.max_seq_length = max_seq_length self.batch_size = batch_size - self.layer_name = layer + self.layer = layer self.output_file = output_file def get_output_stats(self, input_stats) -> RepresentationStats: @@ -184,14 +189,19 @@ def transform(self, modality, aggregation=None): def get_activation(name): def hook(model, input, output): - self.bert_output = output.detach().cpu().numpy() + if isinstance(output, tuple): + self.bert_output = output[0] + elif hasattr(output, "last_hidden_state"): + self.bert_output = output.last_hidden_state + else: + self.bert_output = output return hook aggregate_dim = (0,) - if self.layer_name != "cls": + if self.layer != "cls": for name, layer in self.model.named_modules(): - if name == self.layer_name: + if name == self.layer: layer.register_forward_hook(get_activation(name)) break if ModalityType.TEXT.has_field(modality.metadata, "text_spans"): @@ -211,7 +221,6 @@ def hook(model, input, output): embeddings = self.create_embeddings( modality.data, self.model, tokenizer, aggregation ) - if self.output_file is not None: save_embeddings(embeddings, self.output_file) @@ -274,11 +283,15 @@ def create_embeddings(self, data, model, tokenizer, aggregation=None): with torch.no_grad(): outputs = model(**inputs) - if self.layer_name == "cls": + if self.layer == "cls": cls_embedding = outputs.last_hidden_state.detach().cpu().numpy() else: cls_embedding = self.bert_output.cpu().numpy() - if aggregation is not None: + if ( + aggregation is not None + and self.layer != "pooler" + and self.layer != "pooler.activation" + ): cls_embedding = aggregation.execute(cls_embedding) cls_embeddings.extend(cls_embedding) @@ -298,7 +311,7 @@ def __init__( self.set_parameters(params, max_seq_length, batch_size, layer, output_file) parameters = { - "layer_name": [ + "layer": [ "cls", "encoder.layer.0", "encoder.layer.1", @@ -342,7 +355,7 @@ def __init__( self.set_parameters(params, max_seq_length, batch_size, layer, output_file) parameters = { - "layer_name": [ + "layer": [ "cls", "encoder.layer.0", "encoder.layer.1", @@ -386,7 +399,7 @@ def __init__( self.set_parameters(params, max_seq_length, batch_size, layer, output_file) parameters = { - "layer_name": [ + "layer": [ "cls", "transformer.layer.0", "transformer.layer.1", @@ -420,7 +433,7 @@ def __init__( params=None, ): self.set_parameters(params, max_seq_length, batch_size, layer, output_file) - parameters = {"layer_name": ["cls", "encoder.albert_layer_groups.0", "pooler"]} + parameters = {"layer": ["cls", "encoder.albert_layer_groups.0", "pooler"]} super().__init__( "ALBERT", "albert-base-v2", @@ -445,7 +458,7 @@ def __init__( params=None, ): parameters = { - "layer_name": [ + "layer": [ "cls", "encoder.layer.0", "encoder.layer.1", diff --git a/src/main/python/systemds/scuro/representations/window_aggregation.py b/src/main/python/systemds/scuro/representations/window_aggregation.py index 541a7b68fb2..5d63e4790dd 100644 --- a/src/main/python/systemds/scuro/representations/window_aggregation.py +++ b/src/main/python/systemds/scuro/representations/window_aggregation.py @@ -19,6 +19,9 @@ # # ------------------------------------------------------------- +from concurrent.futures import ThreadPoolExecutor +import inspect +import os import numpy as np import math @@ -33,6 +36,37 @@ ) +def nested_aggregation_param_names(agg_cls): + if not inspect.isclass(agg_cls): + return set() + if agg_cls is Aggregation or agg_cls.__name__ == "Aggregation": + return {"aggregation_function", "pad_modality"} + try: + return set(agg_cls().parameters.keys()) + except Exception: + return set() + + +def instantiate_nested_aggregation(agg_cls, nested): + if not inspect.isclass(agg_cls): + return agg_cls + if not nested: + return agg_cls() + + if agg_cls is Aggregation or agg_cls.__name__ == "Aggregation": + return Aggregation(params=nested) + + allowed = nested_aggregation_param_names(agg_cls) + filtered = {key: value for key, value in nested.items() if key in allowed} + if not filtered: + return agg_cls() + + init_params = inspect.signature(agg_cls.__init__).parameters + if "params" in init_params: + return agg_cls(params=filtered) + return agg_cls(**filtered) + + class Window(Context): def __init__(self, name, aggregation_function): self.aggregation_function = aggregation_function @@ -117,23 +151,52 @@ def _rest_numel(shape): ) class WindowAggregation(Window): def __init__( - self, aggregation_function="mean", window_size=10, pad=True, params=None + self, + aggregation_function="mean", + window_size=10, + pad=True, + params=None, ): if params is not None: - aggregation_function = params["aggregation_function"] - try: - aggregation_function = aggregation_function() - except: - pass + if isinstance( + params.get("aggregation_function"), (Aggregation, Representation) + ): + aggregation_function = params["aggregation_function"] + else: + nested_agg = { + key[len("aggregation_function_") :]: value + for key, value in params.items() + if key.startswith("aggregation_function_") + } + agg_value = params.get("aggregation_function") + if nested_agg and inspect.isclass(agg_value): + aggregation_function = instantiate_nested_aggregation( + agg_value, nested_agg + ) + elif inspect.isclass(agg_value): + aggregation_function = agg_value() + else: + aggregation_function = params.get( + "aggregation_function", aggregation_function + ) window_size = params["window_size"] - pad = True + pad = params.get("pad", True) super().__init__("WindowAggregation", aggregation_function) - self.parameters["window_size"] = [5, 10, 15, 25, 50, 100] + self.parameters["window_size"] = (4, 128) self.window_size = int(window_size) self.pad = pad def get_output_stats(self, input_stats: RepresentationStats) -> tuple: - in_shape = tuple(int(s) for s in input_stats.output_shape) + if not isinstance(self.aggregation_function, Aggregation): + windowed_input_stats = RepresentationStats( + input_stats.num_instances, (self.window_size,) + ) + in_shape = self.aggregation_function.get_output_stats( + windowed_input_stats + ).output_shape + in_shape = (input_stats.output_shape[0], *in_shape) + else: + in_shape = tuple(int(s) for s in input_stats.output_shape) if len(in_shape) == 1: self.stats = RepresentationStats( input_stats.num_instances, @@ -163,6 +226,14 @@ def estimate_peak_memory_bytes(self, input_stats: RepresentationStats) -> dict: if len(in_shape) == 0: return {"cpu_peak_bytes": 0, "gpu_peak_bytes": 0} + out_stats = self.get_output_stats(input_stats) + out_shape = out_stats.output_shape + output_bytes = ( + input_stats.num_instances + * np.prod(out_shape) + * np.dtype(self.data_type).itemsize + ) + effective_seq_len = in_shape[0] in_numel = effective_seq_len * self._rest_numel(in_shape) output_bytes = self.estimate_output_memory_bytes(input_stats) @@ -188,7 +259,7 @@ def execute(self, modality): for instance in modality.data: new_length = math.ceil(len(instance) / self.window_size) if modality.get_data_layout() == DataLayout.SINGLE_LEVEL: - instance = np.array(instance) + instance = np.asarray(instance) instance.setflags(write=False) windowed_instance = self.window_aggregate_single_level( instance, new_length @@ -199,7 +270,7 @@ def execute(self, modality): windowed_instance = self.window_aggregate_nested_level( instance, new_length ) - original_lengths.append(new_length) + original_lengths.append(windowed_instance.shape[0]) windowed_data.append(windowed_instance) if self.pad and not isinstance(windowed_data, np.ndarray): From 35b17a99f090a8ff7862fe756d049760fb49a3c7 Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Tue, 2 Jun 2026 17:01:03 +0200 Subject: [PATCH 026/132] [SYSTEMDS-3835] Improve window aggregation efficiency in Scuro In this patch we improve the runtime efficiency of the window aggregation operator in Scuro. The problem with the latest approach was the iteration over each window instead of vecortized execution. This change neede a couple of adaptions in subsequent representations. --- .../scuro/dataloader/timeseries_loader.py | 46 ++++--- .../scuro/representations/aggregate.py | 4 +- .../systemds/scuro/representations/average.py | 8 +- .../scuro/representations/concatenation.py | 38 +++--- .../scuro/representations/hadamard.py | 30 +++-- .../systemds/scuro/representations/lstm.py | 2 +- .../systemds/scuro/representations/max.py | 2 +- .../scuro/representations/mlp_averaging.py | 21 +++- .../multimodal_attention_fusion.py | 1 + .../scuro/representations/spectrogram.py | 12 +- .../systemds/scuro/representations/sum.py | 32 +++-- .../timeseries_representations.py | 109 +++++++++------- .../representations/window_aggregation.py | 116 +++++++++--------- .../scuro/test_unimodal_representations.py | 3 +- 14 files changed, 241 insertions(+), 183 deletions(-) diff --git a/src/main/python/systemds/scuro/dataloader/timeseries_loader.py b/src/main/python/systemds/scuro/dataloader/timeseries_loader.py index 7131b55db11..6b697e6a7ad 100644 --- a/src/main/python/systemds/scuro/dataloader/timeseries_loader.py +++ b/src/main/python/systemds/scuro/dataloader/timeseries_loader.py @@ -64,15 +64,7 @@ def __init__( def extract(self, file: str, index: Optional[Union[str, List[str]]] = None): self.file_sanity_check(file) - if self.file_format == "npy": - data = self._load_npy(file) - elif self.file_format in ["txt", "csv"]: - with open(file, "r") as f: - first_line = f.readline() - if any(name in first_line for name in self.signal_names): - data = self._load_csv_with_header(file) - else: - data = self._load_txt(file) + data = self._load_data(file) if data.ndim > 1 and len(self.signal_names) == 1: data = data.flatten() @@ -96,6 +88,18 @@ def extract(self, file: str, index: Optional[Union[str, List[str]]] = None): ) self.data.append(data[i]) + def _load_data(self, file: str) -> np.ndarray: + if self.file_format == "npy": + data = self._load_npy(file) + elif self.file_format in ["txt", "csv"]: + with open(file, "r") as f: + first_line = f.readline() + if any(name in first_line for name in self.signal_names): + data = self._load_csv_with_header(file) + else: + data = self._load_txt(file) + return data + def _normalize_signals(self, data: np.ndarray) -> np.ndarray: if data.ndim == 1: mean = np.mean(data) @@ -145,14 +149,16 @@ def _load_csv_with_header(self, file: str, delimiter: str = None) -> np.ndarray: return data def get_stats(self, source_path: str): - pass # TODO: Implement this - # self.file_sanity_check(source_path) - # max_length = 0 - # num_instances = 0 - # num_signals = 0 - # for file in os.listdir(source_path): - # data = self._load_npy(source_path + file) - # max_length = max(max_length, data.shape[0]) - # num_instances += 1 - # num_signals = max(num_signals, data.shape[1]) - # return TimeseriesStats(max_length, num_instances, num_signals) + self.file_sanity_check(source_path) + max_length = 0 + num_instances = 0 + num_signals = 0 + for file_name in self.indices: + file = source_path + file_name + "." + self.file_format + data = self._load_data(file) + max_length = max(max_length, data.shape[0]) + num_instances += 1 + num_signals = max(num_signals, data.shape[1]) + return TimeseriesStats( + max_length, num_instances, num_signals, (max_length,), True + ) diff --git a/src/main/python/systemds/scuro/representations/aggregate.py b/src/main/python/systemds/scuro/representations/aggregate.py index cf2c371676f..e8a44faa34c 100644 --- a/src/main/python/systemds/scuro/representations/aggregate.py +++ b/src/main/python/systemds/scuro/representations/aggregate.py @@ -126,8 +126,8 @@ def execute(self, modality, aggregate_dim=(0,)): def transform(self, modality): return self.execute(modality) - def compute_feature(self, instance): - return self._aggregation_func(instance) + def compute_feature(self, instance, axis=0): + return self._aggregation_func(instance, axis) def get_aggregation_functions(self): return list(self._aggregation_function.keys()) diff --git a/src/main/python/systemds/scuro/representations/average.py b/src/main/python/systemds/scuro/representations/average.py index ac51f5d1e8d..f58ba0b6802 100644 --- a/src/main/python/systemds/scuro/representations/average.py +++ b/src/main/python/systemds/scuro/representations/average.py @@ -32,7 +32,7 @@ @register_fusion_operator() class Average(Fusion): - def __init__(self): + def __init__(self, params=None): """ Combines modalities using averaging """ @@ -41,10 +41,10 @@ def __init__(self): self.associative = True self.commutative = True - def execute(self, modalities: List[Modality]): - data = copy.deepcopy(modalities[0].data) + def execute(self, modalities: List[Modality], labels=None): + data = np.asarray(copy.deepcopy(modalities[0].data), dtype=float) for i in range(1, len(modalities)): - data += modalities[i].data + data += np.asarray(modalities[i].data, dtype=float) data /= len(modalities) diff --git a/src/main/python/systemds/scuro/representations/concatenation.py b/src/main/python/systemds/scuro/representations/concatenation.py index 5d53690317e..3bdfdb28b1f 100644 --- a/src/main/python/systemds/scuro/representations/concatenation.py +++ b/src/main/python/systemds/scuro/representations/concatenation.py @@ -79,23 +79,27 @@ def get_output_stats(self, input_stats_list) -> RepresentationStats: return RepresentationStats(0, (0,)) num_instances = stats_list[0].num_instances - rank = len(stats_list[0].output_shape) - - if rank == 1: - total_dim = sum(s.output_shape[0] for s in stats_list) - output_shape = (total_dim,) - elif rank == 2: - time_dim = stats_list[0].output_shape[0] - total_dim = sum(s.output_shape[1] for s in stats_list) - output_shape = (time_dim, total_dim) - else: - output_shape = stats_list[0].output_shape + total_dim = sum(s.output_shape[-1] for s in stats_list) + output_shape = (total_dim,) return RepresentationStats(num_instances, output_shape) - def estimate_peak_memory_bytes(self, input_stats) -> dict: - # TODO - return { - "cpu_peak_bytes": 0, - "gpu_peak_bytes": 0, - } + def estimate_peak_memory_bytes(self, input_stats_list) -> dict: + elem_size = np.dtype(np.float32).itemsize + + def stats_bytes(s: RepresentationStats) -> int: + numel = int(np.prod(s.output_shape)) if len(s.output_shape) > 0 else 1 + return int(s.num_instances * numel * elem_size) + + current_output = 0 + peak = 0 + for s in input_stats_list: + chunk = stats_bytes(s) + new_output = current_output + chunk + + step_peak = current_output + chunk + new_output + chunk + peak = max(peak, step_peak) + current_output = new_output + + cpu_peak = int(peak * 1.15 + 16 * 1024 * 1024) + return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} diff --git a/src/main/python/systemds/scuro/representations/hadamard.py b/src/main/python/systemds/scuro/representations/hadamard.py index a29f63e2b24..fc053f9c6dc 100644 --- a/src/main/python/systemds/scuro/representations/hadamard.py +++ b/src/main/python/systemds/scuro/representations/hadamard.py @@ -55,18 +55,22 @@ def get_output_stats(self, input_stats_list) -> RepresentationStats: if not stats_list: return RepresentationStats(0, (0,)) - def num_elements(stats: RepresentationStats) -> int: - n = 1 - for d in stats.output_shape: - n *= d - return n + max_dim = max([stats.output_shape[-1] for stats in stats_list]) + return RepresentationStats(stats_list[0].num_instances, (max_dim,)) - largest = max(stats_list, key=num_elements) - return RepresentationStats(largest.num_instances, largest.output_shape) + def estimate_peak_memory_bytes(self, input_stats_list) -> dict: + elem_size = np.dtype(np.float64).itemsize - def estimate_peak_memory_bytes(self, input_stats) -> dict: - # TODO - return { - "cpu_peak_bytes": 0, - "gpu_peak_bytes": 0, - } + def stats_payload_bytes(s: RepresentationStats) -> int: + numel = int(np.prod(s.output_shape)) if len(s.output_shape) > 0 else 1 + return int(s.num_instances * numel * elem_size) + + stacked_input_bytes = sum(stats_payload_bytes(s) for s in input_stats_list) + out_stats = self.get_output_stats(input_stats_list) + output_bytes = stats_payload_bytes(out_stats) + reduction_workspace_bytes = output_bytes + cpu_peak = int( + (stacked_input_bytes + output_bytes + reduction_workspace_bytes) * 1.15 + + 8 * 1024 * 1024 + ) + return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} diff --git a/src/main/python/systemds/scuro/representations/lstm.py b/src/main/python/systemds/scuro/representations/lstm.py index 104f2727e69..7243b65966a 100644 --- a/src/main/python/systemds/scuro/representations/lstm.py +++ b/src/main/python/systemds/scuro/representations/lstm.py @@ -233,7 +233,7 @@ def execute(self, modalities: List[Modality], labels: np.ndarray = None): TensorDataset(X_tensor), batch_size=self.batch_size, shuffle=False ) for (batch_X,) in inference_dataloader: - batch_X = batch_X.to(device) + batch_X = batch_X.to(self.device) features, _ = self.model(batch_X) all_features.append(features.cpu()) diff --git a/src/main/python/systemds/scuro/representations/max.py b/src/main/python/systemds/scuro/representations/max.py index 39f5069c2b5..2dadc497c78 100644 --- a/src/main/python/systemds/scuro/representations/max.py +++ b/src/main/python/systemds/scuro/representations/max.py @@ -30,7 +30,7 @@ @register_fusion_operator() class RowMax(Fusion): - def __init__(self): + def __init__(self, params=None): """ Combines modalities by computing the outer product of a modality combination and taking the row max diff --git a/src/main/python/systemds/scuro/representations/mlp_averaging.py b/src/main/python/systemds/scuro/representations/mlp_averaging.py index 8c8d67a06ec..46fe04899aa 100644 --- a/src/main/python/systemds/scuro/representations/mlp_averaging.py +++ b/src/main/python/systemds/scuro/representations/mlp_averaging.py @@ -54,8 +54,12 @@ def __init__(self, output_dim=512, batch_size=32, params=None): "batch_size": [8, 16, 32, 64, 128], } super().__init__("MLPAveraging", parameters) - self.output_dim = output_dim - self.batch_size = batch_size + if params is not None: + self.output_dim = params.get("output_dim", output_dim) + self.batch_size = params.get("batch_size", batch_size) + else: + self.output_dim = output_dim + self.batch_size = batch_size self.device = None self.data_type = np.float32 self.gpu_id = None @@ -70,7 +74,10 @@ def gpu_id(self, gpu_id): self.device = get_device(gpu_id) def get_output_stats(self, input_stats: RepresentationStats) -> RepresentationStats: - if len(input_stats.output_shape) > 1: + if ( + len(input_stats.output_shape) > 1 + and np.prod(input_stats.output_shape) > self.output_dim + ): return RepresentationStats( input_stats.num_instances, (self.output_dim,), @@ -87,7 +94,13 @@ def get_output_stats(self, input_stats: RepresentationStats) -> RepresentationSt ) return RepresentationStats( input_stats.num_instances, - (self.output_dim,), + ( + ( + np.prod(input_stats.output_shape) + if np.prod(input_stats.output_shape) < self.output_dim + else self.output_dim + ), + ), output_shape_is_known=input_stats.output_shape_is_known, ) diff --git a/src/main/python/systemds/scuro/representations/multimodal_attention_fusion.py b/src/main/python/systemds/scuro/representations/multimodal_attention_fusion.py index a295eaa267a..066f3432159 100644 --- a/src/main/python/systemds/scuro/representations/multimodal_attention_fusion.py +++ b/src/main/python/systemds/scuro/representations/multimodal_attention_fusion.py @@ -42,6 +42,7 @@ def __init__( batch_size=32, num_epochs=20, learning_rate=0.001, + params=None, ): parameters = { "hidden_dim": [32, 128, 256, 384, 512, 768], diff --git a/src/main/python/systemds/scuro/representations/spectrogram.py b/src/main/python/systemds/scuro/representations/spectrogram.py index ba455cb9808..ab0c8a6c649 100644 --- a/src/main/python/systemds/scuro/representations/spectrogram.py +++ b/src/main/python/systemds/scuro/representations/spectrogram.py @@ -60,10 +60,14 @@ def transform(self, modality, aggregation=None): return transformed_modality def compute_feature(self, instance): + data = np.array(instance) spectrogram = librosa.stft( - y=np.array(np.abs(instance)), hop_length=self.hop_length, n_fft=self.n_fft + y=np.abs(data), hop_length=self.hop_length, n_fft=self.n_fft ) - return librosa.amplitude_to_db(np.abs(spectrogram)).T + if data.ndim == 1: + return librosa.amplitude_to_db(np.abs(spectrogram)).T + + return librosa.amplitude_to_db(np.abs(spectrogram)).transpose(0, 2, 1) def estimate_peak_memory_bytes(self, input_stats) -> dict: # TODO: validate this function @@ -74,7 +78,7 @@ def estimate_peak_memory_bytes(self, input_stats) -> dict: if signal_length < self.n_fft: num_frames = 1 else: - num_frames = 1 + (signal_length - self.n_fft) // self.hop_length + num_frames = 1 + signal_length // self.hop_length num_frames = max(int(num_frames), 1) num_freq_bins = 1 + self.n_fft // 2 @@ -121,7 +125,7 @@ def get_output_stats(self, input_stats) -> RepresentationStats: if signal_length < self.n_fft: num_frames = 1 else: - num_frames = 1 + (signal_length - self.n_fft) // self.hop_length + num_frames = 1 + signal_length // self.hop_length num_frames = max(int(num_frames), 1) num_freq_bins = 1 + self.n_fft // 2 diff --git a/src/main/python/systemds/scuro/representations/sum.py b/src/main/python/systemds/scuro/representations/sum.py index d6c4fe659b2..4f658020f1e 100644 --- a/src/main/python/systemds/scuro/representations/sum.py +++ b/src/main/python/systemds/scuro/representations/sum.py @@ -61,18 +61,24 @@ def get_output_stats(self, input_stats_list) -> RepresentationStats: if not stats_list: return RepresentationStats(0, (0,)) - def num_elements(stats: RepresentationStats) -> int: - n = 1 - for d in stats.output_shape: - n *= d - return n + max_dim = max([stats.output_shape[-1] for stats in stats_list]) + return RepresentationStats(stats_list[0].num_instances, (max_dim,)) - largest = max(stats_list, key=num_elements) - return RepresentationStats(largest.num_instances, largest.output_shape) + def estimate_peak_memory_bytes(self, input_stats_list) -> dict: + elem_size = np.dtype(np.float64).itemsize - def estimate_peak_memory_bytes(self, input_stats) -> dict: - # TODO - return { - "cpu_peak_bytes": 0, - "gpu_peak_bytes": 0, - } + def stats_payload_bytes(s: RepresentationStats) -> int: + numel = int(np.prod(s.output_shape)) if len(s.output_shape) > 0 else 1 + return int(s.num_instances * numel * elem_size) + + first_bytes = stats_payload_bytes(input_stats_list[0]) + max_other_bytes = 0 + if len(input_stats_list) > 1: + max_other_bytes = max(stats_payload_bytes(s) for s in input_stats_list[1:]) + + ufunc_workspace_bytes = int(0.1 * max(first_bytes, max_other_bytes)) + cpu_peak = int( + (first_bytes + max_other_bytes + ufunc_workspace_bytes) * 1.15 + + 8 * 1024 * 1024 + ) + return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} diff --git a/src/main/python/systemds/scuro/representations/timeseries_representations.py b/src/main/python/systemds/scuro/representations/timeseries_representations.py index 80f6880a0b8..26c8c1a8d98 100644 --- a/src/main/python/systemds/scuro/representations/timeseries_representations.py +++ b/src/main/python/systemds/scuro/representations/timeseries_representations.py @@ -77,8 +77,8 @@ class Mean(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("Mean") - def compute_feature(self, signal): - return np.array(np.mean(signal)) + def compute_feature(self, signal, axis=-1): + return np.array(np.mean(signal, axis=axis)) @register_representation([ModalityType.TIMESERIES]) @@ -87,8 +87,8 @@ class Min(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("Min") - def compute_feature(self, signal): - return np.array(np.min(signal)) + def compute_feature(self, signal, axis=-1): + return np.array(np.min(signal, axis=axis)) @register_representation([ModalityType.TIMESERIES]) @@ -97,8 +97,8 @@ class Max(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("Max") - def compute_feature(self, signal): - return np.array(np.max(signal)) + def compute_feature(self, signal, axis=-1): + return np.array(np.max(signal, axis=axis)) @register_representation([ModalityType.TIMESERIES]) @@ -107,8 +107,8 @@ class Sum(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("Sum") - def compute_feature(self, signal): - return np.array(np.sum(signal)) + def compute_feature(self, signal, axis=-1): + return np.array(np.sum(signal, axis=axis)) @register_representation([ModalityType.TIMESERIES]) @@ -117,8 +117,8 @@ class Std(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("Std") - def compute_feature(self, signal): - return np.array(np.std(signal)) + def compute_feature(self, signal, axis=-1): + return np.array(np.std(signal, axis=axis)) @register_representation([ModalityType.TIMESERIES]) @@ -127,8 +127,8 @@ class Skew(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("Skew") - def compute_feature(self, signal): - return np.array(stats.skew(signal)) + def compute_feature(self, signal, axis=-1): + return np.array(stats.skew(signal, axis=axis)) @register_representation([ModalityType.TIMESERIES]) @@ -140,8 +140,8 @@ def __init__(self, quantile=0.9, params=None): ) self.quantile = quantile - def compute_feature(self, signal): - return np.array(np.quantile(signal, self.quantile)) + def compute_feature(self, signal, axis=-1): + return np.array(np.quantile(signal, self.quantile, axis=axis)) @register_representation([ModalityType.TIMESERIES]) @@ -150,8 +150,8 @@ class Kurtosis(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("Kurtosis") - def compute_feature(self, signal): - return np.array(stats.kurtosis(signal, fisher=True, bias=False)) + def compute_feature(self, signal, axis=-1): + return np.array(stats.kurtosis(signal, fisher=True, bias=True, axis=axis)) @register_representation([ModalityType.TIMESERIES]) @@ -160,8 +160,8 @@ class RMS(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("RMS") - def compute_feature(self, signal): - return np.array(np.sqrt(np.mean(np.square(signal)))) + def compute_feature(self, signal, axis=-1): + return np.array(np.sqrt(np.mean(np.square(signal), axis=axis))) @register_representation([ModalityType.TIMESERIES]) @@ -170,8 +170,8 @@ class ZeroCrossingRate(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("ZeroCrossingRate") - def compute_feature(self, signal): - return np.array(np.sum(np.diff(np.signbit(signal)) != 0)) + def compute_feature(self, signal, axis=-1): + return np.array(np.sum(np.diff(np.signbit(signal), axis=axis) != 0, axis=axis)) @register_representation([ModalityType.TIMESERIES]) @@ -181,16 +181,23 @@ def __init__(self, k=1, params=None): super().__init__("ACF", {"k": [1, 2, 5, 10, 20, 25, 50, 100, 200, 500]}) self.k = k - def compute_feature(self, signal): - x = np.asarray(signal) - np.mean(signal) + def compute_feature(self, signal, axis=-1): + x = np.asarray(signal, dtype=np.float64) + x = x - np.mean(x, axis=axis, keepdims=True) k = int(self.k) - if k <= 0 or k >= len(x): - return np.array(0.0) - den = np.dot(x, x) - if not np.isfinite(den) or np.isclose(den, 0.0): - return np.array(0.0) - corr = np.correlate(x[:-k], x[k:])[0] - return np.array(corr / den) + n = x.shape[axis] + if k <= 0 or k >= n: + out_shape = list(x.shape) + del out_shape[axis] + return np.zeros(out_shape) if out_shape else np.array(0.0) + den = np.sum(x * x, axis=axis) + xm = np.moveaxis(x, axis, -1) + corr = np.sum(xm[..., :-k] * xm[..., k:], axis=-1) + with np.errstate(invalid="ignore", divide="ignore"): + out = corr / den + bad = ~np.isfinite(den) | np.isclose(den, 0.0) + out = np.where(bad, 0.0, out) + return np.asarray(out) def get_k_values(self, max_length, percent=0.2, num=10, log=False): # TODO: Probably would be useful to invoke this function while tuning the hyperparameters depending on the max length of the singal @@ -205,11 +212,11 @@ def get_k_values(self, max_length, percent=0.2, num=10, log=False): @register_representation([ModalityType.TIMESERIES]) @register_context_representation_operator(ModalityType.TIMESERIES) class FrequencyMagnitude(TimeSeriesRepresentation): - def __init__(self): + def __init__(self, params=None): super().__init__("FrequencyMagnitude") - def compute_feature(self, signal): - return np.array(np.abs(np.fft.rfft(signal))) + def compute_feature(self, signal, axis=-1): + return np.array(np.abs(np.fft.rfft(signal, axis=axis))) @register_representation([ModalityType.TIMESERIES]) @@ -219,11 +226,17 @@ def __init__(self, fs=1.0, params=None): super().__init__("SpectralCentroid", parameters={"fs": [0.5, 1.0, 2.0]}) self.fs = fs - def compute_feature(self, signal): - frequency_magnitude = FrequencyMagnitude().compute_feature(signal) - freqencies = np.fft.rfftfreq(len(signal), d=1.0 / self.fs) - num = np.sum(freqencies * frequency_magnitude) - den = np.sum(frequency_magnitude) + 1e-12 + def compute_feature(self, signal, axis=-1): + signal = np.asarray(signal, dtype=np.float64) + n = signal.shape[axis] + frequency_magnitude = FrequencyMagnitude().compute_feature(signal, axis=axis) + frequencies = np.fft.rfftfreq(n, d=1.0 / self.fs) + ax = axis if axis >= 0 else frequency_magnitude.ndim + axis + freq_shape = [1] * frequency_magnitude.ndim + freq_shape[ax] = frequencies.size + frequencies = frequencies.reshape(freq_shape) + num = np.sum(frequencies * frequency_magnitude, axis=axis) + den = np.sum(frequency_magnitude, axis=axis) + 1e-12 return np.array(num / den) @@ -239,11 +252,17 @@ def __init__(self, fs=1.0, f1=0.0, f2=0.5, params=None): self.f1 = f1 self.f2 = f2 - def compute_feature( - self, - signal, - ): - frequency_magnitude = FrequencyMagnitude().compute_feature(signal) - freqencies = np.fft.rfftfreq(len(signal), d=1.0 / self.fs) - m = (freqencies >= self.f1) & (freqencies < self.f2) - return np.array(np.sum(frequency_magnitude[m] ** 2)) + def compute_feature(self, signal, axis=-1): + signal = np.asarray(signal, dtype=np.float64) + n = signal.shape[axis] + + frequency_magnitude = FrequencyMagnitude().compute_feature(signal, axis=axis) + frequencies = np.fft.rfftfreq(n, d=1.0 / self.fs) + + ax = axis if axis >= 0 else frequency_magnitude.ndim + axis + freq_shape = [1] * frequency_magnitude.ndim + freq_shape[ax] = frequencies.size + frequencies = frequencies.reshape(freq_shape) + + in_band = (frequencies >= self.f1) & (frequencies < self.f2) + return np.array(np.sum((frequency_magnitude**2) * in_band, axis=axis)) diff --git a/src/main/python/systemds/scuro/representations/window_aggregation.py b/src/main/python/systemds/scuro/representations/window_aggregation.py index 5d63e4790dd..36e12c0cd5c 100644 --- a/src/main/python/systemds/scuro/representations/window_aggregation.py +++ b/src/main/python/systemds/scuro/representations/window_aggregation.py @@ -19,9 +19,7 @@ # # ------------------------------------------------------------- -from concurrent.futures import ThreadPoolExecutor import inspect -import os import numpy as np import math @@ -236,7 +234,6 @@ def estimate_peak_memory_bytes(self, input_stats: RepresentationStats) -> dict: effective_seq_len = in_shape[0] in_numel = effective_seq_len * self._rest_numel(in_shape) - output_bytes = self.estimate_output_memory_bytes(input_stats) one_instance_bytes = in_numel * np.dtype(self.data_type).itemsize input_bytes = one_instance_bytes * input_stats.num_instances @@ -272,36 +269,28 @@ def execute(self, modality): ) original_lengths.append(windowed_instance.shape[0]) windowed_data.append(windowed_instance) - if self.pad and not isinstance(windowed_data, np.ndarray): target_length = max(original_lengths) - sample_shape = windowed_data[0].shape - padded_features = [] + padded_features = np.zeros( + (len(windowed_data), target_length, *windowed_data[0].shape[1:]) + ) for i, features in enumerate(windowed_data): - current_len = original_lengths[i] - - if current_len < target_length: - padding_needed = target_length - current_len - - pad_shape = (padding_needed,) + features.shape[1:] - padding = np.zeros(pad_shape) - padded = np.concatenate([features, padding], axis=0) - - padded_features.append(padded) + if padded_features.ndim == 3: + padded_features[i, : features.shape[0], :] = features else: - padded_features.append(features) + padded_features[i, : features.shape[0]] = features - attention_masks = np.zeros((len(windowed_data), target_length)) - for i, length in enumerate(original_lengths): - actual_length = min(length, target_length) - attention_masks[i, :actual_length] = 1 + # attention_masks = np.zeros((len(windowed_data), target_length)) + # for i, length in enumerate(original_lengths): + # actual_length = min(length, target_length) + # attention_masks[i, :actual_length] = 1 - ModalityType(modality.modality_type).add_field_for_instances( - modality.metadata, "attention_masks", attention_masks - ) + # ModalityType(modality.modality_type).add_field_for_instances( + # modality.metadata, "attention_masks", attention_masks + # ) - windowed_data = np.array(padded_features) + windowed_data = padded_features data_type = modality.metadata[0]["data_layout"]["type"] if data_type != "str": windowed_data = windowed_data.astype(data_type) @@ -313,17 +302,28 @@ def window_aggregate_single_level(self, instance, new_length): if isinstance(instance, str): return instance - result = [] - for i in range(0, new_length): - result.append( - self.aggregation_function.compute_feature( - instance[ - i * self.window_size : i * self.window_size + self.window_size - ] - ) - ) + arr = np.asarray(instance) + cut_length = (new_length - 1) * self.window_size - return np.array(result) + full_batches = arr[:cut_length].reshape( + new_length - 1, self.window_size, *arr.shape[1:] + ) + tail = arr[cut_length:] + + sig = inspect.signature(self.aggregation_function.compute_feature) + if "axis" in sig.parameters: + full_result = self.aggregation_function.compute_feature( + full_batches, axis=1 + ) + if tail.size: + tail_result = self.aggregation_function.compute_feature(tail) + full_result = np.concatenate([full_result, np.array([tail_result])]) + else: + full_result = self.aggregation_function.compute_feature(full_batches) + if tail.size: + tail_result = self.aggregation_function.compute_feature(tail) + full_result = np.concatenate([full_result, tail_result[None, :]]) + return full_result def window_aggregate_nested_level(self, instance, new_length): result = [[] for _ in range(0, new_length)] @@ -339,10 +339,12 @@ def window_aggregate_nested_level(self, instance, new_length): [ModalityType.TIMESERIES, ModalityType.AUDIO, ModalityType.EMBEDDING] ) class StaticWindow(Window): - # TODO def __init__(self, aggregation_function="mean", num_windows=100, params=None): super().__init__("StaticWindow", aggregation_function) - self.parameters["num_windows"] = [10, num_windows] + if params is not None: + num_windows = params.get("num_windows", 100) + + self.parameters["num_windows"] = (5, num_windows) self.num_windows = int(num_windows) def get_output_stats(self, input_stats: RepresentationStats) -> tuple: @@ -387,28 +389,26 @@ def estimate_peak_memory_bytes(self, input_stats: RepresentationStats) -> dict: def execute(self, modality): windowed_data = [] - for instance in modality.data: - window_size = len(instance) // self.num_windows - remainder = len(instance) % self.num_windows - output = [] - start = 0 - for i in range(0, self.num_windows): - extra = 1 if i < remainder else 0 - end = start + window_size + extra - window = instance[start:end] - window.setflags(write=False) - val = ( - self.aggregation_function.compute_feature(window) - if len(window) > 0 - else np.zeros_like(output[i - 1]) - ) - output.append(val) - start = end + window_size = int(np.ceil(len(instance) / self.num_windows)) + padding_size = int(window_size * self.num_windows - len(instance)) + pad_width = [(0, 0)] * instance.ndim + pad_width[0] = (0, padding_size) + instance = np.pad( + instance, pad_width=pad_width, mode="constant", constant_values=0 + ) + full_batches = instance.reshape( + self.num_windows, window_size, *instance.shape[1:] + ) - windowed_data.append(output) + sig = inspect.signature(self.aggregation_function.compute_feature) + if "axis" in sig.parameters: + f = self.aggregation_function.compute_feature(full_batches, axis=1) + else: + f = self.aggregation_function.compute_feature(full_batches) + + windowed_data.append(f) windowed_data = np.array(windowed_data) - self.assert_output_stats(windowed_data) return windowed_data @@ -418,7 +418,9 @@ def execute(self, modality): class DynamicWindow(Window): def __init__(self, aggregation_function="mean", num_windows=100, params=None): super().__init__("DynamicWindow", aggregation_function) - self.parameters["num_windows"] = [10, num_windows] + if params is not None: + num_windows = params.get("num_windows", 100) + self.parameters["num_windows"] = (5, num_windows) self.num_windows = int(num_windows) def get_output_stats(self, input_stats: RepresentationStats) -> tuple: diff --git a/src/main/python/tests/scuro/test_unimodal_representations.py b/src/main/python/tests/scuro/test_unimodal_representations.py index a4e18743090..2f474be7fd9 100644 --- a/src/main/python/tests/scuro/test_unimodal_representations.py +++ b/src/main/python/tests/scuro/test_unimodal_representations.py @@ -86,7 +86,7 @@ class TestUnimodalRepresentations(unittest.TestCase): @classmethod def setUpClass(cls): - cls.num_instances = 100 + cls.num_instances = 2 cls.indices = np.array(range(cls.num_instances)) def _create_audio_modality(self, signal_length=1000): @@ -118,7 +118,6 @@ def test_audio_representation_transform_output_shapes(self): for representation, expected_shape_signature in audio_representations: with self.subTest(representation=representation.name): transformed_modality = representation.transform(audio) - print(representation.name) self.assertIsNotNone(transformed_modality.data) self.assertEqual(len(transformed_modality.data), self.num_instances) From fd5c556c9e931e11f19f81356549ec8d4e568812 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jun 2026 15:04:34 +0200 Subject: [PATCH 027/132] Bump codecov/codecov-action from 6.0.1 to 7.0.0 (#2477) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 6.0.1 to 7.0.0. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v6.0.1...v7.0.0) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/javaTests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml index ac69a6eb5c2..da1ceeca23d 100644 --- a/.github/workflows/javaTests.yml +++ b/.github/workflows/javaTests.yml @@ -151,7 +151,7 @@ jobs: run: mvn jacoco:report - name: Upload coverage to Codecov - uses: codecov/codecov-action@v6.0.1 + uses: codecov/codecov-action@v7.0.0 if: github.repository_owner == 'apache' with: fail_ci_if_error: false From f0d59dbb7ea0a60082d3eb2c371d8441368ace24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Pupier?= Date: Mon, 8 Jun 2026 16:04:42 +0200 Subject: [PATCH 028/132] [CI] Configure GitHub workflows to use concurrency cancel-in-progress Based on recommended best practices at Apache: https://cwiki.apache.org/confluence/pages/viewpage.action?spaceKey=INFRA&title=GitHub+Actions+Recommended+Practices This commit change the CI jobs to fail-fast based on subsequent comments. --- .github/workflows/build.yml | 4 ++++ .github/workflows/documentation.yml | 4 ++++ .github/workflows/javaCodestyle.yml | 4 ++++ .github/workflows/javaTests.yml | 4 ++++ .github/workflows/license.yml | 4 ++++ .github/workflows/monitoringUITests.yml | 4 ++++ .github/workflows/python.yml | 4 ++++ .github/workflows/pythonFormatting.yml | 4 ++++ 8 files changed, 32 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d401263bb7d..77d492a2e22 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,6 +41,10 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: build: name: ${{ matrix.os }} Java ${{ matrix.java }} ${{ matrix.javadist }} diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 75b74228f67..78b5ce0458e 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -35,6 +35,10 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: doc1: runs-on: ${{ matrix.os }} diff --git a/.github/workflows/javaCodestyle.yml b/.github/workflows/javaCodestyle.yml index 50c970023c1..2649edcbd0c 100644 --- a/.github/workflows/javaCodestyle.yml +++ b/.github/workflows/javaCodestyle.yml @@ -41,6 +41,10 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: java_codestyle: name: Java Checkstyle diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml index da1ceeca23d..0d6ac02fb82 100644 --- a/.github/workflows/javaTests.yml +++ b/.github/workflows/javaTests.yml @@ -43,6 +43,10 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: java_tests: runs-on: ubuntu-24.04 diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml index 4f7b02ee42f..d42e073c77b 100644 --- a/.github/workflows/license.yml +++ b/.github/workflows/license.yml @@ -41,6 +41,10 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: build: name: ${{ matrix.os }} diff --git a/.github/workflows/monitoringUITests.yml b/.github/workflows/monitoringUITests.yml index 9389b394828..2fcfc90651b 100644 --- a/.github/workflows/monitoringUITests.yml +++ b/.github/workflows/monitoringUITests.yml @@ -43,6 +43,10 @@ on: # enable manual workflow trigger workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: build: runs-on: ubuntu-24.04 diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 035965cf550..ee4e771937d 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -42,6 +42,10 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: test: runs-on: ${{ matrix.os }} diff --git a/.github/workflows/pythonFormatting.yml b/.github/workflows/pythonFormatting.yml index cbdd5e84578..aca79c10f41 100644 --- a/.github/workflows/pythonFormatting.yml +++ b/.github/workflows/pythonFormatting.yml @@ -33,6 +33,10 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: black: runs-on: ubuntu-latest From c465a3a4c4a0e56c426934e82b4abf89278a3cb9 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Tue, 9 Jun 2026 01:03:35 +0200 Subject: [PATCH 029/132] [MINOR] Reduce surefire per-fork test timeout from 1380s to 600s (#2485) The per-fork timeout existed to kill a hung test fork before the 30 min CI cap, so the offending test is named in the log instead of GitHub cancelling the whole job. At 1380s the timeout fired ~23 min after a fork started, but earlier forks already consumed ~8 min of the run, so the timeout would only trigger after the 30 min cap had cancelled the job. As a result hung forks were never reported. Lowering to 600s keeps a large margin below the cap regardless of when in the run a fork hangs, while staying well above the slowest observed test class (~35s), so legitimate tests are not killed. --- pom.xml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index dc9783f8571..5762dc2289e 100644 --- a/pom.xml +++ b/pom.xml @@ -76,8 +76,11 @@ classes 2 1C - - 1380 + + 600 2 false true From 56533374eace0563c74db8deecc98033752a36ab Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Tue, 9 Jun 2026 12:53:10 +0200 Subject: [PATCH 030/132] [MINOR][CI] Retry Maven test-compile on transient repository download errors CI jobs intermittently fail at startup when Maven Central returns a transient error (e.g. HTTP 403 while resolving the Apache parent POM), which is unrelated to the code under test. Retry test-compile once after a short pause when the failure is a "Could not transfer artifact" download error, while genuine compilation and test failures still fail fast. Also point the test action at the entrypoint script in the mounted workspace so changes to docker/entrypoint.sh take effect immediately without rebuilding and republishing apache/systemds:testing-latest. Failing job that motivated this change: https://github.com/apache/systemds/actions/runs/27167144065/job/80197161169 --- .github/action/action.yml | 4 ++++ docker/entrypoint.sh | 23 ++++++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/action/action.yml b/.github/action/action.yml index 57454cf0510..406367567a0 100644 --- a/.github/action/action.yml +++ b/.github/action/action.yml @@ -28,5 +28,9 @@ inputs: runs: using: 'docker' image: 'Dockerfile' + # Run the entrypoint from the mounted workspace rather than the copy baked + # into the image, so changes to docker/entrypoint.sh take effect immediately + # without rebuilding and republishing apache/systemds:testing-latest. + entrypoint: '/github/workspace/docker/entrypoint.sh' args: - ${{ inputs.test-to-run }} diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 9a77766b3a3..53dfabb96e6 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -28,8 +28,29 @@ cd /github/workspace export MAVEN_OPTS="-Xmx512m" +# Printed when Maven fails to download an artifact (transient repo/network +# error), unlike genuine compilation or test failures which fail fast. +transient_mvn_error="Could not transfer artifact" + log="/tmp/sysdstest.log" -mvn -ntp -B test-compile 2>&1 | stdbuf -oL grep -E "BUILD|Total time:|---|Building SystemDS" +compile_log="$(mktemp)" +# test-compile downloads all dependencies; retry once on a transient repo +# error so the test run below can resolve them from the local cache. +mvn -ntp -B test-compile 2>&1 | tee "$compile_log" | stdbuf -oL grep -E "BUILD|Total time:|---|Building SystemDS" +compile_status=${PIPESTATUS[0]} + +# True only when test-compile failed because of a transient repository download. +compile_transient_failure=false +[ "$compile_status" -ne 0 ] && grep -qE "$transient_mvn_error" "$compile_log" && compile_transient_failure=true +rm -f "$compile_log" + +if [ "$compile_transient_failure" = true ]; then + echo "Transient Maven repository error; retrying test-compile in 15s..." + sleep 15 + mvn -ntp -B test-compile 2>&1 | stdbuf -oL grep -E "BUILD|Total time:|---|Building SystemDS" +else + echo "No transient Maven repository error detected; no retry needed." +fi mvn -ntp -B test -D maven.test.skip=false -D automatedtestbase.outputbuffering=true -D test=$1 2>&1 \ | stdbuf -oL grep -Ev "already exists in destination.|Using incubator" \ | tee $log From 3f44187566b619a7c415e565f287a7bf99bc7e59 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Tue, 9 Jun 2026 13:45:03 +0200 Subject: [PATCH 031/132] [MINOR] Fix CholeskyTest crash when residual is exactly zero (#2487) CholeskyTest reconstructs A from its Cholesky factor and asserts that the 1x1 residual D = sum(A-B) is approximately zero. The output was read back with dmlOut.keySet().iterator().next(), which assumes at least one cell is present. When the residual is exactly 0.0, the sparse text writer omits the cell entirely, so the result map comes back empty and the iterator throws NoSuchElementException. A perfect reconstruction therefore caused the test to error out instead of pass. This is not data-dependent flakiness: the input matrix is already seeded, so A is identical on every run. The variability comes from the reduction order of sum(A-B), which differs across Spark partitions and CP threads. Because floating-point addition is not associative, the residual lands on either an exact 0.0 (empty output) or a tiny non-zero value depending on execution, which is why only some runs (notably testLargeCholeskyDenseSP) failed. The fix treats an empty output as 0.0, making the assertion robust to both outcomes, and drops the now-unused MatrixValue import. --- .../sysds/test/functions/unary/matrix/CholeskyTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/apache/sysds/test/functions/unary/matrix/CholeskyTest.java b/src/test/java/org/apache/sysds/test/functions/unary/matrix/CholeskyTest.java index a9f89bc202e..e027801e950 100644 --- a/src/test/java/org/apache/sysds/test/functions/unary/matrix/CholeskyTest.java +++ b/src/test/java/org/apache/sysds/test/functions/unary/matrix/CholeskyTest.java @@ -23,7 +23,6 @@ import org.junit.Test; import org.apache.sysds.api.DMLScript; import org.apache.sysds.common.Types.ExecMode; -import org.apache.sysds.runtime.matrix.data.MatrixValue; import org.apache.sysds.runtime.matrix.data.MatrixValue.CellIndex; import org.apache.sysds.runtime.meta.MatrixCharacteristics; import org.apache.sysds.test.AutomatedTestBase; @@ -110,8 +109,9 @@ private void runTestCholesky( int rows, int cols, ExecMode rt) { //run tests and compare results runTest(true, false, null, -1); HashMap dmlOut = readDMLMatrixFromOutputDir("D"); - MatrixValue.CellIndex index = dmlOut.keySet().iterator().next(); - double d = dmlOut.get(index); + // D is the 1x1 residual sum(A-B); an exact 0.0 result is not written to + // the sparse output, so an empty map corresponds to a perfect residual. + double d = dmlOut.isEmpty() ? 0.0 : dmlOut.values().iterator().next(); Assert.assertEquals(0, d, 1e-5); } finally { From 3a59d44b331f5bc3b53271bed3af6397fb5b8afd Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Tue, 9 Jun 2026 14:29:47 +0200 Subject: [PATCH 032/132] [BWARE] Add HashMapIntToInt primitive int-to-int hash map (#2478) * Add HashMapIntToInt primitive int-to-int hash map Introduce a specialized open-addressing map storing primitive int keys and values, avoiding boxing for integer-to-integer lookups used by compressed column-group operations. * Add component tests for HashMapIntToInt and HashMapLongInt Cover put/get, putIfAbsent variants, collision chains, resize, iteration, and the primitive -1 absent-sentinel contract. * Fix HashMapIntToInt iterator skipping buckets and add randomized tests The entry iterator advanced bucketId twice per empty bucket (pre-increment in the loop condition plus the post-increment step), skipping entries when buckets were unevenly filled. Dense sequential keys masked it because the first probe always hit a non-null bucket. Add randomized tests that cross-check against java.util.HashMap to cover imbalanced bucket layouts, plus a chained-collision resize test, reaching full branch coverage. --- .../compress/utils/HashMapIntToInt.java | 380 +++++++++++++++ .../compress/util/HashMapIntToIntTest.java | 458 ++++++++++++++++++ .../compress/util/HashMapLongIntTest.java | 59 +++ 3 files changed, 897 insertions(+) create mode 100644 src/main/java/org/apache/sysds/runtime/compress/utils/HashMapIntToInt.java create mode 100644 src/test/java/org/apache/sysds/test/component/compress/util/HashMapIntToIntTest.java diff --git a/src/main/java/org/apache/sysds/runtime/compress/utils/HashMapIntToInt.java b/src/main/java/org/apache/sysds/runtime/compress/utils/HashMapIntToInt.java new file mode 100644 index 00000000000..0bf8ac82174 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/compress/utils/HashMapIntToInt.java @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.compress.utils; + +import java.util.AbstractSet; +import java.util.Collection; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; + +public class HashMapIntToInt implements Map { + + static final int DEFAULT_INITIAL_CAPACITY = 1 << 4; + static final float DEFAULT_LOAD_FACTOR = 0.75f; + + protected Node[] buckets; + + protected int size; + + public HashMapIntToInt(int capacity) { + alloc(Math.max(capacity, DEFAULT_INITIAL_CAPACITY)); + } + + protected void alloc(int size) { + Node[] tmp = (Node[]) new Node[size]; + buckets = tmp; + } + + @Override + public int size() { + return size; + } + + @Override + public boolean isEmpty() { + return size == 0; + } + + @Override + public boolean containsKey(Object key) { + return getI((Integer) key) != -1; + } + + @Override + public boolean containsValue(Object value) { + if(value instanceof Integer) { + for(Entry v : this.entrySet()) { + if(v.getValue().equals(value)) + return true; + } + } + return false; + + } + + @Override + public Integer get(Object key) { + final int i = getI((Integer) key); + if(i != -1) + return i; + else + return null; + } + + public int getI(int key) { + + final int ix = hash(key); + Node b = buckets[ix]; + if(b != null) { + do { + if(key == b.key) + return b.value; + } + while((b = b.next) != null); + } + return -1; + + } + + public int hash(int key) { + return Math.abs(Integer.hashCode(key) % buckets.length); + } + + @Override + public Integer put(Integer key, Integer value) { + int i = putI(key, value); + if(i != -1) + return i; + else + return null; + } + + @Override + public Integer putIfAbsent(Integer key, Integer value) { + int i = putIfAbsentI(key, value); + if(i != -1) + return i; + else + return null; + } + + public int putIfAbsentI(int key, int value) { + + final int ix = hash(key); + Node b = buckets[ix]; + if(b == null) + return createBucket(ix, key, value); + else + return putIfAbsentBucket(ix, key, value); + + } + + public int putIfAbsentReturnVal(int key, int value) { + final int ix = hash(key); + Node b = buckets[ix]; + if(b == null) + return createBucketReturnVal(ix, key, value); + else + return putIfAbsentBucketReturnval(ix, key, value); + } + + public int putIfAbsentReturnValHash(int key, int value) { + + final int ix = hash(key); + Node b = buckets[ix]; + if(b == null) + return createBucketReturnVal(ix, key, value); + else + return putIfAbsentBucketReturnval(ix, key, value); + + } + + private int putIfAbsentBucket(int ix, int key, int value) { + Node b = buckets[ix]; + while(true) { + if(b.key == key) + return b.value; + if(b.next == null) { + b.setNext(new Node(key, value, null)); + size++; + resize(); + return -1; + } + b = b.next; + } + } + + private int putIfAbsentBucketReturnval(int ix, int key, int value) { + Node b = buckets[ix]; + while(true) { + if(b.key == key) + return b.value; + if(b.next == null) { + b.setNext(new Node(key, value, null)); + size++; + resize(); + return value; + } + b = b.next; + } + } + + public int putI(int key, int value) { + + final int ix = hash(key); + Node b = buckets[ix]; + if(b == null) + return createBucket(ix, key, value); + else + return addToBucket(ix, key, value); + + } + + private int createBucket(int ix, int key, int value) { + buckets[ix] = new Node(key, value, null); + size++; + return -1; + } + + private int createBucketReturnVal(int ix, int key, int value) { + buckets[ix] = new Node(key, value, null); + size++; + return value; + } + + private int addToBucket(int ix, int key, int value) { + Node b = buckets[ix]; + while(true) { + if(key == b.key) { + int tmp = b.getValue(); + b.setValue(value); + return tmp; + } + if(b.next == null) { + b.setNext(new Node(key, value, null)); + size++; + resize(); + return -1; + } + b = b.next; + } + } + + private void resize() { + if(size > buckets.length * DEFAULT_LOAD_FACTOR) { + + Node[] tmp = (Node[]) new Node[buckets.length * 2]; + Node[] oldBuckets = buckets; + buckets = tmp; + size = 0; + for(Node n : oldBuckets) { + if(n != null) + do { + put(n.key, n.value); + } + while((n = n.next) != null); + } + + } + } + + @Override + public Integer remove(Object key) { + throw new UnsupportedOperationException("Unimplemented method 'remove'"); + } + + @Override + public void putAll(Map m) { + throw new UnsupportedOperationException("Unimplemented method 'putAll'"); + } + + @Override + public void clear() { + throw new UnsupportedOperationException("Unimplemented method 'clear'"); + } + + @Override + public Set keySet() { + throw new UnsupportedOperationException("Unimplemented method 'keySet'"); + } + + @Override + public Collection values() { + throw new UnsupportedOperationException("Unimplemented method 'values'"); + } + + @Override + public Set> entrySet() { + return new EntrySet(); + } + + @Override + public void forEach(BiConsumer action) { + + for(Node n : buckets) { + if(n != null) { + do { + action.accept(n.key, n.value); + } + while((n = n.next) != null); + } + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(size() * 3); + this.forEach((k, v) -> { + sb.append("(" + k + "→" + v + ")"); + }); + return sb.toString(); + } + + private static class Node implements Entry { + final int key; + int value; + Node next; + + Node(int key, int value, Node next) { + this.key = key; + this.value = value; + this.next = next; + } + + public final void setNext(Node n) { + next = n; + } + + @Override + public Integer getKey() { + return key; + } + + @Override + public Integer getValue() { + return value; + } + + @Override + public Integer setValue(Integer value) { + return this.value = value; + } + } + + private final class EntrySet extends AbstractSet> { + + @Override + public int size() { + return size; + } + + @Override + public Iterator> iterator() { + return new EntryIterator(); + } + + } + + private final class EntryIterator implements Iterator> { + Node next; + int bucketId = 0; + + protected EntryIterator() { + + for(; bucketId < buckets.length; bucketId++) { + if(buckets[bucketId] != null) { + next = buckets[bucketId]; + break; + } + } + + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public Entry next() { + + Node e = next; + + if(e.next != null) + next = e.next; + else { + for(bucketId++; bucketId < buckets.length; bucketId++) { + if(buckets[bucketId] != null) { + next = buckets[bucketId]; + break; + } + } + if(bucketId >= buckets.length) + next = null; + } + + return e; + } + + } + +} diff --git a/src/test/java/org/apache/sysds/test/component/compress/util/HashMapIntToIntTest.java b/src/test/java/org/apache/sysds/test/component/compress/util/HashMapIntToIntTest.java new file mode 100644 index 00000000000..15c78ddb3f7 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/compress/util/HashMapIntToIntTest.java @@ -0,0 +1,458 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.compress.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; +import java.util.Set; + +import org.apache.sysds.runtime.compress.utils.HashMapIntToInt; +import org.junit.Test; + +public class HashMapIntToIntTest { + + @Test + public void basic1() { + basic(new HashMapIntToInt(1)); + } + + @Test + public void basic2() { + basic(new HashMapIntToInt(2)); + } + + @Test + public void basic16() { + basic(new HashMapIntToInt(16)); + } + + @Test + public void basic100() { + basic(new HashMapIntToInt(100)); + } + + private void basic(HashMapIntToInt a) { + assertTrue(a.isEmpty()); + assertEquals(0, a.size()); + + // first insert via putIfAbsentI returns the absent sentinel -1 + assertEquals(-1, a.putIfAbsentI(1, 10)); + assertFalse(a.isEmpty()); + // second insert of same key keeps the existing value and returns it + assertEquals(10, a.putIfAbsentI(1, 99)); + assertEquals(1, a.size()); + + for(int i = 2; i < 10; i++) + assertEquals(-1, a.putIfAbsentI(i, i * 10)); + assertEquals(9, a.size()); + + // lookups + assertEquals(10, a.getI(1)); + assertEquals(90, a.getI(9)); + assertEquals(-1, a.getI(13)); // absent + assertEquals(Integer.valueOf(40), a.get(4)); + assertNull(a.get(13)); // absent via boxed accessor + + // containment + assertTrue(a.containsKey(4)); + assertFalse(a.containsKey(42)); + assertTrue(a.containsValue(40)); + assertFalse(a.containsValue(41)); + + // iteration covers exactly the inserted entries + Set keys = new HashSet<>(); + Set vals = new HashSet<>(); + for(Entry e : a.entrySet()) { + keys.add(e.getKey()); + vals.add(e.getValue()); + } + assertEquals(9, keys.size()); + for(int i = 1; i < 10; i++) { + assertTrue(keys.contains(i)); + assertTrue(vals.contains(i * 10)); + } + } + + @Test + public void putReturnsPreviousValue() { + HashMapIntToInt a = new HashMapIntToInt(4); + // putI returns -1 when the key is new + assertEquals(-1, a.putI(5, 50)); + // putI overwrites and returns the previous value + assertEquals(50, a.putI(5, 51)); + assertEquals(51, a.getI(5)); + assertEquals(1, a.size()); // overwrite does not grow the map + } + + @Test + public void putBoxedReturnsNullThenPrevious() { + HashMapIntToInt a = new HashMapIntToInt(4); + assertNull(a.put(7, 70)); + assertEquals(Integer.valueOf(70), a.put(7, 71)); + assertEquals(Integer.valueOf(71), a.get(7)); + } + + @Test + public void putIfAbsentDoesNotOverwrite() { + HashMapIntToInt a = new HashMapIntToInt(4); + assertEquals(-1, a.putIfAbsentI(3, 30)); + assertEquals(30, a.putIfAbsentI(3, 31)); // returns existing, keeps 30 + assertEquals(30, a.getI(3)); + assertNull(a.putIfAbsent(8, 80)); + assertEquals(Integer.valueOf(80), a.putIfAbsent(8, 81)); + assertEquals(80, a.getI(8)); + } + + @Test + public void putIfAbsentReturnValSemantics() { + HashMapIntToInt a = new HashMapIntToInt(4); + // when absent: inserts and returns the newly stored value + assertEquals(40, a.putIfAbsentReturnVal(4, 40)); + // when present: returns the existing value, does not overwrite + assertEquals(40, a.putIfAbsentReturnVal(4, 99)); + assertEquals(40, a.getI(4)); + + // the *Hash variant has identical semantics + assertEquals(50, a.putIfAbsentReturnValHash(5, 50)); + assertEquals(50, a.putIfAbsentReturnValHash(5, 99)); + assertEquals(50, a.getI(5)); + } + + @Test + public void absentSignaledByMinusOneSentinel() { + // Design contract: the primitive int accessors (getI / putI / putIfAbsentI) + // signal "absent" or "no previous value" with the sentinel -1 instead of a + // nullable Integer. This is a deliberate performance choice to avoid boxing + // and null handling on the hot path, so the tests pin the -1 behavior down. + HashMapIntToInt a = new HashMapIntToInt(16); + + // lookup of an absent key returns the sentinel (not null, no exception) + assertEquals(-1, a.getI(1)); + assertEquals(-1, a.getI(Integer.MAX_VALUE)); + + // inserting a previously-absent key returns the sentinel (no prior value) + assertEquals(-1, a.putI(1, 100)); + assertEquals(-1, a.putIfAbsentI(2, 200)); + + // a populated map still returns the sentinel for any missing key + assertEquals(-1, a.getI(3)); + + // Consequence of the sentinel: -1 is reserved and must not be stored as a + // value. A stored -1 is indistinguishable from "absent" through both the + // primitive and the boxed accessors, which callers are required to respect. + HashMapIntToInt b = new HashMapIntToInt(16); + b.putI(7, -1); + assertEquals(1, b.size()); // the entry really is stored + assertEquals(-1, b.getI(7)); // ...but reads back as the absent sentinel + assertNull(b.get(7)); // and the boxed accessor reports null as well + } + + @Test + public void resizeRetainsAllEntries() { + // start small to force several resizes (load factor 0.75) + HashMapIntToInt a = new HashMapIntToInt(1); + final int n = 1000; + for(int i = 0; i < n; i++) + assertEquals(-1, a.putI(i, i * 2)); + assertEquals(n, a.size()); + for(int i = 0; i < n; i++) + assertEquals(i * 2, a.getI(i)); + assertEquals(-1, a.getI(n)); // still absent after resizing + + // iteration still sees every entry after resizing + Set keys = new HashSet<>(); + for(Entry e : a.entrySet()) + keys.add(e.getKey()); + assertEquals(n, keys.size()); + } + + @Test + public void negativeAndBoundaryKeys() { + HashMapIntToInt a = new HashMapIntToInt(8); + int[] keys = {-1000, -1, 0, 1, Integer.MIN_VALUE, Integer.MAX_VALUE}; + for(int i = 0; i < keys.length; i++) + a.putI(keys[i], i + 100); + assertEquals(keys.length, a.size()); + for(int i = 0; i < keys.length; i++) { + assertEquals(i + 100, a.getI(keys[i])); + assertTrue(a.containsKey(keys[i])); + } + } + + @Test + public void forEachVisitsAllEntries() { + HashMapIntToInt a = new HashMapIntToInt(4); + for(int i = 0; i < 50; i++) + a.putI(i, i + 1); + int[] count = new int[] {0}; + long[] sum = new long[] {0}; + a.forEach((k, v) -> { + count[0]++; + sum[0] += (v - k); // each entry contributes exactly 1 + }); + assertEquals(50, count[0]); + assertEquals(50, sum[0]); + } + + @Test + public void collisionChainsPutAndGet() { + // capacity 16 -> 16 buckets; staying <= 12 entries avoids a resize, so + // keys that are congruent mod 16 deterministically share one bucket. + HashMapIntToInt a = new HashMapIntToInt(16); + assertEquals(-1, a.putI(1, 100)); + assertEquals(-1, a.putI(17, 200)); // appended as 2nd node in the chain + assertEquals(-1, a.putI(33, 300)); // traverses node1 -> node2, then appends + assertEquals(3, a.size()); + + // overwrite a node deep in the chain returns the previous value + assertEquals(300, a.putI(33, 333)); + assertEquals(333, a.getI(33)); // getI walks the chain to the last node + // a miss whose key maps to a populated bucket walks the chain, then -1 + assertEquals(-1, a.getI(49)); + + // iterating a multi-node bucket exercises the iterator chain advance + int cnt = 0; + int sum = 0; + for(Entry e : a.entrySet()) { + cnt++; + sum += e.getValue(); + } + assertEquals(3, cnt); + assertEquals(100 + 200 + 333, sum); + } + + @Test + public void collisionChainsPutIfAbsent() { + HashMapIntToInt a = new HashMapIntToInt(16); + // putIfAbsentI into a shared bucket: create, then append into the chain + assertEquals(-1, a.putIfAbsentI(1, 10)); + assertEquals(-1, a.putIfAbsentI(17, 20)); + assertEquals(10, a.putIfAbsentI(1, 99)); // match first node, keep 10 + assertEquals(20, a.putIfAbsentI(17, 99)); // match deeper node, keep 20 + + // the *ReturnVal variants append into the same non-empty bucket + assertEquals(30, a.putIfAbsentReturnVal(33, 30)); // appended -> new value + assertEquals(30, a.putIfAbsentReturnVal(33, 99)); // present -> existing + assertEquals(40, a.putIfAbsentReturnValHash(49, 40)); // appended -> new value + assertEquals(40, a.putIfAbsentReturnValHash(49, 99)); // present -> existing + assertEquals(4, a.size()); + } + + @Test + public void toStringContainsEntries() { + HashMapIntToInt a = new HashMapIntToInt(16); + a.putI(2, 20); + a.putI(3, 30); + String s = a.toString(); + assertTrue(s.contains("(2\u219220)")); // (2->20) + assertTrue(s.contains("(3\u219230)")); // (3->30) + } + + @Test + public void entrySetSize() { + HashMapIntToInt a = new HashMapIntToInt(16); + for(int i = 0; i < 5; i++) + a.putI(i, i); + assertEquals(5, a.entrySet().size()); + } + + @Test + public void resizeWithEmptyAndChainedBuckets() { + // 16 buckets, load factor 0.75 -> a resize fires once size exceeds 12. + // Pre-seed a colliding chain in bucket 1 (keys congruent mod 16), then add + // distinct keys so that at resize time oldBuckets holds both a multi-node + // chain (if(n != null) true + chain re-put) and empty buckets (false side). + HashMapIntToInt a = new HashMapIntToInt(16); + a.putI(1, 1); + a.putI(17, 17); // chain node in bucket 1 + for(int i = 2; i <= 12; i++) // distinct buckets, drives size past 12 + a.putI(i, i); + assertEquals(13, a.size()); // triggers exactly one resize + + assertEquals(1, a.getI(1)); + assertEquals(17, a.getI(17)); + for(int i = 2; i <= 12; i++) + assertEquals(i, a.getI(i)); + } + + @Test + public void resizeWithEmptyBucketsInOldTable() { + // A resize only fires from the collision-append path, so dense sequential + // keys fill every bucket before the load factor is exceeded and the old + // table is always full at rehash time. Here we instead pile keys that are + // all congruent mod 16 into a single chain: the load factor (0.75) is + // crossed while 15 of the 16 buckets stay empty, exercising the + // n == null (skip empty bucket) branch of resize()'s rehash loop. + HashMapIntToInt a = new HashMapIntToInt(16); + final int n = 13; // 13 > 16 * 0.75 triggers exactly one resize + for(int i = 0; i < n; i++) + assertEquals(-1, a.putI(i * 16, i)); // every key maps to bucket 0 at capacity 16 + assertEquals(n, a.size()); + + // every entry survived the rehash over a table that contained empty buckets + for(int i = 0; i < n; i++) + assertEquals(i, a.getI(i * 16)); + } + + @Test + public void emptyEntrySetIteration() { + HashMapIntToInt a = new HashMapIntToInt(16); + int cnt = 0; + for(Entry e : a.entrySet()) + cnt += e.getValue(); + assertEquals(0, cnt); + assertFalse(a.entrySet().iterator().hasNext()); + assertEquals(0, a.entrySet().size()); + } + + @Test + public void forEachOverChain() { + // colliding keys (congruent mod 16) build a multi-node bucket so forEach + // walks the linked list within a bucket + HashMapIntToInt a = new HashMapIntToInt(16); + a.putI(1, 1); + a.putI(17, 1); + a.putI(33, 1); + int[] count = new int[] {0}; + a.forEach((k, v) -> count[0]++); + assertEquals(3, count[0]); + } + + @Test + public void containsValueNonInteger() { + HashMapIntToInt a = new HashMapIntToInt(4); + a.putI(1, 1); + assertFalse(a.containsValue("not-an-integer")); + assertFalse(a.containsValue(null)); + } + + @Test(expected = UnsupportedOperationException.class) + public void putAllUnsupported() { + new HashMapIntToInt(4).putAll(new java.util.HashMap()); + } + + @Test(expected = UnsupportedOperationException.class) + public void removeUnsupported() { + new HashMapIntToInt(4).remove(1); + } + + @Test(expected = UnsupportedOperationException.class) + public void clearUnsupported() { + new HashMapIntToInt(4).clear(); + } + + @Test(expected = UnsupportedOperationException.class) + public void keySetUnsupported() { + new HashMapIntToInt(4).keySet(); + } + + @Test(expected = UnsupportedOperationException.class) + public void valuesUnsupported() { + new HashMapIntToInt(4).values(); + } + + @Test + public void randomKeysMatchReference1() { + randomKeysMatchReference(1, 1, 2000); + } + + @Test + public void randomKeysMatchReference16() { + randomKeysMatchReference(2, 16, 2000); + } + + @Test + public void randomKeysMatchReferenceSmallDomain() { + // a small key domain relative to the entry count forces many overwrites + // and unevenly loaded buckets rather than a clean one-key-per-bucket layout + randomKeysMatchReference(3, 4, 3000); + } + + private void randomKeysMatchReference(long seed, int capacity, int inserts) { + // Cross-check against java.util.HashMap under randomized, value-shifted keys + // so bucket load is genuinely uneven (collisions, chains, and empty buckets + // coexist) instead of the perfectly balanced layout that dense keys produce. + // Values are kept >= 0 because the primitive accessors reserve -1 as the + // "absent" sentinel. + Random r = new Random(seed); + HashMapIntToInt a = new HashMapIntToInt(capacity); + Map ref = new HashMap<>(); + + for(int i = 0; i < inserts; i++) { + int key = r.nextInt(inserts); // domain may be smaller than #inserts -> overwrites + int value = r.nextInt(Integer.MAX_VALUE); // never -1 + Integer prev = ref.put(key, value); + int prevI = a.putI(key, value); + if(prev == null) + assertEquals(-1, prevI); // first time we see the key + else + assertEquals(prev.intValue(), prevI); // overwrite returns previous value + } + + assertEquals(ref.size(), a.size()); + for(Entry e : ref.entrySet()) { + assertEquals(e.getValue().intValue(), a.getI(e.getKey())); + assertTrue(a.containsKey(e.getKey())); + } + + // iteration visits exactly the reference entries, nothing more or less + Map seen = new HashMap<>(); + for(Entry e : a.entrySet()) + assertNull("duplicate key from iterator: " + e.getKey(), seen.put(e.getKey(), e.getValue())); + assertEquals(ref, seen); + + // a few keys outside the inserted domain must report absent + for(int i = 0; i < 50; i++) + assertEquals(-1, a.getI(inserts + r.nextInt(inserts) + 1)); + } + + @Test + public void randomPutIfAbsentKeepsFirstValue() { + // putIfAbsentI must preserve the first value stored for a key even under + // randomized, colliding inserts; mirror that contract with a reference map. + Random r = new Random(7); + HashMapIntToInt a = new HashMapIntToInt(2); + Map ref = new HashMap<>(); + final int inserts = 2000; + + for(int i = 0; i < inserts; i++) { + int key = r.nextInt(inserts / 4); // small domain -> frequent collisions + int value = r.nextInt(Integer.MAX_VALUE); // never -1 + if(ref.containsKey(key)) + assertEquals(ref.get(key).intValue(), a.putIfAbsentI(key, value)); // keep first + else { + assertEquals(-1, a.putIfAbsentI(key, value)); + ref.put(key, value); + } + } + + assertEquals(ref.size(), a.size()); + for(Entry e : ref.entrySet()) + assertEquals(e.getValue().intValue(), a.getI(e.getKey())); + } +} diff --git a/src/test/java/org/apache/sysds/test/component/compress/util/HashMapLongIntTest.java b/src/test/java/org/apache/sysds/test/component/compress/util/HashMapLongIntTest.java index 404380816af..d7d314ce0e9 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/util/HashMapLongIntTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/util/HashMapLongIntTest.java @@ -20,6 +20,7 @@ package org.apache.sysds.test.component.compress.util; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.util.HashSet; @@ -81,4 +82,62 @@ public void addSize(HashMapLongInt a) { assertEquals(4, a.get(4)); assertEquals(-1, a.get(13)); } + + @Test + public void absentSignaledByMinusOneSentinel() { + // Design contract: get / putIfAbsent signal "absent" or "no previous value" + // with the primitive sentinel -1 rather than a nullable Integer. This is a + // deliberate performance choice to avoid boxing on the hot path, so the + // tests pin the -1 behavior down. + HashMapLongInt a = new HashMapLongInt(16); + + // lookup of an absent key returns the sentinel + assertEquals(-1, a.get(1)); + assertEquals(-1, a.get(Long.MAX_VALUE)); + + // inserting a previously-absent key returns the sentinel (no prior value) + assertEquals(-1, a.putIfAbsent(1, 100)); + + // a populated map still returns the sentinel for any missing key + assertEquals(-1, a.get(2)); + + // Consequence of the sentinel: -1 is reserved and must not be stored as a + // value, since a stored -1 reads back as the absent sentinel. + HashMapLongInt b = new HashMapLongInt(16); + b.putIfAbsent(7, -1); + assertEquals(1, b.size()); // the entry really is stored + assertEquals(-1, b.get(7)); // ...but reads back as the absent sentinel + } + + @Test + public void emptyIterator() { + HashMapLongInt a = new HashMapLongInt(4); + assertFalse(a.iterator().hasNext()); + } + + @Test + public void reallocateBucket() { + // capacity 1 forces every key into a single bucket, growing it past the + // initial 4 cells and exercising reallocateBucket. + HashMapLongInt a = new HashMapLongInt(1); + for(int i = 1; i <= 10; i++) + assertEquals(-1, a.putIfAbsent(i, i * 2)); + assertEquals(10, a.size()); + for(int i = 1; i <= 10; i++) + assertEquals(i * 2, a.get(i)); + // re-insert an existing key returns its stored value without growing + assertEquals(20, a.putIfAbsent(10, 999)); + assertEquals(10, a.size()); + } + + @Test + public void toStringContainsEntries() { + HashMapLongInt a = new HashMapLongInt(16); + a.putIfAbsent(2, 20); + a.putIfAbsent(3, 30); + String s = a.toString(); + assertTrue(s.contains("HashMapLongInt")); + assertTrue(s.contains("2->20")); + assertTrue(s.contains("3->30")); + } } From efe8cbdce54d5c9f39bb408800786feddae3630d Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Tue, 9 Jun 2026 14:46:03 +0200 Subject: [PATCH 033/132] [BWARE] Speed up frame-to-matrix conversion and harden number parsing (#2480) * Speed up frame-to-matrix conversion and harden number parsing Tightens the hot path that converts FrameBlocks of arbitrary schema into MatrixBlocks, with a defensive fallback for malformed cells. - DoubleParser.parseFloatingPointLiteral: replace the >= 'I' / >= 'a' character guards with a single 0-9 range check on the last char. The previous guards over-matched and pushed too many strings into the slow Double.parseDouble path - DoubleArray.parseDouble: stop wrapping the parse failure as a DMLRuntimeException so callers can distinguish format errors - MatrixBlockFromFrame: - turn the interface into a class with a private constructor so Jacoco can measure it cleanly - on NumberFormatException / DMLRuntimeException during a bulk block convert, log once and fall back to convertSafeCast which writes NaN per offending cell instead of failing the whole job - add convertSafeCast / convertBlockSafeCast helpers * Gate frame-to-matrix NaN cast fallback behind config flag Frame-to-matrix conversion silently fell back to writing NaN for cells that cannot be cast to double, logging only a single warning. This changed behavior for callers that previously failed fast on incompatible data. Make the lenient behavior opt-in. - Add sysds.frame.tomatrix.warncast (default false): when false the conversion fails fast on number format errors; when true it warns once and writes NaN for the incompatible cells - Read the flag once on the calling thread and pass it down, since the thread-local config is not visible to pool workers - Extract convertStrict to share the contiguous/generic dispatch between the strict and warn-only paths - Add tests for the warn-only fallback, the strict fail-fast default, and the tightened DoubleParser trailing-character guard * Add tests for frame-to-matrix warn-cast and double parse error path - Cover the warn-cast success path where a fully valid frame converts without triggering the NaN fallback - Cover the zero-value branch of the safe-cast non-zero count and an all-invalid frame becoming all NaN - Add parallel fail-fast coverage for the strict (default) path - Assert DoubleArray.parseDouble surfaces the raw NumberFormatException instead of a wrapped DMLRuntimeException --- conf/SystemDS-config.xml.template | 4 + .../java/org/apache/sysds/conf/DMLConfig.java | 4 +- .../frame/data/columns/DoubleArray.java | 2 +- .../frame/data/lib/MatrixBlockFromFrame.java | 80 +++++- .../org/apache/sysds/utils/DoubleParser.java | 9 +- .../frame/MatrixFromFrameSafeCastTest.java | 246 ++++++++++++++++++ .../frame/array/CustomArrayTests.java | 7 + .../test/component/misc/DoubleParserTest.java | 12 + 8 files changed, 352 insertions(+), 12 deletions(-) create mode 100644 src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java diff --git a/conf/SystemDS-config.xml.template b/conf/SystemDS-config.xml.template index 88a1c5947ed..153dcb6ef2d 100644 --- a/conf/SystemDS-config.xml.template +++ b/conf/SystemDS-config.xml.template @@ -57,6 +57,10 @@ 64 + + false + false diff --git a/src/main/java/org/apache/sysds/conf/DMLConfig.java b/src/main/java/org/apache/sysds/conf/DMLConfig.java index e1b7b0bb530..a6339656fb0 100644 --- a/src/main/java/org/apache/sysds/conf/DMLConfig.java +++ b/src/main/java/org/apache/sysds/conf/DMLConfig.java @@ -78,6 +78,7 @@ public class DMLConfig public static final String PARALLEL_ENCODE_NUM_THREADS = "sysds.parallel.encode.numThreads"; public static final String PARALLEL_TOKENIZE = "sysds.parallel.tokenize"; public static final String PARALLEL_TOKENIZE_NUM_BLOCKS = "sysds.parallel.tokenize.numBlocks"; + public static final String FRAME_TO_MATRIX_WARN_CAST = "sysds.frame.tomatrix.warncast"; public static final String COMPRESSED_LINALG = "sysds.compressed.linalg"; public static final String COMPRESSED_LINALG_INTERMEDIATE = "sysds.compressed.linalg.intermediate"; public static final String COMPRESSED_LOSSY = "sysds.compressed.lossy"; @@ -159,6 +160,7 @@ public class DMLConfig _defaultVals.put(IO_COMPRESSION_CODEC, "none"); _defaultVals.put(PARALLEL_TOKENIZE, "false"); _defaultVals.put(PARALLEL_TOKENIZE_NUM_BLOCKS, "64"); + _defaultVals.put(FRAME_TO_MATRIX_WARN_CAST, "false"); _defaultVals.put(PARALLEL_ENCODE, "true" ); _defaultVals.put(PARALLEL_ENCODE_STAGED, "false" ); _defaultVals.put(PARALLEL_ENCODE_APPLY_BLOCKS, "-1"); @@ -456,7 +458,7 @@ public static DMLConfig readConfigurationFile(String configPath) public String getConfigInfo() { String[] tmpConfig = new String[] { LOCAL_TMP_DIR,SCRATCH_SPACE,OPTIMIZATION_LEVEL, DEFAULT_BLOCK_SIZE, - CP_PARALLEL_OPS, CP_PARALLEL_IO, PARALLEL_ENCODE, NATIVE_BLAS, NATIVE_BLAS_DIR, + CP_PARALLEL_OPS, CP_PARALLEL_IO, PARALLEL_ENCODE, FRAME_TO_MATRIX_WARN_CAST, NATIVE_BLAS, NATIVE_BLAS_DIR, COMPRESSED_LINALG, COMPRESSED_LOSSY, COMPRESSED_VALID_COMPRESSIONS, COMPRESSED_OVERLAPPING, COMPRESSED_SAMPLING_RATIO, COMPRESSED_SOFT_REFERENCE_COUNT, COMPRESSED_COCODE, COMPRESSED_TRANSPOSE, COMPRESSED_TRANSFORMENCODE, DAG_LINEARIZATION, diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/DoubleArray.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/DoubleArray.java index 99cce9f9e97..972a2893fd8 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/DoubleArray.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/DoubleArray.java @@ -377,7 +377,7 @@ public static double parseDouble(String value) { return Double.POSITIVE_INFINITY; else if(len == 4 && value.compareToIgnoreCase("-Inf") == 0) return Double.NEGATIVE_INFINITY; - throw new DMLRuntimeException(e); + throw e; } } diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java b/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java index 032afe2cd7c..9ff58065d97 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java @@ -25,6 +25,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.frame.data.FrameBlock; @@ -32,11 +34,17 @@ import org.apache.sysds.runtime.util.CommonThreadPool; import org.apache.sysds.utils.stats.InfrastructureAnalyzer; -public interface MatrixBlockFromFrame { +public class MatrixBlockFromFrame { public static final Log LOG = LogFactory.getLog(MatrixBlockFromFrame.class.getName()); public static final int blocksizeIJ = 32; + public static Boolean WARNED_FOR_FAILED_CAST = false; + + private MatrixBlockFromFrame(){ + // private constructor for code coverage. + } + /** * Converts a frame block with arbitrary schema into a matrix block. Since matrix block only supports value type * double, we do a best effort conversion of non-double types which might result in errors for non-numerical data. @@ -68,11 +76,15 @@ public static MatrixBlock convertToMatrixBlock(FrameBlock frame, MatrixBlock ret if(k == -1) k = InfrastructureAnalyzer.getLocalParallelism(); + // Read once on the calling thread: the thread-local config is not visible to pool workers. + final boolean warnCast = ConfigurationManager.getDMLConfig() + .getBooleanValue(DMLConfig.FRAME_TO_MATRIX_WARN_CAST); + long nnz = 0; if(k == 1) - nnz = convert(frame, ret, n, 0, m); + nnz = convert(frame, ret, n, 0, m, warnCast); else - nnz = convertParallel(frame, ret, m, n, k); + nnz = convertParallel(frame, ret, m, n, k, warnCast); ret.setNonZeros(nnz); ret.examSparsity(); @@ -93,14 +105,37 @@ else if(ret.getNumRows() != m || ret.getNumColumns() != n || ret.isInSparseForma return ret; } - private static long convert(FrameBlock frame, MatrixBlock mb, int n, int rl, int ru) { + private static long convert(FrameBlock frame, MatrixBlock mb, int n, int rl, int ru, boolean warnCast) { + // Strict (default): let number format errors propagate and fail the conversion. + if(!warnCast) + return convertStrict(frame, mb, n, rl, ru); + + // Warn-only: on number format errors fall back to writing NaN for the incompatible cells. + try { + return convertStrict(frame, mb, n, rl, ru); + } + catch(NumberFormatException | DMLRuntimeException e) { + synchronized(WARNED_FOR_FAILED_CAST){ + if(!WARNED_FOR_FAILED_CAST) { + LOG.error( + "Failed to convert to Matrix because of number format errors, falling back to NaN on incompatible cells", + e); + WARNED_FOR_FAILED_CAST = true; + } + } + return convertSafeCast(frame, mb, n, rl, ru); + } + } + + private static long convertStrict(FrameBlock frame, MatrixBlock mb, int n, int rl, int ru) { if(mb.getDenseBlock().isContiguous()) return convertContiguous(frame, mb, n, rl, ru); else return convertGeneric(frame, mb, n, rl, ru); } - private static long convertParallel(FrameBlock frame, MatrixBlock mb, int m, int n, int k) throws Exception { + private static long convertParallel(FrameBlock frame, MatrixBlock mb, int m, int n, int k, boolean warnCast) + throws Exception { ExecutorService pool = CommonThreadPool.get(k); try { List> tasks = new ArrayList<>(); @@ -109,7 +144,7 @@ private static long convertParallel(FrameBlock frame, MatrixBlock mb, int m, int for(int i = 0; i < m; i += blkz) { final int start = i; final int end = Math.min(i + blkz, m); - tasks.add(pool.submit(() -> convert(frame, mb, n, start, end))); + tasks.add(pool.submit(() -> convert(frame, mb, n, start, end, warnCast))); } long nnz = 0; @@ -169,4 +204,37 @@ private static long convertBlockGeneric(final FrameBlock frame, long lnnz, final } return lnnz; } + + private static long convertSafeCast(final FrameBlock frame, final MatrixBlock mb, final int n, final int rl, + final int ru) { + final DenseBlock c = mb.getDenseBlock(); + long lnnz = 0; + for(int bi = rl; bi < ru; bi += blocksizeIJ) { + for(int bj = 0; bj < n; bj += blocksizeIJ) { + int bimin = Math.min(bi + blocksizeIJ, ru); + int bjmin = Math.min(bj + blocksizeIJ, n); + lnnz = convertBlockSafeCast(frame, lnnz, c, bi, bj, bimin, bjmin); + } + } + return lnnz; + } + + private static long convertBlockSafeCast(final FrameBlock frame, long lnnz, final DenseBlock c, final int rl, + final int cl, final int ru, final int cu) { + for(int i = rl; i < ru; i++) { + final double[] cvals = c.values(i); + final int cpos = c.pos(i); + for(int j = cl; j < cu; j++) { + try { + lnnz += (cvals[cpos + j] = frame.getDoubleNaN(i, j)) != 0 ? 1 : 0; + } + catch(NumberFormatException | DMLRuntimeException e) { + lnnz += 1; + cvals[cpos + j] = Double.NaN; + } + } + } + return lnnz; + } + } diff --git a/src/main/java/org/apache/sysds/utils/DoubleParser.java b/src/main/java/org/apache/sysds/utils/DoubleParser.java index 9c77a3e95c8..c0122f8061f 100644 --- a/src/main/java/org/apache/sysds/utils/DoubleParser.java +++ b/src/main/java/org/apache/sysds/utils/DoubleParser.java @@ -184,7 +184,7 @@ public interface DoubleParser { 0x8e679c2f5e44ff8fL}; public static double parseFloatingPointLiteral(String str, int offset, int endIndex) { - if(endIndex > 100) + if(endIndex > 100)// long string return Double.parseDouble(str); // Skip leading whitespace int index = skipWhitespace(str, offset, endIndex); @@ -197,9 +197,10 @@ public static double parseFloatingPointLiteral(String str, int offset, int endIn } // Parse NaN or Infinity (this occurs rarely) - if(ch >= 'I') - return Double.parseDouble(str); - else if(str.charAt(endIndex - 1) >= 'a') + // : is the first character after numbers. + // 0 is the first number. + // we use the last position, since this is not allowed to be other values than a number. + if(str.charAt(endIndex - 1) > '9' || str.charAt(endIndex - 1) < '0') return Double.parseDouble(str); final double val = parseDecFloatLiteral(str, index, offset, endIndex); diff --git a/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java b/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java new file mode 100644 index 00000000000..43a53879f17 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.frame; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Modifier; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.log4j.spi.LoggingEvent; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.data.DenseBlock; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.frame.data.columns.Array; +import org.apache.sysds.runtime.frame.data.columns.ArrayFactory; +import org.apache.sysds.runtime.frame.data.lib.MatrixBlockFromFrame; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.test.LoggingUtils; +import org.apache.sysds.test.LoggingUtils.TestAppender; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Exercises the defensive NaN fallback in {@link MatrixBlockFromFrame} that triggers when a frame contains cells that + * cannot be parsed into doubles. The fallback is gated behind {@link DMLConfig#FRAME_TO_MATRIX_WARN_CAST}. + */ +public class MatrixFromFrameSafeCastTest { + protected static final Log LOG = LogFactory.getLog(MatrixFromFrameSafeCastTest.class.getName()); + + /** Captures the expected fallback LOG.error so it does not pollute test output. */ + private TestAppender appender; + + private void setWarnCast(boolean enabled) { + ConfigurationManager.getDMLConfig().setTextValue(DMLConfig.FRAME_TO_MATRIX_WARN_CAST, String.valueOf(enabled)); + } + + @Before + public void setUp() { + appender = LoggingUtils.overwrite(); + MatrixBlockFromFrame.WARNED_FOR_FAILED_CAST = false; + setWarnCast(true); + } + + @After + public void tearDown() { + LoggingUtils.reinsert(appender); + // restore the strict (default) behavior to avoid leaking into other tests + setWarnCast(false); + MatrixBlockFromFrame.WARNED_FOR_FAILED_CAST = false; + } + + private static final double NA = Double.NaN; + + /** Expected matrix for {@link #mixedFrame()}: parseable cells keep their value, unparseable cells become NaN. */ + private static final double[][] EXPECTED = {{1.0, 4.0}, {NA, 5.0}, {3.0, NA}}; + + /** + * Build a string frame mixing parseable numbers with values that cannot be cast to double. The non-numeric cells + * force the conversion onto the safe-cast path. + */ + private static FrameBlock mixedFrame() { + Array c1 = ArrayFactory.create(new String[] {"1.0", "abc", "3.0"}); + Array c2 = ArrayFactory.create(new String[] {"4.0", "5.0", "xyz"}); + return new FrameBlock(new Array[] {c1, c2}); + } + + @Test + public void safeCastSingleThread() { + FrameBlock fb = mixedFrame(); + MatrixBlock mb = MatrixBlockFromFrame.convertToMatrixBlock(fb, 1); + compareSafeCast(mb); + } + + @Test + public void safeCastParallel() { + FrameBlock fb = mixedFrame(); + MatrixBlock mb = MatrixBlockFromFrame.convertToMatrixBlock(fb, 4); + compareSafeCast(mb); + } + + @Test + public void safeCastProvidedOutput() { + FrameBlock fb = mixedFrame(); + MatrixBlock mb = MatrixBlockFromFrame.convertToMatrixBlock(fb, new MatrixBlock(3, 2, false), 1); + compareSafeCast(mb); + } + + @Test + public void safeCastNonContiguous() { + FrameBlock fb = mixedFrame(); + MatrixBlock mb = new MatrixBlock(fb.getNumRows(), fb.getNumColumns(), false); + mb.allocateBlock(); + DenseBlock spy = spy(mb.getDenseBlock()); + when(spy.isContiguous()).thenReturn(false); + mb.setDenseBlock(spy); + + mb = MatrixBlockFromFrame.convertToMatrixBlock(fb, mb, 1); + compareSafeCast(mb); + } + + @Test + public void safeCastWarnsOnlyOnce() { + FrameBlock fb = mixedFrame(); + + MatrixBlock first = MatrixBlockFromFrame.convertToMatrixBlock(fb, 1); + assertTrue("Conversion should flag that it fell back to NaN casting", + MatrixBlockFromFrame.WARNED_FOR_FAILED_CAST); + compareSafeCast(first); + + // second conversion takes the already-warned branch + MatrixBlock second = MatrixBlockFromFrame.convertToMatrixBlock(fb, 1); + compareSafeCast(second); + + // the fallback warning must be logged exactly once across both conversions + final List log = LoggingUtils.reinsert(appender); + long warnings = log.stream() + .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")) + .count(); + assertEquals(1, warnings); + } + + @Test + public void strictThrowsWhenWarnCastDisabled() { + // default behavior: incompatible cells fail the whole conversion + setWarnCast(false); + FrameBlock fb = mixedFrame(); + + Exception e = assertThrows(DMLRuntimeException.class, + () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 1)); + assertTrue(e.getMessage().contains("Failed to convert FrameBlock to MatrixBlock")); + } + + @Test + public void strictThrowsParallelWhenWarnCastDisabled() { + // default behavior must also fail fast on the multi-threaded path + setWarnCast(false); + FrameBlock fb = mixedFrame(); + + Exception e = assertThrows(DMLRuntimeException.class, + () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 4)); + assertTrue(e.getMessage().contains("Failed to convert FrameBlock to MatrixBlock")); + } + + @Test + public void warnCastValidFrameConvertsWithoutFallback() { + // warn-cast enabled but every cell is parseable: the strict path succeeds and the NaN + // fallback must never trigger (covers the try-succeeds branch of convert). + Array c1 = ArrayFactory.create(new String[] {"1.0", "2.0", "3.0"}); + Array c2 = ArrayFactory.create(new String[] {"4.0", "5.0", "6.0"}); + FrameBlock fb = new FrameBlock(new Array[] {c1, c2}); + + MatrixBlock mb = MatrixBlockFromFrame.convertToMatrixBlock(fb, 1); + + compare(new double[][] {{1.0, 4.0}, {2.0, 5.0}, {3.0, 6.0}}, mb); + assertFalse("No cells failed to parse, so the fallback must not have been used", + MatrixBlockFromFrame.WARNED_FOR_FAILED_CAST); + + final List log = LoggingUtils.reinsert(appender); + long warnings = log.stream() + .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")) + .count(); + assertEquals(0, warnings); + } + + @Test + public void safeCastZeroValues() { + // zero-valued parseable cells must not contribute to the non-zero count even on the safe-cast + // path (covers the ': 0' branch of the nnz ternary), while unparseable cells still become NaN. + Array c1 = ArrayFactory.create(new String[] {"0.0", "abc"}); + Array c2 = ArrayFactory.create(new String[] {"2.0", "0.0"}); + FrameBlock fb = new FrameBlock(new Array[] {c1, c2}); + + MatrixBlock mb = MatrixBlockFromFrame.convertToMatrixBlock(fb, 1); + + compare(new double[][] {{0.0, 2.0}, {NA, 0.0}}, mb); + // non-zeros: 2.0 and the NaN cell count, the two explicit zeros do not + assertEquals(2, mb.getNonZeros()); + } + + @Test + public void safeCastAllInvalid() { + // every cell fails to parse: the whole matrix becomes NaN and each NaN counts as a non-zero + Array c1 = ArrayFactory.create(new String[] {"abc", "def"}); + Array c2 = ArrayFactory.create(new String[] {"ghi", "jkl"}); + FrameBlock fb = new FrameBlock(new Array[] {c1, c2}); + + MatrixBlock mb = MatrixBlockFromFrame.convertToMatrixBlock(fb, 1); + + compare(new double[][] {{NA, NA}, {NA, NA}}, mb); + assertEquals(4, mb.getNonZeros()); + } + + @Test + public void privateConstructor() throws Exception { + Constructor c = MatrixBlockFromFrame.class.getDeclaredConstructor(); + assertTrue("Constructor should be private", Modifier.isPrivate(c.getModifiers())); + c.setAccessible(true); + c.newInstance(); + } + + /** + * Verify that every parseable cell matches its expected value and every unparseable cell became NaN. + */ + private static void compareSafeCast(MatrixBlock mb) { + compare(EXPECTED, mb); + } + + /** + * Verify that the matrix matches the expected values cell by cell, treating NaN cells as expected NaN. + */ + private static void compare(double[][] expected, MatrixBlock mb) { + assertEquals(expected.length, mb.getNumRows()); + assertEquals(expected[0].length, mb.getNumColumns()); + for(int i = 0; i < expected.length; i++) + for(int j = 0; j < expected[i].length; j++) + assertEquals("cell (" + i + "," + j + ")", expected[i][j], mb.get(i, j), 0.0); + } +} diff --git a/src/test/java/org/apache/sysds/test/component/frame/array/CustomArrayTests.java b/src/test/java/org/apache/sysds/test/component/frame/array/CustomArrayTests.java index 73d04f32435..df386d4659d 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/array/CustomArrayTests.java +++ b/src/test/java/org/apache/sysds/test/component/frame/array/CustomArrayTests.java @@ -1899,6 +1899,13 @@ public void parseDoubleInvalid3() { assertEquals(Double.POSITIVE_INFINITY, DoubleArray.parseDouble("iff"), 0.0); } + @Test(expected = NumberFormatException.class) + public void parseDoubleThrowsRawNumberFormatException() { + // the parse failure must surface as the raw NumberFormatException, not a wrapped DMLRuntimeException, + // so callers can distinguish a format error from other runtime failures + DoubleArray.parseDouble("not_a_number"); + } + @Test(expected = Exception.class) public void setDDCArrayWithDDCArray() { Array c = FrameCompressTestUtils.generateArray(100, 32, 5, ValueType.INT32); diff --git a/src/test/java/org/apache/sysds/test/component/misc/DoubleParserTest.java b/src/test/java/org/apache/sysds/test/component/misc/DoubleParserTest.java index 08aa5a94e93..546d6ec1387 100644 --- a/src/test/java/org/apache/sysds/test/component/misc/DoubleParserTest.java +++ b/src/test/java/org/apache/sysds/test/component/misc/DoubleParserTest.java @@ -152,6 +152,18 @@ public void parseWithWhitespace() { compareToDoubleParser(" 132.14"); } + @Test + public void parseTrailingDot() { + // last char '.' is below '0', forcing the slow Double.parseDouble path + compareToDoubleParser("132."); + } + + @Test + public void parseTrailingWhitespace() { + // last char ' ' is below '0', forcing the slow Double.parseDouble path + compareToDoubleParser("132.14 "); + } + @Test public void parsePowerOf10(){ compareToDoubleParser("132e10"); From 503146d694c212c51bac60b31512265096e59e2e Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Mon, 15 Jun 2026 12:37:48 +0200 Subject: [PATCH 034/132] [MINOR] Fix Flaky Tests from Blocking Thread Pools (#2489) --- .../ooc/AggregateUnaryOOCInstruction.java | 11 +++++------ .../runtime/instructions/ooc/OOCInstruction.java | 5 +---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/AggregateUnaryOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/AggregateUnaryOOCInstruction.java index f0d4fd29af7..38b228ccb1d 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/AggregateUnaryOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/AggregateUnaryOOCInstruction.java @@ -35,7 +35,6 @@ import org.apache.sysds.runtime.matrix.operators.AggregateUnaryOperator; import org.apache.sysds.runtime.matrix.operators.Operator; import org.apache.sysds.runtime.meta.DataCharacteristics; -import org.apache.sysds.runtime.ooc.stream.StreamContext; import org.apache.sysds.runtime.util.IndexRange; import java.util.HashMap; @@ -119,9 +118,10 @@ public void processInstruction( ExecutionContext ec ) { }); // global reduce - submitOOCTask(() -> { - IndexedMatrixValue partial; - while ((partial = qLocal.dequeue()) != LocalTaskQueue.NO_MORE_TASKS) { + addOutStream(qOut); + submitOOCTasks(qLocal, callback -> { + IndexedMatrixValue partial = callback.get(); + synchronized(aggTracker) { long idx = aggun.isRowAggregate() ? partial.getIndexes().getRowIndex() : partial.getIndexes() .getColumnIndex(); @@ -150,8 +150,7 @@ public void processInstruction( ExecutionContext ec ) { corrs.remove(idx); } } - qOut.closeInput(); - }, new StreamContext().addOutStream(qOut)); + }).thenRun(qOut::closeInput); } // full aggregation else { diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java index 679e7187e5e..859bca42dfe 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java @@ -1196,7 +1196,7 @@ protected CompletableFuture submitOOCTasks(OOCStream queue, Consume } protected CompletableFuture submitOOCTask(Runnable r, StreamContext ctx) { - ExecutorService pool = CommonThreadPool.get(); + ExecutorService pool = CommonThreadPool.getDynamicPool(); final CompletableFuture future = new CompletableFuture<>(); try { COMPUTE_IN_FLIGHT.incrementAndGet(); @@ -1220,9 +1220,6 @@ protected CompletableFuture submitOOCTask(Runnable r, StreamContext ctx) { COMPUTE_IN_FLIGHT.decrementAndGet(); throw new DMLRuntimeException(ex); } - finally { - pool.shutdown(); - } return future; } From dbe321a44683541918b9006ffea8462abb989ac1 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Mon, 15 Jun 2026 13:07:21 +0200 Subject: [PATCH 035/132] [BWARE] Track more compressed-friendly ops in FederatedWorkloadAnalyzer (#2481) * Track more compressed-friendly ops in FederatedWorkloadAnalyzer Extends the federated workload counter so that compression decisions account for additional instruction shapes beyond AggregateBinary. - Pass the right-hand column count to incOverlappingDecompressions so the cost model reflects the actual decompression size rather than counting a single column - Count MMChainCPInstruction as one LMM and one RMM contribution per invocation - Count AggregateUnaryCPInstruction: when reducing columns with a sum/mean operator, treat it as a dict-op (compression-friendly); otherwise count it as a decompression - Minor formatting cleanup in compressRun * Add unit tests for FederatedWorkloadAnalyzer workload tracking Cover the instruction-shape branches in incrementWorkload that drive federated compression decisions, which previously had no direct tests: - AggregateBinary: RMM/LMM counting, overlapping-decompress sizing by the right-hand column count, and the validSize row/column guards - MMChain: one LMM and one RMM contribution per invocation - AggregateUnary: dict-op vs decompression classification across ReduceAll/ReduceRow/ReduceCol with sum, mean, product, and max operators - Instance-level dispatch and compressRun threshold behavior, asserting async compression materializes when the cost model would compress --- .../federated/FederatedWorkloadAnalyzer.java | 50 ++- .../FederatedWorkloadAnalyzerTest.java | 343 ++++++++++++++++++ 2 files changed, 388 insertions(+), 5 deletions(-) create mode 100644 src/test/java/org/apache/sysds/test/component/federated/FederatedWorkloadAnalyzerTest.java diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkloadAnalyzer.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkloadAnalyzer.java index fc0aa3b1a29..e9f451397bf 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkloadAnalyzer.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkloadAnalyzer.java @@ -27,9 +27,18 @@ import org.apache.sysds.runtime.compress.cost.InstructionTypeCounter; import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; import org.apache.sysds.runtime.controlprogram.context.ExecutionContext; +import org.apache.sysds.runtime.functionobjects.IndexFunction; +import org.apache.sysds.runtime.functionobjects.KahanPlus; +import org.apache.sysds.runtime.functionobjects.Mean; +import org.apache.sysds.runtime.functionobjects.Plus; +import org.apache.sysds.runtime.functionobjects.ReduceCol; import org.apache.sysds.runtime.instructions.Instruction; import org.apache.sysds.runtime.instructions.cp.AggregateBinaryCPInstruction; +import org.apache.sysds.runtime.instructions.cp.AggregateUnaryCPInstruction; import org.apache.sysds.runtime.instructions.cp.ComputationCPInstruction; +import org.apache.sysds.runtime.instructions.cp.MMChainCPInstruction; +import org.apache.sysds.runtime.matrix.operators.AggregateUnaryOperator; +import org.apache.sysds.runtime.matrix.operators.Operator; public class FederatedWorkloadAnalyzer { protected static final Log LOG = LogFactory.getLog(FederatedWorkloadAnalyzer.class.getName()); @@ -55,7 +64,7 @@ public void incrementWorkload(ExecutionContext ec, long tid, Instruction ins) { } public void compressRun(ExecutionContext ec, long tid) { - if(counter >= compressRunFrequency ){ + if(counter >= compressRunFrequency) { counter = 0; get(tid).forEach((K, V) -> CompressedMatrixBlockFactory.compressAsync(ec, Long.toString(K), V)); } @@ -68,6 +77,7 @@ private void incrementWorkload(ExecutionContext ec, long tid, ComputationCPInstr public void incrementWorkload(ExecutionContext ec, ConcurrentHashMap mm, ComputationCPInstruction cpIns) { // TODO: Count transitive closure via lineage + // TODO: add more operations if(cpIns instanceof AggregateBinaryCPInstruction) { final String n1 = cpIns.input1.getName(); MatrixObject d1 = (MatrixObject) ec.getCacheableData(n1); @@ -81,15 +91,45 @@ public void incrementWorkload(ExecutionContext ec, ConcurrentHashMap mm, long id) { @@ -117,8 +157,8 @@ private static boolean validSize(int nRow, int nCol) { return nRow > 90 && nRow >= nCol; } - @Override - public String toString(){ + @Override + public String toString() { StringBuilder sb = new StringBuilder(); sb.append(this.getClass().getSimpleName()); sb.append(" Counter: "); diff --git a/src/test/java/org/apache/sysds/test/component/federated/FederatedWorkloadAnalyzerTest.java b/src/test/java/org/apache/sysds/test/component/federated/FederatedWorkloadAnalyzerTest.java new file mode 100644 index 00000000000..21532caa771 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/federated/FederatedWorkloadAnalyzerTest.java @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.federated; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.sysds.common.Opcodes; +import org.apache.sysds.common.Types.FileFormat; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.compress.CompressedMatrixBlock; +import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory; +import org.apache.sysds.runtime.compress.cost.InstructionTypeCounter; +import org.apache.sysds.runtime.controlprogram.LocalVariableMap; +import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; +import org.apache.sysds.runtime.controlprogram.context.ExecutionContext; +import org.apache.sysds.runtime.controlprogram.federated.FederatedWorkloadAnalyzer; +import org.apache.sysds.runtime.instructions.Instruction; +import org.apache.sysds.runtime.instructions.InstructionUtils; +import org.apache.sysds.runtime.instructions.cp.AggregateBinaryCPInstruction; +import org.apache.sysds.runtime.instructions.cp.AggregateUnaryCPInstruction; +import org.apache.sysds.runtime.instructions.cp.ComputationCPInstruction; +import org.apache.sysds.runtime.instructions.cp.MMChainCPInstruction; +import org.apache.sysds.runtime.instructions.cp.ReorgCPInstruction; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.meta.MatrixCharacteristics; +import org.apache.sysds.runtime.meta.MetaDataFormat; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; + +public class FederatedWorkloadAnalyzerTest { + protected static final Log LOG = LogFactory.getLog(FederatedWorkloadAnalyzerTest.class.getName()); + + /** Async compression triggered by compressRun runs on a worker thread, so poll instead of sleeping. */ + private static final int COMPRESS_TIMEOUT_MS = 10000; + + private final FederatedWorkloadAnalyzer analyzer = new FederatedWorkloadAnalyzer(); + + // -------------------------------------------------------------------------------------------- + // AggregateBinary (matrix multiply) + // -------------------------------------------------------------------------------------------- + + @Test + public void aggregateBinaryBothSidesCounted() { + // left 100x100 (valid), right 100x50 (valid) + ExecutionContext ec = ec("1", mo(100, 100), "2", mo(100, 50)); + ConcurrentHashMap mm = new ConcurrentHashMap<>(); + + analyzer.incrementWorkload(ec, mm, mm("1", "2")); + + // left side: RMM with the right-hand column count, plus overlapping decompress sized by c2 + InstructionTypeCounter left = mm.get(1L); + assertEquals(50, left.getRightMultiplications()); + assertEquals(50, left.getOverlappingDecompressions()); + // right side: LMM with the left-hand row count + InstructionTypeCounter right = mm.get(2L); + assertEquals(100, right.getLeftMultiplications()); + } + + @Test + public void aggregateBinaryOnlyLeftCountedWhenRightTooSmall() { + // left 100x10 (valid), right 10x5 (too few rows -> invalid) + ExecutionContext ec = ec("1", mo(100, 10), "2", mo(10, 5)); + ConcurrentHashMap mm = new ConcurrentHashMap<>(); + + analyzer.incrementWorkload(ec, mm, mm("1", "2")); + + InstructionTypeCounter left = mm.get(1L); + assertEquals(5, left.getRightMultiplications()); + assertEquals(5, left.getOverlappingDecompressions()); + // right side never tracked because it does not pass validSize + assertFalse(mm.containsKey(2L)); + } + + @Test + public void aggregateBinaryNeitherCountedWhenBothTooSmall() { + ExecutionContext ec = ec("1", mo(10, 10), "2", mo(10, 10)); + ConcurrentHashMap mm = new ConcurrentHashMap<>(); + + analyzer.incrementWorkload(ec, mm, mm("1", "2")); + + assertTrue(mm.isEmpty()); + } + + @Test + public void aggregateBinaryWideOperandNotCounted() { + // 100x200: enough rows (>90) but more columns than rows -> validSize false on the second clause + ExecutionContext ec = ec("1", mo(100, 200), "2", mo(10, 5)); + ConcurrentHashMap mm = new ConcurrentHashMap<>(); + + analyzer.incrementWorkload(ec, mm, mm("1", "2")); + + assertTrue(mm.isEmpty()); + } + + // -------------------------------------------------------------------------------------------- + // MMChain + // -------------------------------------------------------------------------------------------- + + @Test + public void mmChainCountsOneLeftAndOneRight() { + ConcurrentHashMap mm = new ConcurrentHashMap<>(); + + analyzer.incrementWorkload(null, mm, mmchain("1")); + + InstructionTypeCounter c = mm.get(1L); + assertEquals(1, c.getRightMultiplications()); + assertEquals(1, c.getLeftMultiplications()); + } + + // -------------------------------------------------------------------------------------------- + // AggregateUnary + // -------------------------------------------------------------------------------------------- + + @Test + public void aggregateUnaryColSumsIsDictOp() { + // colSums -> ReduceRow -> compression friendly (2 dict ops, no decompress) + assertDictOpsAndDecompress(Opcodes.UACKP.toString(), 2, 0); + } + + @Test + public void aggregateUnaryFullSumIsDictOp() { + // sum -> ReduceAll -> compression friendly (2 dict ops, no decompress) + assertDictOpsAndDecompress(Opcodes.UAKP.toString(), 2, 0); + } + + @Test + public void aggregateUnaryRowSumsIsDictOp() { + // rowSums -> ReduceCol with KahanPlus -> compression friendly (2 dict ops, no decompress) + assertDictOpsAndDecompress(Opcodes.UARKP.toString(), 2, 0); + } + + @Test + public void aggregateUnaryRowMeansIsDictOp() { + // rowMeans -> ReduceCol with Mean -> compression friendly (2 dict ops, no decompress) + assertDictOpsAndDecompress(Opcodes.UARMEAN.toString(), 2, 0); + } + + @Test + public void aggregateUnaryRowProductsForcesDecompress() { + // rowProds -> ReduceCol with Multiply -> not friendly (1 dict op + 1 decompress) + assertDictOpsAndDecompress(Opcodes.UARM.toString(), 1, 1); + } + + @Test + public void aggregateUnaryRowSumsPlusIsDictOp() { + // rowSums (plain Plus, no Kahan) -> ReduceCol with Plus -> compression friendly (2 dict ops) + assertDictOpsAndDecompress(Opcodes.UARP.toString(), 2, 0); + } + + @Test + public void aggregateUnaryRowMaxForcesDecompress() { + // rowMax -> ReduceCol with Builtin max -> not friendly (1 dict op + 1 decompress) + assertDictOpsAndDecompress(Opcodes.UARMAX.toString(), 1, 1); + } + + @Test + public void aggregateUnaryNonAggregateOperatorIgnored() { + // nrow uses a SimpleOperator (not an AggregateUnaryOperator) so nothing is tracked + ConcurrentHashMap mm = new ConcurrentHashMap<>(); + + analyzer.incrementWorkload(null, mm, uagg(Opcodes.NROW.toString(), "1")); + + assertTrue(mm.isEmpty()); + } + + private void assertDictOpsAndDecompress(String opcode, int expectedDictOps, int expectedDecompress) { + ConcurrentHashMap mm = new ConcurrentHashMap<>(); + + analyzer.incrementWorkload(null, mm, uagg(opcode, "1")); + + InstructionTypeCounter c = mm.get(1L); + assertEquals("Unexpected dict-ops for " + opcode, expectedDictOps, c.getDictionaryOps()); + assertEquals("Unexpected decompressions for " + opcode, expectedDecompress, c.getDecompressions()); + } + + // -------------------------------------------------------------------------------------------- + // Instance level dispatch + async compress trigger + // -------------------------------------------------------------------------------------------- + + @Test + public void compressRunCompressesAfterEnoughWorkload() { + final long tid = 1; + final int dim = 100, iter = 10; + // Right operand is left-multiplied each matmul, accumulating LMM = leftRows (=dim) per + // invocation, so iter=10 yields LMM=1000 on a 100x100 rounded block. This mirrors the shape + // and counter that FedWorkerMatrixMultiplyWorkload relies on to trigger compression. + MatrixBlock rightBlock = TestUtils.round(TestUtils.generateTestMatrixBlock(dim, dim, 0.5, 2.5, 1.0, 222)); + MatrixBlock probeBlock = new MatrixBlock(); + probeBlock.copy(rightBlock); + + MatrixObject left = compressibleMO(dim, dim, 7); + MatrixObject right = wrap(rightBlock); + ExecutionContext ec = ec("1", left, "2", right); + + // each matmul with two valid sides increments the counter twice; reaching the + // compressRunFrequency threshold of 10 schedules an async compression pass + ComputationCPInstruction ins = mm("1", "2"); + for(int i = 0; i < iter; i++) + analyzer.incrementWorkload(ec, tid, ins); + + analyzer.compressRun(ec, tid); + + // Only assert the async compression materialized if the cost model would compress this shape + // locally; otherwise the workload pass legitimately leaves it uncompressed (matches the skip + // pattern in FedWorkerMatrixMultiplyWorkload). + InstructionTypeCounter probe = new InstructionTypeCounter(0, 0, 0, dim * iter, 0, 0, 0, 0, false); + boolean locallyCompressible = CompressedMatrixBlockFactory.compress(probeBlock, probe) + .getLeft() instanceof CompressedMatrixBlock; + if(locallyCompressible) + assertCompressedWithinTimeout(right); + } + + @Test + public void compressRunNoOpBelowThreshold() { + final long tid = 2; + MatrixObject left = compressibleMO(500, 10, 7); + MatrixObject right = compressibleMO(500, 10, 13); + ExecutionContext ec = ec("1", left, "2", right); + + // only two invocations -> counter = 4, below threshold, so nothing compresses + ComputationCPInstruction ins = mm("1", "2"); + analyzer.incrementWorkload(ec, tid, ins); + analyzer.incrementWorkload(ec, tid, ins); + + analyzer.compressRun(ec, tid); + + assertFalse(left.acquireReadAndRelease() instanceof CompressedMatrixBlock); + assertFalse(right.acquireReadAndRelease() instanceof CompressedMatrixBlock); + } + + @Test + public void nonComputationInstructionIgnored() { + // the public entry point silently ignores non-CP / non-computation instructions + analyzer.incrementWorkload(null, 99, (Instruction) null); + analyzer.compressRun(null, 99); + } + + @Test + public void unhandledComputationInstructionIgnored() { + // a transpose is a ComputationCPInstruction but none of the tracked shapes -> no counters + ConcurrentHashMap mm = new ConcurrentHashMap<>(); + + analyzer.incrementWorkload(null, mm, reorg("1")); + + assertTrue(mm.isEmpty()); + } + + @Test + public void toStringReportsState() { + String s = analyzer.toString(); + assertTrue(s.contains(FederatedWorkloadAnalyzer.class.getSimpleName())); + assertTrue(s.contains("Counter")); + } + + // -------------------------------------------------------------------------------------------- + // helpers + // -------------------------------------------------------------------------------------------- + + private static void assertCompressedWithinTimeout(MatrixObject mo) { + final long deadline = System.currentTimeMillis() + COMPRESS_TIMEOUT_MS; + while(System.currentTimeMillis() < deadline) { + if(mo.acquireReadAndRelease() instanceof CompressedMatrixBlock) + return; + try { + Thread.sleep(50); + } + catch(InterruptedException e) { + Thread.currentThread().interrupt(); + fail("Interrupted while waiting for async compression"); + } + } + fail("Matrix was not compressed by the workload analyzer within " + COMPRESS_TIMEOUT_MS + "ms"); + } + + private static ExecutionContext ec(String n1, MatrixObject m1, String n2, MatrixObject m2) { + LocalVariableMap vars = new LocalVariableMap(); + ExecutionContext ec = new ExecutionContext(vars); + ec.setVariable(n1, m1); + ec.setVariable(n2, m2); + return ec; + } + + /** Build a MatrixObject of the requested shape (data content irrelevant for the counters). */ + private static MatrixObject mo(int rows, int cols) { + return wrap(new MatrixBlock(rows, cols, 0.0)); + } + + private static MatrixObject compressibleMO(int rows, int cols, int seed) { + return wrap(TestUtils.round(TestUtils.generateTestMatrixBlock(rows, cols, 0, 3, 1.0, seed))); + } + + private static MatrixObject wrap(MatrixBlock mb) { + MatrixCharacteristics mc = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), -1, mb.getNonZeros()); + MetaDataFormat md = new MetaDataFormat(mc, FileFormat.BINARY); + MatrixObject mo = new MatrixObject(ValueType.FP64, "/dev/null", md, mb); + mo.getDataCharacteristics().setDimension(mb.getNumRows(), mb.getNumColumns()); + return mo; + } + + private static ComputationCPInstruction mm(String in1, String in2) { + String str = InstructionUtils.concatOperands("CP", Opcodes.MMULT.toString(), in1, in2, "3", "16"); + return AggregateBinaryCPInstruction.parseInstruction(str); + } + + private static ComputationCPInstruction mmchain(String in1) { + String str = InstructionUtils.concatOperands("CP", Opcodes.MMCHAIN.toString(), in1, "2", "3", "XtXv", "16"); + return MMChainCPInstruction.parseInstruction(str); + } + + private static ComputationCPInstruction uagg(String opcode, String in1) { + String str = InstructionUtils.concatOperands("CP", opcode, in1, "2", "16"); + return AggregateUnaryCPInstruction.parseInstruction(str); + } + + private static ComputationCPInstruction reorg(String in1) { + String str = InstructionUtils.concatOperands("CP", Opcodes.TRANSPOSE.toString(), in1, "2", "16"); + return ReorgCPInstruction.parseInstruction(str); + } +} From d77096f52a22678802fac6b63c72cbe19e689703 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Tue, 16 Jun 2026 13:54:52 +0200 Subject: [PATCH 036/132] [MINOR][CI] Fix leaked threads hanging Java test forks (#2488) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several Java test suites (most visibly **.component.c** and data.misc/lineage) intermittently ran until the GitHub Actions job timeout even though the tests themselves had completed. The cause was leaked non-daemon threads keeping the surefire fork JVM alive, so the fork never exited and the job stalled until cancelled. There were two sources: in-JVM federated workers (FederatedWorker's Netty event loops and the test-side worker wrapper threads were non-daemon), and CommonThreadPool's fallback pools — when called off the main thread, it returned Executors.newFixedThreadPool/newCachedThreadPool, which default to non-daemon threads, while only the ForkJoinPool-backed variants were already daemon. This PR makes those threads daemon at the source: FederatedWorker now creates its Netty event-loop groups with a daemon thread factory, and CommonThreadPool routes its fixed/cached fallbacks through one too, so daemon behavior is uniform across all pool variants. On the test side, AutomatedTestBase marks spawned worker threads as daemon, TestUtils.shutdownThread bounds its join (30s, warns on stragglers, restores the interrupt flag), and the lineage tests (LineageFedReuseAlg, FedFullReuseTest, FedUDFReuseTest) now shut workers down in a finally block so failures no longer leak workers (the large line counts there are just reindentation from the try/finally wrap). The javaTests.yml job cap stays at 30 minutes, with a comment documenting why it sits above the 600s per-fork surefire timeout, which remains the backstop for genuine hangs. --- .github/workflows/javaTests.yml | 2 + .../org/apache/sysds/validation/Utility.java | 1 + .../federated/FederatedWorker.java | 7 +- .../runtime/ooc/cache/OOCMatrixIOHandler.java | 6 +- .../sysds/runtime/util/CommonThreadPool.java | 20 ++++- .../performance/generators/GenMatrices.java | 2 +- .../apache/sysds/test/AutomatedTestBase.java | 6 ++ .../java/org/apache/sysds/test/TestUtils.java | 15 +++- .../FederatedBackendPerformanceTest.java | 7 +- .../FederatedMatrixScalarOperationsTest.java | 2 +- .../functions/lineage/FedFullReuseTest.java | 88 ++++++++++--------- .../functions/lineage/FedUDFReuseTest.java | 74 ++++++++-------- .../functions/lineage/LineageFedReuseAlg.java | 6 +- .../org/apache/sysds/test/usertest/Base.java | 2 +- 14 files changed, 142 insertions(+), 96 deletions(-) diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml index 0d6ac02fb82..0d4c71e946b 100644 --- a/.github/workflows/javaTests.yml +++ b/.github/workflows/javaTests.yml @@ -50,6 +50,8 @@ concurrency: jobs: java_tests: runs-on: ubuntu-24.04 + # Job cap kept above the per-fork surefire timeout (test-forkedProcessTimeout, + # 600s) so surefire can kill a hung fork before GitHub Actions cancels the job. timeout-minutes: 30 strategy: fail-fast: false diff --git a/dev/release/src/test/java/org/apache/sysds/validation/Utility.java b/dev/release/src/test/java/org/apache/sysds/validation/Utility.java index 34f5eeaf92b..da23b8e40d0 100644 --- a/dev/release/src/test/java/org/apache/sysds/validation/Utility.java +++ b/dev/release/src/test/java/org/apache/sysds/validation/Utility.java @@ -185,6 +185,7 @@ public static int runCommand(String [] command, String strCurDir, String strOutp try { exitValue = process.waitFor(); } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); debugPrint(Constants.DEBUG_ERROR, "Program interrunpted: " + ie); } debugPrint(Constants.DEBUG_CODE, "Program '" + String.join(" ", command) + "' exited with exit status " + exitValue, strOutputFile); diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java index 55f2f17cd8a..fc8989053bc 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java @@ -65,6 +65,7 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.util.SelfSignedCertificate; +import io.netty.util.concurrent.DefaultThreadFactory; @SuppressWarnings("deprecation") public class FederatedWorker { @@ -99,9 +100,11 @@ private void run() { LOG.info("Setting up Federated Worker on port " + _port); int par_conn = ConfigurationManager.getDMLConfig().getIntValue(DMLConfig.FEDERATED_PAR_CONN); final int EVENT_LOOP_THREADS = (par_conn > 0) ? par_conn : InfrastructureAnalyzer.getLocalParallelism(); - NioEventLoopGroup bossGroup = new NioEventLoopGroup(1); + // Daemon event loops so a leaked in-JVM (test) worker cannot block JVM exit. + NioEventLoopGroup bossGroup = new NioEventLoopGroup(1, + new DefaultThreadFactory("fed-worker-boss", true)); ThreadPoolExecutor workerTPE = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 10, TimeUnit.SECONDS, - new SynchronousQueue(true)); + new SynchronousQueue(true), new DefaultThreadFactory("fed-worker-pool", true)); NioEventLoopGroup workerGroup = new NioEventLoopGroup(EVENT_LOOP_THREADS, workerTPE); final boolean ssl = ConfigurationManager.isFederatedSSL(); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCMatrixIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCMatrixIOHandler.java index 3146439165f..7509b669701 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCMatrixIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCMatrixIOHandler.java @@ -148,7 +148,8 @@ public void shutdown() { _q[i].close(); } } - catch(InterruptedException ignored) { + catch(InterruptedException e) { + Thread.currentThread().interrupt(); } } _writeExec.getQueue().clear(); @@ -174,7 +175,8 @@ public CompletableFuture scheduleEviction(BlockEntry block) { int i = (int)(q % WRITER_SIZE); _q[i].enqueueIfOpen(new Tuple2<>(block, future)); } - catch(InterruptedException ignored) { + catch(InterruptedException e) { + Thread.currentThread().interrupt(); } return future; diff --git a/src/main/java/org/apache/sysds/runtime/util/CommonThreadPool.java b/src/main/java/org/apache/sysds/runtime/util/CommonThreadPool.java index 3ee08da0def..156a7820f86 100644 --- a/src/main/java/org/apache/sysds/runtime/util/CommonThreadPool.java +++ b/src/main/java/org/apache/sysds/runtime/util/CommonThreadPool.java @@ -29,6 +29,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.Future; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -141,11 +142,26 @@ else if(mainThread || threadName.contains("PARFOR") || threadName.contains("FedE incorrectPoolUse = true; } - return Executors.newFixedThreadPool(k); + return Executors.newFixedThreadPool(k, daemonThreadFactory()); } } + /** + * Thread factory that produces daemon threads. The ForkJoinPool-backed pools already use daemon + * threads; the fallback {@link Executors#newFixedThreadPool} and {@link Executors#newCachedThreadPool} + * pools default to non-daemon threads, which can keep the JVM (e.g. a surefire test fork) alive + * if a caller forgets to shut the pool down. Making them daemon keeps that behavior uniform. + */ + private static ThreadFactory daemonThreadFactory() { + final ThreadFactory base = Executors.defaultThreadFactory(); + return r -> { + Thread t = base.newThread(r); + t.setDaemon(true); + return t; + }; + } + /** * Invoke the collection of tasks and shutdown the pool upon job termination. * @@ -180,7 +196,7 @@ public synchronized static ExecutorService getDynamicPool() { // It is guaranteed not to be shut down because of the synchronized barrier return asyncPool; else { - asyncPool = Executors.newCachedThreadPool(); + asyncPool = Executors.newCachedThreadPool(daemonThreadFactory()); return asyncPool; } } diff --git a/src/test/java/org/apache/sysds/performance/generators/GenMatrices.java b/src/test/java/org/apache/sysds/performance/generators/GenMatrices.java index f96233ae6fe..9ae1a0f1048 100644 --- a/src/test/java/org/apache/sysds/performance/generators/GenMatrices.java +++ b/src/test/java/org/apache/sysds/performance/generators/GenMatrices.java @@ -72,7 +72,7 @@ public void generate(int N) throws InterruptedException { } } catch(InterruptedException e) { - e.printStackTrace(); + Thread.currentThread().interrupt(); } }); } diff --git a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java index 150a358bdf0..2ff98c921ea 100644 --- a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java @@ -1939,6 +1939,9 @@ private static Thread spawnLocalFedWorkerThread(int port, String[] otherArgs) { LOG.error("Exception in startup of federated worker", e); } }); + // Daemon so a worker left running by a failed/forgetful test cannot keep the + // surefire fork JVM alive and stall CI until the job-level timeout. + t.setDaemon(true); t.start(); return t; } @@ -1979,6 +1982,9 @@ public static Thread startLocalFedWorkerWithArgs(String[] args) { LOG.error("Exception in startup of federated worker on port " + port, e); } }); + // Daemon so a worker left running by a failed/forgetful test cannot keep the + // surefire fork JVM alive and stall CI until the job-level timeout. + t.setDaemon(true); t.start(); FederatedWorkerUtils.waitForWorker(t, port, FED_WORKER_WAIT); return t; diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java index 5ebc243dd44..683d355e05c 100644 --- a/src/test/java/org/apache/sysds/test/TestUtils.java +++ b/src/test/java/org/apache/sysds/test/TestUtils.java @@ -3489,15 +3489,23 @@ public static void shutdownThreads(Process... ts) { } } + /** Upper bound (ms) on how long {@link #shutdownThread(Thread)} waits for a worker to stop. */ + private static final long THREAD_SHUTDOWN_JOIN_MS = 30_000; + public static void shutdownThread(Thread t) { // kill the worker if( t != null ) { t.interrupt(); try { - t.join(); + // Bounded join: workers are daemon threads, so even if one ignores the interrupt + // we must not block cleanup (and the JVM) indefinitely waiting for it. + t.join(THREAD_SHUTDOWN_JOIN_MS); + if( t.isAlive() ) + LOG.warn("Federated worker thread " + t.getName() + + " did not stop within " + THREAD_SHUTDOWN_JOIN_MS + "ms; leaving it as a daemon."); } catch (InterruptedException e) { - e.printStackTrace(); + Thread.currentThread().interrupt(); } } } @@ -3514,7 +3522,8 @@ public static void shutdownThread(Process t) { forciblyDestroyed.waitFor(); // Wait until it's definitely terminated } } catch (InterruptedException e) { - e.printStackTrace(); + LOG.warn("Interrupted while shutting down federated worker process", e); + Thread.currentThread().interrupt(); } } } diff --git a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java index df886fc0086..5de429a3c53 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java @@ -91,8 +91,11 @@ public void testBackendPerformance() throws InterruptedException { taskFutures.forEach(res -> { try { Assert.assertEquals("Stats parsed correctly", res.get().statusCode(), 200); - } catch (InterruptedException | ExecutionException e) { - e.printStackTrace(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + Assert.fail("Interrupted while fetching statistics: " + e.getMessage()); + } catch (ExecutionException e) { + Assert.fail("Failed to fetch statistics: " + e.getMessage()); } }); diff --git a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part5/FederatedMatrixScalarOperationsTest.java b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part5/FederatedMatrixScalarOperationsTest.java index 84b906b9a49..4c5cc1682cf 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part5/FederatedMatrixScalarOperationsTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part5/FederatedMatrixScalarOperationsTest.java @@ -209,7 +209,7 @@ private void runGenericTest(String dmlFile, int scalar) { compareResults(); } catch(InterruptedException e) { - e.printStackTrace(); + Thread.currentThread().interrupt(); assert (false); } finally { diff --git a/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java b/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java index 0c46cd68ea5..4852220861e 100644 --- a/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java @@ -105,50 +105,52 @@ public void federatedReuse(String test) { Lineage.resetInternalState(); Thread[] workers = startLocalFedWorkerThreads(new int[] {port1, port2}, otherargs, FED_WORKER_WAIT); - TestConfiguration config = availableTestConfigurations.get(test); - loadTestConfiguration(config); - - // Run reference dml script with normal matrix. Reuse of ba+*. - fullDMLScriptName = HOME + test + "Reference.dml"; - programArgs = new String[] {"-stats", "-lineage", "reuse_full", - "-nvargs", "X1=" + input("X1"), "X2=" + input("X2"), "Y1=" + input("Y1"), - "Y2=" + input("Y2"), "Z=" + expected("Z")}; - runTest(true, false, null, -1); - long mmCount = Statistics.getCPHeavyHitterCount(Opcodes.MMULT.toString()); - - // Run actual dml script with federated matrix - // The fed workers reuse ba+* - fullDMLScriptName = HOME + test + ".dml"; - programArgs = new String[] {"-stats","-lineage", "reuse_full", - "-nvargs", "X1=" + TestUtils.federatedAddress(port1, input("X1")), - "X2=" + TestUtils.federatedAddress(port2, input("X2")), - "Y1=" + TestUtils.federatedAddress(port1, input("Y1")), - "Y2=" + TestUtils.federatedAddress(port2, input("Y2")), "r=" + rows, "c=" + cols, "Z=" + output("Z")}; - runTest(true, false, null, -1); - long mmCount_fed = Statistics.getCPHeavyHitterCount(Opcodes.MMULT.toString()); - long fedMMCount = Statistics.getCPHeavyHitterCount("fed_ba+*"); - - // compare results - compareResults(1e-9); - // compare matrix multiplication count - // #federated execution of ba+* = #threads times #non-federated execution of ba+* (after reuse) - Assert.assertTrue("Violated reuse count: "+mmCount_fed+" == "+mmCount*2, - mmCount_fed == mmCount * 2); // #threads = 2 - switch(test) { - case TEST_NAME1: - // If the o/p is federated, fed_ba+* will be called everytime - // but the workers should be able to reuse ba+* - assertTrue(fedMMCount > mmCount_fed); - break; - case TEST_NAME2: - // If the o/p is non-federated, fed_ba+* will be called once - // and each worker will call ba+* once. - assertTrue(fedMMCount < mmCount_fed); - break; + try { + TestConfiguration config = availableTestConfigurations.get(test); + loadTestConfiguration(config); + + // Run reference dml script with normal matrix. Reuse of ba+*. + fullDMLScriptName = HOME + test + "Reference.dml"; + programArgs = new String[] {"-stats", "-lineage", "reuse_full", + "-nvargs", "X1=" + input("X1"), "X2=" + input("X2"), "Y1=" + input("Y1"), + "Y2=" + input("Y2"), "Z=" + expected("Z")}; + runTest(true, false, null, -1); + long mmCount = Statistics.getCPHeavyHitterCount(Opcodes.MMULT.toString()); + + // Run actual dml script with federated matrix + // The fed workers reuse ba+* + fullDMLScriptName = HOME + test + ".dml"; + programArgs = new String[] {"-stats","-lineage", "reuse_full", + "-nvargs", "X1=" + TestUtils.federatedAddress(port1, input("X1")), + "X2=" + TestUtils.federatedAddress(port2, input("X2")), + "Y1=" + TestUtils.federatedAddress(port1, input("Y1")), + "Y2=" + TestUtils.federatedAddress(port2, input("Y2")), "r=" + rows, "c=" + cols, "Z=" + output("Z")}; + runTest(true, false, null, -1); + long mmCount_fed = Statistics.getCPHeavyHitterCount(Opcodes.MMULT.toString()); + long fedMMCount = Statistics.getCPHeavyHitterCount("fed_ba+*"); + + // compare results + compareResults(1e-9); + // compare matrix multiplication count + // #federated execution of ba+* = #threads times #non-federated execution of ba+* (after reuse) + Assert.assertTrue("Violated reuse count: "+mmCount_fed+" == "+mmCount*2, + mmCount_fed == mmCount * 2); // #threads = 2 + switch(test) { + case TEST_NAME1: + // If the o/p is federated, fed_ba+* will be called everytime + // but the workers should be able to reuse ba+* + assertTrue(fedMMCount > mmCount_fed); + break; + case TEST_NAME2: + // If the o/p is non-federated, fed_ba+* will be called once + // and each worker will call ba+* once. + assertTrue(fedMMCount < mmCount_fed); + break; + } + } + finally { + TestUtils.shutdownThreads(workers); } - - - TestUtils.shutdownThreads(workers); } } diff --git a/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java b/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java index 0cf9d972719..eca3628a89b 100644 --- a/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java @@ -110,43 +110,45 @@ private void runTriUDFReuse(ExecMode execMode) { Lineage.resetInternalState(); Thread[] workers = startLocalFedWorkerThreads(new int[] {port1, port2, port3, port4}, otherargs, FED_WORKER_WAIT); - rtplatform = execMode; - if(rtplatform == ExecMode.SPARK) { - System.out.println(7); - DMLScript.USE_LOCAL_SPARK_CONFIG = true; + try { + rtplatform = execMode; + if(rtplatform == ExecMode.SPARK) { + System.out.println(7); + DMLScript.USE_LOCAL_SPARK_CONFIG = true; + } + TestConfiguration config = availableTestConfigurations.get(TEST_NAME); + loadTestConfiguration(config); + + // Run reference dml script with normal matrix + fullDMLScriptName = HOME + TEST_NAME + "Reference.dml"; + programArgs = new String[] {"-lineage", "reuse_full", "-stats", "100", "-args", + input("X1"), input("X2"), input("X3"), input("X4"), + Boolean.toString(rowPartitioned).toUpperCase(), expected("S")}; + runTest(null); + + // Run actual dml script with federated matrix + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + programArgs = new String[] {"-lineage", "reuse_full", "-stats", "100", "-nvargs", + "in_X1=" + TestUtils.federatedAddress(port1, input("X1")), + "in_X2=" + TestUtils.federatedAddress(port2, input("X2")), + "in_X3=" + TestUtils.federatedAddress(port3, input("X3")), + "in_X4=" + TestUtils.federatedAddress(port4, input("X4")), "rows=" + rows, "cols=" + cols, + "rP=" + Boolean.toString(rowPartitioned).toUpperCase(), "out_S=" + output("S")}; + + runTest(null); + + // compare via files + compareResults(1e-9); + // check if lowertri is federated + Assert.assertTrue(heavyHittersContainsString("fed_lowertri")); + // assert reuse count + Assert.assertTrue(LineageCacheStatistics.getInstHits() > 0); + } + finally { + TestUtils.shutdownThreads(workers); + rtplatform = platformOld; + DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; } - TestConfiguration config = availableTestConfigurations.get(TEST_NAME); - loadTestConfiguration(config); - - // Run reference dml script with normal matrix - fullDMLScriptName = HOME + TEST_NAME + "Reference.dml"; - programArgs = new String[] {"-lineage", "reuse_full", "-stats", "100", "-args", - input("X1"), input("X2"), input("X3"), input("X4"), - Boolean.toString(rowPartitioned).toUpperCase(), expected("S")}; - runTest(null); - - // Run actual dml script with federated matrix - fullDMLScriptName = HOME + TEST_NAME + ".dml"; - programArgs = new String[] {"-lineage", "reuse_full", "-stats", "100", "-nvargs", - "in_X1=" + TestUtils.federatedAddress(port1, input("X1")), - "in_X2=" + TestUtils.federatedAddress(port2, input("X2")), - "in_X3=" + TestUtils.federatedAddress(port3, input("X3")), - "in_X4=" + TestUtils.federatedAddress(port4, input("X4")), "rows=" + rows, "cols=" + cols, - "rP=" + Boolean.toString(rowPartitioned).toUpperCase(), "out_S=" + output("S")}; - - runTest(null); - - // compare via files - compareResults(1e-9); - // check if lowertri is federated - Assert.assertTrue(heavyHittersContainsString("fed_lowertri")); - // assert reuse count - Assert.assertTrue(LineageCacheStatistics.getInstHits() > 0); - - TestUtils.shutdownThreads(workers); - - rtplatform = platformOld; - DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; } } diff --git a/src/test/java/org/apache/sysds/test/functions/lineage/LineageFedReuseAlg.java b/src/test/java/org/apache/sysds/test/functions/lineage/LineageFedReuseAlg.java index b8a66196670..4fe5cc0921a 100644 --- a/src/test/java/org/apache/sysds/test/functions/lineage/LineageFedReuseAlg.java +++ b/src/test/java/org/apache/sysds/test/functions/lineage/LineageFedReuseAlg.java @@ -69,6 +69,7 @@ public void federatedLmPipeline(ExecMode execMode, boolean contSplits, String TE getAndLoadTestConfiguration(TEST_NAME); String HOME = SCRIPT_DIR + TEST_DIR; + Thread[] workers = null; try { // generated lm data MatrixBlock X = MatrixBlock.randOperations(rows, cols, 1.0, 0, 1, "uniform", 7); @@ -93,7 +94,7 @@ public void federatedLmPipeline(ExecMode execMode, boolean contSplits, String TE int port3 = getRandomAvailablePort(); int port4 = getRandomAvailablePort(); String[] otherargs = new String[] {"-lineage", "reuse_full"}; - Thread[] workers = startLocalFedWorkerThreads(new int[] {port1, port2}, otherargs, FED_WORKER_WAIT); + workers = startLocalFedWorkerThreads(new int[] {port1, port2}, otherargs, FED_WORKER_WAIT); TestConfiguration config = availableTestConfigurations.get(TEST_NAME); loadTestConfiguration(config); @@ -134,10 +135,9 @@ public void federatedLmPipeline(ExecMode execMode, boolean contSplits, String TE assertTrue(fed_tsmmCount > fed_tsmmCount_reuse); assertTrue(mmCount > mmCount_reuse); assertTrue(fed_mmCount > fed_mmCount_reuse); - - TestUtils.shutdownThreads(workers); } finally { + TestUtils.shutdownThreads(workers); resetExecMode(oldExec); ColumnEncoderRecode.SORT_RECODE_MAP = oldSort; } diff --git a/src/test/java/org/apache/sysds/test/usertest/Base.java b/src/test/java/org/apache/sysds/test/usertest/Base.java index 4a7f64824d5..fd9ac743eeb 100644 --- a/src/test/java/org/apache/sysds/test/usertest/Base.java +++ b/src/test/java/org/apache/sysds/test/usertest/Base.java @@ -98,7 +98,7 @@ public static Pair runThread(String[] args) { t.join(); } catch(InterruptedException e) { - e.printStackTrace(); + Thread.currentThread().interrupt(); } System.setOut(old); From db4f60d62b33a1471571b470fc4d32cfae7ecd25 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Tue, 16 Jun 2026 23:09:24 +0200 Subject: [PATCH 037/132] [BWARE] Add getCategoricalMask DML builtin (#2482) * Add getCategoricalMask DML builtin Adds a new builtin that, given a transform-encode metadata frame and the encoding JSON spec, returns a 1xN matrix mask marking which output columns are categorical (1) versus continuous (0). Useful when callers need to know the category boundary in transformed output without re-deriving it from the spec. --- .../org/apache/sysds/common/Builtins.java | 1 + .../java/org/apache/sysds/common/Opcodes.java | 2 + .../java/org/apache/sysds/common/Types.java | 1 + .../java/org/apache/sysds/hops/BinaryOp.java | 5 +- .../parser/BuiltinFunctionExpression.java | 16 + .../apache/sysds/parser/DMLTranslator.java | 3 + .../runtime/functionobjects/Builtin.java | 3 +- .../instructions/cp/BinaryCPInstruction.java | 2 + .../cp/BinaryFrameScalarCPInstruction.java | 206 ++++++++++++ .../java/org/apache/sysds/test/TestUtils.java | 19 ++ .../GetCategoricalMaskInstructionTest.java | 306 ++++++++++++++++++ .../transform/GetCategoricalMaskTest.java | 167 ++++++++++ .../transform/GetCategoricalMaskTest.dml | 37 +++ 13 files changed, 766 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java create mode 100644 src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java create mode 100644 src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java create mode 100644 src/test/scripts/functions/transform/GetCategoricalMaskTest.dml diff --git a/src/main/java/org/apache/sysds/common/Builtins.java b/src/main/java/org/apache/sysds/common/Builtins.java index 62145124d82..c77a2e9d866 100644 --- a/src/main/java/org/apache/sysds/common/Builtins.java +++ b/src/main/java/org/apache/sysds/common/Builtins.java @@ -154,6 +154,7 @@ public enum Builtins { GARCH("garch", true), GAUSSIAN_CLASSIFIER("gaussianClassifier", true), GET_ACCURACY("getAccuracy", true), + GET_CATEGORICAL_MASK("getCategoricalMask", false), GLM("glm", true), GLM_PREDICT("glmPredict", true), GLOVE("glove", true), diff --git a/src/main/java/org/apache/sysds/common/Opcodes.java b/src/main/java/org/apache/sysds/common/Opcodes.java index 1b0536416d6..9a894dde13b 100644 --- a/src/main/java/org/apache/sysds/common/Opcodes.java +++ b/src/main/java/org/apache/sysds/common/Opcodes.java @@ -215,6 +215,8 @@ public enum Opcodes { TRANSFORMMETA("transformmeta", InstructionType.ParameterizedBuiltin), TRANSFORMENCODE("transformencode", InstructionType.MultiReturnParameterizedBuiltin, InstructionType.MultiReturnBuiltin), + GET_CATEGORICAL_MASK("get_categorical_mask", InstructionType.Binary), + //Ternary instruction opcodes PM("+*", InstructionType.Ternary), MINUSMULT("-*", InstructionType.Ternary), diff --git a/src/main/java/org/apache/sysds/common/Types.java b/src/main/java/org/apache/sysds/common/Types.java index 2e3543882d2..c2832aeb8cd 100644 --- a/src/main/java/org/apache/sysds/common/Types.java +++ b/src/main/java/org/apache/sysds/common/Types.java @@ -639,6 +639,7 @@ public enum OpOp2 { MINUS_NZ(false), //sparse-safe minus: X-(mean*ppred(X,0,!=)) LOG_NZ(false), //sparse-safe log; ppred(X,0,"!=")*log(X,0.5) MINUS1_MULT(false), //1-X*Y + GET_CATEGORICAL_MASK(false), // get transformation mask QUANTIZE_COMPRESS(false), //quantization-fused compression UNION_DISTINCT(false); diff --git a/src/main/java/org/apache/sysds/hops/BinaryOp.java b/src/main/java/org/apache/sysds/hops/BinaryOp.java index 2b803a053c1..dc7edf76e50 100644 --- a/src/main/java/org/apache/sysds/hops/BinaryOp.java +++ b/src/main/java/org/apache/sysds/hops/BinaryOp.java @@ -853,7 +853,10 @@ else if( (op == OpOp2.CBIND && getDataType().isList()) || (op == OpOp2.RBIND && getDataType().isList())) { _etype = ExecType.CP; } - + + if( op == OpOp2.GET_CATEGORICAL_MASK) + _etype = ExecType.CP; + //mark for recompile (forever) setRequiresRecompileIfNecessary(); diff --git a/src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java b/src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java index 28f6949f722..ab0c7993b4e 100644 --- a/src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java +++ b/src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java @@ -2018,6 +2018,15 @@ else if(this.getOpCode() == Builtins.MAX_POOL || this.getOpCode() == Builtins.AV else raiseValidateError("The compress or decompress instruction is not allowed in dml scripts"); break; + case GET_CATEGORICAL_MASK: + checkNumParameters(2); + checkFrameParam(getFirstExpr()); + checkScalarParam(getSecondExpr()); + output.setDataType(DataType.MATRIX); + output.setDimensions(1, -1); + output.setBlocksize( id.getBlocksize()); + output.setValueType(ValueType.FP64); + break; case QUANTIZE_COMPRESS: if(OptimizerUtils.ALLOW_SCRIPT_LEVEL_QUANTIZE_COMPRESS_COMMAND) { checkNumParameters(2); @@ -2383,6 +2392,13 @@ protected void checkMatrixFrameParam(Expression e) { //always unconditional raiseValidateError("Expecting matrix or frame parameter for function "+ getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS); } } + + protected void checkFrameParam(Expression e) { + if(e.getOutput().getDataType() != DataType.FRAME) { + raiseValidateError("Expecting frame parameter for function " + getOpCode(), false, + LanguageErrorCodes.UNSUPPORTED_PARAMETERS); + } + } protected void checkMatrixScalarParam(Expression e) { //always unconditional if (e.getOutput().getDataType() != DataType.MATRIX && e.getOutput().getDataType() != DataType.SCALAR) { diff --git a/src/main/java/org/apache/sysds/parser/DMLTranslator.java b/src/main/java/org/apache/sysds/parser/DMLTranslator.java index c6e7188d7bc..e14cfd31388 100644 --- a/src/main/java/org/apache/sysds/parser/DMLTranslator.java +++ b/src/main/java/org/apache/sysds/parser/DMLTranslator.java @@ -2821,6 +2821,9 @@ else if ( in.length == 2 ) DataType.MATRIX, target.getValueType(), AggOp.COUNT_DISTINCT, Direction.Col, expr); break; + case GET_CATEGORICAL_MASK: + currBuiltinOp = new BinaryOp(target.getName(), DataType.MATRIX, ValueType.FP64, OpOp2.GET_CATEGORICAL_MASK, expr, expr2); + break; default: throw new ParseException("Unsupported builtin function type: "+source.getOpCode()); } diff --git a/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java b/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java index 39735be62e0..eed2c58f78c 100644 --- a/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java +++ b/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java @@ -54,7 +54,7 @@ public enum BuiltinCode { AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, MAX, ABS, SIGN, SQRT, EXP, PLOGP, PRINT, PRINTF, NROW, NCOL, LENGTH, LINEAGE, ROUND, MAXINDEX, MININDEX, STOP, CEIL, FLOOR, CUMSUM, ROWCUMSUM, CUMPROD, CUMMIN, CUMMAX, CUMSUMPROD, INVERSE, SPROP, SIGMOID, EVAL, LIST, TYPEOF, APPLY_SCHEMA, DETECTSCHEMA, ISNA, ISNAN, ISINF, DROP_INVALID_TYPE, - DROP_INVALID_LENGTH, VALUE_SWAP, FRAME_ROW_REPLICATE, + DROP_INVALID_LENGTH, VALUE_SWAP, FRAME_ROW_REPLICATE, GET_CATEGORICAL_MASK, MAP, COUNT_DISTINCT, COUNT_DISTINCT_APPROX, UNIQUE} private static final VectorSpecies SPECIES = DoubleVector.SPECIES_PREFERRED; @@ -120,6 +120,7 @@ public enum BuiltinCode { AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, String2BuiltinCode.put( "_map", BuiltinCode.MAP); String2BuiltinCode.put( "valueSwap", BuiltinCode.VALUE_SWAP); String2BuiltinCode.put( "applySchema", BuiltinCode.APPLY_SCHEMA); + String2BuiltinCode.put( "get_categorical_mask", BuiltinCode.GET_CATEGORICAL_MASK); } protected Builtin(BuiltinCode bf) { diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java index 28b8775ebd5..86184f47be6 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java @@ -59,6 +59,8 @@ else if (in1.getDataType() == DataType.TENSOR && in2.getDataType() == DataType.T return new BinaryTensorTensorCPInstruction(operator, in1, in2, out, opcode, str); else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.FRAME) return new BinaryFrameFrameCPInstruction(operator, in1, in2, out, opcode, str); + else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.SCALAR) + return new BinaryFrameScalarCPInstruction(operator, in1, in2, out, opcode, str); else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.MATRIX) return new BinaryFrameMatrixCPInstruction(operator, in1, in2, out, opcode, str); else diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java new file mode 100644 index 00000000000..193894fd9bc --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.instructions.cp; + +import java.util.Arrays; + +import org.apache.sysds.common.Builtins; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.controlprogram.context.ExecutionContext; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.frame.data.columns.ColumnMetadata; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.operators.MultiThreadedOperator; +import org.apache.sysds.runtime.transform.TfUtils.TfMethod; +import org.apache.sysds.runtime.util.UtilFunctions; +import org.apache.wink.json4j.JSONException; +import org.apache.wink.json4j.JSONObject; + +public class BinaryFrameScalarCPInstruction extends BinaryCPInstruction { + // private static final Log LOG = LogFactory.getLog(BinaryFrameFrameCPInstruction.class.getName()); + + private static final TfMethod[] UNSUPPORTED_MASK_METHODS = new TfMethod[] {TfMethod.BIN, + TfMethod.WORD_EMBEDDING, TfMethod.BAG_OF_WORDS, TfMethod.UDF}; + + protected BinaryFrameScalarCPInstruction(MultiThreadedOperator op, CPOperand in1, CPOperand in2, CPOperand out, + String opcode, String istr) { + super(CPType.Binary, op, in1, in2, out, opcode, istr); + } + + @Override + public void processInstruction(ExecutionContext ec) { + // get input frames + FrameBlock inBlock1 = ec.getFrameInput(input1.getName()); + ScalarObject spec = ec.getScalarInput(input2.getName(), ValueType.STRING, true); + if(getOpcode().equals(Builtins.GET_CATEGORICAL_MASK.toString().toLowerCase())) { + processGetCategorical(ec, inBlock1, spec); + } + else { + throw new DMLRuntimeException("Unsupported operation"); + } + + // Release the memory occupied by input frames + ec.releaseFrameInput(input1.getName()); + } + + private static void validate(JSONObject jSpec) { + try { + if(!jSpec.containsKey("ids") || !jSpec.getBoolean("ids")) + throw new DMLRuntimeException("not supported non ID based spec for get_categorical_mask"); + + for(TfMethod m : UNSUPPORTED_MASK_METHODS) + if(jSpec.containsKey(m.toString())) + throw new DMLRuntimeException("unsupported transform method '" + m + "' for get_categorical_mask"); + } + catch(JSONException e) { + throw new DMLRuntimeException(e); + } + } + + public void processGetCategorical(ExecutionContext ec, FrameBlock f, ScalarObject spec) { + try { + // 1. extract the spec, 2. validate it + JSONObject jSpec = new JSONObject(spec.getStringValue()); + validate(jSpec); + + // 3.-5. fold each supported transform method into the per-column mask state + CategoricalMask mask = new CategoricalMask(f, jSpec); + mask.hash(); + mask.recode(); + mask.dummycode(); + + // 6.-7. size and materialize the output mask + ec.setMatrixOutput(output.getName(), mask.toMatrixBlock()); + } + catch(Exception e) { + throw new DMLRuntimeException(e); + } + } + + /** + * Accumulates, per input column, how many output columns it expands to (lengths) and whether those + * output columns are categorical (categorical). The arrays are allocated lazily: a column that no + * method touches keeps the implicit default of a single, non-categorical output column. + */ + private static final class CategoricalMask { + private final FrameBlock f; + private final JSONObject jSpec; + private final int nCol; + + private int[] lengths = null; + private boolean[] categorical = null; + + // feature-hashed columns map to K buckets; a plain hashed column produces a single + // (categorical) bucket-id column, while a hashed column that is additionally dummycoded + // expands to K columns. + private boolean[] hashed = null; + private int K = 0; + + private CategoricalMask(FrameBlock f, JSONObject jSpec) { + this.f = f; + this.jSpec = jSpec; + this.nCol = f.getNumColumns(); + } + + private void hash() throws JSONException { + String hash = TfMethod.HASH.toString(); + if(!jSpec.containsKey(hash)) + return; + K = jSpec.getInt("K"); + hashed = new boolean[nCol]; + ensureCategorical(); + for(Object aa : jSpec.getJSONArray(hash)) { + int av = (Integer) aa - 1; + hashed[av] = true; + categorical[av] = true; + } + } + + private void recode() throws JSONException { + String recode = TfMethod.RECODE.toString(); + if(!jSpec.containsKey(recode)) + return; + ensureCategorical(); + for(Object aa : jSpec.getJSONArray(recode)) { + int av = (Integer) aa - 1; + categorical[av] = true; + } + } + + private void dummycode() throws JSONException { + String dummycode = TfMethod.DUMMYCODE.toString(); + if(!jSpec.containsKey(dummycode)) + return; + ensureCategorical(); + ensureLengths(); + for(Object aa : jSpec.getJSONArray(dummycode)) { + int av = (Integer) aa - 1; + lengths[av] = distinctCount(av); + categorical[av] = true; + } + } + + private int distinctCount(int av) { + if(hashed != null && hashed[av]) + // feature hashing followed by dummycoding yields K columns + return K; + ColumnMetadata d = f.getColumnMetadata()[av]; + String v = f.getString(0, av); + if(v.length() > 1 && v.charAt(0) == '¿') + return UtilFunctions.parseToInt(v.substring(1)); + return d.isDefault() ? 0 : (int) d.getNumDistinct(); + } + + private int sumLengths() { + if(lengths == null) + return nCol; + int sum = 0; + for(int i = 0; i < nCol; i++) + sum += lengths[i]; + return sum; + } + + private MatrixBlock toMatrixBlock() { + MatrixBlock ret = new MatrixBlock(1, sumLengths(), false); + ret.allocateDenseBlock(); + int off = 0; + for(int i = 0; i < nCol; i++) { + int len = (lengths == null) ? 1 : lengths[i]; + double val = (categorical != null && categorical[i]) ? 1 : 0; + for(int j = 0; j < len; j++) + ret.set(0, off++, val); + } + return ret; + } + + private void ensureCategorical() { + if(categorical == null) + categorical = new boolean[nCol]; + } + + private void ensureLengths() { + if(lengths == null) { + lengths = new int[nCol]; + Arrays.fill(lengths, 1); + } + } + } +} diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java index 683d355e05c..f14a614b583 100644 --- a/src/test/java/org/apache/sysds/test/TestUtils.java +++ b/src/test/java/org/apache/sysds/test/TestUtils.java @@ -2941,6 +2941,25 @@ public static void writeTestScalar(String file, double value) { } } + + /** + * Write scalar to file + * + * @param file File to write to + * @param value Value to write + */ + public static void writeTestScalar(String file, String value) { + try { + DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); + try(PrintWriter pw = new PrintWriter(out)) { + pw.println(value); + } + } + catch(IOException e) { + fail("unable to write test scalar (" + file + "): " + e.getMessage()); + } + } + /** * Write scalar to file * diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java new file mode 100644 index 00000000000..d9c540f54c5 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.frame.transform; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.sysds.common.Types.DataType; +import org.apache.sysds.common.Types.FileFormat; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.controlprogram.caching.CacheableData; +import org.apache.sysds.runtime.controlprogram.caching.FrameObject; +import org.apache.sysds.runtime.controlprogram.context.ExecutionContext; +import org.apache.sysds.runtime.controlprogram.context.ExecutionContextFactory; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.frame.data.columns.ColumnMetadata; +import org.apache.sysds.runtime.instructions.InstructionUtils; +import org.apache.sysds.runtime.instructions.cp.BinaryCPInstruction; +import org.apache.sysds.runtime.instructions.cp.BinaryFrameScalarCPInstruction; +import org.apache.sysds.runtime.instructions.cp.StringObject; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.meta.MatrixCharacteristics; +import org.apache.sysds.runtime.meta.MetaDataFormat; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Unit tests that drive the get_categorical_mask instruction directly to exercise the defensive code + * paths (distinct-count prefix in the metadata frame, default column metadata, non id-based specs and + * the unsupported opcode guard) that the script-level transform tests cannot reach. + */ +public class GetCategoricalMaskInstructionTest { + protected static final Log LOG = LogFactory.getLog(GetCategoricalMaskInstructionTest.class.getName()); + + private static final String MASK_OPCODE = "get_categorical_mask"; + + @BeforeClass + public static void init() throws java.io.IOException { + CacheableData.initCaching("get_categorical_mask_instruction_test"); + } + + @Test + public void dummycodeReadsDistinctCountFromMetadataPrefix() { + // a metadata cell prefixed with '¿' encodes the number of distinct values inline + FrameBlock meta = new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"¿3"}}); + MatrixBlock res = run(meta, "{\"ids\": true, \"dummycode\": [1]}"); + + assertEquals(1, res.getNumRows()); + assertEquals(3, res.getNumColumns()); + assertArrayEquals(new double[] {1, 1, 1}, res.getDenseBlockValues(), 0.0); + } + + @Test + public void dummycodeDefaultMetadataContributesNoColumns() { + // first column is dummycoded but carries default metadata (no distinct count) -> 0 columns, + // the trailing pass-through column keeps the output non-empty + FrameBlock meta = new FrameBlock(new ValueType[] {ValueType.STRING, ValueType.STRING}, + new String[][] {{"x", "y"}}); + MatrixBlock res = run(meta, "{\"ids\": true, \"dummycode\": [1]}"); + + assertEquals(1, res.getNumRows()); + assertEquals(1, res.getNumColumns()); + assertEquals(0.0, res.get(0, 0), 0.0); + } + + @Test + public void noMethodAllColumnsPassThrough() { + // a spec with only "ids" touches no column: every column is a single, non-categorical output + FrameBlock meta = metaWithDistinct(3, new int[] {0, 0, 0}); + MatrixBlock res = run(meta, "{\"ids\": true}"); + + assertMask(res, new double[] {0, 0, 0}); + } + + @Test + public void recodeInterleavedWithPassThrough() { + // categorical (recode, 1 col each) interleaved with continuous pass-through columns + FrameBlock meta = metaWithDistinct(5, new int[] {0, 0, 0, 0, 0}); + MatrixBlock res = run(meta, "{\"ids\": true, \"recode\": [1, 4]}"); + + assertMask(res, new double[] {1, 0, 0, 1, 0}); + } + + @Test + public void leadingPassThroughThenDummycodeOffsets() { + // the dummycode expansion must start at the correct offset after three continuous columns + FrameBlock meta = metaWithDistinct(4, new int[] {0, 0, 0, 3}); + MatrixBlock res = run(meta, "{\"ids\": true, \"dummycode\": [4]}"); + + assertMask(res, new double[] {0, 0, 0, 1, 1, 1}); + } + + @Test + public void multipleDummycodeVaryingDistinctCounts() { + // several dummycoded columns of different widths, all categorical, no pass-through + FrameBlock meta = metaWithDistinct(3, new int[] {2, 4, 1}); + MatrixBlock res = run(meta, "{\"ids\": true, \"dummycode\": [1, 2, 3]}"); + + assertMask(res, new double[] {1, 1, 1, 1, 1, 1, 1}); + } + + @Test + public void dummycodeAndPassThroughAndRecodeInterleaved() { + // dummycode(3) | pass-through | recode | dummycode(2): exercises every offset transition + FrameBlock meta = metaWithDistinct(4, new int[] {3, 0, 0, 2}); + MatrixBlock res = run(meta, "{\"ids\": true, \"recode\": [3], \"dummycode\": [1, 4]}"); + + assertMask(res, new double[] {1, 1, 1, 0, 1, 1, 1}); + } + + @Test + public void recodeAndDummycodeOnSameColumnExpands() { + // a column listed in both recode and dummycode must expand to its dummycode width, not collapse + FrameBlock meta = metaWithDistinct(2, new int[] {4, 0}); + MatrixBlock res = run(meta, "{\"ids\": true, \"recode\": [1], \"dummycode\": [1]}"); + + assertMask(res, new double[] {1, 1, 1, 1, 0}); + } + + @Test + public void hashOnlyColumnStaysSingleCategorical() { + // a hashed-but-not-dummycoded column is a single categorical column; K must not widen it + FrameBlock meta = metaWithDistinct(3, new int[] {0, 0, 0}); + MatrixBlock res = run(meta, "{\"ids\": true, \"hash\": [2], \"K\": 5}"); + + assertMask(res, new double[] {0, 1, 0}); + } + + @Test + public void hashDummycodeRecodePassThroughMixed() { + // col1: hash+dummycode -> K=3 (metadata ignored); col2: pass-through; col3: dummycode(9); + // col4: pass-through; col5: recode. Verifies hashed columns use K while plain dummycode uses + // the metadata distinct count, with correct offsets across the whole row. + FrameBlock meta = metaWithDistinct(5, new int[] {0, 0, 9, 0, 0}); + MatrixBlock res = run(meta, "{\"ids\": true, \"recode\": [5], \"dummycode\": [1, 3], \"hash\": [1], \"K\": 3}"); + + assertMask(res, new double[] {1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1}); + } + + @Test + public void nonIdSpecMissingIdsKeyThrows() { + // a spec without the "ids" key must be rejected, not silently mis-interpreted + FrameBlock meta = new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}); + assertThrowsMessage("non ID based spec", () -> run(meta, "{\"recode\": [1]}")); + } + + @Test + public void nonIdSpecIdsFalseThrows() { + // "ids": false is equally unsupported + FrameBlock meta = new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}); + assertThrowsMessage("non ID based spec", () -> run(meta, "{\"ids\": false, \"recode\": [1]}")); + } + + @Test + public void unsupportedBinMethodThrows() { + // bin expands to bin-count columns under dummycode, which the mask does not model + FrameBlock meta = new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}); + assertThrowsMessage("unsupported transform method 'bin'", + () -> run(meta, "{\"ids\": true, \"bin\": [{\"id\": 1, \"method\": \"equi-width\", \"numbins\": 3}]}")); + } + + @Test + public void unsupportedWordEmbeddingMethodThrows() { + // word_embedding maps a column to an embedding vector (many columns), not a single mask entry + FrameBlock meta = new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}); + assertThrowsMessage("unsupported transform method 'word_embedding'", + () -> run(meta, "{\"ids\": true, \"word_embedding\": [1]}")); + } + + @Test + public void unsupportedBagOfWordsMethodThrows() { + // bag_of_words expands to one column per dictionary token + FrameBlock meta = new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}); + assertThrowsMessage("unsupported transform method 'bag_of_words'", + () -> run(meta, "{\"ids\": true, \"bag_of_words\": [1]}")); + } + + @Test + public void unsupportedUdfMethodThrows() { + // udf output arity is user-defined and cannot be inferred from the spec + FrameBlock meta = new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}); + assertThrowsMessage("unsupported transform method 'udf'", + () -> run(meta, "{\"ids\": true, \"udf\": {\"name\": \"f\", \"ids\": [1]}}")); + } + + @Test + public void imputeAndOmitAreAccepted() { + // impute and omit do not change the output column count or categorical flag, so a spec that + // only adds them on top of a recoded column must still succeed and mark that column categorical + FrameBlock meta = new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}); + MatrixBlock res = run(meta, "{\"ids\": true, \"recode\": [1], \"impute\": [{\"id\": 1, \"method\": \"global_mode\"}], \"omit\": [1]}"); + + assertEquals(1, res.getNumRows()); + assertEquals(1, res.getNumColumns()); + assertEquals(1.0, res.get(0, 0), 0.0); + } + + @Test + public void malformedSpecWrapsJsonException() { + // "ids" present but not a boolean makes spec parsing throw a JSONException, which must be + // wrapped as a DMLRuntimeException rather than propagating raw + FrameBlock meta = new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}); + assertThrowsMessage("was not a boolean", () -> run(meta, "{\"ids\": 5, \"recode\": [1]}")); + } + + @Test + public void unsupportedOpcodeThrows() { + // any frame-scalar binary opcode other than get_categorical_mask must be rejected + ExecutionContext ec = ExecutionContextFactory.createContext(); + ec.setAutoCreateVars(true); + ec.setVariable("F", frameObject(new FrameBlock(new ValueType[] {ValueType.STRING}, + new String[][] {{"a"}}))); + assertThrowsMessage("Unsupported operation", () -> maskInstruction("+").processInstruction(ec)); + } + + /** Assert the action throws a DMLRuntimeException whose message chain contains the expected text. */ + private static void assertThrowsMessage(String expected, Runnable action) { + try { + action.run(); + fail("Expected DMLRuntimeException containing \"" + expected + "\" but nothing was thrown"); + } + catch(DMLRuntimeException e) { + StringBuilder chain = new StringBuilder(); + for(Throwable t = e; t != null; t = t.getCause()) + chain.append(t.getMessage()).append(" | "); + assertTrue("Exception chain [" + chain + "] should contain \"" + expected + "\"", + chain.toString().contains(expected)); + } + } + + /** Assert the mask is a single row equal to the expected values (which also fixes its width). */ + private static void assertMask(MatrixBlock res, double[] expected) { + assertEquals(1, res.getNumRows()); + assertEquals(expected.length, res.getNumColumns()); + // compare per cell rather than via getDenseBlockValues(): an all-zero mask has nnz == 0 and + // therefore no materialized dense block + double[] actual = new double[expected.length]; + for(int i = 0; i < expected.length; i++) + actual[i] = res.get(0, i); + assertArrayEquals(expected, actual, 0.0); + } + + /** + * Build a single-row metadata frame of nCol string columns. A positive distinct[i] is written to + * that column's metadata as the recode distinct count (the path real transformencode uses), while + * a zero leaves the column with default metadata (a continuous / non-dummycoded column). + */ + private static FrameBlock metaWithDistinct(int nCol, int[] distinct) { + ValueType[] schema = new ValueType[nCol]; + String[][] data = new String[1][nCol]; + for(int i = 0; i < nCol; i++) { + schema[i] = ValueType.STRING; + data[0][i] = "v"; + } + FrameBlock fb = new FrameBlock(schema, data); + for(int i = 0; i < nCol; i++) + if(distinct[i] > 0) + fb.setColumnMetadata(i, new ColumnMetadata(distinct[i])); + return fb; + } + + private static MatrixBlock run(FrameBlock meta, String spec) { + ExecutionContext ec = ExecutionContextFactory.createContext(); + ec.setAutoCreateVars(true); + maskInstruction(MASK_OPCODE).processGetCategorical(ec, meta, new StringObject(spec)); + return ec.getMatrixObject("out").acquireReadAndRelease(); + } + + private static BinaryFrameScalarCPInstruction maskInstruction(String opcode) { + String in1 = InstructionUtils.concatOperandParts("F", DataType.FRAME.name(), ValueType.STRING.name(), "false"); + String in2 = InstructionUtils.concatOperandParts("spec", DataType.SCALAR.name(), ValueType.STRING.name(), "true"); + String out = InstructionUtils.concatOperandParts("out", DataType.MATRIX.name(), ValueType.FP64.name(), "false"); + String str = InstructionUtils.concatOperands("CP", opcode, in1, in2, out); + return (BinaryFrameScalarCPInstruction) BinaryCPInstruction.parseInstruction(str); + } + + private static FrameObject frameObject(FrameBlock fb) { + MatrixCharacteristics mc = new MatrixCharacteristics(fb.getNumRows(), fb.getNumColumns(), -1, -1); + FrameObject fo = new FrameObject("F", new MetaDataFormat(mc, FileFormat.BINARY), fb.getSchema()); + fo.acquireModify(fb); + fo.release(); + return fo; + } +} diff --git a/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java b/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java new file mode 100644 index 00000000000..30681f373e4 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.transform; + +import static org.junit.Assert.fail; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.sysds.common.Types.FileFormat; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; + +public class GetCategoricalMaskTest extends AutomatedTestBase { + protected static final Log LOG = LogFactory.getLog(GetCategoricalMaskTest.class.getName()); + + private final static String TEST_NAME1 = "GetCategoricalMaskTest"; + private final static String TEST_DIR = "functions/transform/"; + private final static String TEST_CLASS_DIR = TEST_DIR + TransformFrameEncodeApplyTest.class.getSimpleName() + "/"; + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"y"})); + } + + @Test + public void testRecode() throws Exception { + FrameBlock fb = TestUtils.generateRandomFrameBlock(10, new ValueType[] {ValueType.UINT8}, 32); + MatrixBlock expected = new MatrixBlock(1, 1, 1.0); + String spec = "{\"ids\": true, \"recode\": [1]}"; + runTransformTest(fb, spec, expected); + + } + + @Test + public void testRecode2() throws Exception { + FrameBlock fb = TestUtils.generateRandomFrameBlock(10, new ValueType[] {ValueType.UINT8, ValueType.UINT8}, 32); + MatrixBlock expected = new MatrixBlock(1, 2, new double[] {0, 1}); + + String spec = "{\"ids\": true, \"recode\": [2]}"; + runTransformTest(fb, spec, expected); + + } + + @Test + public void testDummy1() throws Exception { + FrameBlock fb = TestUtils.generateRandomFrameBlock(5, new ValueType[] {ValueType.UINT8, ValueType.INT64}, 32); + MatrixBlock expected = new MatrixBlock(1, 6, new double[] {0, 1, 1, 1, 1, 1}); + + String spec = "{\"ids\": true, \"dummycode\": [2]}"; + runTransformTest(fb, spec, expected); + + } + + @Test + public void testDummy2() throws Exception { + FrameBlock fb = TestUtils.generateRandomFrameBlock(5, new ValueType[] {ValueType.UINT8, ValueType.INT64}, 32); + MatrixBlock expected = new MatrixBlock(1, 6, new double[] {1, 1, 1, 1, 1, 0}); + + String spec = "{\"ids\": true, \"dummycode\": [1]}"; + runTransformTest(fb, spec, expected); + + } + + @Test + public void testHash1() throws Exception { + FrameBlock fb = TestUtils.generateRandomFrameBlock(5, new ValueType[] {ValueType.UINT8, ValueType.INT64}, 32); + MatrixBlock expected = new MatrixBlock(1, 4, new double[] {1, 1, 1, 0}); + + String spec = "{\"ids\": true, \"dummycode\": [1], \"hash\": [1], \"K\": 3}"; + runTransformTest(fb, spec, expected); + + } + + @Test + public void testHash2() throws Exception { + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.INT64}, 32); + MatrixBlock expected = new MatrixBlock(1, 4, new double[] {1, 1, 1, 0}); + + String spec = "{\"ids\": true, \"dummycode\": [1], \"hash\": [1], \"K\": 3}"; + runTransformTest(fb, spec, expected); + + } + + @Test + public void testHash3() throws Exception { + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.INT64,ValueType.UINT8}, 32); + MatrixBlock expected = new MatrixBlock(1, 7, new double[] {1, 1, 1, 0, 1, 1, 1}); + + String spec = "{\"ids\": true, \"dummycode\": [1,3], \"hash\": [1,3], \"K\": 3}"; + runTransformTest(fb, spec, expected); + + } + + + @Test + public void testHybrid1() throws Exception { + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.INT64,ValueType.UINT8, ValueType.BOOLEAN}, 32); + MatrixBlock expected = new MatrixBlock(1, 9, new double[] {1, 1, 1, 0, 1, 1, 1,1,1}); + + String spec = "{\"ids\": true, \"dummycode\": [1,3,4], \"hash\": [1,3], \"K\": 3}"; + runTransformTest(fb, spec, expected); + + } + + @Test + public void testHybrid2() throws Exception { + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.BOOLEAN,ValueType.UINT8, ValueType.BOOLEAN}, 32); + MatrixBlock expected = new MatrixBlock(1, 10, new double[] {1, 1, 1, 1,1, 1, 1, 1,1,1}); + + String spec = "{\"ids\": true, \"dummycode\": [1,2,3,4], \"hash\": [1,3], \"K\": 3}"; + runTransformTest(fb, spec, expected); + + } + + private void runTransformTest(FrameBlock fb, String spec, MatrixBlock expected) throws Exception { + try { + + getAndLoadTestConfiguration(TEST_NAME1); + + String inF = input("F-In"); + String inS = input("spec"); + + TestUtils.writeTestFrame(inF, fb, fb.getSchema(), FileFormat.CSV); + TestUtils.writeTestScalar(input("spec"), spec); + + String out = output("ret"); + + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + TEST_NAME1 + ".dml"; + programArgs = new String[] {"-args", inF, inS, out, expected.getNumColumns() + ""}; + + runTest(true, false, null, -1); + + MatrixBlock result = TestUtils.readBinary(out); + + TestUtils.compareMatrices(expected, result, 0.0); + } + catch(Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + +} diff --git a/src/test/scripts/functions/transform/GetCategoricalMaskTest.dml b/src/test/scripts/functions/transform/GetCategoricalMaskTest.dml new file mode 100644 index 00000000000..5d7bb35a250 --- /dev/null +++ b/src/test/scripts/functions/transform/GetCategoricalMaskTest.dml @@ -0,0 +1,37 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +F1 = read($1, data_type="frame", format="csv"); + +jspec = read($2, data_type="scalar", value_type="string"); + +[X, M] = transformencode(target=F1, spec=jspec); + +Cm = getCategoricalMask(M, jspec) +expectedColumns = $4 +if(ncol(Cm) != expectedColumns){ + stop("Wrong number of metadata columns in categorical mask") +} +# print mean to verify that Cm is a matrix, not a Frame according to compiler +print(mean(Cm)) + +write(Cm, $3, format="csv"); + From 0680df85334ada3e2fa959d0a7387199902af222 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Wed, 17 Jun 2026 16:57:26 +0200 Subject: [PATCH 038/132] [MINOR] Deduplicate not-implemented logs in ColGroupTest (#2491) The parameterized leftMultNoPreAgg test logged the same "not implemented: X or: Y" error for every base/other column group combination, producing hundreds of duplicate lines for shared type pairs. Track already-logged type pairs in a thread-safe set and log each unique permutation at most once per run. --- .../component/compress/colgroup/ColGroupTest.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupTest.java b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupTest.java index 4c5fc3bd8a7..71c9c0ce549 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupTest.java @@ -29,6 +29,8 @@ import java.util.Collections; import java.util.List; import java.util.Random; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.logging.Log; @@ -92,6 +94,9 @@ public class ColGroupTest extends ColGroupBase { protected static final Log LOG = LogFactory.getLog(ColGroupTest.class.getName()); + /** Tracks already-logged "not implemented" column group type pairs to avoid duplicate log spam. */ + private static final Set loggedNotImplemented = ConcurrentHashMap.newKeySet(); + public ColGroupTest(AColGroup base, AColGroup other, int nRow) { super(base, other, nRow); } @@ -1282,7 +1287,7 @@ public void leftMultNoPreAgg(int nRowLeft, int rl, int ru, int cl, int cu, Matri compare(bt, ot); } catch(NotImplementedException e) { - LOG.error("not implemented: " + base.getClass().getSimpleName() + " or: " + other.getClass().getSimpleName()); + logNotImplementedOnce(); } catch(Exception e) { e.printStackTrace(); @@ -1290,6 +1295,12 @@ public void leftMultNoPreAgg(int nRowLeft, int rl, int ru, int cl, int cu, Matri } } + private void logNotImplementedOnce() { + final String pair = base.getClass().getSimpleName() + " or: " + other.getClass().getSimpleName(); + if(loggedNotImplemented.add(pair)) + LOG.error("not implemented: " + pair); + } + @Test public void sparseSelection() { MatrixBlock mb = CLALibSelectionMultTest.createSelectionMatrix(nRow, 5, false); From e71501b45b1570fee9054ec5a6ed88b55ff93231 Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Tue, 23 Jun 2026 14:37:11 +0200 Subject: [PATCH 039/132] [MINOR] Improve efficiency of Scuro tests --- src/main/python/tests/scuro/test_hp_tuner.py | 9 ++- .../tests/scuro/test_multimodal_join.py | 8 +-- .../tests/scuro/test_unimodal_optimizer.py | 36 ++++------ .../scuro/test_unimodal_representations.py | 70 +++++-------------- .../tests/scuro/test_window_operations.py | 22 +++--- 5 files changed, 53 insertions(+), 92 deletions(-) diff --git a/src/main/python/tests/scuro/test_hp_tuner.py b/src/main/python/tests/scuro/test_hp_tuner.py index c418cefcae8..03ffc1c2dad 100644 --- a/src/main/python/tests/scuro/test_hp_tuner.py +++ b/src/main/python/tests/scuro/test_hp_tuner.py @@ -24,6 +24,7 @@ import numpy as np +from systemds.scuro import Mean from systemds.scuro.drsearch.multimodal_optimizer import MultimodalOptimizer from systemds.scuro.representations.average import Average from systemds.scuro.representations.color_histogram import ColorHistogram @@ -128,7 +129,7 @@ def run_hp_for_modality( { ModalityType.TEXT: [BoW, W2V], ModalityType.AUDIO: [Spectrogram, ZeroCrossing, Spectral, Pitch], - ModalityType.TIMESERIES: [ResNet], + ModalityType.TIMESERIES: [Mean], ModalityType.VIDEO: [ResNet], ModalityType.IMAGE: [ResNet, ColorHistogram], ModalityType.EMBEDDING: [], @@ -136,7 +137,9 @@ def run_hp_for_modality( ): registry = Registry() registry._fusion_operators = [LSTM] - unimodal_optimizer = UnimodalOptimizer(modalities, self.tasks, False) + unimodal_optimizer = UnimodalOptimizer( + modalities, self.tasks, False, k=2, max_num_workers=1 + ) unimodal_optimizer.optimize() hp = HyperparameterTuner( @@ -165,7 +168,7 @@ def run_hp_for_modality( ) else: - hp.tune_unimodal_representations(max_eval_per_rep=10) + hp.tune_unimodal_representations(max_eval_per_rep=2) assert len(hp.optimization_results.results) == len(self.tasks) if multimodal: diff --git a/src/main/python/tests/scuro/test_multimodal_join.py b/src/main/python/tests/scuro/test_multimodal_join.py index 14ce9376be1..4a53129db33 100644 --- a/src/main/python/tests/scuro/test_multimodal_join.py +++ b/src/main/python/tests/scuro/test_multimodal_join.py @@ -47,7 +47,7 @@ def setUpClass(cls): cls.num_instances = 4 cls.indices = np.array(range(cls.num_instances)) cls.audio_data, cls.audio_md = ModalityRandomDataGenerator().create_audio_data( - cls.num_instances, 32000 + cls.num_instances, 500 ) cls.video_data, cls.video_md = ( @@ -104,7 +104,7 @@ def _prepare_data(self, l_chunk_size=None, r_chunk_size=None): l_chunk_size, ModalityType.VIDEO, copy.deepcopy(self.video_data), - np.float32, + np.uint8, copy.deepcopy(self.video_md), ) ) @@ -118,9 +118,7 @@ def _join(self, left_modality, right_modality, window_size): left_modality.join( right_modality, JoinCondition("timestamp", "timestamp", "<") ) - .apply_representation( - ResNet(layer_name="layer1.0.conv2", model_name="ResNet18") - ) + .apply_representation(ResNet()) .window_aggregation(window_size, "mean") .combine("concat") ) diff --git a/src/main/python/tests/scuro/test_unimodal_optimizer.py b/src/main/python/tests/scuro/test_unimodal_optimizer.py index ad824b0335f..11c3aa29ea6 100644 --- a/src/main/python/tests/scuro/test_unimodal_optimizer.py +++ b/src/main/python/tests/scuro/test_unimodal_optimizer.py @@ -23,17 +23,17 @@ import unittest import numpy as np -from systemds.scuro.representations.clip import CLIPText, CLIPVisual from systemds.scuro.representations.color_histogram import ColorHistogram from systemds.scuro.drsearch.operator_registry import Registry from systemds.scuro.drsearch.unimodal_optimizer import UnimodalOptimizer -from systemds.scuro.representations.mfcc import MFCC +from systemds.scuro.representations.covarep_audio_features import ZeroCrossing + +from systemds.scuro.representations.resnet import ResNet from systemds.scuro.representations.mel_spectrogram import MelSpectrogram -from systemds.scuro.representations.word2vec import W2V +from systemds.scuro.representations.tfidf import TfIdf from systemds.scuro.representations.bow import BoW from systemds.scuro.representations.bert import Bert from systemds.scuro.modality.unimodal_modality import UnimodalModality -from systemds.scuro.representations.resnet import ResNet from tests.scuro.data_generator import ( ModalityRandomDataGenerator, TestDataLoader, @@ -53,6 +53,15 @@ from unittest.mock import patch +LIGHTWEIGHT_REGISTRY = { + ModalityType.TEXT: [BoW, TfIdf], + ModalityType.AUDIO: [MelSpectrogram, ZeroCrossing], + ModalityType.VIDEO: [ResNet], + ModalityType.IMAGE: [ColorHistogram], + ModalityType.TIMESERIES: [], + ModalityType.EMBEDDING: [], +} + class TestUnimodalRepresentationOptimizer(unittest.TestCase): data_generator = None @@ -198,24 +207,7 @@ def optimize_unimodal_representation_for_modality(self, modalities): with patch.object( Registry, "_representations", - { - ModalityType.TEXT: [ - W2V, - BoW, - Bert, - CLIPText, - ], - ModalityType.AUDIO: [ - MFCC, - MelSpectrogram, - ], - ModalityType.VIDEO: [ - ResNet, - CLIPVisual, - ], - ModalityType.IMAGE: [ColorHistogram, CLIPVisual], - ModalityType.EMBEDDING: [], - }, + LIGHTWEIGHT_REGISTRY, ): registry = Registry() diff --git a/src/main/python/tests/scuro/test_unimodal_representations.py b/src/main/python/tests/scuro/test_unimodal_representations.py index 2f474be7fd9..59bef40ef64 100644 --- a/src/main/python/tests/scuro/test_unimodal_representations.py +++ b/src/main/python/tests/scuro/test_unimodal_representations.py @@ -19,18 +19,10 @@ # # ------------------------------------------------------------- -import time import unittest import copy import numpy as np -from systemds.scuro.representations.bert import ( - Bert, - ALBERT, - ELECTRA, - RoBERTa, - DistillBERT, -) -from systemds.scuro.representations.clip import CLIPVisual, CLIPText + from systemds.scuro.representations.bow import BoW from systemds.scuro.representations.covarep_audio_features import ( Spectral, @@ -38,20 +30,13 @@ Pitch, ZeroCrossing, ) -from systemds.scuro.representations.glove import GloVe -from systemds.scuro.representations.wav2vec import Wav2Vec +from systemds.scuro.representations.color_histogram import ColorHistogram from systemds.scuro.representations.spectrogram import Spectrogram -from systemds.scuro.representations.window_aggregation import WindowAggregation -from systemds.scuro.representations.word2vec import W2V from systemds.scuro.representations.tfidf import TfIdf -from systemds.scuro.representations.x3d import X3D -from systemds.scuro.representations.x3d import I3D -from systemds.scuro.representations.color_histogram import ColorHistogram +from systemds.scuro.representations.resnet import ResNet from systemds.scuro.modality.unimodal_modality import UnimodalModality from systemds.scuro.representations.mel_spectrogram import MelSpectrogram from systemds.scuro.representations.mfcc import MFCC -from systemds.scuro.representations.resnet import ResNet -from systemds.scuro.representations.swin_video_transformer import SwinVideoTransformer from tests.scuro.data_generator import ( TestDataLoader, ModalityRandomDataGenerator, @@ -72,7 +57,6 @@ ZeroCrossingRate, BandpowerFFT, ) -from systemds.scuro.representations.vgg import VGG19 class TestUnimodalRepresentations(unittest.TestCase): @@ -103,12 +87,11 @@ def _create_audio_modality(self, signal_length=1000): return audio def test_audio_representation_transform_output_shapes(self): - audio = self._create_audio_modality() + audio = self._create_audio_modality(signal_length=200) audio_representations = [ (MFCC(), (2, 12)), (MelSpectrogram(), (2, 128)), (Spectrogram(), (2, 1025)), - (Wav2Vec(), (1, None)), (Spectral(), (2, 4)), (ZeroCrossing(), (2, None)), (RMSE(), (2, None)), @@ -138,14 +121,13 @@ def test_audio_representations(self): MFCC(), MelSpectrogram(), Spectrogram(), - Wav2Vec(), Spectral(), ZeroCrossing(), RMSE(), Pitch(), ] audio_data, audio_md = ModalityRandomDataGenerator().create_audio_data( - self.num_instances, 1000 + self.num_instances, 200 ) audio = UnimodalModality( @@ -181,7 +163,7 @@ def test_timeseries_representations(self): BandpowerFFT(), ] ts_data, ts_md = ModalityRandomDataGenerator().create_timeseries_data( - self.num_instances, 1000 + self.num_instances, 100 ) ts = UnimodalModality( @@ -201,10 +183,8 @@ def test_timeseries_representations(self): assert (ts.data[i] == original_data[i]).all() def test_image_representations(self): - image_representations = [ColorHistogram(), CLIPVisual(), ResNet()] - image_data, image_md = ModalityRandomDataGenerator().create_visual_modality( - self.num_instances, 1 + self.num_instances, 1, height=8, width=8 ) image = UnimodalModality( @@ -213,10 +193,9 @@ def test_image_representations(self): ) ) - for representation in image_representations: - r = image.apply_representation(representation) - assert r.data is not None - assert len(r.data) == self.num_instances + r = image.apply_representation(ColorHistogram()) + assert r.data is not None + assert len(r.data) == self.num_instances # def test_video_representations(self): # video_representations = [ @@ -241,47 +220,34 @@ def test_image_representations(self): # assert len(r.data) == self.num_instances def test_text_representations(self): - test_representations = [ - CLIPText(), - Bert(), - BoW(2, 2), - TfIdf(), - W2V(), - GloVe(), - ALBERT(), - ELECTRA(), - RoBERTa(), - DistillBERT(), - ] text_data, text_md = ModalityRandomDataGenerator().create_text_data( - self.num_instances, 100 + self.num_instances, 3 ) text = UnimodalModality( TestDataLoader( self.indices, None, ModalityType.TEXT, text_data, str, text_md ) ) - for representation in test_representations: + for representation in [BoW(2, 2), TfIdf()]: r = text.apply_representation(representation) assert r.data is not None assert len(r.data) == self.num_instances def test_chunked_video_representations(self): - video_representations = [ResNet()] video_data, video_md = ModalityRandomDataGenerator().create_visual_modality( - self.num_instances, 25 + self.num_instances, 30 ) video = UnimodalModality( TestDataLoader( self.indices, None, ModalityType.VIDEO, video_data, np.float32, video_md ) ) - for representation in video_representations: - r = video.apply_representation(representation) - assert r.data is not None - assert len(r.data) == self.num_instances - assert len(r.metadata) == self.num_instances + r = video.apply_representation(ResNet(model_name="ResNet18")) + assert r.data is not None + assert len(r.data) == self.num_instances + assert len(r.metadata) == self.num_instances +# TODO: add unit tests for the other representations if __name__ == "__main__": unittest.main() diff --git a/src/main/python/tests/scuro/test_window_operations.py b/src/main/python/tests/scuro/test_window_operations.py index 2eaf5985db1..a8c86374801 100644 --- a/src/main/python/tests/scuro/test_window_operations.py +++ b/src/main/python/tests/scuro/test_window_operations.py @@ -39,13 +39,13 @@ class TestWindowOperations(unittest.TestCase): @classmethod def setUpClass(cls): - cls.num_instances = 40 + cls.num_instances = 4 cls.data_generator = ModalityRandomDataGenerator() cls.aggregations = ["mean", "sum", "max", "min"] def test_static_window(self): num_windows = 5 - data, md = self.data_generator.create_visual_modality(self.num_instances, 50) + data, md = self.data_generator.create_visual_modality(self.num_instances, 10) modality = UnimodalModality( TestDataLoader( [i for i in range(0, self.num_instances)], @@ -63,7 +63,7 @@ def test_static_window(self): def test_dynamic_window(self): num_windows = 5 - data, md = self.data_generator.create_visual_modality(self.num_instances, 50) + data, md = self.data_generator.create_visual_modality(self.num_instances, 10) modality = UnimodalModality( TestDataLoader( [i for i in range(0, self.num_instances)], @@ -93,19 +93,21 @@ def test_window_operations_on_text_representations(self): self.run_window_aggregation_for_modality(ModalityType.TEXT, window_size) def run_window_aggregation_for_modality(self, modality_type, window_size): - r = self.data_generator.create1DModality(40, 5000, modality_type) + r = self.data_generator.create1DModality(self.num_instances, 200, modality_type) for aggregation in self.aggregations: windowed_modality = r.window_aggregation(window_size, aggregation) self.verify_window_operation(aggregation, r, windowed_modality, window_size) def test_window_aggregation_on_3d_modality(self): - data, _ = self.data_generator.create_3d_modality(40, (100, 28, 28)) + data, _ = self.data_generator.create_3d_modality( + self.num_instances, (100, 8, 8) + ) embedding_modality = TransformedModality( self.data_generator, "test_transformation" ) embedding_modality.data = data - embedding_modality.stats = RepresentationStats(40, (100, 28, 28)) + embedding_modality.stats = RepresentationStats(self.num_instances, (100, 8, 8)) num_windows = 10 for window_operator in [ @@ -115,17 +117,17 @@ def test_window_aggregation_on_3d_modality(self): ]: stats = window_operator.get_output_stats(embedding_modality.stats) assert stats.num_instances == self.num_instances - assert stats.output_shape == (num_windows, 28, 28) + assert stats.output_shape == (num_windows, 8, 8) windowed_modality = embedding_modality.context(window_operator) def test_window_aggregation_on_2d_modality(self): - data, _ = self.data_generator.create_2d_modality(40, (100, 28)) + data, _ = self.data_generator.create_2d_modality(self.num_instances, (100, 8)) embedding_modality = TransformedModality( self.data_generator, "test_transformation" ) embedding_modality.data = data - embedding_modality.stats = RepresentationStats(40, (100, 28)) + embedding_modality.stats = RepresentationStats(self.num_instances, (100, 8)) num_windows = 10 for window_operator in [ @@ -135,7 +137,7 @@ def test_window_aggregation_on_2d_modality(self): ]: stats = window_operator.get_output_stats(embedding_modality.stats) assert stats.num_instances == self.num_instances - assert stats.output_shape == (num_windows, 28) + assert stats.output_shape == (num_windows, 8) windowed_modality = embedding_modality.context(window_operator) From aa1c718d3576daf6620925ff8e834872fee595f7 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Tue, 23 Jun 2026 14:50:41 +0200 Subject: [PATCH 040/132] [MINOR] Fix class-init deadlock between AOffset and its subclasses (#2502) AOffset initialized a cached empty slice in its static initializer by instantiating its own OffsetEmpty subclass. Since OffsetEmpty (and the other offset subclasses) depend on AOffset being initialized first, this formed a superclass/subclass class-initialization cycle. When two threads first touched the offset classes concurrently (e.g. parallel test execution), each could hold one class's init monitor while waiting for the other, deadlocking on the JVM class-initialization monitors. Such a deadlock is invisible to the JVM deadlock detector and cannot be interrupted, so the affected JVM hangs indefinitely. It only manifests under concurrent first-touch, which is why it never reproduced in single-threaded local runs. Defer the empty slice to a lazy holder accessed via emptySlice(), so AOffset's static initializer no longer references any subclass. By the time the holder is touched, AOffset is already initialized, so no cycle exists. Add a regression test that forces concurrent first-initialization of the offset classes through a dedicated class loader across repeated rounds and fails if it does not complete promptly. --- .../compress/colgroup/offset/AOffset.java | 25 +++- .../compress/colgroup/offset/OffsetEmpty.java | 2 +- .../colgroup/offset/OffsetSingle.java | 2 +- .../compress/colgroup/offset/OffsetTwo.java | 4 +- .../OffsetClassInitConcurrencyTest.java | 137 ++++++++++++++++++ 5 files changed, 162 insertions(+), 8 deletions(-) create mode 100644 src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java index 8930074eb0e..a961c1188bf 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java @@ -55,8 +55,25 @@ public abstract class AOffset implements Serializable { protected static final Log LOG = LogFactory.getLog(AOffset.class.getName()); - /** Cached final empty slice to return in cases of empty slice returns to avoid object allocation */ - protected static final OffsetSliceInfo EMPTY_SLICE = new OffsetSliceInfo(-1, -1, new OffsetEmpty()); + /** + * Lazy holder for the cached empty slice. The empty slice is built on first use rather than in AOffset's + * static initializer: instantiating the OffsetEmpty subclass from AOffset's {@code } forms a + * superclass/subclass class-initialization cycle that deadlocks when several threads first touch the offset + * classes concurrently (e.g. parallel tests). Deferring it to first use guarantees AOffset is already + * initialized by the time OffsetEmpty is loaded, so no cycle exists. + */ + private static final class EmptySliceHolder { + static final OffsetSliceInfo EMPTY_SLICE = new OffsetSliceInfo(-1, -1, new OffsetEmpty()); + } + + /** + * Get the cached empty slice, returned for empty slice results to avoid object allocation. + * + * @return the shared empty {@link OffsetSliceInfo} + */ + protected static OffsetSliceInfo emptySlice() { + return EmptySliceHolder.EMPTY_SLICE; + } /** The skip list stride size, aka how many indexes skipped for each index. */ protected static final int SKIP_STRIDE = 1000; @@ -570,12 +587,12 @@ public OffsetSliceInfo slice(int l, int u) { return new OffsetSliceInfo(0, s, moveIndex(l)); } else if (u < first) - return EMPTY_SLICE; + return emptySlice(); final AIterator it = getIteratorSkipCache(l); if(it == null || it.value() >= u) - return EMPTY_SLICE; + return emptySlice(); if(u >= last) // If including the last do not iterate. return constructSliceReturn(l, u, it.getDataIndex(), s - 1, it.getOffsetsIndex(), getLength(), it.value(), diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java index 73264c84767..acd3b0d04eb 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java @@ -93,7 +93,7 @@ public int getSize() { @Override public OffsetSliceInfo slice(int l, int u) { - return EMPTY_SLICE; + return emptySlice(); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetSingle.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetSingle.java index 98dab591bf9..66b9010371a 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetSingle.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetSingle.java @@ -93,7 +93,7 @@ public OffsetSliceInfo slice(int l, int u) { if(l <= off && u > off) return new OffsetSliceInfo(0, 1, new OffsetSingle(off - l)); else - return EMPTY_SLICE; + return emptySlice(); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetTwo.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetTwo.java index 48ce65f171f..d18c66188e4 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetTwo.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetTwo.java @@ -98,7 +98,7 @@ public static OffsetTwo readFields(DataInput in) throws IOException { public OffsetSliceInfo slice(int l, int u) { if(l <= first) { if(u <= first) - return EMPTY_SLICE; + return emptySlice(); else if(u > last) return new OffsetSliceInfo(0, 2, moveIndex(l)); else @@ -107,7 +107,7 @@ else if(u > last) else if(l <= last && u > last) return new OffsetSliceInfo(1, 2, new OffsetSingle(last - l)); else - return EMPTY_SLICE; + return emptySlice(); } @Override diff --git a/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java b/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java new file mode 100644 index 00000000000..8907c82f1d1 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.compress.offset; + +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.Test; + +/** + * Regression guard for a superclass/subclass class-initialization deadlock in the offset hierarchy. + *

+ * {@code AOffset} previously instantiated its {@code OffsetEmpty} subclass from a {@code static final} field, so + * {@code AOffset.} depended on {@code OffsetEmpty} while {@code OffsetEmpty} (being a subclass) depends on + * {@code AOffset}. Initializing the two classes from different threads at the same time deadlocked on the JVM class + * initialization monitors -- which only happens under concurrent first-touch (e.g. parallel tests) and is invisible to + * the JVM deadlock detector, so it hangs forever. + *

+ * This test forces a fresh, concurrent first-initialization of the offset classes through a dedicated class loader and + * fails if it does not complete promptly. + */ +public class OffsetClassInitConcurrencyTest { + + private static final String PKG = "org.apache.sysds.runtime.compress.colgroup.offset."; + + /** Classes whose static initializers participate in the (former) cycle. */ + private static final String[] INIT_TARGETS = {PKG + "AOffset", PKG + "OffsetEmpty", PKG + "OffsetChar", + PKG + "OffsetByte", PKG + "OffsetSingle", PKG + "OffsetTwo"}; + + /** Whether a class-init cycle deadlocks depends on thread timing, so repeat to make a regression reliable to catch. */ + private static final int ROUNDS = 20; + + /** A real init deadlock never resolves; a healthy round finishes in milliseconds, so this bound is generous. */ + private static final long ROUND_TIMEOUT_MS = 10000; + + @Test(timeout = 60000) + public void concurrentFirstInitDoesNotDeadlock() throws Exception { + for(int round = 0; round < ROUNDS; round++) + runConcurrentInitRound(round); + } + + private static void runConcurrentInitRound(int round) throws Exception { + // A fresh loader per round so the offset classes initialize from scratch (rather than reusing state from + // an earlier round or earlier test), reproducing the concurrent first-touch race. + final ClassLoader loader = new OffsetPackageClassLoader(OffsetClassInitConcurrencyTest.class.getClassLoader()); + final CyclicBarrier startLine = new CyclicBarrier(INIT_TARGETS.length); + final List threads = new ArrayList<>(); + final AtomicReference failure = new AtomicReference<>(); + + for(String target : INIT_TARGETS) { + final Thread t = new Thread(() -> { + try { + startLine.await(); + // init=true forces the static initializer to run on this thread. + Class.forName(target, true, loader); + } + catch(Throwable e) { + failure.compareAndSet(null, e); + } + }, "init-" + target.substring(PKG.length())); + // Daemon so a regression (deadlock) cannot keep the JVM alive after the test times out. + t.setDaemon(true); + threads.add(t); + t.start(); + } + + final long deadline = System.currentTimeMillis() + ROUND_TIMEOUT_MS; + for(Thread t : threads) { + final long remaining = deadline - System.currentTimeMillis(); + if(remaining > 0) + t.join(remaining); + if(t.isAlive()) + fail("Concurrent class initialization deadlocked in round " + round + " (thread " + t.getName() + + " did not finish); likely a static-init cycle between AOffset and a subclass."); + } + + if(failure.get() != null) + fail("Concurrent class initialization failed in round " + round + ": " + failure.get()); + } + + /** Loads the offset package classes itself (delegating everything else) so they initialize fresh. */ + private static final class OffsetPackageClassLoader extends ClassLoader { + OffsetPackageClassLoader(ClassLoader parent) { + super(parent); + } + + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + if(!name.startsWith(PKG)) + return super.loadClass(name, resolve); + synchronized(getClassLoadingLock(name)) { + Class c = findLoadedClass(name); + if(c == null) + c = defineFromParentResource(name); + if(resolve) + resolveClass(c); + return c; + } + } + + private Class defineFromParentResource(String name) throws ClassNotFoundException { + final String path = name.replace('.', '/') + ".class"; + try(InputStream is = getParent().getResourceAsStream(path)) { + if(is == null) + throw new ClassNotFoundException(name); + final byte[] b = is.readAllBytes(); + return defineClass(name, b, 0, b.length); + } + catch(IOException e) { + throw new ClassNotFoundException(name, e); + } + } + } +} From 9fba045476fa6ffdb35567e966e6c612ded8b3bd Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Tue, 23 Jun 2026 19:49:55 +0200 Subject: [PATCH 041/132] [BWARE] Tune compressed matmul fast paths and Spark execution decisions (#2483) Mixes two related performance changes: refined compressed multiply heuristics, and a Spark-vs-CP decision refresh on the Hop layer. CLALib matmul changes: - CLALibMMChain: for XtXv with few col groups and a wide-enough matrix, compute X' * X via leftMultByTransposeSelf and finish with a regular matrix multiply against v. Cheaper than chaining when the X' * X path can stay compressed - CLALibTSMM: refactor leftMultByTransposeSelf into a package-private helper so MMChain can call it; widen the ColGroupUncompressed handling - CLALibRightMultBy: stop forcing decompression for ASDC / ASDCZero inputs; they have working preAggregate paths that beat the dense fallback - CLALibCompAgg: fix blklen rounding so the last partition is not short by k rows on parallel aggregates Spark/CP exec-decision refresh (Hop, UnaryOp, BinaryOp): - Hop: new helpers hasSparkOutput() and isScalarOrVectorBelowBlockSize() shared between unary and binary decision points - UnaryOp.optFindExecType: replace the inline chain of negations with isDisallowedSparkOps(), allow Frame outputs, and pull unary ops into Spark when the input already has a Spark output; gated on the ALLOW_TRANSITIVE_SPARK_EXEC_TYPE flag so it shares a kill-switch with the binary path - BinaryOp.optFindExecType: same kind of restructuring; allow matrix-or-frame outputs to be pulled into Spark when exactly one operand is a scalar or small vector Instruction-side adjustments: - VariableCPInstruction (CAST_AS_MATRIX from frame): use the parallel MatrixBlockFromFrame.convertToMatrixBlock(fin, k) path instead of the single-threaded DataConverter helper - ParameterizedBuiltinCPInstruction (transformdecode): call the parallel decoder.decode(data, out, k) overload using InfrastructureAnalyzer.getLocalParallelism() - DecoderComposite (parallel decode): parallelize over row blocks instead of over decoders, so the sub-decoders still run in order within each block (e.g. recode-on-output depends on the category indexes from the preceding dummycode decoder); fall back to the sequential path for k <= 1 Testing: - New CompilerTestBase harness for compile-time exec-type assertions, plus SparkTransitiveExecTypeCompileTest and an end-to-end SparkTransitiveExecTypeTest (with DML) covering the unary/binary transitive Spark pull, its multi-consumer guard, the cumulative-op exclusion, and the flag-off kill-switch - CLALibMMChainTest, CLALibRightMultBySDCTest, and DecoderCompositeTest for the compressed matmul fast paths, ASDC right-multiply, and parallel composite decode; shared helpers added to CompressedTestBase --- .../java/org/apache/sysds/hops/BinaryOp.java | 36 ++- src/main/java/org/apache/sysds/hops/Hop.java | 11 + .../java/org/apache/sysds/hops/UnaryOp.java | 34 ++- .../runtime/compress/lib/CLALibCompAgg.java | 2 +- .../runtime/compress/lib/CLALibMMChain.java | 6 + .../compress/lib/CLALibRightMultBy.java | 6 +- .../runtime/compress/lib/CLALibTSMM.java | 39 ++- .../cp/ParameterizedBuiltinCPInstruction.java | 2 +- .../cp/VariableCPInstruction.java | 3 +- .../transform/decode/DecoderComposite.java | 15 +- .../component/compile/CompilerTestBase.java | 189 ++++++++++++ .../SparkTransitiveExecTypeCompileTest.java | 138 +++++++++ .../compress/CompressedTestBase.java | 46 +++ .../compress/lib/CLALibMMChainTest.java | 273 ++++++++++++++++++ .../lib/CLALibRightMultBySDCTest.java | 116 ++++++++ .../frame/transform/DecoderCompositeTest.java | 132 +++++++++ .../SparkTransitiveExecTypeTest.java | 104 +++++++ .../sparkexectype/SparkExecTypeBinary.dml | 33 +++ .../sparkexectype/SparkExecTypeUnary.dml | 31 ++ 19 files changed, 1173 insertions(+), 43 deletions(-) create mode 100644 src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java create mode 100644 src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java create mode 100644 src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java create mode 100644 src/test/java/org/apache/sysds/test/component/compress/lib/CLALibRightMultBySDCTest.java create mode 100644 src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java create mode 100644 src/test/java/org/apache/sysds/test/functions/sparkexectype/SparkTransitiveExecTypeTest.java create mode 100644 src/test/scripts/functions/sparkexectype/SparkExecTypeBinary.dml create mode 100644 src/test/scripts/functions/sparkexectype/SparkExecTypeUnary.dml diff --git a/src/main/java/org/apache/sysds/hops/BinaryOp.java b/src/main/java/org/apache/sysds/hops/BinaryOp.java index dc7edf76e50..8c5ccb31809 100644 --- a/src/main/java/org/apache/sysds/hops/BinaryOp.java +++ b/src/main/java/org/apache/sysds/hops/BinaryOp.java @@ -763,8 +763,8 @@ protected ExecType optFindExecType(boolean transitive) { checkAndSetForcedPlatform(); - DataType dt1 = getInput().get(0).getDataType(); - DataType dt2 = getInput().get(1).getDataType(); + final DataType dt1 = getInput(0).getDataType(); + final DataType dt2 = getInput(1).getDataType(); if( _etypeForced != null ) { setExecType(_etypeForced); @@ -812,18 +812,28 @@ else if ( dt1 == DataType.SCALAR && dt2 == DataType.MATRIX ) { checkAndSetInvalidCPDimsAndSize(); } - //spark-specific decision refinement (execute unary scalar w/ spark input and + // spark-specific decision refinement (execute unary scalar w/ spark input and // single parent also in spark because it's likely cheap and reduces intermediates) - if(transitive && _etype == ExecType.CP && _etypeForced != ExecType.CP && _etypeForced != ExecType.FED && - getDataType().isMatrix() // output should be a matrix - && (dt1.isScalar() || dt2.isScalar()) // one side should be scalar - && supportsMatrixScalarOperations() // scalar operations - && !(getInput().get(dt1.isScalar() ? 1 : 0) instanceof DataOp) // input is not checkpoint - && getInput().get(dt1.isScalar() ? 1 : 0).getParent().size() == 1 // unary scalar is only parent - && !HopRewriteUtils.isSingleBlock(getInput().get(dt1.isScalar() ? 1 : 0)) // single block triggered exec - && getInput().get(dt1.isScalar() ? 1 : 0).optFindExecType() == ExecType.SPARK) { - // pull unary scalar operation into spark - _etype = ExecType.SPARK; + if(transitive // we allow transitive Spark operations. continue sequences of spark operations + && _etype == ExecType.CP // The instruction is currently in CP + && _etypeForced != ExecType.CP // not forced CP + && _etypeForced != ExecType.FED // not federated + && (getDataType().isMatrix() || getDataType().isFrame()) // output should be a matrix or frame + ) { + final boolean v1 = getInput(0).isScalarOrVectorBelowBlockSize(); + final boolean v2 = getInput(1).isScalarOrVectorBelowBlockSize(); + final boolean left = v1; // left side is the vector or scalar + final Hop sparkIn = getInput(left ? 1 : 0); + if((v1 ^ v2) // XOR only one side is allowed to be a vector or a scalar. + && (supportsMatrixScalarOperations() || op == OpOp2.APPLY_SCHEMA) // supported operation + && sparkIn.getParent().size() == 1 // only one parent + && !HopRewriteUtils.isSingleBlock(sparkIn) // single block triggered exec + && sparkIn.hasSparkOutput() // input was spark op. + && !(sparkIn instanceof DataOp) // input is not checkpoint + ) { + // pull operation into spark + _etype = ExecType.SPARK; + } } if( OptimizerUtils.ALLOW_BINARY_UPDATE_IN_PLACE && diff --git a/src/main/java/org/apache/sysds/hops/Hop.java b/src/main/java/org/apache/sysds/hops/Hop.java index 86749d44c1c..19f499e5b81 100644 --- a/src/main/java/org/apache/sysds/hops/Hop.java +++ b/src/main/java/org/apache/sysds/hops/Hop.java @@ -1045,6 +1045,12 @@ public final String toString() { // ======================================================================================== + protected boolean isScalarOrVectorBelowBlockSize(){ + return getDataType().isScalar() || (dimsKnown() && + (( _dc.getRows() == 1 && _dc.getCols() < ConfigurationManager.getBlocksize()) + || ( _dc.getCols() == 1 && _dc.getRows() < ConfigurationManager.getBlocksize()))); + } + protected boolean isVector() { return (dimsKnown() && (_dc.getRows() == 1 || _dc.getCols() == 1) ); } @@ -1629,6 +1635,11 @@ protected void setMemoryAndComputeEstimates(Lop lop) { lop.setComputeEstimate(ComputeCost.getHOPComputeCost(this)); } + protected boolean hasSparkOutput(){ + return (this.optFindExecType() == ExecType.SPARK + || (this instanceof DataOp && ((DataOp)this).hasOnlyRDD())); + } + /** * Set parse information. * diff --git a/src/main/java/org/apache/sysds/hops/UnaryOp.java b/src/main/java/org/apache/sysds/hops/UnaryOp.java index b3475edfbae..1ba5b75db57 100644 --- a/src/main/java/org/apache/sysds/hops/UnaryOp.java +++ b/src/main/java/org/apache/sysds/hops/UnaryOp.java @@ -366,7 +366,11 @@ protected double computeOutputMemEstimate( long dim1, long dim2, long nnz ) } else { sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz); } - return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity, getDataType()); + + if(getDataType() == DataType.FRAME) + return OptimizerUtils.estimateSizeExactFrame(dim1, dim2); + else + return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity); } @Override @@ -463,6 +467,13 @@ public boolean isMetadataOperation() { || _op == OpOp1.CAST_AS_LIST; } + private boolean isDisallowedSparkOps(){ + return isCumulativeUnaryOperation() + || isCastUnaryOperation() + || _op==OpOp1.MEDIAN + || _op==OpOp1.IQM; + } + @Override protected ExecType optFindExecType(boolean transitive) { @@ -493,19 +504,22 @@ else if ( getInput().get(0).areDimsBelowThreshold() || getInput().get(0).isVecto checkAndSetInvalidCPDimsAndSize(); } + //spark-specific decision refinement (execute unary w/ spark input and //single parent also in spark because it's likely cheap and reduces intermediates) - if( _etype == ExecType.CP && _etypeForced != ExecType.CP - && getInput().get(0).optFindExecType() == ExecType.SPARK - && getDataType().isMatrix() - && !isCumulativeUnaryOperation() && !isCastUnaryOperation() - && _op!=OpOp1.MEDIAN && _op!=OpOp1.IQM - && !(getInput().get(0) instanceof DataOp) //input is not checkpoint - && getInput().get(0).getParent().size()==1 ) //unary is only parent - { + if(transitive // transitive refinement enabled + && _etype == ExecType.CP // currently CP instruction + && _etypeForced != ExecType.CP // not forced as CP instruction + && getInput(0).hasSparkOutput() // input is a spark instruction + && (getDataType().isMatrix() || getDataType().isFrame()) // output is a matrix or frame + && !isDisallowedSparkOps() // op is allowed to run on spark + && !(getInput(0) instanceof DataOp) // input is not checkpoint + && getInput(0).getParent().size() == 1 // unary is only parent + ) { //pull unary operation into spark _etype = ExecType.SPARK; } + //mark for recompile (forever) setRequiresRecompileIfNecessary(); @@ -520,7 +534,7 @@ && getInput().get(0).getParent().size()==1 ) //unary is only parent } else { setRequiresRecompileIfNecessary(); } - + return _etype; } diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCompAgg.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCompAgg.java index 99693635a9b..948a78f96af 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCompAgg.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCompAgg.java @@ -486,7 +486,7 @@ private static List> generateUnaryAggregateOverlappingFuture final ArrayList tasks = new ArrayList<>(); final int nCol = m1.getNumColumns(); final int nRow = m1.getNumRows(); - final int blklen = Math.max(64, nRow / k); + final int blklen = Math.max(64, (nRow + k) / k); final List groups = m1.getColGroups(); final boolean shouldFilter = CLALibUtils.shouldPreFilter(groups); if(shouldFilter) { diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java index d82d58e323e..cc7953f8c5d 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java @@ -30,6 +30,7 @@ import org.apache.sysds.runtime.compress.colgroup.AColGroup; import org.apache.sysds.runtime.compress.colgroup.ColGroupConst; import org.apache.sysds.runtime.functionobjects.Multiply; +import org.apache.sysds.runtime.instructions.InstructionUtils; import org.apache.sysds.runtime.matrix.data.LibMatrixBincell; import org.apache.sysds.runtime.matrix.data.LibMatrixReorg; import org.apache.sysds.runtime.matrix.data.MatrixBlock; @@ -95,6 +96,11 @@ public static MatrixBlock mmChain(CompressedMatrixBlock x, MatrixBlock v, Matrix if(x.isEmpty()) return returnEmpty(x, out); + if(ctype == ChainType.XtXv && x.getColGroups().size() < 5 && x.getNumColumns()> 30){ + MatrixBlock tmp = CLALibTSMM.leftMultByTransposeSelf(x, k); + return tmp.aggregateBinaryOperations(tmp, v, out, InstructionUtils.getMatMultOperator(k)); + } + // Morph the columns to efficient types for the operation. x = filterColGroups(x); double preFilterTime = t.stop(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java index f14d6833d95..642b57124f1 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java @@ -31,6 +31,8 @@ import org.apache.sysds.conf.DMLConfig; import org.apache.sysds.runtime.compress.CompressedMatrixBlock; import org.apache.sysds.runtime.compress.colgroup.AColGroup; +import org.apache.sysds.runtime.compress.colgroup.ASDC; +import org.apache.sysds.runtime.compress.colgroup.ASDCZero; import org.apache.sysds.runtime.compress.colgroup.ColGroupConst; import org.apache.sysds.runtime.compress.colgroup.ColGroupUncompressed; import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory; @@ -143,7 +145,9 @@ private static MatrixBlock decompressingMatrixMult(CompressedMatrixBlock m1, Mat private static boolean betterIfDecompressed(CompressedMatrixBlock m) { for(AColGroup g : m.getColGroups()) { - if(!(g instanceof ColGroupUncompressed) && g.getNumValues() * 2 >= m.getNumRows()) { + // TODO add subpport for decompressing RMM to ASDC and ASDCZero + if(!(g instanceof ColGroupUncompressed || g instanceof ASDC || g instanceof ASDCZero) && + g.getNumValues() * 2 >= m.getNumRows()) { return true; } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibTSMM.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibTSMM.java index a1d47a9b150..25cac42caec 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibTSMM.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibTSMM.java @@ -31,6 +31,7 @@ import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.compress.CompressedMatrixBlock; import org.apache.sysds.runtime.compress.colgroup.AColGroup; +import org.apache.sysds.runtime.compress.colgroup.ColGroupUncompressed; import org.apache.sysds.runtime.matrix.data.LibMatrixMult; import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.runtime.util.CommonThreadPool; @@ -42,6 +43,10 @@ private CLALibTSMM() { // private constructor } + public static MatrixBlock leftMultByTransposeSelf(CompressedMatrixBlock cmb, int k) { + return leftMultByTransposeSelf(cmb, new MatrixBlock(), k); + } + /** * Self left Matrix multiplication (tsmm) * @@ -51,17 +56,25 @@ private CLALibTSMM() { * @param ret The output matrix to put the result into * @param k The parallelization degree allowed */ - public static void leftMultByTransposeSelf(CompressedMatrixBlock cmb, MatrixBlock ret, int k) { + public static MatrixBlock leftMultByTransposeSelf(CompressedMatrixBlock cmb, MatrixBlock ret, int k) { + final int numColumns = cmb.getNumColumns(); + final int numRows = cmb.getNumRows(); + if(cmb.isEmpty()) + return new MatrixBlock(numColumns, numColumns, true); + // create output matrix block + if(ret == null) + ret = new MatrixBlock(numColumns, numColumns, false); + else + ret.reset(numColumns, numColumns, false); + ret.allocateDenseBlock(); final List groups = cmb.getColGroups(); - final int numColumns = cmb.getNumColumns(); - if(groups.size() >= numColumns) { + if(groups.size() >= numColumns || containsUncompressedColGroup(groups)) { MatrixBlock m = cmb.getUncompressed("TSMM to many columngroups", k); LibMatrixMult.matrixMultTransposeSelf(m, ret, true, k); - return; + return ret; } - final int numRows = cmb.getNumRows(); final boolean shouldFilter = CLALibUtils.shouldPreFilter(groups); final boolean overlapping = cmb.isOverlapping(); if(shouldFilter) { @@ -77,6 +90,14 @@ public static void leftMultByTransposeSelf(CompressedMatrixBlock cmb, MatrixBloc ret.setNonZeros(LibMatrixMult.copyUpperToLowerTriangle(ret)); ret.examSparsity(); + return ret; + } + + private static boolean containsUncompressedColGroup(List groups) { + for(AColGroup g : groups) + if(g instanceof ColGroupUncompressed) + return true; + return false; } private static void addCorrectionLayer(List filteredGroups, MatrixBlock result, int nRows, int nCols, @@ -86,8 +107,6 @@ private static void addCorrectionLayer(List filteredGroups, MatrixBlo addCorrectionLayer(constV, filteredColSum, nRows, retV); } - - private static void tsmmColGroups(List groups, MatrixBlock ret, int nRows, boolean overlapping, int k) { if(k <= 1) tsmmColGroupsSingleThread(groups, ret, nRows); @@ -136,12 +155,12 @@ private static void tsmmColGroupsMultiThread(List groups, MatrixBlock public static void addCorrectionLayer(double[] constV, double[] filteredColSum, int nRow, double[] ret) { final int nColRow = constV.length; - for(int row = 0; row < nColRow; row++){ + for(int row = 0; row < nColRow; row++) { int offOut = nColRow * row; final double v1l = constV[row]; final double v2l = filteredColSum[row] + constV[row] * nRow; - for(int col = row; col < nColRow; col++){ - ret[offOut + col] += v1l * filteredColSum[col] + v2l * constV[col]; + for(int col = row; col < nColRow; col++) { + ret[offOut + col] += v1l * filteredColSum[col] + v2l * constV[col]; } } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java index 119589a3033..e53958ac4b8 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java @@ -352,7 +352,7 @@ else if(opcode.equalsIgnoreCase(Opcodes.TRANSFORMDECODE.toString())) { // compute transformdecode Decoder decoder = DecoderFactory .createDecoder(getParameterMap().get("spec"), colnames, null, meta, data.getNumColumns()); - FrameBlock fbout = decoder.decode(data, new FrameBlock(decoder.getSchema())); + FrameBlock fbout = decoder.decode(data, new FrameBlock(decoder.getSchema()), InfrastructureAnalyzer.getLocalParallelism()); fbout.setColumnNames(Arrays.copyOfRange(colnames, 0, fbout.getNumColumns())); // release locks diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java index 359df747e7b..0f707b74412 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java @@ -44,6 +44,7 @@ import org.apache.sysds.runtime.controlprogram.parfor.util.IDSequence; import org.apache.sysds.runtime.data.TensorBlock; import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.frame.data.lib.MatrixBlockFromFrame; import org.apache.sysds.runtime.instructions.Instruction; import org.apache.sysds.runtime.instructions.InstructionUtils; import org.apache.sysds.runtime.instructions.ooc.TeeOOCInstruction; @@ -923,7 +924,7 @@ private void processCastAsMatrixVariableInstruction(ExecutionContext ec) { switch( getInput1().getDataType() ) { case FRAME: { FrameBlock fin = ec.getFrameInput(getInput1().getName()); - MatrixBlock out = DataConverter.convertToMatrixBlock(fin); + MatrixBlock out = MatrixBlockFromFrame.convertToMatrixBlock(fin, k); ec.releaseFrameInput(getInput1().getName()); ec.setMatrixOutput(output.getName(), out); break; diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderComposite.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderComposite.java index f4bc9f8b216..f1afcfac194 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderComposite.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderComposite.java @@ -62,17 +62,20 @@ public FrameBlock decode(MatrixBlock in, FrameBlock out) { @Override public FrameBlock decode(final MatrixBlock in, final FrameBlock out, final int k) { + if(k <= 1) + return decode(in, out); final ExecutorService pool = CommonThreadPool.get(k); out.ensureAllocatedColumns(in.getNumRows()); try { final List> tasks = new ArrayList<>(); int blz = Math.max(in.getNumRows() / k, 1000); - for(Decoder decoder : _decoders){ - for(int i = 0; i < in.getNumRows(); i += blz){ - final int start = i; - final int end = Math.min(in.getNumRows(), i + blz); - tasks.add(pool.submit(() -> decoder.decode(in, out, start, end))); - } + // Parallelize over row blocks (not over decoders): all decoders must + // run in order within a block, e.g. recode-on-output depends on the + // category indexes produced by the preceding dummycode decoder. + for(int i = 0; i < in.getNumRows(); i += blz){ + final int start = i; + final int end = Math.min(in.getNumRows(), i + blz); + tasks.add(pool.submit(() -> decode(in, out, start, end))); } for(Future f : tasks) f.get(); diff --git a/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java b/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java new file mode 100644 index 00000000000..07ec9752928 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.compile; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.sysds.api.DMLScript; +import org.apache.sysds.common.Types.ExecMode; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; +import org.apache.sysds.hops.OptimizerUtils; +import org.apache.sysds.hops.recompile.Recompiler; +import org.apache.sysds.parser.DMLProgram; +import org.apache.sysds.parser.DMLTranslator; +import org.apache.sysds.parser.ParserFactory; +import org.apache.sysds.parser.ParserWrapper; +import org.apache.sysds.runtime.controlprogram.BasicProgramBlock; +import org.apache.sysds.runtime.controlprogram.ForProgramBlock; +import org.apache.sysds.runtime.controlprogram.FunctionProgramBlock; +import org.apache.sysds.runtime.controlprogram.IfProgramBlock; +import org.apache.sysds.runtime.controlprogram.Program; +import org.apache.sysds.runtime.controlprogram.ProgramBlock; +import org.apache.sysds.runtime.controlprogram.WhileProgramBlock; +import org.apache.sysds.runtime.instructions.Instruction; +import org.apache.sysds.runtime.instructions.cp.CPInstruction; +import org.apache.sysds.runtime.instructions.spark.SPInstruction; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.utils.Explain; +import org.apache.sysds.utils.stats.InfrastructureAnalyzer; +import org.junit.Assert; + +/** + * Base class for compilation-verification tests: compile a DML script into a runtime {@link Program} and inspect the + * resulting plan (instructions and their exec types) without ever executing it. + */ +public abstract class CompilerTestBase extends AutomatedTestBase { + + /** A small default local memory budget (8 MB) that forces large operations onto Spark in HYBRID mode. */ + public static final long SMALL_MEM_BUDGET = 8L * 1024 * 1024; + + @Override + public void setUp() { + // no test-configuration setup needed; scripts are compiled from in-memory strings + } + + /** + * Compile a DML script string into a runtime {@link Program} without executing it. + * + * @param dmlScript the DML source + * @param args named command-line arguments ($name -> value), may be null + * @param mode the global execution mode (e.g. {@link ExecMode#HYBRID}) + * @param localMaxMem the local memory budget in bytes used for memory-based exec-type decisions + * @return the compiled runtime program + */ + protected Program compile(String dmlScript, Map args, ExecMode mode, long localMaxMem) { + final ExecMode oldMode = DMLScript.getGlobalExecMode(); + final long oldMem = InfrastructureAnalyzer.getLocalMaxMemory(); + final DMLConfig oldConfig = ConfigurationManager.getDMLConfig(); + try { + ConfigurationManager.setGlobalConfig(new DMLConfig()); + DMLScript.setGlobalExecMode(mode); + InfrastructureAnalyzer.setLocalMaxMemory(localMaxMem); + OptimizerUtils.resetDefaultSize(); + + Map argVals = (args == null) ? new HashMap<>() : new HashMap<>(args); + ParserWrapper parser = ParserFactory.createParser(); + DMLProgram prog = parser.parse(null, dmlScript, argVals); + DMLTranslator dmlt = new DMLTranslator(prog); + dmlt.liveVariableAnalysis(prog); + dmlt.validateParseTree(prog); + dmlt.constructHops(prog); + dmlt.rewriteHopsDAG(prog); + dmlt.constructLops(prog); + dmlt.rewriteLopDAG(prog); + return dmlt.getRuntimeProgram(prog, ConfigurationManager.getDMLConfig()); + } + catch(Exception e) { + throw new RuntimeException("Failed to compile DML script:\n" + dmlScript, e); + } + finally { + DMLScript.setGlobalExecMode(oldMode); + InfrastructureAnalyzer.setLocalMaxMemory(oldMem); + ConfigurationManager.setGlobalConfig(oldConfig); + Recompiler.reinitRecompiler(); + } + } + + /** Recursively collect every instruction in the program, including control-flow predicates and function bodies. */ + protected List getInstructions(Program prog) { + List out = new ArrayList<>(); + for(ProgramBlock pb : prog.getProgramBlocks()) + collect(pb, out); + for(FunctionProgramBlock fpb : prog.getFunctionProgramBlocks(false).values()) + collect(fpb, out); + return out; + } + + private void collect(ProgramBlock pb, List out) { + if(pb instanceof BasicProgramBlock) { + out.addAll(((BasicProgramBlock) pb).getInstructions()); + } + else if(pb instanceof IfProgramBlock) { + IfProgramBlock ipb = (IfProgramBlock) pb; + out.addAll(ipb.getPredicate()); + ipb.getChildBlocksIfBody().forEach(c -> collect(c, out)); + ipb.getChildBlocksElseBody().forEach(c -> collect(c, out)); + } + else if(pb instanceof WhileProgramBlock) { + WhileProgramBlock wpb = (WhileProgramBlock) pb; + out.addAll(wpb.getPredicate()); + wpb.getChildBlocks().forEach(c -> collect(c, out)); + } + else if(pb instanceof ForProgramBlock) { // incl. ParForProgramBlock + ForProgramBlock fpb = (ForProgramBlock) pb; + out.addAll(fpb.getFromInstructions()); + out.addAll(fpb.getToInstructions()); + out.addAll(fpb.getIncrementInstructions()); + fpb.getChildBlocks().forEach(c -> collect(c, out)); + } + else if(pb instanceof FunctionProgramBlock) { + ((FunctionProgramBlock) pb).getChildBlocks().forEach(c -> collect(c, out)); + } + } + + /** All instructions whose opcode equals {@code opcode} (exact match). */ + protected List getByOpcode(Program prog, String opcode) { + return getInstructions(prog).stream().filter(i -> opcode.equals(i.getOpcode())) + .collect(Collectors.toList()); + } + + protected static boolean isSpark(Instruction inst) { + return inst instanceof SPInstruction; + } + + protected static boolean isCP(Instruction inst) { + return inst instanceof CPInstruction; + } + + /** Assert that at least one instruction with the given opcode exists and that all such instructions are Spark. */ + protected void assertSpark(Program prog, String opcode) { + assertExecType(prog, opcode, true); + } + + /** Assert that at least one instruction with the given opcode exists and that all such instructions are CP. */ + protected void assertCP(Program prog, String opcode) { + assertExecType(prog, opcode, false); + } + + private void assertExecType(Program prog, String opcode, boolean expectSpark) { + List matches = getByOpcode(prog, opcode); + Assert.assertFalse("Expected at least one '" + opcode + "' instruction but found none.\n" + + Explain.explain(prog), matches.isEmpty()); + for(Instruction inst : matches) { + boolean spark = isSpark(inst); + Assert.assertEquals("Instruction '" + opcode + "' expected exec type " + + (expectSpark ? "SPARK" : "CP") + " but was " + (spark ? "SPARK" : "CP") + ".\n" + + Explain.explain(prog), expectSpark, spark); + } + } + + protected long countSpark(Program prog) { + return getInstructions(prog).stream().filter(CompilerTestBase::isSpark).count(); + } + + protected String explain(Program prog) { + return Explain.explain(prog); + } +} diff --git a/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java b/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java new file mode 100644 index 00000000000..0b3889db908 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.compile; + +import org.apache.sysds.common.Types.ExecMode; +import org.apache.sysds.hops.OptimizerUtils; +import org.apache.sysds.runtime.controlprogram.Program; +import org.junit.Test; + +/** + * Verifies the transitive Spark exec-type refinement in {@link org.apache.sysds.hops.UnaryOp} and + * {@link org.apache.sysds.hops.BinaryOp}: a CP-by-estimate unary or matrix-scalar binary on a Spark-resident input is + * pulled into Spark only when it is the sole consumer ({@code getParent().size() == 1}) and the operation is eligible. + * Cumulative (and cast) operations are excluded and stay CP. The {@code ALLOW_TRANSITIVE_SPARK_EXEC_TYPE} flag gates + * the pull, so disabling it keeps an otherwise-pullable op in CP. + */ +public class SparkTransitiveExecTypeCompileTest extends CompilerTestBase { + + private static final String DML_HEADER = + "X = rand(rows=20000000, cols=8, seed=1);\n" + // ~1.2GB -> rand and colSums run on Spark + "v = colSums(X);\n"; // 1x8 Spark-resident vector (opcode uack+) + + @Test + public void singleConsumerUnaryPulledIntoSpark() { + String dml = DML_HEADER + + "r = round(v);\n" + // sole consumer of the Spark-resident vector -> pulled into Spark + "print(sum(r));\n"; + Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); + + assertSpark(prog, "uack+"); // input genuinely has a Spark output + assertSpark(prog, "round"); // unary pulled into Spark (CP by mem estimate, single consumer) + } + + @Test + public void multiConsumerUnaryStaysCP() { + String dml = DML_HEADER + + "a = round(v);\n" + // v now has two consumers (round + abs) ... + "b = abs(v);\n" + + "print(sum(a) + sum(b));\n"; + Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); + + assertSpark(prog, "uack+"); // input still has a Spark output ... + assertCP(prog, "round"); // ... but the multi-parent guard keeps both unaries in CP + assertCP(prog, "abs"); + } + + // A tall, Spark-resident column vector that is still small enough (40 KB) to be CP by memory + // estimate: rowSums over a very wide matrix runs on Spark, but its 1-column result fits in CP. + private static final String TALL_VECTOR_HEADER = + "X = rand(rows=5000, cols=200000, seed=1);\n" + // ~8GB -> rand and rowSums run on Spark + "c = rowSums(X);\n"; // 5000x1 Spark-resident vector (opcode uark+) + + @Test + public void cumulativeUnaryStaysCP() { + String dml = TALL_VECTOR_HEADER + + "r = cumsum(c);\n" + // sole consumer of the Spark-resident vector, CP by estimate ... + "print(as.scalar(r[2500,1]));\n"; // ... consume via indexing (avoids the sum(cumsum) rewrite) + Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); + + assertSpark(prog, "uark+"); // input genuinely has a Spark output + assertCP(prog, "ucumk+"); // ... but cumulative ops are excluded from the transitive pull + } + + @Test + public void singleConsumerBinaryPulledIntoSpark() { + String dml = TALL_VECTOR_HEADER + + "r = c + 2.0;\n" + // matrix-scalar on the Spark-resident vector, sole consumer -> pulled into Spark + "print(as.scalar(r[2500,1]));\n"; + Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); + + assertSpark(prog, "uark+"); // input genuinely has a Spark output (multi-block column vector) + assertSpark(prog, "+"); // matrix-scalar binary pulled into Spark (CP by estimate, single consumer) + } + + @Test + public void multiConsumerBinaryStaysCP() { + String dml = TALL_VECTOR_HEADER + + "a = c + 2.0;\n" + // c now has two consumers (+ and *) ... + "b = c * 3.0;\n" + + "print(as.scalar(a[2500,1]) + as.scalar(b[2500,1]));\n"; + Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); + + assertSpark(prog, "uark+"); // input still has a Spark output ... + assertCP(prog, "+"); // ... but the multi-parent guard keeps both binaries in CP + assertCP(prog, "*"); + } + + @Test + public void transitiveDisabledUnaryStaysCP() { + String dml = DML_HEADER + + "r = round(v);\n" + // pullable unary, but flag is off + "print(sum(r));\n"; + Program prog = compileWithTransitive(dml, false); + + assertSpark(prog, "uack+"); + assertCP(prog, "round"); // flag off keeps the unary in CP + } + + @Test + public void transitiveDisabledBinaryStaysCP() { + String dml = TALL_VECTOR_HEADER + + "r = c + 2.0;\n" + // pullable matrix-scalar, but flag is off + "print(as.scalar(r[2500,1]));\n"; + Program prog = compileWithTransitive(dml, false); + + assertSpark(prog, "uark+"); + assertCP(prog, "+"); // flag off keeps the binary in CP + } + + /** Compile with {@code ALLOW_TRANSITIVE_SPARK_EXEC_TYPE} forced to {@code enabled}, restoring it afterwards. */ + private Program compileWithTransitive(String dml, boolean enabled) { + final boolean old = OptimizerUtils.ALLOW_TRANSITIVE_SPARK_EXEC_TYPE; + OptimizerUtils.ALLOW_TRANSITIVE_SPARK_EXEC_TYPE = enabled; + try { + return compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); + } + finally { + OptimizerUtils.ALLOW_TRANSITIVE_SPARK_EXEC_TYPE = old; + } + } +} diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java index c1fb10d211a..a6ad0d4ee0d 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java +++ b/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java @@ -60,6 +60,7 @@ import org.apache.sysds.runtime.compress.estim.CompressedSizeInfo; import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup; import org.apache.sysds.runtime.compress.lib.CLALibCBind; +import org.apache.sysds.runtime.compress.lib.CLALibTSMM; import org.apache.sysds.runtime.functionobjects.Builtin; import org.apache.sysds.runtime.functionobjects.Builtin.BuiltinCode; import org.apache.sysds.runtime.functionobjects.Divide; @@ -503,6 +504,51 @@ public void testMatrixMultChain(ChainType ctype) { } } + @Test + public void testTransposeSelfLeftMultOverload() { + // Exercises the package-public CLALibTSMM.leftMultByTransposeSelf(cmb, k) entry point (used by the + // XtXv mm-chain fast path) across all compression configurations. + if(!(cmb instanceof CompressedMatrixBlock)) + return; + try { + MatrixBlock ret2 = CLALibTSMM.leftMultByTransposeSelf((CompressedMatrixBlock) cmb, _k); + MatrixBlock ucRet2 = mb.transposeSelfMatrixMultOperations(new MatrixBlock(), MMTSJType.LEFT, _k); + compareResultMatrices(ucRet2, ret2, overlappingType != OverLapping.NONE ? 256 : 2); + } + catch(Exception e) { + e.printStackTrace(); + throw new RuntimeException(bufferedToString + "\n" + e.getMessage(), e); + } + } + + @Test + public void testMatrixMultChainXtXvWide() { + // Widen the input beyond 30 columns so the XtXv fast path triggers, validating it against the + // uncompressed result for whatever compression the current configuration produces. + if(!(cmb instanceof CompressedMatrixBlock)) + return; + try { + final int nCol = mb.getNumColumns(); + final int reps = (int) Math.ceil(31.0 / nCol) + 1; + MatrixBlock wide = mb; + for(int i = 1; i < reps; i++) + wide = wide.append(mb, new MatrixBlock(), true); + + MatrixBlock wideC = CompressedMatrixBlockFactory.compress(wide, _k).getLeft(); + if(!(wideC instanceof CompressedMatrixBlock)) + return; // not compressible in this configuration + + MatrixBlock vector1 = TestUtils.generateTestMatrixBlock(wide.getNumColumns(), 1, 0.9, 1.5, 1.0, 3); + MatrixBlock ucRet2 = wide.chainMatrixMultOperations(vector1, null, new MatrixBlock(), ChainType.XtXv, _k); + MatrixBlock ret2 = wideC.chainMatrixMultOperations(vector1, null, new MatrixBlock(), ChainType.XtXv, _k); + compareResultMatricesPercentDistance(ucRet2, ret2, 0.99, 0.99); + } + catch(Exception e) { + e.printStackTrace(); + throw new RuntimeException(bufferedToString + "\n" + e.getMessage(), e); + } + } + @Test public void testVectorMatrixMult() { MatrixBlock vector = TestUtils.generateTestMatrixBlock(1, rows, 0, 5, 1.0, 3); diff --git a/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java new file mode 100644 index 00000000000..833128ad9f0 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.compress.lib; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.sysds.lops.MapMultChain.ChainType; +import org.apache.sysds.runtime.compress.CompressedMatrixBlock; +import org.apache.sysds.runtime.compress.colgroup.AColGroup; +import org.apache.sysds.runtime.compress.colgroup.ColGroupDDC; +import org.apache.sysds.runtime.compress.colgroup.ColGroupEmpty; +import org.apache.sysds.runtime.compress.colgroup.ColGroupUncompressed; +import org.apache.sysds.runtime.compress.colgroup.dictionary.Dictionary; +import org.apache.sysds.runtime.compress.colgroup.dictionary.IDictionary; +import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory; +import org.apache.sysds.runtime.compress.colgroup.mapping.AMapToData; +import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory; +import org.apache.sysds.runtime.compress.lib.CLALibTSMM; +import org.apache.sysds.lops.MMTSJ.MMTSJType; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.test.TestUtils; +import org.apache.sysds.test.component.compress.mapping.MappingTestUtil; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Targeted tests for the compressed transpose-self multiply ({@link CLALibTSMM}) and the XtXv mm-chain fast path that + * was added in {@code CLALibMMChain}. The fast path triggers when the input has fewer than five column groups and more + * than thirty columns, in which case the chain is computed as {@code (t(X) %*% X) %*% v}. + */ +public class CLALibMMChainTest { + protected static final Log LOG = LogFactory.getLog(CLALibMMChainTest.class.getName()); + + @BeforeClass + public static void setup() { + Thread.currentThread().setName("main_test_" + Thread.currentThread().getId()); + } + + /** + * Build a compressed matrix backed by a single DDC column group spanning all {@code nCol} columns. This guarantees a + * single (non-uncompressed) column group, which is what triggers the mm-chain fast path for wide enough matrices. + */ + private static CompressedMatrixBlock singleDDC(int nRow, int nCol, int nVal, int seed) { + Random r = new Random(seed); + double[] dictValues = new double[nVal * nCol]; + for(int i = 0; i < dictValues.length; i++) + dictValues[i] = Math.round(r.nextDouble() * 20 - 10); + IDictionary dict = Dictionary.create(dictValues); + AMapToData data = MappingTestUtil.createRandomMap(nRow, nVal, r); + AColGroup g = ColGroupDDC.create(ColIndexFactory.create(nCol), dict, data, null); + CompressedMatrixBlock cmb = new CompressedMatrixBlock(nRow, nCol); + cmb.allocateColGroup(g); + cmb.recomputeNonZeros(); + return cmb; + } + + private static CompressedMatrixBlock uncompressedGroup(int nRow, int nCol, int seed) { + MatrixBlock mb = TestUtils.round(TestUtils.generateTestMatrixBlock(nRow, nCol, -10, 10, 1.0, seed)); + CompressedMatrixBlock cmb = new CompressedMatrixBlock(nRow, nCol); + cmb.allocateColGroup(ColGroupUncompressed.create(mb)); + cmb.recomputeNonZeros(); + return cmb; + } + + private static CompressedMatrixBlock empty(int nRow, int nCol) { + CompressedMatrixBlock cmb = new CompressedMatrixBlock(nRow, nCol); + cmb.allocateColGroup(new ColGroupEmpty(ColIndexFactory.create(nCol))); + cmb.recomputeNonZeros(); + return cmb; + } + + @Test + public void tsmmWideSingleThread() { + execTSMM(singleDDC(200, 40, 6, 1), 1); + } + + @Test + public void tsmmWideParallel() { + execTSMM(singleDDC(200, 40, 6, 2), 4); + } + + @Test + public void tsmmNarrowSingleThread() { + execTSMM(singleDDC(200, 8, 4, 3), 1); + } + + @Test + public void tsmmNarrowParallel() { + execTSMM(singleDDC(200, 8, 4, 4), 4); + } + + @Test + public void tsmmUncompressedGroupSingleThread() { + // A compressed block holding an uncompressed column group must fall back to the dense tsmm path. + execTSMM(uncompressedGroup(150, 12, 5), 1); + } + + @Test + public void tsmmUncompressedGroupParallel() { + execTSMM(uncompressedGroup(150, 12, 6), 4); + } + + @Test + public void tsmmEmpty() { + CompressedMatrixBlock cmb = empty(100, 13); + MatrixBlock ret = CLALibTSMM.leftMultByTransposeSelf(cmb, 1); + assertEquals(13, ret.getNumRows()); + assertEquals(13, ret.getNumColumns()); + assertTrue("empty input must produce an empty result", ret.isEmptyBlock(false)); + } + + @Test + public void tsmmRetReused() { + // A non-null ret must be reset and reused, producing the same result as a fresh allocation. + CompressedMatrixBlock cmb = singleDDC(120, 36, 5, 7); + MatrixBlock preAllocated = new MatrixBlock(3, 3, 99.0); + preAllocated.allocateDenseBlock(); + MatrixBlock cRet = CLALibTSMM.leftMultByTransposeSelf(cmb, preAllocated, 4); + MatrixBlock uRet = CompressedMatrixBlock.getUncompressed(cmb) + .transposeSelfMatrixMultOperations(new MatrixBlock(), MMTSJType.LEFT, 4); + TestUtils.compareMatricesBitAvgDistance(uRet, cRet, 0, 0); + } + + @Test + public void tsmmRetNull() { + // Explicitly exercise the null-ret allocation branch of the helper. + CompressedMatrixBlock cmb = singleDDC(120, 36, 5, 8); + MatrixBlock cRet = CLALibTSMM.leftMultByTransposeSelf(cmb, null, 1); + MatrixBlock uRet = CompressedMatrixBlock.getUncompressed(cmb) + .transposeSelfMatrixMultOperations(new MatrixBlock(), MMTSJType.LEFT, 1); + TestUtils.compareMatricesBitAvgDistance(uRet, cRet, 0, 0); + } + + private static void execTSMM(CompressedMatrixBlock cmb, int k) { + try { + MatrixBlock cRet = CLALibTSMM.leftMultByTransposeSelf(cmb, k); + MatrixBlock uRet = CompressedMatrixBlock.getUncompressed(cmb) + .transposeSelfMatrixMultOperations(new MatrixBlock(), MMTSJType.LEFT, k); + assertEquals(cmb.getNumColumns(), cRet.getNumRows()); + assertEquals(cmb.getNumColumns(), cRet.getNumColumns()); + TestUtils.compareMatricesBitAvgDistance(uRet, cRet, 0, 0); + } + catch(Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void mmChainFastPathSingleThread() { + // 40 columns, single column group -> XtXv fast path. + execMMChain(singleDDC(200, 40, 6, 11), 1); + } + + @Test + public void mmChainFastPathParallel() { + execMMChain(singleDDC(200, 40, 6, 12), 4); + } + + @Test + public void mmChainFastPathFewGroups() { + // Two column groups (< 5) over 40 columns still triggers the fast path. + execMMChain(twoGroups(200, 40, 13), 4); + } + + @Test + public void mmChainRegularPathNarrow() { + // Only 20 columns -> below the width threshold, exercises the regular (non fast) chain path. + execMMChain(singleDDC(200, 20, 6, 14), 4); + } + + private static CompressedMatrixBlock twoGroups(int nRow, int nCol, int seed) { + final int half = nCol / 2; + Random r = new Random(seed); + List gs = new ArrayList<>(); + gs.add(ddcGroup(nRow, ColIndexFactory.create(0, half), 5, r)); + gs.add(ddcGroup(nRow, ColIndexFactory.create(half, nCol), 5, r)); + CompressedMatrixBlock cmb = new CompressedMatrixBlock(nRow, nCol); + cmb.allocateColGroupList(gs); + cmb.recomputeNonZeros(); + return cmb; + } + + private static AColGroup ddcGroup(int nRow, org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex cols, + int nVal, Random r) { + int nCol = cols.size(); + double[] dictValues = new double[nVal * nCol]; + for(int i = 0; i < dictValues.length; i++) + dictValues[i] = Math.round(r.nextDouble() * 20 - 10); + IDictionary dict = Dictionary.create(dictValues); + AMapToData data = MappingTestUtil.createRandomMap(nRow, nVal, r); + return ColGroupDDC.create(cols, dict, data, null); + } + + @Test + public void mmChainWideRecompressedDDC() { + // Mirrors the e2e CompressedTestBase#testMatrixMultChainXtXvWide flow: tile a narrow matrix until it + // exceeds the 30-column fast-path threshold, recompress it, then validate XtXv against uncompressed. + execMMChainWide(TestUtils.round(TestUtils.generateTestMatrixBlock(300, 4, -10, 10, 1.0, 21)), 1); + } + + @Test + public void mmChainWideRecompressedSparse() { + execMMChainWide(TestUtils.round(TestUtils.generateTestMatrixBlock(300, 3, 1, 5, 0.2, 22)), 4); + } + + private static void execMMChainWide(MatrixBlock base, int k) { + try { + final int nCol = base.getNumColumns(); + final int reps = (int) Math.ceil(31.0 / nCol) + 1; + MatrixBlock wide = base; + for(int i = 1; i < reps; i++) + wide = wide.append(base, new MatrixBlock(), true); + assertTrue("widened matrix must exceed the fast-path threshold", wide.getNumColumns() > 30); + + MatrixBlock wideC = CompressedMatrixBlockFactory.compress(wide, k).getLeft(); + assertTrue("tiled matrix should compress", wideC instanceof CompressedMatrixBlock); + + MatrixBlock v = TestUtils.generateTestMatrixBlock(wide.getNumColumns(), 1, 0.9, 1.5, 1.0, 3); + MatrixBlock uRet = wide.chainMatrixMultOperations(v, null, new MatrixBlock(), ChainType.XtXv, k); + MatrixBlock cRet = wideC.chainMatrixMultOperations(v, null, new MatrixBlock(), ChainType.XtXv, k); + TestUtils.compareMatrices(uRet, cRet, 1e-6, "wide recompressed mm-chain result mismatch"); + } + catch(Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private static void execMMChain(CompressedMatrixBlock cmb, int k) { + try { + final int cols = cmb.getNumColumns(); + MatrixBlock v = TestUtils.round(TestUtils.generateTestMatrixBlock(cols, 1, -3, 3, 1.0, 42)); + MatrixBlock uncompressed = CompressedMatrixBlock.getUncompressed(cmb); + + MatrixBlock cRet = cmb.chainMatrixMultOperations(v, null, new MatrixBlock(), ChainType.XtXv, k); + MatrixBlock uRet = uncompressed.chainMatrixMultOperations(v, null, new MatrixBlock(), ChainType.XtXv, k); + + assertEquals(cols, cRet.getNumRows()); + assertEquals(1, cRet.getNumColumns()); + TestUtils.compareMatrices(uRet, cRet, 1e-6, "mm-chain result mismatch"); + } + catch(Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibRightMultBySDCTest.java b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibRightMultBySDCTest.java new file mode 100644 index 00000000000..0aa4064b5a7 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibRightMultBySDCTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.compress.lib; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.sysds.runtime.compress.CompressedMatrixBlock; +import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory; +import org.apache.sysds.runtime.compress.colgroup.AColGroup; +import org.apache.sysds.runtime.compress.colgroup.ASDC; +import org.apache.sysds.runtime.compress.colgroup.ASDCZero; +import org.apache.sysds.runtime.compress.lib.CLALibRightMultBy; +import org.apache.sysds.runtime.matrix.data.LibMatrixMult; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.test.TestUtils; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Right matrix multiply on compressed inputs that contain SDC / SDC-zeros column groups. + * + *

+ * The PR stops forcing a decompressing right multiply for {@link ASDC} / {@link ASDCZero} backed inputs (they have + * working pre-aggregate paths). These tests build such inputs and verify the compressed right multiply still matches + * the uncompressed reference for both single-threaded and parallel execution. + *

+ */ +public class CLALibRightMultBySDCTest { + protected static final Log LOG = LogFactory.getLog(CLALibRightMultBySDCTest.class.getName()); + + @BeforeClass + public static void setup() { + Thread.currentThread().setName("main_test_" + Thread.currentThread().getId()); + } + + /** + * Build a compressed matrix dominated by a single value with a handful of exceptions per column, which compresses + * into SDC / SDC-zeros column groups. + */ + private static CompressedMatrixBlock sdcBlock(int rows, int cols, double sparsity, int seed) { + MatrixBlock mb = TestUtils.round(TestUtils.generateTestMatrixBlock(rows, cols, 1, 5, sparsity, seed)); + CompressedMatrixBlock cmb = (CompressedMatrixBlock) CompressedMatrixBlockFactory.compress(mb, 1).getLeft(); + return cmb; + } + + private static boolean containsSDC(CompressedMatrixBlock cmb) { + for(AColGroup g : cmb.getColGroups()) + if(g instanceof ASDC || g instanceof ASDCZero) + return true; + return false; + } + + @Test + public void rightMultVectorSparseSingleThread() { + execRightMult(sdcBlock(500, 6, 0.2, 21), 1, 1); + } + + @Test + public void rightMultVectorSparseParallel() { + execRightMult(sdcBlock(500, 6, 0.2, 22), 1, 4); + } + + @Test + public void rightMultMatrixSparseSingleThread() { + execRightMult(sdcBlock(500, 6, 0.2, 23), 4, 1); + } + + @Test + public void rightMultMatrixSparseParallel() { + execRightMult(sdcBlock(500, 6, 0.2, 24), 4, 4); + } + + @Test + public void rightMultWideSparseParallel() { + execRightMult(sdcBlock(500, 6, 0.2, 27), 12, 4); + } + + private static void execRightMult(CompressedMatrixBlock cmb, int rhsCols, int k) { + try { + assertTrue("test input should contain an SDC/SDCZeros column group", containsSDC(cmb)); + + final int cols = cmb.getNumColumns(); + MatrixBlock right = TestUtils.round(TestUtils.generateTestMatrixBlock(cols, rhsCols, -3, 3, 1.0, 99)); + MatrixBlock uncompressed = CompressedMatrixBlock.getUncompressed(cmb); + + MatrixBlock cRet = CLALibRightMultBy.rightMultByMatrix(cmb, right, null, k); + MatrixBlock uRet = LibMatrixMult.matrixMult(uncompressed, right, k); + + TestUtils.compareMatricesBitAvgDistance(uRet, CompressedMatrixBlock.getUncompressed(cRet), 1024, 1); + } + catch(Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java new file mode 100644 index 00000000000..ccba674707b --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.frame.transform; + +import static org.junit.Assert.fail; + +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.transform.decode.Decoder; +import org.apache.sysds.runtime.transform.decode.DecoderComposite; +import org.apache.sysds.runtime.transform.decode.DecoderFactory; +import org.apache.sysds.runtime.transform.encode.EncoderFactory; +import org.apache.sysds.runtime.transform.encode.MultiColumnEncoder; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; + +/** + * Tests for the multi-threaded {@link DecoderComposite#decode(MatrixBlock, FrameBlock, int)} path. + * + *

+ * The parallel decode partitions over row blocks and runs all sub decoders in order within each block. This is + * important for the dummycode+recode case: the recode-on-output decoder reads the category indexes written by the + * preceding dummycode decoder, so running them out of order produces wrong (or null) values. These tests verify the + * parallel result equals the single-threaded result and reconstructs the original frame, and they also exercise the + * {@code k <= 1} short-circuit to the sequential path. + *

+ */ +public class DecoderCompositeTest { + protected static final Log LOG = LogFactory.getLog(DecoderCompositeTest.class.getName()); + + /** Enough rows that the parallel path forms multiple row blocks (block size is max(rows/k, 1000)). */ + private static final int ROWS = 8000; + + private static FrameBlock categoricalFrame(int rows, int nCol, int nCat, int seed) { + ValueType[] schema = new ValueType[nCol]; + for(int c = 0; c < nCol; c++) + schema[c] = ValueType.STRING; + String[][] data = new String[rows][nCol]; + Random r = new Random(seed); + for(int i = 0; i < rows; i++) + for(int c = 0; c < nCol; c++) + data[i][c] = "v" + r.nextInt(nCat); + return new FrameBlock(schema, data); + } + + private static Decoder buildDecoder(FrameBlock data, String spec, MultiColumnEncoder encoder) { + FrameBlock meta = encoder.getMetaData(new FrameBlock(data.getNumColumns(), ValueType.STRING)); + return DecoderFactory.createDecoder(spec, data.getColumnNames(), data.getSchema(), meta); + } + + private void runDecode(String spec, int nCol, int nCat) { + try { + FrameBlock data = categoricalFrame(ROWS, nCol, nCat, 17); + + MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), + data.getNumColumns(), null); + MatrixBlock encoded = encoder.encode(data, 1); + + Decoder decoder = buildDecoder(data, spec, encoder); + + FrameBlock single = decoder.decode(encoded, new FrameBlock(decoder.getSchema()), 1); + FrameBlock parallel = decoder.decode(encoded, new FrameBlock(decoder.getSchema()), 4); + + // Parallel decode must match the single-threaded decode exactly. + TestUtils.compareFrames(single, parallel, false); + // And both must reconstruct the original categorical values. + TestUtils.compareFrames(data, parallel, false); + } + catch(Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void recodeOnly() { + runDecode("{recode:[C1,C2,C3]}", 3, 6); + } + + @Test + public void dummycodeAndRecode() { + // dummycode implies recode-on-output: the composite decoder is [Dummycode, Recode-on-output] + // and the recode step depends on the indexes the dummycode step writes. This is exactly the + // ordering the parallel fix protects against breaking. + runDecode("{dummycode:[C1,C2,C3]}", 3, 5); + } + + @Test + public void dummycodeAndRecodeSameColumns() { + // recode and dummycode listed on the same columns -> recoded then dummycoded, decoded in order. + runDecode("{recode:[C1,C2], dummycode:[C1,C2]}", 2, 4); + } + + @Test + public void singleThreadEqualsParallelManyCategories() { + runDecode("{dummycode:[C1,C2]}", 2, 25); + } + + @Test + public void decoderIsComposite() { + FrameBlock data = categoricalFrame(100, 2, 3, 1); + String spec = "{recode:[C1], dummycode:[C2]}"; + MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), + data.getNumColumns(), null); + encoder.encode(data, 1); + Decoder decoder = buildDecoder(data, spec, encoder); + if(!(decoder instanceof DecoderComposite)) + fail("expected a DecoderComposite but got " + decoder.getClass().getSimpleName()); + } +} diff --git a/src/test/java/org/apache/sysds/test/functions/sparkexectype/SparkTransitiveExecTypeTest.java b/src/test/java/org/apache/sysds/test/functions/sparkexectype/SparkTransitiveExecTypeTest.java new file mode 100644 index 00000000000..04e1e2e0be6 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/sparkexectype/SparkTransitiveExecTypeTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.sparkexectype; + +import java.util.HashMap; + +import org.apache.sysds.common.Types.ExecMode; +import org.apache.sysds.hops.OptimizerUtils; +import org.apache.sysds.hops.recompile.Recompiler; +import org.apache.sysds.runtime.matrix.data.MatrixValue.CellIndex; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.apache.sysds.utils.Statistics; +import org.apache.sysds.utils.stats.InfrastructureAnalyzer; +import org.junit.Assert; +import org.junit.Test; + +/** + * Exercises the transitive Spark exec-type refinement in {@link org.apache.sysds.hops.UnaryOp} and + * {@link org.apache.sysds.hops.BinaryOp}: cheap unary / matrix-scalar / matrix-vector operations whose input already + * has a Spark output are pulled into Spark. + * + *

+ * Each script is run in HYBRID mode with a constrained memory budget, once with the transitive decision enabled and + * once disabled. The results must match (correctness regardless of placement), and the transitive run must actually + * execute Spark instructions. + *

+ */ +public class SparkTransitiveExecTypeTest extends AutomatedTestBase { + + private static final String TEST_DIR = "functions/sparkexectype/"; + private static final String TEST_CLASS_DIR = TEST_DIR + SparkTransitiveExecTypeTest.class.getSimpleName() + "/"; + private static final String TEST_UNARY = "SparkExecTypeUnary"; + private static final String TEST_BINARY = "SparkExecTypeBinary"; + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_UNARY, new TestConfiguration(TEST_CLASS_DIR, TEST_UNARY, new String[] {"R"})); + addTestConfiguration(TEST_BINARY, new TestConfiguration(TEST_CLASS_DIR, TEST_BINARY, new String[] {"R"})); + } + + @Test + public void testUnaryPulledIntoSpark() { + runTransitiveExecTypeTest(TEST_UNARY); + } + + @Test + public void testBinaryPulledIntoSpark() { + runTransitiveExecTypeTest(TEST_BINARY); + } + + private void runTransitiveExecTypeTest(String testname) { + final boolean oldTransitive = OptimizerUtils.ALLOW_TRANSITIVE_SPARK_EXEC_TYPE; + final ExecMode oldPlatform = setExecMode(ExecMode.HYBRID); + final long oldMem = InfrastructureAnalyzer.getLocalMaxMemory(); + // Small memory budget so the large operations are placed on Spark. + InfrastructureAnalyzer.setLocalMaxMemory(1024 * 1024 * 8); + + try { + getAndLoadTestConfiguration(testname); + fullDMLScriptName = getScript(); + programArgs = new String[] {"-args", output("R")}; + + // Reference run with the transitive Spark decision disabled. + OptimizerUtils.ALLOW_TRANSITIVE_SPARK_EXEC_TYPE = false; + runTest(true, false, null, -1); + HashMap expected = readDMLScalarFromOutputDir("R"); + + // Run with the transitive Spark decision enabled (the path under test). + OptimizerUtils.ALLOW_TRANSITIVE_SPARK_EXEC_TYPE = true; + runTest(true, false, null, -1); + HashMap actual = readDMLScalarFromOutputDir("R"); + + TestUtils.compareScalars(expected.get(new CellIndex(1, 1)), actual.get(new CellIndex(1, 1)), 1e-8); + Assert.assertTrue("Expected Spark instructions to be executed in the transitive run.", + Statistics.getNoOfExecutedSPInst() > 0); + } + finally { + OptimizerUtils.ALLOW_TRANSITIVE_SPARK_EXEC_TYPE = oldTransitive; + resetExecMode(oldPlatform); + InfrastructureAnalyzer.setLocalMaxMemory(oldMem); + Recompiler.reinitRecompiler(); + } + } +} diff --git a/src/test/scripts/functions/sparkexectype/SparkExecTypeBinary.dml b/src/test/scripts/functions/sparkexectype/SparkExecTypeBinary.dml new file mode 100644 index 00000000000..b15391d5c60 --- /dev/null +++ b/src/test/scripts/functions/sparkexectype/SparkExecTypeBinary.dml @@ -0,0 +1,33 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# Binary operations where exactly one operand is a scalar or a small vector and +# the other operand has a Spark output. These should be pulled into Spark by the +# transitive exec-type decision (matrix-scalar / matrix-vector broadcasting). +X = rand(rows=10000, cols=200, seed=42); +v = rand(rows=1, cols=200, seed=7); # small row vector (below block size) +c = rand(rows=10000, cols=1, seed=9); # tall column vector + +sp1 = X * 2.0; # matrix-scalar, spark input +sp2 = sp1 + v; # matrix + small row vector, spark input +sp3 = sp2 - c; # matrix - column vector, spark input +R = sum(sp3); +write(R, $1, format="text"); diff --git a/src/test/scripts/functions/sparkexectype/SparkExecTypeUnary.dml b/src/test/scripts/functions/sparkexectype/SparkExecTypeUnary.dml new file mode 100644 index 00000000000..7c2c6e3c03d --- /dev/null +++ b/src/test/scripts/functions/sparkexectype/SparkExecTypeUnary.dml @@ -0,0 +1,31 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# Large input forces a Spark-resident output. The following unary operations +# are individually cheap but their input already has a Spark output, so the +# transitive exec-type decision should pull them into Spark. +X = rand(rows=10000, cols=200, seed=42); +sp1 = X + ceil(X); # spark transformation -> spark output +sp2 = round(sp1); # unary on spark input +sp3 = abs(sp2); # unary on spark input +sp4 = sp3 * 2.0; # binary matrix-scalar on spark input +R = sum(sp4); +write(R, $1, format="text"); From c2ae705e8a8c0adf356969c0ec49eb3fbf0754b7 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Wed, 24 Jun 2026 00:57:29 +0200 Subject: [PATCH 042/132] [BWARE] Handle hash columns in transform decoders and tighten decode metadata (#2479) Reworks the transform decoders so feature-hashed columns survive the inverse-transform path and decode metadata is initialized consistently. The dummycode hash domain size K is now recovered from the meta cell instead of numDistinct, and each decoder builds its output->source column mapping in initMetaData via a shared helper that accounts for dummycode expansion. DecoderRecode skips hash columns (their bucket codes pass through unchanged), applies recode-on-output when dummycoding is present, and reports a bounded error on malformed recode entries. The parallel row-block decode is hoisted into the base Decoder so all decoders share it (DecoderComposite drops its own override); DecoderDummycode gains separate dense/sparse paths; DecoderBin serialization is fixed to persist the column mappings and the dead _numBins field is removed; DecoderFactory routes the hash/dummycode/bin/passthrough column sets accordingly. StringArray.getAsDouble also gets a behavior-equivalent boolean-token fast path (drive-by perf cleanup). Testing: a new TransformDecodeRoundTripTest covers exact-inverse round trips (dense/sparse/parallel), hash + dummycode + bin combinations, federated sub-range decoding, and error/edge paths (corrupt recode meta, parallel-decode interruption, base sub-range rejection); TransformDecodeTest adds dense/sparse hash + dummycode consistency cases. --- .../frame/data/columns/StringArray.java | 18 +- .../runtime/transform/decode/Decoder.java | 90 ++- .../runtime/transform/decode/DecoderBin.java | 51 +- .../transform/decode/DecoderComposite.java | 33 - .../transform/decode/DecoderDummycode.java | 114 ++-- .../transform/decode/DecoderFactory.java | 43 +- .../transform/decode/DecoderPassThrough.java | 45 +- .../transform/decode/DecoderRecode.java | 38 +- .../encode/ColumnEncoderFeatureHash.java | 2 + .../frame/array/CustomArrayTests.java | 25 + .../TransformDecodeRoundTripTest.java | 590 ++++++++++++++++++ .../frame/transform/TransformDecodeTest.java | 186 ++++++ 12 files changed, 1084 insertions(+), 151 deletions(-) create mode 100644 src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java create mode 100644 src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/StringArray.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/StringArray.java index 1fc582924e4..1541f16c96d 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/StringArray.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/StringArray.java @@ -607,17 +607,23 @@ public double getAsNaNDouble(int i) { private static double getAsDouble(String s) { try { - return DoubleArray.parseDouble(s); } catch(Exception e) { - String ls = s.toLowerCase(); - if(ls.equals("true") || ls.equals("t")) + // fallback for boolean-like tokens, without allocating a lower-cased copy + final int len = s.length(); + if(len == 1) { + final char c = s.charAt(0); + if(c == 't' || c == 'T') + return 1; + else if(c == 'f' || c == 'F') + return 0; + } + else if(len == 4 && s.compareToIgnoreCase("true") == 0) return 1; - else if(ls.equals("false") || ls.equals("f")) + else if(len == 5 && s.compareToIgnoreCase("false") == 0) return 0; - else - throw new DMLRuntimeException("Unable to change to double: " + s, e); + throw new DMLRuntimeException("Unable to change to double: " + s, e); } } diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java b/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java index 724af1be630..1f731fc3aa5 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java @@ -23,13 +23,22 @@ import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.sysds.common.Types.ValueType; import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.frame.data.columns.ColumnMetadata; import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.util.CommonThreadPool; +import org.apache.sysds.runtime.util.UtilFunctions; /** * Base class for all transform decoders providing both a row and block @@ -43,11 +52,61 @@ public abstract class Decoder implements Externalizable{ protected ValueType[] _schema; protected int[] _colList; protected String[] _colnames = null; + // dummycoded columns that were feature-hashed: domain size K is read from the meta cell, not + // numDistinct. Only used during initMetaData (driver side), so not serialized. + protected transient int[] _dcHashCols = null; + protected Decoder(ValueType[] schema, int[] colList) { _schema = schema; _colList = colList; } + protected boolean isHashCol(int colID) { + return ArrayUtils.contains(_dcHashCols, colID); + } + + /** + * Domain size of a dummycoded source column: the hash domain K from the meta cell for + * feature-hashed columns, otherwise the column's {@code numDistinct} (0 when unset). + * + * @param meta transform meta frame + * @param colID 1-based column id of the dummycoded source column + * @param isHash whether the column was feature-hashed + * @return the domain size, never negative + */ + protected static int getNumDummycodeDistinct(FrameBlock meta, int colID, boolean isHash) { + if(isHash) { + Object o = meta.get(0, colID - 1); + return (o == null) ? 0 : (int) UtilFunctions.parseToLong(o.toString()); + } + ColumnMetadata d = meta.getColumnMetadata()[colID - 1]; + int ndist = d.isDefault() ? 0 : (int) d.getNumDistinct(); + return Math.max(ndist, 0); + } + + /** + * Maps output column ids ({@code _colList}) to source positions in the encoded matrix, shifting past the column + * expansion of any dummycoded columns that precede them. Returns {@code _colList} directly when none apply. + */ + protected int[] buildSrcCols(FrameBlock meta, int[] dcCols) { + if(dcCols == null || dcCols.length == 0) + return _colList; + int[] srcCols = new int[_colList.length]; + int ix1 = 0, ix2 = 0, off = 0; + while(ix1 < _colList.length) { + if(ix2 >= dcCols.length || _colList[ix1] < dcCols[ix2]) { + srcCols[ix1] = _colList[ix1] + off; + ix1++; + } + else { // skip past the dummycode expansion + int dcCol = dcCols[ix2]; + off += getNumDummycodeDistinct(meta, dcCol, isHashCol(dcCol)) - 1; + ix2++; + } + } + return srcCols; + } + public ValueType[] getSchema() { return _schema; } @@ -77,8 +136,35 @@ public String[] getColnames() { * @param k Parallelization degree * @return returns the given output frame block for convenience */ - public FrameBlock decode(MatrixBlock in, FrameBlock out, int k) { - return decode(in, out); + public FrameBlock decode(final MatrixBlock in, final FrameBlock out, final int k) { + if(k <= 1) + return decode(in, out); + final ExecutorService pool = CommonThreadPool.get(k); + out.ensureAllocatedColumns(in.getNumRows()); + try { + final List> tasks = new ArrayList<>(); + int blz = Math.max((in.getNumRows() + k) / k, 1000); + + for(int i = 0; i < in.getNumRows(); i += blz){ + final int start = i; + final int end = Math.min(in.getNumRows(), i + blz); + tasks.add(pool.submit(() -> decode(in, out, start, end))); + } + + for(Future f : tasks) + f.get(); + return out; + } + catch(InterruptedException e) { + Thread.currentThread().interrupt(); + throw new DMLRuntimeException("Parallel decode interrupted", e); + } + catch(ExecutionException e) { + throw new DMLRuntimeException("Parallel decode failed", e); + } + finally { + pool.shutdown(); + } } /** diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java index edee095f612..a286c03dce8 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java @@ -41,8 +41,9 @@ public class DecoderBin extends Decoder { private static final long serialVersionUID = -3784249774608228805L; - // a) column bin boundaries - private int[] _numBins; + // dummycoded source columns and the resulting output->source column mapping + private int[] _dcCols = null; + private int[] _srcCols = null; private double[][] _binMins = null; private double[][] _binMaxs = null; @@ -50,8 +51,10 @@ public DecoderBin() { super(null, null); } - protected DecoderBin(ValueType[] schema, int[] binCols) { + protected DecoderBin(ValueType[] schema, int[] binCols, int[] dcCols, int[] hashCols) { super(schema, binCols); + _dcCols = dcCols; + _dcHashCols = hashCols; } @Override @@ -66,14 +69,19 @@ public void decode(MatrixBlock in, FrameBlock out, int rl, int ru) { for( int i=rl; i< ru; i++ ) { for( int j=0; j<_colList.length; j++ ) { final Array a = out.getColumn(_colList[j] - 1); - final double val = in.get(i, _colList[j] - 1); + final double val = in.get(i, _srcCols[j] - 1); if(!Double.isNaN(val)){ final int key = (int) Math.round(val); - double bmin = _binMins[j][key - 1]; - double bmax = _binMaxs[j][key - 1]; - double oval = bmin + (bmax - bmin) / 2 // bin center - + (val - key) * (bmax - bmin); // bin fractions - a.set(i, oval); + if(key == 0){ + a.set(i, _binMins[j][key]); + } + else{ + double bmin = _binMins[j][key - 1]; + double bmax = _binMaxs[j][key - 1]; + double oval = bmin + (bmax - bmin) / 2 // bin center + + (val - key) * (bmax - bmin); // bin fractions + a.set(i, oval); + } } else a.set(i, val); // NaN @@ -90,7 +98,6 @@ public Decoder subRangeDecoder(int colStart, int colEnd, int dummycodedOffset) { @Override public void initMetaData(FrameBlock meta) { //initialize bin boundaries - _numBins = new int[_colList.length]; _binMins = new double[_colList.length][]; _binMaxs = new double[_colList.length][]; @@ -111,34 +118,52 @@ public void initMetaData(FrameBlock meta) { _binMaxs[j][i] = Double.parseDouble(parts[1]); } } + + _srcCols = buildSrcCols(meta, _dcCols); } @Override public void writeExternal(ObjectOutput out) throws IOException { super.writeExternal(out); + // bin boundaries; the per-column bin count is the length of the boundary arrays for( int i=0; i<_colList.length; i++ ) { - int len = _numBins[i]; + int len = _binMins[i].length; out.writeInt(len); for(int j=0; j> tasks = new ArrayList<>(); - int blz = Math.max(in.getNumRows() / k, 1000); - // Parallelize over row blocks (not over decoders): all decoders must - // run in order within a block, e.g. recode-on-output depends on the - // category indexes produced by the preceding dummycode decoder. - for(int i = 0; i < in.getNumRows(); i += blz){ - final int start = i; - final int end = Math.min(in.getNumRows(), i + blz); - tasks.add(pool.submit(() -> decode(in, out, start, end))); - } - for(Future f : tasks) - f.get(); - return out; - } - catch(Exception e) { - throw new RuntimeException(e); - } - finally { - pool.shutdown(); - } - } - @Override public void decode(MatrixBlock in, FrameBlock out, int rl, int ru){ for( Decoder decoder : _decoders ) diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java index 0c4c6b42690..ee1a33c49fd 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java @@ -27,31 +27,33 @@ import java.util.List; import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.frame.data.FrameBlock; -import org.apache.sysds.runtime.frame.data.columns.ColumnMetadata; import org.apache.sysds.runtime.matrix.data.MatrixBlock; -import org.apache.sysds.runtime.util.UtilFunctions; /** - * Simple atomic decoder for dummycoded columns. This decoder builds internally - * inverted column mappings from the given frame meta data. - * + * Simple atomic decoder for dummycoded columns. This decoder builds internally inverted column mappings from the given + * frame meta data. + * */ -public class DecoderDummycode extends Decoder -{ +public class DecoderDummycode extends Decoder { private static final long serialVersionUID = 4758831042891032129L; - + private int[] _clPos = null; private int[] _cuPos = null; - + protected DecoderDummycode(ValueType[] schema, int[] dcCols) { - //dcCols refers to column IDs in output (non-dc) + this(schema, dcCols, null); + } + + protected DecoderDummycode(ValueType[] schema, int[] dcCols, int[] hashCols) { + // dcCols refers to column IDs in output (non-dc) super(schema, dcCols); + _dcHashCols = hashCols; } @Override public FrameBlock decode(MatrixBlock in, FrameBlock out) { - //TODO perf (exploit sparse representation for better asymptotic behavior) out.ensureAllocatedColumns(in.getNumRows()); decode(in, out, 0, in.getNumRows()); return out; @@ -59,59 +61,97 @@ public FrameBlock decode(MatrixBlock in, FrameBlock out) { @Override public void decode(MatrixBlock in, FrameBlock out, int rl, int ru) { - //TODO perf (exploit sparse representation for better asymptotic behavior) - // out.ensureAllocatedColumns(in.getNumRows()); - for( int i=rl; i= low && aix[h] < high) { + int k = aix[h]; + int col = _colList[j] - 1; + out.getColumn(col).set(i, k - low + 1); + } + // limit the binary search. + apos = h; + } + + } + @Override public Decoder subRangeDecoder(int colStart, int colEnd, int dummycodedOffset) { List dcList = new ArrayList<>(); List clPosList = new ArrayList<>(); List cuPosList = new ArrayList<>(); - + // get the column IDs for the sub range of the dummycode columns and their destination positions, // where they will be decoded to - for( int j=0; j<_colList.length; j++ ) { + for(int j = 0; j < _colList.length; j++) { int colID = _colList[j]; - if (colID >= colStart && colID < colEnd) { + if(colID >= colStart && colID < colEnd) { dcList.add(colID - (colStart - 1)); clPosList.add(_clPos[j] - dummycodedOffset); cuPosList.add(_cuPos[j] - dummycodedOffset); } } - if (dcList.isEmpty()) + if(dcList.isEmpty()) return null; // create sub-range decoder int[] colList = dcList.stream().mapToInt(i -> i).toArray(); - DecoderDummycode subRangeDecoder = new DecoderDummycode( - Arrays.copyOfRange(_schema, colStart - 1, colEnd - 1), colList); + DecoderDummycode subRangeDecoder = new DecoderDummycode(Arrays.copyOfRange(_schema, colStart - 1, colEnd - 1), + colList); subRangeDecoder._clPos = clPosList.stream().mapToInt(i -> i).toArray(); subRangeDecoder._cuPos = cuPosList.stream().mapToInt(i -> i).toArray(); return subRangeDecoder; } - + @Override public void updateIndexRanges(long[] beginDims, long[] endDims) { if(_colList == null) return; - + long lowerColDest = beginDims[1]; long upperColDest = endDims[1]; for(int i = 0; i < _colList.length; i++) { long numDistinct = _cuPos[i] - _clPos[i]; - + if(_cuPos[i] <= beginDims[1] + 1) if(numDistinct > 0) lowerColDest -= numDistinct - 1; - + if(_cuPos[i] <= endDims[1] + 1) if(numDistinct > 0) upperColDest -= numDistinct - 1; @@ -119,16 +159,16 @@ public void updateIndexRanges(long[] beginDims, long[] endDims) { beginDims[1] = lowerColDest; endDims[1] = upperColDest; } - + @Override public void initMetaData(FrameBlock meta) { - _clPos = new int[_colList.length]; //col lower pos - _cuPos = new int[_colList.length]; //col upper pos - for( int j=0, off=0; j<_colList.length; j++ ) { + _clPos = new int[_colList.length]; // col lower pos + _cuPos = new int[_colList.length]; // col upper pos + for(int j = 0, off = 0; j < _colList.length; j++) { int colID = _colList[j]; - ColumnMetadata d = meta.getColumnMetadata()[colID-1]; - int ndist = d.isDefault() ? 0 : (int)d.getNumDistinct(); - ndist = ndist < -1 ? 0: ndist; + // hash columns store the domain size K in the meta cell; others use numDistinct + int ndist = getNumDummycodeDistinct(meta, colID, isHashCol(colID)); + _clPos[j] = off + colID; _cuPos[j] = _clPos[j] + ndist; off += ndist - 1; diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java index 0a400e6da92..8f6c45d63e8 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java @@ -64,41 +64,67 @@ public static Decoder createDecoder(String spec, String[] colnames, ValueType[] try { //parse transform specification JSONObject jSpec = new JSONObject(spec); - List ldecoders = new ArrayList<>(); - //create decoders 'bin', 'recode', 'dummy' and 'pass-through' + //create decoders 'bin', 'recode', 'hash', 'dummy', and 'pass-through' List binIDs = TfMetaUtils.parseBinningColIDs(jSpec, colnames, minCol, maxCol); List rcIDs = Arrays.asList(ArrayUtils.toObject( TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.RECODE.toString(), minCol, maxCol))); + List hcIDs = Arrays.asList(ArrayUtils.toObject( + TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.HASH.toString(), minCol, maxCol))); List dcIDs = Arrays.asList(ArrayUtils.toObject( TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.DUMMYCODE.toString(), minCol, maxCol))); + // only specially treat the columns with both recode and dictionary rcIDs = unionDistinct(rcIDs, dcIDs); + // hashing is a lossy, one-way transform with no inverse recode map, so hash columns + // are never recode-decoded; exclude them from the recode set + rcIDs = except(rcIDs, hcIDs); + + // dummycoded hash columns: domain size K lives in the meta cell, so the decoders + // need to know which dummycoded columns to read it from + List hcdcIDs = new ArrayList<>(dcIDs); + hcdcIDs.retainAll(hcIDs); + int[] hashCols = ArrayUtils.toPrimitive(hcdcIDs.toArray(new Integer[0])); + int len = dcIDs.isEmpty() ? Math.min(meta.getNumColumns(), clen) : meta.getNumColumns(); - List ptIDs = except(except(UtilFunctions.getSeqList(1, len, 1), rcIDs), binIDs); - + + // set the remaining columns to passthrough. + List ptIDs = UtilFunctions.getSeqList(1, len, 1); + // except recoded columns + ptIDs = except(ptIDs, rcIDs); + // binned columns + ptIDs = except(ptIDs, binIDs); + // dummycoded columns (incl. dummycoded hash) are rebuilt by the dummycode decoder; + // hash columns without dummycode stay in passthrough so their bucket code survives + ptIDs = except(ptIDs, dcIDs); + //create default schema if unspecified (with double columns for pass-through) if( schema == null ) { schema = UtilFunctions.nCopies(len, ValueType.STRING); for( Integer col : ptIDs ) schema[col-1] = ValueType.FP64; } + + // collect all the decoders in one list. + List ldecoders = new ArrayList<>(); if( !binIDs.isEmpty() ) { ldecoders.add(new DecoderBin(schema, - ArrayUtils.toPrimitive(binIDs.toArray(new Integer[0])))); + ArrayUtils.toPrimitive(binIDs.toArray(new Integer[0])), + ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); } if( !dcIDs.isEmpty() ) { ldecoders.add(new DecoderDummycode(schema, - ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])))); + ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); } if( !rcIDs.isEmpty() ) { + // recode on output (after dummycode rebuilds the categorical columns) when dummycoding is present ldecoders.add(new DecoderRecode(schema, !dcIDs.isEmpty(), ArrayUtils.toPrimitive(rcIDs.toArray(new Integer[0])))); } if( !ptIDs.isEmpty() ) { ldecoders.add(new DecoderPassThrough(schema, ArrayUtils.toPrimitive(ptIDs.toArray(new Integer[0])), - ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])))); + ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); } //create composite decoder of all created decoders @@ -121,6 +147,8 @@ else if( decoder instanceof DecoderRecode ) return DecoderType.Recode.ordinal(); else if( decoder instanceof DecoderPassThrough ) return DecoderType.PassThrough.ordinal(); + else if( decoder instanceof DecoderBin ) + return DecoderType.Bin.ordinal(); throw new DMLRuntimeException("Unsupported decoder type: " + decoder.getClass().getCanonicalName()); } @@ -130,6 +158,7 @@ public static Decoder createInstance(int type) { // create instance switch(dtype) { + case Bin: return new DecoderBin(); case Dummycode: return new DecoderDummycode(null, null); case PassThrough: return new DecoderPassThrough(null, null, null); case Recode: return new DecoderRecode(null, false, null); diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderPassThrough.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderPassThrough.java index 5b6bf7a093e..9b134601419 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderPassThrough.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderPassThrough.java @@ -28,9 +28,7 @@ import org.apache.sysds.common.Types.ValueType; import org.apache.sysds.runtime.frame.data.FrameBlock; -import org.apache.sysds.runtime.frame.data.columns.ColumnMetadata; import org.apache.sysds.runtime.matrix.data.MatrixBlock; -import org.apache.sysds.runtime.util.UtilFunctions; /** * Simple atomic decoder for passing through numeric columns to the output. @@ -45,8 +43,13 @@ public class DecoderPassThrough extends Decoder private int[] _srcCols = null; protected DecoderPassThrough(ValueType[] schema, int[] ptCols, int[] dcCols) { + this(schema, ptCols, dcCols, null); + } + + protected DecoderPassThrough(ValueType[] schema, int[] ptCols, int[] dcCols, int[] hashCols) { super(schema, ptCols); _dcCols = dcCols; + _dcHashCols = hashCols; } public DecoderPassThrough() { super(null, null); } @@ -61,13 +64,12 @@ public FrameBlock decode(MatrixBlock in, FrameBlock out) { @Override public void decode(MatrixBlock in, FrameBlock out, int rl, int ru) { int clen = Math.min(_colList.length, out.getNumColumns()); - for( int i=rl; i 0 ) { - //prepare source column id mapping w/ dummy coding - _srcCols = new int[_colList.length]; - int ix1 = 0, ix2 = 0, off = 0; - while( ix1<_colList.length ) { - if( ix2>=_dcCols.length || _colList[ix1] < _dcCols[ix2] ) { - _srcCols[ix1] = _colList[ix1] + off; - ix1 ++; - } - else { //_colList[ix1] > _dcCols[ix2] - ColumnMetadata d =meta.getColumnMetadata()[_dcCols[ix2]-1]; - off += d.isDefault() ? -1 : d.getNumDistinct() - 1; - ix2 ++; - } - } - } - else { - //prepare direct source column mapping - _srcCols = _colList; - } + _srcCols = buildSrcCols(meta, _dcCols); } @Override @@ -134,8 +117,8 @@ public void writeExternal(ObjectOutput os) for(int i = 0; i < _srcCols.length; i++) os.writeInt(_srcCols[i]); - os.writeInt(_dcCols.length); - for(int i = 0; i < _dcCols.length; i++) + os.writeInt(_dcCols == null ? 0 : _dcCols.length); + for(int i = 0; _dcCols != null && i < _dcCols.length; i++) os.writeInt(_dcCols[i]); } diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java index 33459a1c4f9..11dd2c7faa5 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java @@ -29,6 +29,7 @@ import java.util.Map.Entry; import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.frame.data.FrameBlock; import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.runtime.matrix.data.Pair; @@ -46,7 +47,6 @@ public class DecoderRecode extends Decoder private static final long serialVersionUID = -3784249774608228805L; private HashMap[] _rcMaps = null; - private Object[][] _rcMapsDirect = null; private boolean _onOut = false; public DecoderRecode() { @@ -59,8 +59,7 @@ protected DecoderRecode(ValueType[] schema, boolean onOut, int[] rcCols) { } public Object getRcMapValue(int i, long key) { - return (_rcMapsDirect != null && key > 0) ? - _rcMapsDirect[i][(int)key-1] : _rcMaps[i].get(key); + return _rcMaps[i].get(key); } @Override @@ -125,31 +124,26 @@ public Decoder subRangeDecoder(int colStart, int colEnd, int dummycodedOffset) { public void initMetaData(FrameBlock meta) { //initialize recode maps according to schema _rcMaps = new HashMap[_colList.length]; - long[] max = new long[_colList.length]; for( int j=0; j<_colList.length; j++ ) { HashMap map = new HashMap<>(); for( int i=0; i v < Integer.MAX_VALUE) ) { - _rcMapsDirect = new Object[_rcMaps.length][]; - for( int i=0; i<_rcMaps.length; i++ ) { - Object[] arr = new Object[(int)max[i]]; - for(Entry e1 : _rcMaps[i].entrySet()) - arr[e1.getKey().intValue()-1] = e1.getValue(); - _rcMapsDirect[i] = arr; - } - } } /** diff --git a/src/main/java/org/apache/sysds/runtime/transform/encode/ColumnEncoderFeatureHash.java b/src/main/java/org/apache/sysds/runtime/transform/encode/ColumnEncoderFeatureHash.java index 400b7f64ffc..cd9a583d60f 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/encode/ColumnEncoderFeatureHash.java +++ b/src/main/java/org/apache/sysds/runtime/transform/encode/ColumnEncoderFeatureHash.java @@ -146,7 +146,9 @@ public FrameBlock getMetaData(FrameBlock meta) { return meta; meta.ensureAllocatedColumns(1); + // store the hash domain size K in the single meta cell meta.set(0, _colID - 1, String.valueOf(_K)); + return meta; } diff --git a/src/test/java/org/apache/sysds/test/component/frame/array/CustomArrayTests.java b/src/test/java/org/apache/sysds/test/component/frame/array/CustomArrayTests.java index df386d4659d..5119cbadd65 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/array/CustomArrayTests.java +++ b/src/test/java/org/apache/sysds/test/component/frame/array/CustomArrayTests.java @@ -2859,4 +2859,29 @@ public void stringArrayGetDoubleNaN(){ assertTrue(Double.isNaN(s.getAsNaNDouble(i))); } } + + @Test + public void stringArrayGetDoubleBooleanTokens() { + // non-numeric boolean-like tokens fall back to 1/0 (case insensitive, single char or full word) + String[] truthy = new String[] {"true", "True", "TRUE", "t", "T"}; + String[] falsy = new String[] {"false", "False", "FALSE", "f", "F"}; + Array t = ArrayFactory.create(truthy); + for(int i = 0; i < t.size(); i++) + assertEquals(1.0, t.getAsDouble(i), 0.0); + Array f = ArrayFactory.create(falsy); + for(int i = 0; i < f.size(); i++) + assertEquals(0.0, f.getAsDouble(i), 0.0); + } + + @Test(expected = DMLRuntimeException.class) + public void stringArrayGetDoubleInvalidThrows() { + // a token that is neither numeric nor a boolean word/char must throw + ArrayFactory.create(new String[] {"notabool"}).getAsDouble(0); + } + + @Test(expected = DMLRuntimeException.class) + public void stringArrayGetDoubleAmbiguousLengthThrows() { + // length matches neither 1, 4, nor 5 boolean tokens -> reject + ArrayFactory.create(new String[] {"tru"}).getAsDouble(0); + } } diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java new file mode 100644 index 00000000000..b2d31f43b83 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java @@ -0,0 +1,590 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.frame.transform; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.concurrent.CountDownLatch; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.transform.decode.Decoder; +import org.apache.sysds.runtime.transform.decode.DecoderFactory; +import org.apache.sysds.runtime.transform.encode.EncoderFactory; +import org.apache.sysds.runtime.transform.encode.MultiColumnEncoder; +import org.apache.sysds.test.TestUtils; +import org.junit.Before; +import org.junit.Test; + +/** + * Exact inverse correctness tests for the transform decoders. Recode and dummycode are lossless category encodings, so a + * decode of the encoded matrix must reconstruct the original categorical frame. These tests assert exact reconstruction + * for the dense path, the sparse path, and the parallel path so that the dummycode sparse binary search and the parallel + * block split are validated against ground truth rather than only against each other. + */ +public class TransformDecodeRoundTripTest { + protected static final Log LOG = LogFactory.getLog(TransformDecodeRoundTripTest.class.getName()); + + @Before + public void setUp() { + // name must contain "main" so the parallel decode path reuses the shared thread pool + Thread.currentThread().setName("main_test_decode"); + } + + private static FrameBlock categoricalFrame() { + final String[] values = new String[] { + "apple", "banana", "apple", "cherry", "banana", "date", "apple", "cherry", "date", "banana", "elderberry", + "apple", "fig", "banana", "cherry", "apple", "date", "fig", "elderberry", "banana"}; + final FrameBlock f = new FrameBlock(new ValueType[] {ValueType.STRING}); + f.ensureAllocatedColumns(values.length); + for(int i = 0; i < values.length; i++) + f.set(i, 0, values[i]); + return f; + } + + @Test + public void recodeReconstructsOriginalDense() { + roundTrip("{ids:true, recode:[1]}", false, 1); + } + + @Test + public void recodeReconstructsOriginalSparse() { + roundTrip("{ids:true, recode:[1]}", true, 1); + } + + @Test + public void recodeReconstructsOriginalParallel() { + roundTrip("{ids:true, recode:[1]}", false, 4); + } + + @Test + public void dummycodeReconstructsOriginalDense() { + roundTrip("{ids:true, recode:[1], dummycode:[1]}", false, 1); + } + + @Test + public void dummycodeReconstructsOriginalSparse() { + // the one-hot encoded matrix is sparse, so this drives the dummycode sparse binary-search decode path + roundTrip("{ids:true, recode:[1], dummycode:[1]}", true, 1); + } + + @Test + public void dummycodeReconstructsOriginalParallel() { + roundTrip("{ids:true, recode:[1], dummycode:[1]}", false, 4); + } + + /** + * Binning a column while a different column is dummycoded shifts the bin column's source position in the encoded + * matrix. The bin decoder must rebuild that source-column mapping from the dummycode domain sizes. This asserts the + * dense, sparse, and parallel decode paths agree for that layout (bin output is lossy, so exact reconstruction is + * not asserted, only cross-mode consistency and dimensions). + */ + @Test + public void binWithDummycodeOnOtherColumnConsistency() { + // bin column (1) precedes the dummycode column (2): the bin decoder takes the direct + // source-column path because no expanded column sits before it + final FrameBlock original = TestUtils.generateRandomFrameBlock(150, + new ValueType[] {ValueType.FP32, ValueType.UINT4, ValueType.UINT8}, 4242); + binConsistency("{ids:true, bin:[{id:1, method:equi-width, numbins:4}], dummycode:[2]}", original); + } + + /** + * Dummycode on an earlier column (1) shifts the bin column (2) to the right in the encoded matrix. The bin decoder + * must walk the dummycode domain sizes to recover the bin column's true source position. This drives the + * non-magic offset branch of the bin source-column mapping. + */ + @Test + public void binAfterDummycodeOnEarlierColumnConsistency() { + final FrameBlock original = TestUtils.generateRandomFrameBlock(150, + new ValueType[] {ValueType.UINT4, ValueType.FP32, ValueType.UINT8}, 4242); + binConsistency("{ids:true, recode:[1], dummycode:[1], bin:[{id:2, method:equi-width, numbins:4}]}", original); + } + + /** + * Same right-shift as above, but the earlier column is feature-hashed before being dummycoded. The hash domain + * size K is stored as a plain integer in the single meta cell, so the bin source-column mapping reads it (instead + * of numDistinct) to compute the offset. + */ + @Test + public void binAfterHashDummycodeOnEarlierColumnConsistency() { + final FrameBlock original = TestUtils.generateRandomFrameBlock(150, + new ValueType[] {ValueType.UINT4, ValueType.FP32, ValueType.UINT8}, 4242); + binConsistency("{ids:true, hash:[1], K:6, dummycode:[1], bin:[{id:2, method:equi-width, numbins:4}]}", + original); + } + + /** + * Encode then decode the dense, parallel and sparse paths and assert they agree. Bin output is lossy, so only + * cross-mode consistency and row count are asserted (not exact reconstruction). + */ + private void binConsistency(String spec, FrameBlock original) { + try { + final String[] colnames = original.getColumnNames(); + + final MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, colnames, original.getNumColumns(), + null); + final MatrixBlock encoded = encoder.encode(original, 1); + final FrameBlock meta = encoder.getMetaData(null); + + final MatrixBlock dense = new MatrixBlock(); + dense.copy(encoded); + if(dense.isInSparseFormat()) + dense.sparseToDense(); + + final MatrixBlock sparse = new MatrixBlock(); + sparse.copy(encoded); + if(!sparse.isInSparseFormat()) + sparse.denseToSparse(); + + final FrameBlock reference = decodeOnce(spec, colnames, meta, dense, 1); + final FrameBlock parallel = decodeOnce(spec, colnames, meta, dense, 4); + final FrameBlock fromSparse = decodeOnce(spec, colnames, meta, sparse, 1); + + org.junit.Assert.assertEquals(original.getNumRows(), reference.getNumRows()); + TestUtils.compareFrames(reference, parallel, false); + TestUtils.compareFrames(reference, fromSparse, false); + } + catch(Exception e) { + e.printStackTrace(); + fail(spec + " : " + e.getMessage()); + } + } + + /** + * The bin encoder always emits codes >= 1, but the decoder defensively handles a 0 code by mapping it to the + * first bin's lower boundary. Inject a 0 into an otherwise validly encoded matrix to exercise that branch. + */ + @Test + public void binDecodeZeroCodeUsesFirstBinBoundary() { + final String spec = "{ids:true, bin:[{id:1, method:equi-width, numbins:4}]}"; + try { + final FrameBlock original = TestUtils.generateRandomFrameBlock(50, new ValueType[] {ValueType.FP32}, 13); + final String[] colnames = original.getColumnNames(); + final MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, colnames, original.getNumColumns(), + null); + final MatrixBlock encoded = encoder.encode(original, 1); + if(encoded.isInSparseFormat()) + encoded.sparseToDense(); + final FrameBlock meta = encoder.getMetaData(null); + + encoded.set(0, 0, 0); // force a 0 bin code + + final Decoder decoder = DecoderFactory.createDecoder(spec, colnames, null, meta, encoded.getNumColumns()); + final FrameBlock decoded = decoder.decode(encoded, new FrameBlock(decoder.getSchema()), 1); + + final double first = Double.parseDouble(decoded.get(0, 0).toString()); + final double second = Double.parseDouble(decoded.get(1, 0).toString()); + // the 0-coded row decodes to the first bin lower bound, which is <= any properly binned center + org.junit.Assert.assertTrue("0-code must map to the lowest bin boundary", first <= second); + } + catch(Exception e) { + e.printStackTrace(); + fail(spec + " : " + e.getMessage()); + } + } + + /** + * Spark broadcasts the decoder to executors via Java serialization without re-running initMetaData, so the + * decoder must round-trip all of its decode state through writeExternal/readExternal. Decode with a freshly + * deserialized decoder and assert it matches the in-memory decode. Covers plain bin and bin-with-dummycode + * (the latter exercises the serialized _srcCols/_dcCols source-column mapping). + */ + @Test + public void binDecoderSurvivesSerialization() { + final FrameBlock original = TestUtils.generateRandomFrameBlock(80, new ValueType[] {ValueType.FP32}, 21); + serializeRoundTrip("{ids:true, bin:[{id:1, method:equi-width, numbins:4}]}", original); + } + + @Test + public void binWithDummycodeDecoderSurvivesSerialization() { + final FrameBlock original = TestUtils.generateRandomFrameBlock(80, + new ValueType[] {ValueType.UINT4, ValueType.FP32}, 21); + serializeRoundTrip("{ids:true, recode:[1], dummycode:[1], bin:[{id:2, method:equi-width, numbins:4}]}", + original); + } + + private void serializeRoundTrip(String spec, FrameBlock original) { + try { + final String[] colnames = original.getColumnNames(); + final MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, colnames, original.getNumColumns(), + null); + final MatrixBlock encoded = encoder.encode(original, 1); + if(encoded.isInSparseFormat()) + encoded.sparseToDense(); + final FrameBlock meta = encoder.getMetaData(null); + + final Decoder decoder = DecoderFactory.createDecoder(spec, colnames, null, meta, encoded.getNumColumns()); + final FrameBlock expected = decoder.decode(encoded, new FrameBlock(decoder.getSchema()), 1); + + final Decoder restored = serializeDeserialize(decoder); + final FrameBlock actual = restored.decode(encoded, new FrameBlock(restored.getSchema()), 1); + + TestUtils.compareFrames(expected, actual, false); + } + catch(Exception e) { + e.printStackTrace(); + fail(spec + " : " + e.getMessage()); + } + } + + private static Decoder serializeDeserialize(Decoder decoder) throws Exception { + final ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try(ObjectOutputStream oos = new ObjectOutputStream(bos)) { + oos.writeObject(decoder); + } + try(ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bos.toByteArray()))) { + return (Decoder) ois.readObject(); + } + } + + /** + * Feature hashing is non-invertible, so the decode contract for a hash column that is NOT dummycoded is that the + * encoded bucket code passes through unchanged. Regression test: a hash-only column must not be dropped from the + * decoded frame (it previously was, because hash columns were excluded from passthrough). + */ + @Test + public void hashWithoutDummycodeDecodesToBucketCode() { + final String spec = "{ids:true, hash:[1], K:8}"; + try { + final FrameBlock original = categoricalFrame(); + final String[] colnames = original.getColumnNames(); + final MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, colnames, original.getNumColumns(), + null); + final MatrixBlock encoded = encoder.encode(original, 1); + if(encoded.isInSparseFormat()) + encoded.sparseToDense(); + final FrameBlock meta = encoder.getMetaData(null); + + final Decoder decoder = DecoderFactory.createDecoder(spec, colnames, null, meta, encoded.getNumColumns()); + final FrameBlock decoded = decoder.decode(encoded, new FrameBlock(decoder.getSchema()), 1); + + org.junit.Assert.assertEquals(1, decoded.getNumColumns()); + for(int i = 0; i < original.getNumRows(); i++) { + final Object v = decoded.get(i, 0); + org.junit.Assert.assertNotNull("hash column must survive decode at row " + i, v); + org.junit.Assert.assertEquals("hash bucket code must pass through at row " + i, encoded.get(i, 0), + Double.parseDouble(v.toString()), 0.0); + } + } + catch(Exception e) { + e.printStackTrace(); + fail(spec + " : " + e.getMessage()); + } + } + + /** + * A corrupt recode meta entry (no token/code separator) must surface as a {@link DMLRuntimeException} during + * meta-data initialization rather than a raw parsing exception, so callers get an actionable error. Covers the + * defensive try/catch added around the recode-map reconstruction. + */ + @Test + public void recodeInitMetaDataRejectsCorruptEntry() { + final String spec = "{ids:true, recode:[1]}"; + try { + final FrameBlock original = categoricalFrame(); + final String[] colnames = original.getColumnNames(); + final MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, colnames, original.getNumColumns(), + null); + encoder.encode(original, 1); + final FrameBlock meta = encoder.getMetaData(null); + // overwrite the first recode entry with a value lacking the token/code separator + meta.set(0, 0, "corrupt-entry-without-separator"); + + try { + DecoderFactory.createDecoder(spec, colnames, null, meta, original.getNumColumns()); + fail("expected a corrupt recode entry to be rejected"); + } + catch(DMLRuntimeException expected) { + assertTrue("error should identify the recode map reinitialization, got: " + messageChain(expected), + messageChain(expected).contains("recode map")); + } + } + catch(Exception e) { + e.printStackTrace(); + fail(spec + " : " + e.getMessage()); + } + } + + /** + * Federated transform-decode slices a global decoder per worker via {@link Decoder#updateIndexRanges} and + * {@link Decoder#subRangeDecoder}. For a single worker covering the whole matrix, the dummycode expansion must + * collapse the encoded column count down to the decoded column count, and the resulting sub-range decoder must + * reproduce the global decode exactly. Exercises the dummycode index-range and sub-range mapping. + */ + @Test + public void dummycodeSubRangeFullRangeMatchesGlobalDecode() { + final String spec = "{ids:true, recode:[1], dummycode:[1]}"; + try { + final FrameBlock original = TestUtils.generateRandomFrameBlock(60, + new ValueType[] {ValueType.UINT4, ValueType.FP32}, 91); + final String[] colnames = original.getColumnNames(); + final MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, colnames, original.getNumColumns(), + null); + final MatrixBlock encoded = encoder.encode(original, 1); + if(encoded.isInSparseFormat()) + encoded.sparseToDense(); + final FrameBlock meta = encoder.getMetaData(null); + + final Decoder global = DecoderFactory.createDecoder(spec, colnames, null, meta, encoded.getNumColumns()); + final FrameBlock full = global.decode(encoded, new FrameBlock(global.getSchema()), 1); + + // single worker covering the whole matrix: map encoded column range to decoded column range + final long[] beginDims = {0, 0}; + final long[] endDims = {encoded.getNumRows(), encoded.getNumColumns()}; + global.updateIndexRanges(beginDims, endDims); + + org.junit.Assert.assertEquals("begin column must stay at 0", 0, beginDims[1]); + org.junit.Assert.assertEquals("dummycode expansion must collapse to the decoded column count", + full.getNumColumns(), (int) endDims[1]); + + final Decoder sub = global.subRangeDecoder(1, (int) endDims[1] + 1, 0); + final FrameBlock subDecoded = sub.decode(encoded, new FrameBlock(sub.getSchema()), 1); + TestUtils.compareFrames(full, subDecoded, false); + } + catch(Exception e) { + e.printStackTrace(); + fail(spec + " : " + e.getMessage()); + } + } + + /** + * A federated worker holding only the columns after a dummycoded column must shift its index range left by the + * dummycode expansion and receive a sub-range decoder containing just the trailing pass-through columns (the + * dummycode and recode decoders drop out). Mirrors the {@code updateIndexRanges} + {@code subRangeDecoder} call + * sequence in federated transform-decode, covering the index-range shift for a fully-preceding dummycode column and + * the empty sub-range branch. + */ + @Test + public void dummycodeSubRangeExcludingDummycodedColumnKeepsRemaining() { + final String spec = "{ids:true, recode:[1], dummycode:[1]}"; + try { + final FrameBlock original = TestUtils.generateRandomFrameBlock(40, + new ValueType[] {ValueType.UINT4, ValueType.FP32, ValueType.FP32}, 73); + final String[] colnames = original.getColumnNames(); + final MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, colnames, original.getNumColumns(), + null); + final MatrixBlock encoded = encoder.encode(original, 1); + if(encoded.isInSparseFormat()) + encoded.sparseToDense(); + final FrameBlock meta = encoder.getMetaData(null); + + final Decoder global = DecoderFactory.createDecoder(spec, colnames, null, meta, encoded.getNumColumns()); + + // the dummycode column expands to (encodedCols - 2) one-hot columns; a worker owning only the two trailing + // pass-through columns starts after that expanded block in encoded column space + final int dcWidth = encoded.getNumColumns() - 2; + final long[] beginDims = {0, dcWidth}; + final long[] endDims = {encoded.getNumRows(), dcWidth + 2}; + final int colStartBefore = (int) beginDims[1]; + global.updateIndexRanges(beginDims, endDims); + + // after collapsing the preceding dummycode expansion, the worker maps to decoded columns 2..3 + org.junit.Assert.assertEquals(1, beginDims[1]); + org.junit.Assert.assertEquals(3, endDims[1]); + + final Decoder sub = global.subRangeDecoder((int) beginDims[1] + 1, (int) endDims[1] + 1, colStartBefore); + org.junit.Assert.assertNotNull("pass-through columns must still yield a decoder", sub); + org.junit.Assert.assertEquals("only the two trailing pass-through columns remain", 2, + sub.getSchema().length); + } + catch(Exception e) { + e.printStackTrace(); + fail(spec + " : " + e.getMessage()); + } + } + + /** + * Two recode columns with different domain sizes leave trailing empty (null) cells in the shorter column's + * recode-map column. Reconstructing that map must stop at the first null rather than read past it. Recode is + * lossless, so the decode must reconstruct the original frame exactly. + */ + @Test + public void recodeMultiColumnWithTrailingNullMapEntries() { + final String spec = "{ids:true, recode:[1, 2]}"; + try { + final FrameBlock original = new FrameBlock(new ValueType[] {ValueType.STRING, ValueType.STRING}); + final String[] high = {"a", "b", "c", "d", "e", "f", "g", "h"}; + final String[] low = {"x", "y"}; + final int n = 16; + original.ensureAllocatedColumns(n); + for(int i = 0; i < n; i++) { + original.set(i, 0, high[i % high.length]); + original.set(i, 1, low[i % low.length]); + } + final String[] colnames = original.getColumnNames(); + final MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, colnames, original.getNumColumns(), + null); + final MatrixBlock encoded = encoder.encode(original, 1); + if(encoded.isInSparseFormat()) + encoded.sparseToDense(); + final FrameBlock meta = encoder.getMetaData(null); + + final Decoder decoder = DecoderFactory.createDecoder(spec, colnames, null, meta, encoded.getNumColumns()); + final FrameBlock decoded = decoder.decode(encoded, new FrameBlock(decoder.getSchema()), 1); + TestUtils.compareFrames(original, decoded, false); + } + catch(Exception e) { + e.printStackTrace(); + fail(spec + " : " + e.getMessage()); + } + } + + /** + * The parallel decode path runs per-row-block decode tasks on a thread pool; a failure inside a worker must not be + * swallowed but resurface as an unchecked exception to the caller. Feeding a matrix with far fewer columns than the + * decoder expects forces an out-of-range access in a worker, which the parallel wrapper must propagate. + */ + @Test + public void parallelDecodeWrapsWorkerException() { + final String spec = "{ids:true, recode:[1], dummycode:[1]}"; + try { + final FrameBlock original = categoricalFrame(); + final String[] colnames = original.getColumnNames(); + final MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, colnames, original.getNumColumns(), + null); + final MatrixBlock encoded = encoder.encode(original, 1); + final FrameBlock meta = encoder.getMetaData(null); + final Decoder decoder = DecoderFactory.createDecoder(spec, colnames, null, meta, encoded.getNumColumns()); + + // far fewer columns than the dummycode decoder reads -> a parallel worker accesses out of range + final MatrixBlock broken = new MatrixBlock(2, 1, false); + broken.allocateDenseBlock(); + try { + decoder.decode(broken, new FrameBlock(decoder.getSchema()), 4); + fail("expected the parallel decode wrapper to propagate the worker failure"); + } + catch(DMLRuntimeException expected) { + assertNotNull("parallel decode wrapper must retain the worker exception as cause", + expected.getCause()); + } + } + catch(Exception e) { + e.printStackTrace(); + fail(spec + " : " + e.getMessage()); + } + } + + /** + * Interrupting a worker mid parallel-decode must restore the caller's interrupt flag (which {@code Future.get} + * clears when it throws) and surface the failure as a {@link DMLRuntimeException}. The same minimal decoder also + * exercises the base sub-range contract, which rejects decoders that do not implement column sub-ranging. + */ + @Test + public void parallelDecodeInterruptionRestoresFlagAndRejectsSubRange() { + final Thread caller = Thread.currentThread(); + final CountDownLatch release = new CountDownLatch(1); + final Decoder decoder = new Decoder(new ValueType[] {ValueType.FP64}, new int[] {1}) { + private static final long serialVersionUID = 1L; + + @Override + public FrameBlock decode(MatrixBlock in, FrameBlock out) { + return out; + } + + @Override + public void decode(MatrixBlock in, FrameBlock out, int rl, int ru) { + // interrupt the thread blocked in Future.get and stay unfinished so the interrupt is observed + caller.interrupt(); + try { + release.await(); + } + catch(InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + @Override + public void initMetaData(FrameBlock meta) { + // no meta data needed + } + }; + + try { + decoder.subRangeDecoder(1, 2, 0); + fail("a decoder without sub-range support must reject the request"); + } + catch(DMLRuntimeException expected) { + assertTrue(messageChain(expected).contains("sub-range")); + } + + final MatrixBlock in = new MatrixBlock(1, 1, false); + in.allocateDenseBlock(); + try { + decoder.decode(in, new FrameBlock(decoder.getSchema()), 2); + fail("an interrupted parallel decode must throw"); + } + catch(DMLRuntimeException expected) { + assertTrue("the interrupt flag must be restored", Thread.currentThread().isInterrupted()); + assertNotNull(expected.getCause()); + } + finally { + release.countDown(); + Thread.interrupted(); // clear so the interrupt does not leak into other tests + } + } + + private static String messageChain(Throwable t) { + final StringBuilder sb = new StringBuilder(); + for(Throwable c = t; c != null; c = c.getCause()) + sb.append(c.getMessage()).append('\n'); + return sb.toString(); + } + + private static FrameBlock decodeOnce(String spec, String[] colnames, FrameBlock meta, MatrixBlock in, int k) { + final Decoder decoder = DecoderFactory.createDecoder(spec, colnames, null, meta, in.getNumColumns()); + return decoder.decode(in, new FrameBlock(decoder.getSchema()), k); + } + + private void roundTrip(String spec, boolean sparse, int k) { + try { + final FrameBlock original = categoricalFrame(); + final String[] colnames = original.getColumnNames(); + + final MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, colnames, original.getNumColumns(), + null); + MatrixBlock encoded = encoder.encode(original, 1); + final FrameBlock meta = encoder.getMetaData(null); + + if(sparse && !encoded.isInSparseFormat()) + encoded.denseToSparse(); + else if(!sparse && encoded.isInSparseFormat()) + encoded.sparseToDense(); + + final Decoder decoder = DecoderFactory.createDecoder(spec, colnames, null, meta, encoded.getNumColumns()); + final FrameBlock decoded = decoder.decode(encoded, new FrameBlock(decoder.getSchema()), k); + + TestUtils.compareFrames(original, decoded, false); + } + catch(Exception e) { + e.printStackTrace(); + fail(spec + " (sparse=" + sparse + ", k=" + k + ") : " + e.getMessage()); + } + } +} diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java new file mode 100644 index 00000000000..54bd1679716 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.frame.transform; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.transform.decode.Decoder; +import org.apache.sysds.runtime.transform.decode.DecoderFactory; +import org.apache.sysds.runtime.transform.encode.EncoderFactory; +import org.apache.sysds.runtime.transform.encode.MultiColumnEncoder; +import org.apache.sysds.runtime.util.CommonThreadPool; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Component tests for the transform decoders. These exercise the row-block and parallel decode paths, the sparse and + * dense dummycode decode paths, the binning source-column offset mapping, and feature-hash column handling end-to-end + * through an encode followed by decode round trip. + */ +@RunWith(value = Parameterized.class) +public class TransformDecodeTest { + protected static final Log LOG = LogFactory.getLog(TransformDecodeTest.class.getName()); + + private final FrameBlock data; + private final int k; + + public TransformDecodeTest(FrameBlock data, int k) { + // name must contain "main" so the parallel decode path reuses the shared thread pool + Thread.currentThread().setName("main_test_decode"); + Logger.getLogger(CommonThreadPool.class.getName()).setLevel(Level.OFF); + this.data = data; + this.k = k; + } + + @Parameters + public static Collection data() { + final ArrayList tests = new ArrayList<>(); + final int[] threads = new int[] {1, 4}; + try { + final FrameBlock[] blocks = new FrameBlock[] { + // single low-cardinality categorical column + TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT4}, 231), + // single categorical column with nulls + TestUtils.generateRandomFrameBlock(64, new ValueType[] {ValueType.UINT4}, 99, 0.2), + // multi column: dummycode/bin on col1 must offset the trailing passthrough columns + TestUtils.generateRandomFrameBlock(120, + new ValueType[] {ValueType.UINT4, ValueType.UINT8, ValueType.FP32}, 17), + // large enough to split into multiple row blocks in the parallel decode path + TestUtils.generateRandomFrameBlock(2500, new ValueType[] {ValueType.UINT4}, 7)}; + + for(FrameBlock block : blocks) + for(int k : threads) + tests.add(new Object[] {block, k}); + } + catch(Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + return tests; + } + + @Test + public void testPassThrough() { + decodeConsistency("{ids:true}"); + } + + @Test + public void testRecode() { + decodeConsistency("{ids:true, recode:[1]}"); + } + + @Test + public void testDummycode() { + decodeConsistency("{ids:true, recode:[1], dummycode:[1]}"); + } + + @Test + public void testBinWidth() { + decodeConsistency("{ids:true, bin:[{id:1, method:equi-width, numbins:4}]}"); + } + + @Test + public void testBinHeight() { + decodeConsistency("{ids:true, bin:[{id:1, method:equi-height, numbins:10}]}"); + } + + @Test + public void testBinSingleBin() { + // numbins:1 collapses every value into a single bin, exercising the degenerate boundary handling + decodeConsistency("{ids:true, bin:[{id:1, method:equi-width, numbins:1}]}"); + } + + @Test + public void testHashToDummy() { + // feature-hash columns store their domain size K as a plain integer in the single meta cell, which the + // dummycode decoder reads (instead of numDistinct) to reconstruct the one-hot column ranges + decodeConsistency("{ids:true, hash:[1], K:8, dummycode:[1]}"); + } + + @Test + public void testHashToDummyDomain1() { + decodeConsistency("{ids:true, hash:[1], K:1, dummycode:[1]}"); + } + + /** + * Encode the data, then decode the encoded matrix in three ways: serial dense, parallel dense, and serial sparse. + * All three must produce identical frames. This jointly exercises the parallel block-decode path in + * {@link Decoder#decode(MatrixBlock, FrameBlock, int)} and the separate sparse / dense dummycode decode paths. + */ + private void decodeConsistency(String spec) { + try { + final String[] colnames = data.getColumnNames(); + final MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, colnames, data.getNumColumns(), null); + final MatrixBlock encoded = encoder.encode(data, 1); + final FrameBlock meta = encoder.getMetaData(null); + + final MatrixBlock dense = forceDense(encoded); + final MatrixBlock sparse = forceSparse(encoded); + + final FrameBlock reference = decode(spec, colnames, meta, dense, 1); + final FrameBlock parallel = decode(spec, colnames, meta, dense, k); + final FrameBlock fromSparse = decode(spec, colnames, meta, sparse, 1); + + assertEquals("decoded rows must match input rows", data.getNumRows(), reference.getNumRows()); + + TestUtils.compareFrames(reference, parallel, false); + TestUtils.compareFrames(reference, fromSparse, false); + } + catch(Exception e) { + e.printStackTrace(); + fail(spec + " : " + e.getMessage()); + } + } + + private static FrameBlock decode(String spec, String[] colnames, FrameBlock meta, MatrixBlock in, int k) { + final Decoder decoder = DecoderFactory.createDecoder(spec, colnames, null, meta, in.getNumColumns()); + return decoder.decode(in, new FrameBlock(decoder.getSchema()), k); + } + + private static MatrixBlock forceDense(MatrixBlock in) { + final MatrixBlock out = new MatrixBlock(); + out.copy(in); + if(out.isInSparseFormat()) + out.sparseToDense(); + return out; + } + + private static MatrixBlock forceSparse(MatrixBlock in) { + final MatrixBlock out = new MatrixBlock(); + out.copy(in); + if(!out.isInSparseFormat()) + out.denseToSparse(); + return out; + } +} From b83f71aecd584265f31ce2eef207f066c3d8a734 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Wed, 24 Jun 2026 01:55:25 +0200 Subject: [PATCH 043/132] [MINOR][CI] Raise federated request timeout for multitenant Spark tests (#2505) * Raise federated request timeout for multitenant Spark tests Set sysds.federated.timeout in the multi-tenant test config from 16s to 60s. The 16s bound was too aggressive for the Spark-backed (SP) variants of the federated multitenant reuse tests: Spark context creation alone takes ~14s, so under shared CI load a single federated request (both the rightIndex/rblk instruction execution and the end-of-run stats collection) routinely exceeded 16s and threw TimeoutException. --- src/test/config/SystemDS-MultiTenant-config.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/config/SystemDS-MultiTenant-config.xml b/src/test/config/SystemDS-MultiTenant-config.xml index 321fcc0b282..ae915177617 100644 --- a/src/test/config/SystemDS-MultiTenant-config.xml +++ b/src/test/config/SystemDS-MultiTenant-config.xml @@ -21,6 +21,6 @@ 30 - 16 + 60 true From 2ba9304c23ac5059d252ff621e1396de3a0dfe8c Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Wed, 24 Jun 2026 11:17:31 +0200 Subject: [PATCH 044/132] [MINOR] Add ToString test for decimal-formatting in toString builtin (#2503) Adds new test cases and a code cleanup that exercise toString(X, rows=, cols=, decimal=) with values that hit common formatting edge cases: integer-valued scalars, mid-range decimals, requested decimals beyond the value's precision, rounding at the last requested digit, and small near-zero values. --- .../sysds/runtime/util/DataConverter.java | 34 ++++--- .../component/frame/FrameToStringTest.java | 37 ++++++++ .../component/tensor/TensorToStringTest.java | 66 +++++++++++++ .../test/functions/misc/ToStringTest.java | 92 +++++++++++++++++++ .../scripts/functions/misc/ToString12.dml | 24 +++++ 5 files changed, 239 insertions(+), 14 deletions(-) create mode 100644 src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java create mode 100644 src/test/scripts/functions/misc/ToString12.dml diff --git a/src/main/java/org/apache/sysds/runtime/util/DataConverter.java b/src/main/java/org/apache/sysds/runtime/util/DataConverter.java index 3373205fc35..b4296e227d3 100644 --- a/src/main/java/org/apache/sysds/runtime/util/DataConverter.java +++ b/src/main/java/org/apache/sysds/runtime/util/DataConverter.java @@ -884,6 +884,23 @@ private static String dfFormat(DecimalFormat df, double value) { } } + /** + * Creates a non-grouping {@link DecimalFormat} for printing values. When {@code decimal >= 0} + * both the minimum and maximum fraction digits are pinned to {@code decimal}, so values are + * printed with exactly that many decimals; otherwise the {@link DecimalFormat} defaults apply. + * @param decimal number of decimal places to print, -1 for default + * @return a configured {@link DecimalFormat} + */ + private static DecimalFormat createDecimalFormat(int decimal) { + DecimalFormat df = new DecimalFormat(); + df.setGroupingUsed(false); + if (decimal >= 0) { + df.setMinimumFractionDigits(decimal); + df.setMaximumFractionDigits(decimal); + } + return df; + } + public static String toString(MatrixBlock mb) { return toString(mb, false, " ", "\n", mb.getNumRows(), mb.getNumColumns(), 3); } @@ -913,11 +930,7 @@ public static String toString(MatrixBlock mb, boolean sparse, String separator, if (colsToPrint >= 0) colLength = colsToPrint < clen ? colsToPrint : clen; - DecimalFormat df = new DecimalFormat(); - df.setGroupingUsed(false); - if (decimal >= 0){ - df.setMinimumFractionDigits(decimal); - } + DecimalFormat df = createDecimalFormat(decimal); if (sparse){ // Sparse Print Format if (mb.isInSparseFormat()){ // Block is in sparse format @@ -997,11 +1010,7 @@ public static String toString(TensorBlock tb, boolean sparse, String separator, if (colsToPrint >= 0) colLength = Math.min(colsToPrint, clen); - DecimalFormat df = new DecimalFormat(); - df.setGroupingUsed(false); - if (decimal >= 0){ - df.setMinimumFractionDigits(decimal); - } + DecimalFormat df = createDecimalFormat(decimal); if (sparse){ // Sparse Print Format // TODO use sparse iterator for sparse block @@ -1147,10 +1156,7 @@ public static String toString(FrameBlock fb, boolean sparse, String separator, S sb.append(lineseparator); //print data - DecimalFormat df = new DecimalFormat(); - df.setGroupingUsed(false); - if (decimal >= 0) - df.setMinimumFractionDigits(decimal); + DecimalFormat df = createDecimalFormat(decimal); Iterator iter = IteratorFactory.getObjectRowIterator(fb, 0, rowLength); while( iter.hasNext() ) { diff --git a/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java b/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java index 2b29214b591..60587bf51a2 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java @@ -19,6 +19,7 @@ package org.apache.sysds.test.component.frame; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import org.apache.sysds.common.Types.ValueType; @@ -38,6 +39,42 @@ public void test100x100() { FrameBlock f = createFrameBlock(); assertTrue(DataConverter.toString(f, false, " ", "\n", 100, 100, 3).length() < 75); } + + @Test + public void testDecimalClampsFractionDigits() { + FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); + f.ensureAllocatedColumns(1); + f.set(0, 0, 5.244058388023880); + // decimal=2 must print exactly two fraction digits, not DecimalFormat's default max of 3 + String out = DataConverter.toString(f, false, " ", "\n", 1, 1, 2); + assertTrue("expected value clamped to 5.24, got: " + out, out.contains("5.24\n")); + assertFalse("decimal=2 must not print three digits: " + out, out.contains("5.244")); + } + + @Test + public void testDecimalPadsAndRounds() { + FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); + f.ensureAllocatedColumns(2); + f.set(0, 0, 22.0); // integer-valued: padded up to the requested digits + f.set(1, 0, 5.244058388023880); // rounded at the last requested digit + String out = DataConverter.toString(f, false, " ", "\n", 2, 1, 4); + assertTrue("expected 22.0000 padded: " + out, out.contains("22.0000\n")); + assertTrue("expected 5.2441 rounded: " + out, out.contains("5.2441\n")); + } + + @Test + public void testNegativeDecimalUsesDefaultFormatting() { + FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); + f.ensureAllocatedColumns(2); + f.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained + f.set(1, 0, 5.244058388023880); // default cap of three fraction digits + // decimal < 0 leaves DecimalFormat unconstrained (no min/max fraction digits set) + String out = DataConverter.toString(f, false, " ", "\n", 2, 1, -1); + assertTrue("expected unpadded 22: " + out, out.contains("22\n")); + assertFalse("integer value must not be padded: " + out, out.contains("22.0")); + assertTrue("expected default 5.244: " + out, out.contains("5.244\n")); + assertFalse("must not print a fourth digit: " + out, out.contains("5.2441")); + } private FrameBlock createFrameBlock() { FrameBlock f = new FrameBlock(new ValueType[]{ValueType.STRING, ValueType.STRING}); diff --git a/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java b/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java new file mode 100644 index 00000000000..5c9ed821e78 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.tensor; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.data.TensorBlock; +import org.apache.sysds.runtime.util.DataConverter; +import org.junit.Test; + +public class TensorToStringTest { + @Test + public void testDecimalClampsFractionDigits() { + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 1}); + tb.allocateBlock(); + tb.set(0, 0, 5.244058388023880); + // decimal=2 must print exactly two fraction digits, not DecimalFormat's default max of 3 + String out = DataConverter.toString(tb, false, " ", "\n", "[", "]", 1, 1, 2); + assertTrue("expected value clamped to 5.24, got: " + out, out.contains("5.24")); + assertFalse("decimal=2 must not print three digits: " + out, out.contains("5.244")); + } + + @Test + public void testDecimalPadsAndRounds() { + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 2}); + tb.allocateBlock(); + tb.set(0, 0, 22.0); // integer-valued: padded up to the requested digits + tb.set(0, 1, 5.244058388023880); // rounded at the last requested digit + String out = DataConverter.toString(tb, false, " ", "\n", "[", "]", 1, 2, 4); + assertTrue("expected 22.0000 padded: " + out, out.contains("22.0000")); + assertTrue("expected 5.2441 rounded: " + out, out.contains("5.2441")); + } + + @Test + public void testNegativeDecimalUsesDefaultFormatting() { + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 2}); + tb.allocateBlock(); + tb.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained + tb.set(0, 1, 5.244058388023880); // default cap of three fraction digits + // decimal < 0 leaves DecimalFormat unconstrained (no min/max fraction digits set) + String out = DataConverter.toString(tb, false, " ", "\n", "[", "]", 1, 2, -1); + assertTrue("expected unpadded 22: " + out, out.contains("22")); + assertFalse("integer value must not be padded: " + out, out.contains("22.0")); + assertTrue("expected default 5.244: " + out, out.contains("5.244")); + assertFalse("must not print a fourth digit: " + out, out.contains("5.2441")); + } +} diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java index ee6a2953980..18ca2fbc454 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java @@ -270,4 +270,96 @@ protected void toStringTestHelper(ExecMode platform, String testName, String exp DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; } } + + @Test + public void testPrintWithDecimal(){ + String testName = "ToString12"; + + String decimalPoints = "2"; + String value = "22"; + String expectedOutput = "22.00\n"; + + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); + toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); + } + + + @Test + public void testPrintWithDecimal2(){ + String testName = "ToString12"; + + String decimalPoints = "2"; + String value = "5.244058388023880"; + String expectedOutput = "5.24\n"; + + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); + toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); + } + + + @Test + public void testPrintWithDecimal3(){ + String testName = "ToString12"; + + String decimalPoints = "10"; + String value = "5.244058388023880"; + String expectedOutput = "5.2440583880\n"; + + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); + toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); + } + + + @Test + public void testPrintWithDecimal4(){ + String testName = "ToString12"; + + String decimalPoints = "4"; + String value = "5.244058388023880"; + String expectedOutput = "5.2441\n"; + + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); + toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); + } + + + @Test + public void testPrintWithDecimal5(){ + String testName = "ToString12"; + + String decimalPoints = "10"; + String value = "0.000000008023880"; + String expectedOutput = "0.0000000080\n"; + + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); + toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); + } + + protected void toStringTestHelper2(ExecMode platform, String testName, String expectedOutput, String decimalPoints, String value) { + ExecMode platformOld = rtplatform; + + rtplatform = platform; + boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; + if (rtplatform == ExecMode.SPARK) + DMLScript.USE_LOCAL_SPARK_CONFIG = true; + try { + // Create and load test configuration + getAndLoadTestConfiguration(testName); + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + testName + ".dml"; + programArgs = new String[]{"-args", output(OUTPUT_NAME), value, decimalPoints}; + + // Run DML and R scripts + runTest(true, false, null, -1); + + // Compare output strings + String output = TestUtils.readDMLString(output(OUTPUT_NAME)); + TestUtils.compareScalars(expectedOutput, output); + } + finally { + // Reset settings + rtplatform = platformOld; + DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; + } + } } diff --git a/src/test/scripts/functions/misc/ToString12.dml b/src/test/scripts/functions/misc/ToString12.dml new file mode 100644 index 00000000000..4f120630b75 --- /dev/null +++ b/src/test/scripts/functions/misc/ToString12.dml @@ -0,0 +1,24 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = matrix($2, rows=1, cols=1) +str = toString(X, rows=3, cols=3, decimal=$3) +write(str, $1) From e177b99c19324d5e7ab3ea82af858ea82edb2009 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Wed, 24 Jun 2026 13:29:16 +0200 Subject: [PATCH 045/132] [BWARE] Add removeEmpty support to compressed column groups (#2504) Implement removeEmptyRows and removeEmptyCols for compressed matrices without full decompression. Add CLALibRemoveEmpty driver and LibMatrixReorg helpers (rmemptyEarlyAbort/rmemptyUnsafe), per-column-group removeEmptyRows/removeEmptyColsSubset, dictionary sliceColumns, and offset/mapping support for index-only row removal. --- .../compress/CompressedMatrixBlock.java | 5 +- .../runtime/compress/colgroup/AColGroup.java | 76 +++++++- .../compress/colgroup/AColGroupValue.java | 3 +- .../compress/colgroup/ADictBasedColGroup.java | 1 + .../runtime/compress/colgroup/ASDCZero.java | 19 ++ .../compress/colgroup/ColGroupConst.java | 13 +- .../compress/colgroup/ColGroupDDC.java | 29 ++- .../compress/colgroup/ColGroupDDCFOR.java | 15 ++ .../compress/colgroup/ColGroupDDCLZW.java | 13 ++ .../compress/colgroup/ColGroupEmpty.java | 12 ++ .../runtime/compress/colgroup/ColGroupIO.java | 4 +- .../colgroup/ColGroupLinearFunctional.java | 10 + .../compress/colgroup/ColGroupOLE.java | 11 +- .../compress/colgroup/ColGroupRLE.java | 10 + .../compress/colgroup/ColGroupSDC.java | 22 ++- .../compress/colgroup/ColGroupSDCFOR.java | 18 ++ .../compress/colgroup/ColGroupSDCSingle.java | 22 ++- .../colgroup/ColGroupSDCSingleZeros.java | 32 ++- .../compress/colgroup/ColGroupSDCZeros.java | 53 +++-- .../colgroup/ColGroupUncompressed.java | 69 ++++--- .../colgroup/ColGroupUncompressedArray.java | 11 ++ .../colgroup/dictionary/DeltaDictionary.java | 6 + .../colgroup/dictionary/Dictionary.java | 7 + .../colgroup/dictionary/IDictionary.java | 11 ++ .../dictionary/IdentityDictionary.java | 7 +- .../dictionary/IdentityDictionarySlice.java | 6 + .../dictionary/MatrixBlockDictionary.java | 48 ++++- .../colgroup/dictionary/PlaceHolderDict.java | 6 + .../colgroup/dictionary/QDictionary.java | 5 + .../compress/colgroup/mapping/AMapToData.java | 34 ++++ .../compress/colgroup/offset/AIterator.java | 4 +- .../compress/colgroup/offset/AOffset.java | 48 ++++- .../compress/colgroup/offset/OffsetEmpty.java | 4 + .../compress/lib/CLALibRemoveEmpty.java | 142 ++++++++++++++ .../runtime/matrix/data/LibMatrixReorg.java | 74 +++++-- .../compress/CompressedMatrixTest.java | 110 +++++++++++ .../CompressedRemoveEmptyColSubsetTest.java | 182 ++++++++++++++++++ .../CompressedRemoveEmptyForcedTest.java | 97 ++++++++++ .../colgroup/ColGroupNegativeTests.java | 25 +++ .../compress/offset/CustomOffsetTest.java | 94 ++++++++- 40 files changed, 1265 insertions(+), 93 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java create mode 100644 src/test/java/org/apache/sysds/test/component/compress/CompressedRemoveEmptyColSubsetTest.java create mode 100644 src/test/java/org/apache/sysds/test/component/compress/CompressedRemoveEmptyForcedTest.java diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java index e08f731e829..58e33a616ca 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java @@ -58,6 +58,7 @@ import org.apache.sysds.runtime.compress.lib.CLALibMMChain; import org.apache.sysds.runtime.compress.lib.CLALibMatrixMult; import org.apache.sysds.runtime.compress.lib.CLALibMerge; +import org.apache.sysds.runtime.compress.lib.CLALibRemoveEmpty; import org.apache.sysds.runtime.compress.lib.CLALibReplace; import org.apache.sysds.runtime.compress.lib.CLALibReorg; import org.apache.sysds.runtime.compress.lib.CLALibReshape; @@ -871,9 +872,7 @@ public MatrixBlock groupedAggOperations(MatrixValue tgt, MatrixValue wghts, Matr @Override public MatrixBlock removeEmptyOperations(MatrixBlock ret, boolean rows, boolean emptyReturn, MatrixBlock select) { - printDecompressWarning("removeEmptyOperations"); - MatrixBlock tmp = getUncompressed(); - return tmp.removeEmptyOperations(ret, rows, emptyReturn, select); + return CLALibRemoveEmpty.rmempty(this, ret, rows, emptyReturn, select); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java index fbe04c732e6..f30cf8b17b2 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java @@ -29,9 +29,9 @@ import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.sysds.runtime.compress.colgroup.ColGroupUtils.P; import org.apache.sysds.runtime.compress.CompressionSettings; import org.apache.sysds.runtime.compress.CompressionSettingsBuilder; +import org.apache.sysds.runtime.compress.colgroup.ColGroupUtils.P; import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory; import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex.SliceResult; @@ -41,6 +41,7 @@ import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup; import org.apache.sysds.runtime.compress.estim.encoding.IEncode; import org.apache.sysds.runtime.compress.lib.CLALibCombineGroups; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.data.SparseBlockMCSR; @@ -401,8 +402,9 @@ public final AColGroup rightMultByMatrix(MatrixBlock right) { * @param cru The right hand side column upper * @param nRows The number of rows in this column group */ - public void rightDecompressingMult(MatrixBlock right, MatrixBlock ret, int rl, int ru, int nRows, int crl, int cru){ - throw new NotImplementedException("not supporting right Decompressing Multiply on class: " + this.getClass().getSimpleName()); + public void rightDecompressingMult(MatrixBlock right, MatrixBlock ret, int rl, int ru, int nRows, int crl, int cru) { + throw new NotImplementedException( + "not supporting right Decompressing Multiply on class: " + this.getClass().getSimpleName()); } /** @@ -806,7 +808,7 @@ public final void selectionMultiply(MatrixBlock selection, P[] points, MatrixBlo else denseSelection(selection, points, ret, rl, ru); } - + /** * Get an approximate sparsity of this column group * @@ -981,4 +983,70 @@ public String toString() { sb.append(_colIndexes); return sb.toString(); } + + /** + * Return a new column group containing only the selected rows in the given boolean vector. + * + * Whenever possible only modify the index structure, not the dictionary of the column groups. + * + * @param selectV The selection vector + * @param rOut The number of rows in the output + * @return The new column group + */ + public abstract AColGroup removeEmptyRows(boolean[] selectV, int rOut); + + /** + * Return a new column group containing only the selected columns in the given boolean vector. + * + * Whenever possible only modify the column index, and reduce the dictionaries of the column groups. + * + * @param selectV The selection vector + * @return The new column group, or {@code null} if no column of this group is selected + */ + public AColGroup removeEmptyCols(boolean[] selectV) { + if(!inSelection(selectV)) + return null; + + final IntArrayList selectedColumns = new IntArrayList(); + final IntArrayList newIDs = new IntArrayList(); + int idx = 0; + int idxOwn = 0; + final int end = Math.min(selectV.length, _colIndexes.get(_colIndexes.size() - 1) + 1); + for(int i = 0; i < end; i++) { + + if(i == _colIndexes.get(idxOwn)) { + if(selectV[i]) { + selectedColumns.appendValue(idxOwn); + newIDs.appendValue(idx); + } + idxOwn++; + } + if(selectV[i]) + idx++; + } + + final IColIndex newColumnIDs = ColIndexFactory.create(newIDs); + if(newColumnIDs.size() == _colIndexes.size()) + return copyAndSet(newColumnIDs); + else + return removeEmptyColsSubset(newColumnIDs, selectedColumns); + } + + /** + * Using the selection of columns, slice out those and return in a new column group with the given column indexes. + * Ideally this method should only modify the dictionaries. + * + * @param newColumnIDs the new column indexes + * @param selectedColumns The selected columns of this column group (guaranteed < current number of columns) + * @return A new Column group + */ + protected abstract AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns); + + private boolean inSelection(boolean[] selection) { + for(int i = 0; i < _colIndexes.size(); i++) { + if(selection[_colIndexes.get(i)]) + return true; + } + return false; + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java index 45358c7ce46..d825b91f089 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java @@ -59,8 +59,6 @@ public int getNumValues() { * produce an overhead in cases where the count is calculated, but the overhead will be limited to number of distinct * tuples in the dictionary. * - * The returned counts always contains the number of zero tuples as well if there are some contained, even if they - * are not materialized. * * @return The count of each value in the MatrixBlock. */ @@ -212,6 +210,7 @@ public void clear() { counts = null; } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictBasedColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictBasedColGroup.java index 8f2f0b46055..d114f029df8 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictBasedColGroup.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictBasedColGroup.java @@ -402,4 +402,5 @@ protected IDictionary combineDictionaries(int nCol, List right) { public double getSparsity() { return _dict.getSparsity(); } + } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java index 3de98a1c23f..30de5e120c5 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java @@ -203,6 +203,22 @@ private final void leftMultByMatrixNoPreAggRowsDense(MatrixBlock mb, double[] re */ protected abstract void multiplyScalar(double v, double[] resV, int offRet, AIterator it); + public void decompressToSparseBlock(SparseBlock sb, int rl, int ru, int offR, int offC, AIterator it) { + if(_dict instanceof MatrixBlockDictionary) { + final MatrixBlockDictionary md = (MatrixBlockDictionary) _dict; + final MatrixBlock mb = md.getMatrixBlock(); + // The dictionary is never empty. + if(mb.isInSparseFormat()) + // TODO make sparse decompression where the iterator is known in argument + decompressToSparseBlockSparseDictionary(sb, rl, ru, offR, offC, mb.getSparseBlock()); + else + decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, mb.getDenseBlockValues(), + it); + } + else + decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, _dict.getValues(), it); + } + public void decompressToDenseBlock(DenseBlock db, int rl, int ru, int offR, int offC, AIterator it) { if(_dict instanceof MatrixBlockDictionary) { final MatrixBlockDictionary md = (MatrixBlockDictionary) _dict; @@ -223,6 +239,9 @@ public void decompressToDenseBlockDenseDictionary(DenseBlock db, int rl, int ru, decompressToDenseBlockDenseDictionaryWithProvidedIterator(db, rl, ru, offR, offC, _dict.getValues(), it); } + public abstract void decompressToSparseBlockDenseDictionaryWithProvidedIterator(SparseBlock db, int rl, int ru, + int offR, int offC, double[] values, AIterator it); + public abstract void decompressToDenseBlockDenseDictionaryWithProvidedIterator(DenseBlock db, int rl, int ru, int offR, int offC, double[] values, AIterator it); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java index 94137eb6381..7d0b2469ec8 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java @@ -46,6 +46,7 @@ import org.apache.sysds.runtime.compress.estim.encoding.EncodingFactory; import org.apache.sysds.runtime.compress.estim.encoding.IEncode; import org.apache.sysds.runtime.compress.lib.CLALibLeftMultBy; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.data.SparseBlockMCSR; @@ -527,7 +528,7 @@ public CmCovObject centralMoment(CMOperator op, int nRows) { @Override public AColGroup rexpandCols(int max, boolean ignore, boolean cast, int nRows) { IDictionary d = _dict.rexpandCols(max, ignore, cast, _colIndexes.size()); - if(d == null){ + if(d == null) { if(max <= 0) return null; return ColGroupEmpty.create(max); @@ -758,4 +759,14 @@ public AColGroup combineWithSameIndex(int nRow, int nCol, List right) protected boolean allowShallowIdentityRightMult() { return true; } + + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + return this; + } + + @Override + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + return ColGroupConst.create(newColumnIDs, _dict.sliceColumns(selectedColumns, getNumCols())); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java index a3fdf1fc89f..6ac1544e61e 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java @@ -26,8 +26,6 @@ import java.util.List; import java.util.concurrent.ExecutorService; -import jdk.incubator.vector.DoubleVector; -import jdk.incubator.vector.VectorSpecies; import org.apache.commons.lang3.NotImplementedException; import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.compress.CompressedMatrixBlock; @@ -56,6 +54,7 @@ import org.apache.sysds.runtime.compress.estim.EstimationFactors; import org.apache.sysds.runtime.compress.estim.encoding.EncodingFactory; import org.apache.sysds.runtime.compress.estim.encoding.IEncode; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.data.SparseBlockMCSR; @@ -71,6 +70,9 @@ import org.apache.sysds.runtime.matrix.operators.UnaryOperator; import org.jboss.netty.handler.codec.compression.CompressionException; +import jdk.incubator.vector.DoubleVector; +import jdk.incubator.vector.VectorSpecies; + /** * Class to encapsulate information about a column group that is encoded with dense dictionary encoding (DDC). */ @@ -672,7 +674,8 @@ private void defaultRightDecompressingMult(MatrixBlock right, MatrixBlock ret, i } } - final void vectMM(double aa, double[] b, double[] c, int endT, int jd, int crl, int cru, int offOut, int k, int vLen, DoubleVector vVec) { + final void vectMM(double aa, double[] b, double[] c, int endT, int jd, int crl, int cru, int offOut, int k, int vLen, + DoubleVector vVec) { vVec = vVec.broadcast(aa); final int offj = k * jd; final int end = endT + offj; @@ -1095,6 +1098,21 @@ public AColGroup[] splitReshapePushDown(int multiplier, int nRow, int nColOrg, E return res; } + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + return ColGroupDDC.create(_colIndexes, _dict, _data.removeEmpty(selectV, rOut), null); + } + + @Override + protected boolean allowShallowIdentityRightMult() { + return true; + } + + @Override + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + return ColGroupDDC.create(newColumnIDs, _dict.sliceColumns(selectedColumns, getNumCols()), _data, null); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -1104,11 +1122,6 @@ public String toString() { return sb.toString(); } - @Override - protected boolean allowShallowIdentityRightMult() { - return true; - } - public AColGroup convertToDeltaDDC() { int numCols = _colIndexes.size(); int numRows = _data.size(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java index d2ee8cd6673..6a4a92469d2 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java @@ -40,6 +40,7 @@ import org.apache.sysds.runtime.compress.estim.EstimationFactors; import org.apache.sysds.runtime.compress.estim.encoding.EncodingFactory; import org.apache.sysds.runtime.compress.estim.encoding.IEncode; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.compress.utils.Util; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.SparseBlock; @@ -546,6 +547,20 @@ protected boolean allowShallowIdentityRightMult() { return false; } + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + return ColGroupDDCFOR.create(_colIndexes, _dict, _data.removeEmpty(selectV, rOut), null, _reference); + } + + @Override + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + double[] ref = new double[selectedColumns.size()]; + for(int i = 0; i < selectedColumns.size(); i++) { + ref[i] = _reference[selectedColumns.get(i)]; + } + return ColGroupDDCFOR.create(newColumnIDs, _dict.sliceColumns(selectedColumns, getNumCols()), _data, null, ref); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCLZW.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCLZW.java index a3926948b83..c820f875a05 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCLZW.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCLZW.java @@ -1009,4 +1009,17 @@ protected void computeRowProduct(double[] c, int rl, int ru, double[] preAgg) { for(int rix = rl; rix < ru; rix++) c[rix] *= preAgg[it.next()]; } + + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + ColGroupDDC g = (ColGroupDDC) convertToDDC(); + return g.removeEmptyRows(selectV, rOut); + } + + @Override + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, + org.apache.sysds.runtime.compress.utils.IntArrayList selectedColumns) { + ColGroupDDC g = (ColGroupDDC) convertToDDC(); + return g.removeEmptyColsSubset(newColumnIDs, selectedColumns); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java index 6d7872fce54..7c0a15e123b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java @@ -44,6 +44,7 @@ import org.apache.sysds.runtime.compress.estim.EstimationFactors; import org.apache.sysds.runtime.compress.estim.encoding.EncodingFactory; import org.apache.sysds.runtime.compress.estim.encoding.IEncode; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.data.SparseBlockMCSR; @@ -476,4 +477,15 @@ public AColGroup combineWithSameIndex(int nRow, int nCol, List right) return new ColGroupEmpty(combinedIndex); } + + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut){ + return this; + } + + + @Override + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + return new ColGroupEmpty(newColumnIDs); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupIO.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupIO.java index f4e9007575c..6add5967fde 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupIO.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupIO.java @@ -94,9 +94,7 @@ public static long getExactSizeOnDisk(List colGroups) { } ret += grp.getExactSizeOnDisk(); } - if(LOG.isWarnEnabled()) - LOG.warn(" duplicate dicts on exact Size on Disk : " + (colGroups.size() - dicts.size()) ); - + return ret; } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java index 4e9fffaf718..5ac168b9406 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java @@ -32,6 +32,7 @@ import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme; import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator; import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.data.SparseBlockMCSR; @@ -740,4 +741,13 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { throw new NotImplementedException("Unimplemented method 'splitReshape'"); } + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + throw new NotImplementedException("Unimplemented method 'removeEmptyRows'"); + } + + @Override + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java index ea6d0f34c2a..5833729c378 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java @@ -26,15 +26,16 @@ import org.apache.commons.lang3.NotImplementedException; import org.apache.sysds.runtime.compress.CompressionSettings; import org.apache.sysds.runtime.compress.bitmap.ABitmap; -import org.apache.sysds.runtime.compress.colgroup.dictionary.IDictionary; import org.apache.sysds.runtime.compress.colgroup.ColGroupUtils.P; import org.apache.sysds.runtime.compress.colgroup.dictionary.Dictionary; import org.apache.sysds.runtime.compress.colgroup.dictionary.DictionaryFactory; +import org.apache.sysds.runtime.compress.colgroup.dictionary.IDictionary; import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory; import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme; import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator; import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.data.SparseBlockMCSR; @@ -731,5 +732,13 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { throw new NotImplementedException("Unimplemented method 'splitReshape'"); } + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + throw new NotImplementedException("Unimplemented method 'removeEmptyRows'"); + } + @Override + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java index 2b4b23792e3..c9fc920a845 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java @@ -39,6 +39,7 @@ import org.apache.sysds.runtime.compress.colgroup.scheme.RLEScheme; import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator; import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.data.SparseBlockMCSR; @@ -1190,4 +1191,13 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { throw new NotImplementedException("Unimplemented method 'splitReshape'"); } + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + throw new NotImplementedException("Unimplemented method 'removeEmptyRows'"); + } + + @Override + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java index 4340637a737..5522a33e3e0 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java @@ -42,6 +42,7 @@ import org.apache.sysds.runtime.compress.colgroup.offset.AIterator; import org.apache.sysds.runtime.compress.colgroup.offset.AOffset; import org.apache.sysds.runtime.compress.colgroup.offset.AOffset.OffsetSliceInfo; +import org.apache.sysds.runtime.compress.colgroup.offset.AOffset.RemoveEmptyOffsetsTmp; import org.apache.sysds.runtime.compress.colgroup.offset.OffsetFactory; import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator; import org.apache.sysds.runtime.compress.estim.encoding.EncodingFactory; @@ -508,10 +509,10 @@ protected static AColGroup rexpandCols(int max, boolean ignore, boolean cast, in AOffset indexes, AMapToData data, int[] counts, int def, int nVal) { if(d == null) { - if(def <= 0){ + if(def <= 0) { if(max > 0) return ColGroupEmpty.create(max); - else + else return null; } else if(def > max && max > 0) @@ -873,6 +874,23 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { return res; } + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + final RemoveEmptyOffsetsTmp offsetTmp = _indexes.removeEmptyRows(selectV, rOut); + final AMapToData nm = _data.removeEmpty(offsetTmp.select); + return ColGroupSDC.create(_colIndexes, rOut, _dict, _defaultTuple, offsetTmp.retOffset, nm, null); + } + + @Override + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + double[] ref = new double[selectedColumns.size()]; + for(int i = 0; i < selectedColumns.size(); i++) { + ref[i] = _defaultTuple[selectedColumns.get(i)]; + } + return ColGroupSDC.create(newColumnIDs, _numRows, _dict.sliceColumns(selectedColumns, getNumCols()), ref, + _indexes, _data, null); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java index 675c1120c38..2ef7f3012bc 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java @@ -39,6 +39,7 @@ import org.apache.sysds.runtime.compress.colgroup.offset.AIterator; import org.apache.sysds.runtime.compress.colgroup.offset.AOffset; import org.apache.sysds.runtime.compress.colgroup.offset.AOffset.OffsetSliceInfo; +import org.apache.sysds.runtime.compress.colgroup.offset.AOffset.RemoveEmptyOffsetsTmp; import org.apache.sysds.runtime.compress.colgroup.offset.OffsetFactory; import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme; import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator; @@ -620,6 +621,23 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { return res; } + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + final RemoveEmptyOffsetsTmp offsetTmp = _indexes.removeEmptyRows(selectV, rOut); + final AMapToData nm = _data.removeEmpty(offsetTmp.select); + return ColGroupSDCFOR.create(_colIndexes, rOut, _dict, offsetTmp.retOffset, nm, null, _reference); + } + + @Override + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + double[] ref = new double[selectedColumns.size()]; + for(int i = 0; i < selectedColumns.size(); i++) { + ref[i] = _reference[selectedColumns.get(i)]; + } + return ColGroupSDCFOR.create(newColumnIDs, _numRows, _dict.sliceColumns(selectedColumns, getNumCols()), _indexes, _data, null, + ref); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java index a954f380a04..0f89e54d975 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java @@ -40,6 +40,7 @@ import org.apache.sysds.runtime.compress.colgroup.offset.AIterator; import org.apache.sysds.runtime.compress.colgroup.offset.AOffset; import org.apache.sysds.runtime.compress.colgroup.offset.AOffset.OffsetSliceInfo; +import org.apache.sysds.runtime.compress.colgroup.offset.AOffset.RemoveEmptyOffsetsTmp; import org.apache.sysds.runtime.compress.colgroup.offset.OffsetEmpty; import org.apache.sysds.runtime.compress.colgroup.offset.OffsetFactory; import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator; @@ -469,10 +470,10 @@ public AColGroup rexpandCols(int max, boolean ignore, boolean cast, int nRows) { IDictionary d = _dict.rexpandCols(max, ignore, cast, _colIndexes.size()); final int def = (int) _defaultTuple[0]; if(d == null) { - if(def <= 0){ + if(def <= 0) { if(max > 0) return ColGroupEmpty.create(max); - else + else return null; } else if(def > max && max > 0) @@ -718,6 +719,23 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { return res; } + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + // TODO optimize by not constructing boolean array. + final RemoveEmptyOffsetsTmp offsetTmp = _indexes.removeEmptyRows(selectV, rOut); + return ColGroupSDCSingle.create(_colIndexes, rOut, _dict, _defaultTuple, offsetTmp.retOffset, null); + } + + @Override + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + double[] ref = new double[selectedColumns.size()]; + for(int i = 0; i < selectedColumns.size(); i++) { + ref[i] = _defaultTuple[selectedColumns.get(i)]; + } + return ColGroupSDCSingle.create(newColumnIDs, _numRows, _dict.sliceColumns(selectedColumns, getNumCols()), ref, + _indexes, null); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java index 9efd0c41098..d9341bb9ea8 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java @@ -40,6 +40,7 @@ import org.apache.sysds.runtime.compress.colgroup.offset.AIterator; import org.apache.sysds.runtime.compress.colgroup.offset.AOffset; import org.apache.sysds.runtime.compress.colgroup.offset.AOffset.OffsetSliceInfo; +import org.apache.sysds.runtime.compress.colgroup.offset.AOffset.RemoveEmptyOffsetsTmp; import org.apache.sysds.runtime.compress.colgroup.offset.AOffsetIterator; import org.apache.sysds.runtime.compress.colgroup.offset.OffsetEmpty; import org.apache.sysds.runtime.compress.colgroup.offset.OffsetFactory; @@ -109,10 +110,8 @@ protected void decompressToDenseBlockDenseDictionary(DenseBlock db, int rl, int return; else if(it.value() >= ru) return; - // _indexes.cacheIterator(it, ru); else { decompressToDenseBlockDenseDictionaryWithProvidedIterator(db, rl, ru, offR, offC, values, it); - // _indexes.cacheIterator(it, ru); } } @@ -238,7 +237,7 @@ protected void decompressToSparseBlockSparseDictionary(SparseBlock ret, int rl, if(it == null) return; else if(it.value() >= ru) - _indexes.cacheIterator(it, ru); + return; else if(ru > last) { final int apos = sb.pos(0); final int alen = sb.size(0) + apos; @@ -277,8 +276,15 @@ protected void decompressToSparseBlockDenseDictionary(SparseBlock ret, int rl, i if(it == null) return; else if(it.value() >= ru) - _indexes.cacheIterator(it, ru); - else if(ru > _indexes.getOffsetToLast()) { + return; + else + decompressToSparseBlockDenseDictionaryWithProvidedIterator(ret, rl, ru, offR, offC, values, it); + } + + @Override + public void decompressToSparseBlockDenseDictionaryWithProvidedIterator(SparseBlock ret, int rl, int ru, int offR, + int offC, double[] values, final AIterator it) { + if(ru > _indexes.getOffsetToLast()) { final int nCol = _colIndexes.size(); final int lastOff = _indexes.getOffsetToLast(); int row = offR + it.value(); @@ -963,7 +969,7 @@ protected void sparseSelection(MatrixBlock selection, P[] points, MatrixBlock re protected void denseSelection(MatrixBlock selection, P[] points, MatrixBlock ret, int rl, int ru) { throw new NotImplementedException(); } - + protected void decompressToDenseBlockTransposedSparseDictionary(DenseBlock db, int rl, int ru, SparseBlock sb) { throw new NotImplementedException(); } @@ -1043,6 +1049,20 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { return res; } + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + // TODO optimize by not constructing boolean array. + final RemoveEmptyOffsetsTmp offsetTmp = _indexes.removeEmptyRows(selectV, rOut); + return ColGroupSDCSingleZeros.create(_colIndexes, rOut, _dict, offsetTmp.retOffset, null); + } + + @Override + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + + return ColGroupSDCSingleZeros.create(newColumnIDs, _numRows, _dict.sliceColumns(selectedColumns, getNumCols()), + _indexes, null); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java index 69e0f776383..86cd9866a75 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java @@ -45,6 +45,7 @@ import org.apache.sysds.runtime.compress.colgroup.offset.AIterator; import org.apache.sysds.runtime.compress.colgroup.offset.AOffset; import org.apache.sysds.runtime.compress.colgroup.offset.AOffset.OffsetSliceInfo; +import org.apache.sysds.runtime.compress.colgroup.offset.AOffset.RemoveEmptyOffsetsTmp; import org.apache.sysds.runtime.compress.colgroup.offset.OffsetFactory; import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator; import org.apache.sysds.runtime.compress.estim.encoding.EncodingFactory; @@ -184,8 +185,7 @@ private final void decompressToDenseBlockDenseDictionaryPostAllCols(DenseBlock d final double[] c = db.values(idx); final int off = db.pos(idx); final int offDict = _data.getIndex(it.getDataIndex()) * nCol; - for(int j = 0; j < nCol; j++) - c[off + j] += values[offDict + j]; + decompressSingleRow(values, nCol, c, off, offDict); if(it.value() == lastOff) return; it.next(); @@ -301,13 +301,19 @@ private void decompressToDenseBlockDenseDictionaryPreAllCols(DenseBlock db, int final double[] c = db.values(idx); final int off = db.pos(idx) + offC; final int offDict = _data.getIndex(it.getDataIndex()) * nCol; - for(int j = 0; j < nCol; j++) - c[off + j] += values[offDict + j]; + decompressSingleRow(values, nCol, c, off, offDict); it.next(); } } + private static void decompressSingleRow(double[] values, final int nCol, final double[] c, final int off, + final int offDict) { + final int end = nCol + off; + for(int j = off, k = offDict; j < end; j++, k++) + c[j] += values[k]; + } + @Override protected void decompressToDenseBlockSparseDictionary(DenseBlock db, int rl, int ru, int offR, int offC, SparseBlock sb) { @@ -438,8 +444,16 @@ protected void decompressToSparseBlockDenseDictionary(SparseBlock ret, int rl, i if(it == null) return; else if(it.value() >= ru) - _indexes.cacheIterator(it, ru); - else if(ru > _indexes.getOffsetToLast()) { + return; + else + decompressToSparseBlockDenseDictionaryWithProvidedIterator(ret, rl, ru, offR, offC, values, it); + + } + + @Override + public void decompressToSparseBlockDenseDictionaryWithProvidedIterator(SparseBlock ret, int rl, int ru, int offR, + int offC, double[] values, final AIterator it) { + if(ru > _indexes.getOffsetToLast()) { final int lastOff = _indexes.getOffsetToLast(); final int nCol = _colIndexes.size(); while(true) { @@ -467,7 +481,6 @@ else if(ru > _indexes.getOffsetToLast()) { } _indexes.cacheIterator(it, ru); } - } @Override @@ -899,7 +912,6 @@ public AColGroup morph(CompressionType ct, int nRow) { return super.morph(ct, nRow); } - @Override public void sparseSelection(MatrixBlock selection, P[] points, MatrixBlock ret, int rl, int ru) { final SparseBlock sr = ret.getSparseBlock(); @@ -942,14 +954,14 @@ protected void denseSelection(MatrixBlock selection, P[] points, MatrixBlock ret of = it.next(); } else if(points[c].o < of) - c++; + c++; else of = it.next(); - } - // increment the c pointer until it is pointing at least to last point or is done. - while(c < points.length && points[c].o < last) - c++; - c = processRowDense(points, dr, nCol, c, of, _data.getIndex(it.getDataIndex())); + } + // increment the c pointer until it is pointing at least to last point or is done. + while(c < points.length && points[c].o < last) + c++; + c = processRowDense(points, dr, nCol, c, of, _data.getIndex(it.getDataIndex())); } private int processRowSparse(P[] points, final SparseBlock sr, final int nCol, int c, int of, final int did) { @@ -1078,6 +1090,19 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { return res; } + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + final RemoveEmptyOffsetsTmp offsetTmp = _indexes.removeEmptyRows(selectV, rOut); + final AMapToData nm = _data.removeEmpty(offsetTmp.select); + return ColGroupSDCZeros.create(_colIndexes, rOut, _dict, offsetTmp.retOffset, nm, null); + } + + @Override + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + return ColGroupSDCZeros.create(newColumnIDs, _numRows, _dict.sliceColumns(selectedColumns, getNumCols()), + _indexes, _data, null); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java index 8d446575975..e4e98da46f2 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java @@ -43,6 +43,7 @@ import org.apache.sysds.runtime.compress.estim.CompressedSizeInfo; import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup; import org.apache.sysds.runtime.compress.estim.EstimationFactors; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.compress.utils.Util; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.SparseBlock; @@ -82,7 +83,8 @@ public class ColGroupUncompressed extends AColGroup { /** * Do not use this constructor of column group uncompressed, instead use the create constructor. - * @param mb The contained data. + * + * @param mb The contained data. * @param colIndexes Column indexes for this Columngroup */ protected ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes) { @@ -92,14 +94,15 @@ protected ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes) { /** * Do not use this constructor of column group quantization-fused uncompressed, instead use the create constructor. - * @param mb The contained data. - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix - * @param colIndexes Column indexes for this Columngroup + * + * @param mb The contained data. + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix + * @param colIndexes Column indexes for this Columngroup */ protected ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes, double[] scaleFactors) { super(colIndexes); - // Apply scaling and flooring - // TODO: Use internal matrix prod + // Apply scaling and flooring + // TODO: Use internal matrix prod for(int r = 0; r < mb.getNumRows(); r++) { double scaleFactor = scaleFactors.length == 1 ? scaleFactors[0] : scaleFactors[r]; for(int c = 0; c < mb.getNumColumns(); c++) { @@ -108,7 +111,8 @@ protected ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes, double[] sc } } _data = mb; - } + } + /** * Create an Uncompressed Matrix Block, where the columns are offset by col indexes. * @@ -130,9 +134,9 @@ public static AColGroup create(MatrixBlock mb, IColIndex colIndexes) { * * It is assumed that the size of the colIndexes and number of columns in mb is matching. * - * @param mb The MB / data to contain in the uncompressed column - * @param colIndexes The column indexes for the group - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix + * @param mb The MB / data to contain in the uncompressed column + * @param colIndexes The column indexes for the group + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix * @return An Uncompressed Column group */ public static AColGroup createQuantized(MatrixBlock mb, IColIndex colIndexes, double[] scaleFactors) { @@ -147,14 +151,15 @@ public static AColGroup createQuantized(MatrixBlock mb, IColIndex colIndexes, do /** * Main constructor for a quantization-fused uncompressed ColGroup. * - * @param colIndexes Indices (relative to the current block) of the columns that this column group represents. - * @param rawBlock The uncompressed block; uncompressed data must be present at the time that the constructor is - * called - * @param transposed Says if the input matrix raw block have been transposed. - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix + * @param colIndexes Indices (relative to the current block) of the columns that this column group represents. + * @param rawBlock The uncompressed block; uncompressed data must be present at the time that the constructor is + * called + * @param transposed Says if the input matrix raw block have been transposed. + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix * @return AColGroup. */ - public static AColGroup createQuantized(IColIndex colIndexes, MatrixBlock rawBlock, boolean transposed, double[] scaleFactors) { + public static AColGroup createQuantized(IColIndex colIndexes, MatrixBlock rawBlock, boolean transposed, + double[] scaleFactors) { // special cases if(rawBlock.isEmptyBlock(false)) // empty input @@ -187,22 +192,24 @@ else if(!transposed && colIndexes.size() == rawBlock.getNumColumns()) final int n = colIndexes.size(); if(transposed) { - if (scaleFactors.length == 1) { + if(scaleFactors.length == 1) { for(int i = 0; i < m; i++) for(int j = 0; j < n; j++) mb.appendValue(i, j, Math.floor(rawBlock.get(i, colIndexes.get(j)) * scaleFactors[0])); - } else { + } + else { for(int i = 0; i < m; i++) for(int j = 0; j < n; j++) mb.appendValue(i, j, Math.floor(rawBlock.get(i, colIndexes.get(j)) * scaleFactors[j])); } } else { - if (scaleFactors.length == 1) { + if(scaleFactors.length == 1) { for(int i = 0; i < m; i++) for(int j = 0; j < n; j++) mb.appendValue(i, j, Math.floor(rawBlock.get(i, colIndexes.get(j)) * scaleFactors[0])); - } else { + } + else { for(int i = 0; i < m; i++) for(int j = 0; j < n; j++) mb.appendValue(i, j, Math.floor(rawBlock.get(i, colIndexes.get(j)) * scaleFactors[i])); @@ -1075,7 +1082,6 @@ public AColGroup morph(CompressionType ct, int nRow) { return comp.get(0).copyAndSet(_colIndexes); } - @Override public void sparseSelection(MatrixBlock selection, P[] points, MatrixBlock ret, int rl, int ru) { if(_data.isInSparseFormat()) @@ -1092,7 +1098,6 @@ protected void denseSelection(MatrixBlock selection, P[] points, MatrixBlock ret denseSelectionDenseColumnGroup(selection, ret, rl, ru); } - private void sparseSelectionSparseColumnGroup(MatrixBlock selection, MatrixBlock ret, int rl, int ru) { final SparseBlock sb = selection.getSparseBlock(); @@ -1192,7 +1197,7 @@ public AColGroup reduceCols() { else return new ColGroupUncompressed(mb, ColIndexFactory.createI(0)); } - + @Override public void decompressToDenseBlockTransposed(DenseBlock db, int rl, int ru) { if(_data.isInSparseFormat()) @@ -1289,11 +1294,25 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { for(int i = 0; i < multiplier; i++) for(int j = 0; j < s; j++) newColumns[i * s + j] = _colIndexes.get(j) + nColOrg * i; - MatrixBlock newData = _data.reshape(nRow/ multiplier, s * multiplier, true); - return new AColGroup[]{create(newData,ColIndexFactory.create(newColumns))}; + MatrixBlock newData = _data.reshape(nRow / multiplier, s * multiplier, true); + return new AColGroup[] {create(newData, ColIndexFactory.create(newColumns))}; // throw new NotImplementedException("Unimplemented method 'splitReshape'"); } + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + MatrixBlock tmp = new MatrixBlock(); + tmp = LibMatrixReorg.removeEmptyRows(_data, tmp, false, false, selectV, rOut); + return ColGroupUncompressed.create(_colIndexes, tmp, false); + } + + @Override + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + double[] vals = MatrixBlockDictionary.sliceColumns(_data, selectedColumns); + MatrixBlock ret = new MatrixBlock(_data.getNumRows(), selectedColumns.size(), vals); + return ColGroupUncompressed.create(newColumnIDs, ret, false); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java index 08cbab30bcc..0c8f07685b6 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java @@ -19,11 +19,13 @@ package org.apache.sysds.runtime.compress.colgroup; +import org.apache.commons.lang3.NotImplementedException; import org.apache.sysds.runtime.compress.colgroup.ColGroupUtils.P; import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme; import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator; import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.data.SparseBlockMCSR; @@ -282,4 +284,13 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { throw new UnsupportedOperationException("Unimplemented method 'splitReshape'"); } + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + throw new NotImplementedException("Unimplemented method 'removeEmptyRows'"); + } + + @Override + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java index d667e76ed5e..c26de004373 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java @@ -24,6 +24,7 @@ import java.io.IOException; import org.apache.commons.lang3.NotImplementedException; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.functionobjects.Divide; import org.apache.sysds.runtime.functionobjects.Multiply; import org.apache.sysds.runtime.matrix.operators.ScalarOperator; @@ -136,4 +137,9 @@ public boolean equals(IDictionary o) { public IDictionary clone() { throw new NotImplementedException(); } + + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ + throw new NotImplementedException(); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java index e94cbd7c570..06bd811b50b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java @@ -31,6 +31,7 @@ import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.runtime.compress.DMLCompressionException; import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.compress.utils.Util; import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.functionobjects.Builtin; @@ -1341,4 +1342,10 @@ public IDictionary append(double[] row) { return new Dictionary(retV); } + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { + // TODO: make specialized version for this. + return getMBDict(nCol).sliceColumns(selectedColumns, nCol); + } + } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java index 49330ba2748..726df96d5c8 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.functionobjects.Builtin; @@ -1051,4 +1052,14 @@ public IDictionary rightMMPreAggSparse(int numVals, SparseBlock b, IColIndex thi * @return The nonzero count of each column in the dictionary. */ public int[] countNNZZeroColumns(int[] counts); + + /** + * Slice out the selected columns given of this encoded group. + * + * @param selectedColumns The columns to slice out and return as a new matrix. + * @param nCol The number of columns in this dictionary. + * @return The returned matrix + */ + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol); + } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java index 40e1b065653..c2540de959a 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java @@ -27,6 +27,7 @@ import org.apache.commons.lang3.NotImplementedException; import org.apache.sysds.runtime.compress.DMLCompressionException; import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.data.SparseBlockFactory; import org.apache.sysds.runtime.data.SparseBlockMCSR; @@ -540,9 +541,13 @@ public String getString(int colIndexes) { return "IdentityMatrix of size: " + nRowCol + " with empty: " + withEmpty; } + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ + return getMBDict().sliceColumns(selectedColumns, nCol); + } + @Override public String toString() { return "IdentityMatrix of size: " + nRowCol + " with empty: " + withEmpty; } - } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java index df702524d55..c7f642edfd0 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java @@ -27,6 +27,7 @@ import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.compress.DMLCompressionException; import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.functionobjects.Builtin; import org.apache.sysds.runtime.matrix.data.MatrixBlock; @@ -310,6 +311,11 @@ public String getString(int colIndexes) { return toString(); } + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ + return getMBDict().sliceColumns(selectedColumns, nCol); + } + @Override public String toString() { return "IdentityMatrixSlice of size: " + nRowCol + " l " + l + " u " + u; diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java index 71a4112f157..c1d2ecc5296 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java @@ -27,8 +27,6 @@ import java.util.Arrays; import java.util.Set; -import jdk.incubator.vector.DoubleVector; -import jdk.incubator.vector.VectorSpecies; import org.apache.commons.lang3.NotImplementedException; import org.apache.sysds.runtime.compress.DMLCompressionException; import org.apache.sysds.runtime.compress.colgroup.indexes.ArrayIndex; @@ -36,6 +34,7 @@ import org.apache.sysds.runtime.compress.colgroup.indexes.RangeIndex; import org.apache.sysds.runtime.compress.colgroup.indexes.SingleIndex; import org.apache.sysds.runtime.compress.colgroup.indexes.TwoIndex; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.compress.utils.Util; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.DenseBlockFP64; @@ -61,6 +60,9 @@ import org.apache.sysds.runtime.matrix.operators.ScalarOperator; import org.apache.sysds.runtime.matrix.operators.UnaryOperator; +import jdk.incubator.vector.DoubleVector; +import jdk.incubator.vector.VectorSpecies; + public class MatrixBlockDictionary extends ADictionary { private static final long serialVersionUID = 2535887782150955098L; @@ -2801,4 +2803,46 @@ private void SparseAdd(int sPos, int sEnd, double[] ret, int offOut, int[] sIdx, } } + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { + + final double[] ret = sliceColumns(_data, selectedColumns); + + return new Dictionary(ret); + } + + public static double[] sliceColumns(MatrixBlock mb, IntArrayList selectedColumns) { + // TODO: Optimize to allow sparse outputs. and change output type to MatrixBlock. + final int outC = selectedColumns.size(); + final int nRow = mb.getNumRows(); + if((long) nRow * outC > (long) Integer.MAX_VALUE) + throw new NotImplementedException("Not supported large output blocks for slicing dictionary columns"); + final double[] ret = new double[nRow * outC]; + if(mb.isEmpty()) + return ret; + + // Read through the current representation without mutating the (shared, immutable) dictionary block. + if(mb.isInSparseFormat()) { + final SparseBlock sb = mb.getSparseBlock(); + for(int i = 0; i < nRow; i++) { + if(sb.isEmpty(i)) + continue; + final int offOut = i * outC; + for(int j = 0; j < outC; j++) + ret[offOut + j] = sb.get(i, selectedColumns.get(j)); + } + } + else { + final DenseBlock db = mb.getDenseBlock(); + for(int i = 0; i < nRow; i++) { + final double[] vals = db.values(i); + final int offIn = db.pos(i); + final int offOut = i * outC; + for(int j = 0; j < outC; j++) + ret[offOut + j] = vals[offIn + selectedColumns.get(j)]; + } + } + return ret; + } + } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/PlaceHolderDict.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/PlaceHolderDict.java index f5746647a37..2d9075f73c9 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/PlaceHolderDict.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/PlaceHolderDict.java @@ -23,6 +23,7 @@ import java.io.DataOutput; import java.io.IOException; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.io.IOUtilFunctions; public class PlaceHolderDict extends ADictionary { @@ -101,4 +102,9 @@ public DictType getDictType() { throw new RuntimeException("invalid to get dictionary type for PlaceHolderDict"); } + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { + throw new RuntimeException("Invalid call"); + } + } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/QDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/QDictionary.java index 6802d920b49..30b9d806c1f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/QDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/QDictionary.java @@ -23,6 +23,7 @@ import java.io.DataOutput; import java.io.IOException; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.functionobjects.Builtin; import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.utils.MemoryEstimates; @@ -277,4 +278,8 @@ public MatrixBlockDictionary createMBDict(int nCol) { return new MatrixBlockDictionary(mb); } + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { + return getMBDict().sliceColumns(selectedColumns, nCol); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java index 5fc2acaea7a..83a74972db7 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java @@ -30,6 +30,7 @@ import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.compress.CompressedMatrixBlock; import org.apache.sysds.runtime.compress.DMLCompressionException; import org.apache.sysds.runtime.compress.colgroup.IMapToDataGroup; @@ -39,6 +40,7 @@ import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE; import org.apache.sysds.runtime.compress.colgroup.offset.AOffset; import org.apache.sysds.runtime.compress.colgroup.offset.AOffsetIterator; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.matrix.data.MatrixBlock; @@ -1041,4 +1043,36 @@ public String toString() { sb.append("]"); return sb.toString(); } + + public AMapToData removeEmpty(final boolean[] selectV, final int rOut) { + final int s = size(); + int trueCount = 0; + for(int i = 0; i < s; i++) + if(selectV[i]) + trueCount++; + if(trueCount != rOut) + throw new DMLRuntimeException( + "Invalid removeEmpty: number of selected rows " + trueCount + " does not match argument rOut " + rOut); + + final AMapToData ret = MapToFactory.create(rOut, getUnique()); + int t = 0; + for(int i = 0; i < s; i++) + if(selectV[i]) + ret.set(t++, getIndex(i)); + return ret; + } + + /** + * Use the offsets of the select vector to choose which values to keep. + * + * @param select The row indexes to keep + * @return A New MapToData + */ + public AMapToData removeEmpty(IntArrayList select) { + final int s = select.size(); + final AMapToData ret = MapToFactory.create(s, getUnique()); + for(int i = 0; i < s; i++) + ret.set(i, getIndex(select.get(i))); + return ret; + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AIterator.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AIterator.java index 45c78dd3abd..a809afccd3d 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AIterator.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AIterator.java @@ -71,8 +71,8 @@ public boolean isNotOver(int ub) { /** * Get the current data index associated with the index returned from value. * - * This index points to a position int the mapToData object, that then inturn can be used to lookup the dictionary - * entry in ADictionary. + * This index points to a position in the AMapToData object, that can be used to lookup the dictionary entry in + * ADictionary. * * @return The Data Index. */ diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java index a961c1188bf..f65876b7f37 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java @@ -586,7 +586,7 @@ public OffsetSliceInfo slice(int l, int u) { else return new OffsetSliceInfo(0, s, moveIndex(l)); } - else if (u < first) + else if(u < first) return emptySlice(); final AIterator it = getIteratorSkipCache(l); @@ -781,6 +781,41 @@ public AOffset reverse(int numRows) { return OffsetFactory.createOffset(newOff); } + public RemoveEmptyOffsetsTmp removeEmptyRows(boolean[] selectV, int rOut) { + IntArrayList newOff = new IntArrayList(); + IntArrayList selectMTmp = new IntArrayList(); + + final AIterator it = getIterator(); + final int last = getOffsetToLast(); + int t = 0; + int o = 0; + while(it.value() < last) { + while(t < it.value()) { + if(selectV[t]) + o++; + t++; + } + if(selectV[it.value()]) { + newOff.appendValue(o); + selectMTmp.appendValue(it.getDataIndex()); + o++; + t++; + } + it.next(); + } + while(t < last) { + if(selectV[t]) + o++; + t++; + } + if(selectV[last]) { + newOff.appendValue(o); + selectMTmp.appendValue(it.getDataIndex()); + } + + return new RemoveEmptyOffsetsTmp(OffsetFactory.createOffset(newOff), selectMTmp); + } + /** * Offset slice info containing the start and end index an offset that contains the slice, and an new AOffset * containing only the sliced elements @@ -810,6 +845,16 @@ public String toString() { } + public static final class RemoveEmptyOffsetsTmp { + public final AOffset retOffset; + public final IntArrayList select; + + protected RemoveEmptyOffsetsTmp(AOffset retOffset, IntArrayList select) { + this.retOffset = retOffset; + this.select = select; + } + } + private static class OffsetCache { private final AIterator it; private final int row; @@ -841,4 +886,5 @@ public String toString() { return "r" + row + " d " + dataIndex + " o " + offIndex + "\n"; } } + } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java index acd3b0d04eb..866168ded2f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java @@ -76,6 +76,10 @@ public int getOffsetToLast() { public long getInMemorySize() { return estimateInMemorySize(); } + @Override + public boolean equals(AOffset b) { + return b instanceof OffsetEmpty; + } public static long estimateInMemorySize() { return 16; // object header diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java new file mode 100644 index 00000000000..3755e4040e7 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.compress.lib; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang3.NotImplementedException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.sysds.runtime.compress.CompressedMatrixBlock; +import org.apache.sysds.runtime.compress.colgroup.AColGroup; +import org.apache.sysds.runtime.matrix.data.LibMatrixReorg; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.util.DataConverter; + +public class CLALibRemoveEmpty { + protected static final Log LOG = LogFactory.getLog(CLALibRemoveEmpty.class.getName()); + + /** + * CP rmempty operation (single input, single output matrix) + * + * @param in The input matrix + * @param ret The output matrix + * @param rows If we are removing based on rows, or columns. + * @param emptyReturn Return row/column of zeros for empty input. + * @param select An optional selection vector, to remove based on rather than empty rows or columns + * @return The result MatrixBlock, can be a different object that the caller used. + */ + public static MatrixBlock rmempty(CompressedMatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, + MatrixBlock select) { + if(ret == null) + ret = new MatrixBlock(); + MatrixBlock ret2 = LibMatrixReorg.rmemptyEarlyAbort(in, ret, rows, emptyReturn, select); + if(ret2 != null) + return ret2; + + if(rows) + return rmEmptyRows(in, ret, emptyReturn, select); + else + return rmEmptyCols(in, ret, emptyReturn, select); + } + + private static MatrixBlock rmEmptyCols(CompressedMatrixBlock in, MatrixBlock ret, boolean emptyReturn, + MatrixBlock select) { + if(select == null) + return fallback(in, false, emptyReturn, select, ret); + + int cOut = (int) select.getNonZeros(); + if(cOut == -1) + cOut = (int) select.recomputeNonZeros(); + if(cOut == 0){ + ret.reset(in.getNumRows(), !emptyReturn ? 0 : 1); + return ret; + } + + final boolean[] selectV = DataConverter + .convertToBooleanVector(CompressedMatrixBlock.getUncompressed(select, "decompressing selection in rmempty")); + + final List inG = in.getColGroups(); + final List retG = new ArrayList<>(inG.size()); + try { + for(int i = 0; i < inG.size(); i++) { + AColGroup tmp = inG.get(i).removeEmptyCols(selectV); + if(tmp != null) + retG.add(tmp); + } + } + catch(NotImplementedException e) { + // Some column-group encodings (e.g. OLE/RLE) do not support index-only column removal; + // decompress and remove on the uncompressed representation instead of failing. + return fallback(in, false, emptyReturn, select, ret); + } + return new CompressedMatrixBlock(in.getNumRows(), cOut, -1, in.isOverlapping(), retG); + + } + + private static MatrixBlock rmEmptyRows(CompressedMatrixBlock in, MatrixBlock ret, boolean emptyReturn, + MatrixBlock select) { + if(select == null) + return fallback(in, true, emptyReturn, select, ret); + + select = CompressedMatrixBlock.getUncompressed(select, "decompressing selection in rmempty"); + + int rOut = (int) select.getNonZeros(); + if(rOut == -1) + rOut = (int) select.recomputeNonZeros(); + if(rOut == 0){ + ret.reset(!emptyReturn ? 0 : 1, in.getNumColumns()); + return ret; + } + + // TODO: add optimization to avoid linear scan and make selectV indexes, if selection is small relative to number + // of rows + // TODO: add decompress to boolean vector. + final boolean[] selectV = DataConverter.convertToBooleanVector(select); + + + + final List inG = in.getColGroups(); + final List retG = new ArrayList<>(inG.size()); + try { + for(int i = 0; i < inG.size(); i++) { + retG.add(inG.get(i).removeEmptyRows(selectV, rOut)); + } + } + catch(NotImplementedException e) { + // Some column-group encodings (e.g. OLE/RLE) do not support index-only row removal; + // decompress and remove on the uncompressed representation instead of failing. + return fallback(in, true, emptyReturn, select, ret); + } + + return new CompressedMatrixBlock(rOut, in.getNumColumns(), -1, in.isOverlapping(), retG); + } + + private static MatrixBlock fallback(CompressedMatrixBlock in, boolean rows, boolean emptyReturn, MatrixBlock select, + MatrixBlock ret) { + if(LOG.isDebugEnabled()) + LOG.debug("Decompressing for removeEmptyOperations with select: " + (select != null) + " rows: " + rows); + MatrixBlock tmp = CompressedMatrixBlock.getUncompressed(in); + MatrixBlock select2 = CompressedMatrixBlock.getUncompressed(select); + return LibMatrixReorg.rmemptyUnsafe(tmp, ret, rows, emptyReturn, select2); + } + +} diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java index 040a4e1dcb1..5f478979104 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java @@ -964,16 +964,45 @@ public static List reshape(IndexedMatrixValue in, DataCharac } /** - * CP rmempty operation (single input, single output matrix) + * CP rmempty operation (single input, single output matrix) * - * @param in input matrix - * @param ret output matrix - * @param rows ? - * @param emptyReturn return row/column of zeros for empty input - * @param select ? - * @return matrix block + * @param in The input matrix + * @param ret The output matrix + * @param rows If we are removing based on rows, or columns. + * @param emptyReturn Return row/column of zeros for empty input + * @param select An optional selection vector, to remove based on rather than empty rows or columns + * @return The result MatrixBlock */ public static MatrixBlock rmempty(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, MatrixBlock select) { + if(ret == null) + ret = new MatrixBlock(); + MatrixBlock ret2 = rmemptyEarlyAbort(in, ret, rows, emptyReturn, select); + if(ret2 != null ) + return ret2; + // core removeEmpty + return rmemptyUnsafe(in, ret, rows, emptyReturn, select); + } + + public static MatrixBlock rmemptyUnsafe(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, + MatrixBlock select) { + if( rows ) + return removeEmptyRows(in, ret, select, emptyReturn); + else // cols + return removeEmptyColumns(in, ret, select, emptyReturn); + } + + /** + * Handle the early-termination cases of removeEmpty that do not require scanning for empty rows/columns. + * + * @param in The input matrix + * @param ret The output matrix, reused for the empty-input case + * @param rows If removing based on rows, or columns + * @param emptyReturn Return a row/column of zeros for empty input + * @param select An optional selection vector + * @return The early-abort result, or {@code null} if no early termination applies and the caller must continue. + * For the select-all case the returned block is the input {@code in} itself (a shallow alias, not a copy). + */ + public static MatrixBlock rmemptyEarlyAbort(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, MatrixBlock select){ //check for empty inputs //(the semantics of removeEmpty are that for an empty m-by-n matrix, the output //is an empty 1-by-n or m-by-1 matrix because we don't allow matrices with dims 0) @@ -990,12 +1019,8 @@ public static MatrixBlock rmempty(MatrixBlock in, MatrixBlock ret, boolean rows, if( select != null && (select.nonZeros == (rows?in.rlen:in.clen)) ) { return in; } - - // core removeEmpty - if( rows ) - return removeEmptyRows(in, ret, select, emptyReturn); - else //cols - return removeEmptyColumns(in, ret, select, emptyReturn); + + return null; } /** @@ -3620,6 +3645,25 @@ private static MatrixBlock removeEmptyRows(MatrixBlock in, MatrixBlock ret, Matr rlen2 = (int)select.getNonZeros(); } + return removeEmptyRows(in, ret, emptyReturn, select == null, flags, rlen2); + } + + /** + * Remove selected rows, based on the boolean array given. Note this function is internal use only, and require a + * boolean vector to be constructed first. + * + * @param in Input to remove rows from + * @param ret Output to assign the result into + * @param emptyReturn If the output is allowed to be empty. + * @param selectNull If the original caller did not have a selection matrix. + * @param flags The boolean selection vector to specify which rows to keep. + * @param rlen2 The number of true values in the flags argument. + * @return Another reference to the ret matrix input argument. + */ + public static MatrixBlock removeEmptyRows(MatrixBlock in, MatrixBlock ret, boolean emptyReturn, boolean selectNull, + boolean[] flags, int rlen2) { + final int m = in.rlen; + final int n = in.clen; //Step 2: reset result and copy rows //dense stays dense if correct input representation (but robust for any input), //sparse might be dense/sparse @@ -3629,7 +3673,7 @@ private static MatrixBlock removeEmptyRows(MatrixBlock in, MatrixBlock ret, Matr if( in.isEmptyBlock(false) ) return ret; - if( SHALLOW_COPY_REORG && m == rlen2 && select == null ) { + if( SHALLOW_COPY_REORG && m == rlen2 && selectNull ) { // the condition m==rlen2 is not enough with non-empty 1-row input but empty // 1-row select vector because if emptyReturn should output a single empty row ret.sparse = in.sparse; @@ -3672,7 +3716,7 @@ else if( !in.sparse && !ret.sparse ) //DENSE <- DENSE } //check sparsity - ret.nonZeros = (select==null) ? + ret.nonZeros = (selectNull) ? in.nonZeros : ret.recomputeNonZeros(); ret.examSparsity(); diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedMatrixTest.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedMatrixTest.java index d36c6167cf7..934a5458557 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/CompressedMatrixTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/CompressedMatrixTest.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.junit.Assume.assumeTrue; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -687,4 +688,113 @@ public void toRDDAndBack(int blen) { fail(e.getMessage()); } } + + @Test + public void removeEmptyOperationsBase1() { + removeEmptyOperations(false, false, null); + } + + @Test + public void removeEmptyOperationsBase2() { + removeEmptyOperations(true, false, null); + } + + @Test + public void removeEmptyOperationsBase3() { + removeEmptyOperations(false, true, null); + } + + @Test + public void removeEmptyOperationsBase4() { + removeEmptyOperations(true, true, null); + } + + @Test + public void removeEmptyOperationsSelect1() { + // limit to smaller row counts to keep the dense selection vector generation cheap + assumeTrue(rows < 5000); + MatrixBlock s = TestUtils.generateTestMatrixBlock(rows, 1, 1, 1, 0.05, 321); + removeEmptyOperations(true, false, s); + } + + @Test + public void removeEmptyOperationsSelect2() { + // limit to smaller row counts to keep the dense selection vector generation cheap + assumeTrue(rows < 5000); + MatrixBlock s = TestUtils.generateTestMatrixBlock(1, cols, 1, 1, 0.5, 321); + removeEmptyOperations(false, false, s); + } + + @Test + public void removeEmptyOperationsSelectRowsEmptyReturn() { + assumeTrue(rows < 5000); + MatrixBlock s = TestUtils.generateTestMatrixBlock(rows, 1, 1, 1, 0.05, 321); + removeEmptyOperations(true, true, s); + } + + @Test + public void removeEmptyOperationsSelectColsEmptyReturn() { + assumeTrue(rows < 5000); + MatrixBlock s = TestUtils.generateTestMatrixBlock(1, cols, 1, 1, 0.5, 321); + removeEmptyOperations(false, true, s); + } + + @Test + public void removeEmptyOperationsSelectRowsDense() { + assumeTrue(rows < 5000); + MatrixBlock s = TestUtils.generateTestMatrixBlock(rows, 1, 1, 1, 0.6, 654); + removeEmptyOperations(true, false, s); + } + + @Test + public void removeEmptyOperationsSelectAllRows() { + assumeTrue(rows < 5000); + MatrixBlock s = TestUtils.generateTestMatrixBlock(rows, 1, 1, 1, 1.0, 13); + removeEmptyOperations(true, false, s); + } + + @Test + public void removeEmptyOperationsSelectAllCols() { + assumeTrue(rows < 5000); + MatrixBlock s = TestUtils.generateTestMatrixBlock(1, cols, 1, 1, 1.0, 13); + removeEmptyOperations(false, false, s); + } + + @Test + public void removeEmptyOperationsSelectNoRows() { + assumeTrue(rows < 5000); + removeEmptyOperations(true, false, new MatrixBlock(rows, 1, true)); + } + + @Test + public void removeEmptyOperationsSelectNoRowsEmptyReturn() { + assumeTrue(rows < 5000); + removeEmptyOperations(true, true, new MatrixBlock(rows, 1, true)); + } + + @Test + public void removeEmptyOperationsSelectNoCols() { + assumeTrue(rows < 5000); + removeEmptyOperations(false, false, new MatrixBlock(1, cols, true)); + } + + @Test + public void removeEmptyOperationsSelectNoColsEmptyReturn() { + assumeTrue(rows < 5000); + removeEmptyOperations(false, true, new MatrixBlock(1, cols, true)); + } + + public void removeEmptyOperations(boolean rows, boolean emptyReturn, MatrixBlock select) { + try { + MatrixBlock a = cmb.removeEmptyOperations(null, rows, emptyReturn, select); + MatrixBlock b = mb.removeEmptyOperations(null, rows, emptyReturn, select); + compareResultMatrices(b, a, 0); + } + catch(Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + } + } diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedRemoveEmptyColSubsetTest.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedRemoveEmptyColSubsetTest.java new file mode 100644 index 00000000000..e6e28016ee2 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/compress/CompressedRemoveEmptyColSubsetTest.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.compress; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.Random; + +import org.apache.sysds.runtime.compress.CompressedMatrixBlock; +import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory; +import org.apache.sysds.runtime.compress.CompressionSettingsBuilder; +import org.apache.sysds.runtime.compress.colgroup.AColGroup; +import org.apache.sysds.runtime.compress.colgroup.AColGroup.CompressionType; +import org.apache.sysds.runtime.compress.colgroup.ColGroupSDC; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; + +/** + * Exercises {@code removeEmptyOperations} with a column-selection vector that keeps a strict subset of a + * multi-column column group. This drives the per-encoding {@code removeEmptyColsSubset} dictionary slicing + * paths (SDC / SDC-single) and the {@link org.apache.commons.lang3.NotImplementedException} fallback for encodings that + * do not implement index-only column removal (RLE/OLE). + */ +public class CompressedRemoveEmptyColSubsetTest { + + private static final int ROWS = 500; + private static final int COLS = 3; + + @Test + public void colSubsetSDC() { + // Multiple distinct non-default tuples -> ColGroupSDC. + runColSubset(buildIdentical(new double[] {3.0, 5.0, 9.0}, 1), CompressionType.SDC); + } + + @Test + public void colSubsetSDCSingle() { + // A single distinct non-default tuple -> ColGroupSDCSingle. + runColSubset(buildIdentical(new double[] {3.0}, 2), CompressionType.SDC); + } + + @Test + public void colSubsetDDC() { + runColSubset(buildIdentical(new double[] {3.0, 5.0, 9.0}, 3), CompressionType.DDC); + } + + @Test + public void colSubsetRLEFallback() { + runColSubset(buildIdentical(new double[] {3.0, 5.0, 9.0}, 4), CompressionType.RLE); + } + + @Test + public void colSubsetOLEFallback() { + runColSubset(buildIdentical(new double[] {3.0, 5.0, 9.0}, 5), CompressionType.OLE); + } + + @Test + public void colSubsetSDCFOR() { + // SDCFOR cannot be forced at the planner level, so build a multi-column SDC group and sparsify it to the + // frame-of-reference variant (the production path) before slicing a strict column subset. + MatrixBlock mb = buildIdentical(new double[] {3.0, 5.0, 9.0}, 8); + CompressedMatrixBlock sdc = compressForced(mb, CompressionType.SDC); + AColGroup g = sdc.getColGroups().get(0); + assumeTrue("Expected a multi-column ColGroupSDC to sparsify", g instanceof ColGroupSDC && g.getNumCols() > 1); + AColGroup forGroup = ((ColGroupSDC) g).sparsifyFOR(); + assumeTrue("Expected an SDCFOR group after sparsify", forGroup.getCompType() == CompressionType.SDCFOR); + + CompressedMatrixBlock cmb = new CompressedMatrixBlock(mb.getNumRows(), mb.getNumColumns(), -1, false, + Collections.singletonList(forGroup)); + + MatrixBlock select = new MatrixBlock(1, COLS, false); + select.set(0, 0, 1); + select.set(0, 2, 1); + MatrixBlock actual = cmb.removeEmptyOperations(null, false, false, select); + select = new MatrixBlock(1, COLS, false); + select.set(0, 0, 1); + select.set(0, 2, 1); + MatrixBlock expected = mb.removeEmptyOperations(null, false, false, select); + TestUtils.compareMatrices(expected, actual, 0.0, "removeEmpty col subset for SDCFOR"); + } + + /** Column selection vector with unknown (-1) non-zero count, forcing the recompute branch. */ + @Test + public void colSubsetUnknownNnz() { + MatrixBlock mb = buildIdentical(new double[] {3.0, 5.0, 9.0}, 6); + CompressedMatrixBlock cmb = compressForced(mb, CompressionType.SDC); + MatrixBlock select = new MatrixBlock(1, COLS, false); + select.set(0, 0, 1); + select.set(0, 2, 1); + select.setNonZeros(-1); + MatrixBlock actual = cmb.removeEmptyOperations(null, false, false, select); + select = new MatrixBlock(1, COLS, false); + select.set(0, 0, 1); + select.set(0, 2, 1); + MatrixBlock expected = mb.removeEmptyOperations(null, false, false, select); + TestUtils.compareMatrices(expected, actual, 0.0, "removeEmpty cols unknown-nnz select"); + } + + /** Row selection vector with unknown (-1) non-zero count, forcing the recompute branch. */ + @Test + public void rowsUnknownNnz() { + MatrixBlock mb = buildIdentical(new double[] {3.0, 5.0, 9.0}, 7); + CompressedMatrixBlock cmb = compressForced(mb, CompressionType.SDC); + MatrixBlock select = rowSelect(); + select.setNonZeros(-1); + MatrixBlock actual = cmb.removeEmptyOperations(null, true, false, select); + MatrixBlock expected = mb.removeEmptyOperations(null, true, false, rowSelect()); + TestUtils.compareMatrices(expected, actual, 0.0, "removeEmpty rows unknown-nnz select"); + } + + private void runColSubset(MatrixBlock mb, CompressionType ct) { + CompressedMatrixBlock cmb = compressForced(mb, ct); + assertTrue("Expected a multi-column " + ct + " group to reach the subset path", + cmb.getColGroups().stream().anyMatch(g -> g.getNumCols() > 1)); + + // Keep a strict subset (drop the middle column) so removeEmptyColsSubset is hit instead of copyAndSet. + MatrixBlock select = new MatrixBlock(1, COLS, false); + select.set(0, 0, 1); + select.set(0, 2, 1); + + MatrixBlock actual = cmb.removeEmptyOperations(null, false, false, select); + select = new MatrixBlock(1, COLS, false); + select.set(0, 0, 1); + select.set(0, 2, 1); + MatrixBlock expected = mb.removeEmptyOperations(null, false, false, select); + TestUtils.compareMatrices(expected, actual, 0.0, "removeEmpty col subset for " + ct); + } + + private static MatrixBlock rowSelect() { + MatrixBlock select = new MatrixBlock(ROWS, 1, false); + for(int i = 0; i < ROWS; i += 2) + select.set(i, 0, 1); + return select; + } + + private static CompressedMatrixBlock compressForced(MatrixBlock mb, CompressionType ct) { + CompressionSettingsBuilder csb = new CompressionSettingsBuilder().setMinimumCompressionRatio(0.0) + .setValidCompressions(EnumSet.of(ct)); + MatrixBlock c = CompressedMatrixBlockFactory.compress(mb, 1, csb).getLeft(); + assertTrue("Expected the input to compress into a " + ct + " backed block", c instanceof CompressedMatrixBlock); + return (CompressedMatrixBlock) c; + } + + /** + * Builds a {@code ROWS x COLS} matrix whose columns are identical so column co-coding merges them into a single + * multi-column group, with one dominant value plus a few off-values. + */ + private static MatrixBlock buildIdentical(double[] others, int seed) { + MatrixBlock mb = new MatrixBlock(ROWS, COLS, false); + mb.allocateDenseBlock(); + Random r = new Random(seed); + for(int i = 0; i < ROWS; i++) { + double v = 7.0; + if(r.nextDouble() < 0.2) + v = others[r.nextInt(others.length)]; + for(int j = 0; j < COLS; j++) + mb.set(i, j, v); + } + mb.recomputeNonZeros(); + return mb; + } +} diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedRemoveEmptyForcedTest.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedRemoveEmptyForcedTest.java new file mode 100644 index 00000000000..f08cbd92f42 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/compress/CompressedRemoveEmptyForcedTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.compress; + +import static org.junit.Assert.assertTrue; + +import java.util.EnumSet; + +import org.apache.sysds.runtime.compress.CompressedMatrixBlock; +import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory; +import org.apache.sysds.runtime.compress.CompressionSettingsBuilder; +import org.apache.sysds.runtime.compress.colgroup.AColGroup; +import org.apache.sysds.runtime.compress.colgroup.AColGroup.CompressionType; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; + +/** + * Verifies that {@code removeEmptyOperations} with a selection vector degrades gracefully (decompresses) for + * column-group encodings that do not implement index-only row/column removal (e.g. OLE/RLE), rather than throwing + * {@link org.apache.commons.lang3.NotImplementedException}. + */ +public class CompressedRemoveEmptyForcedTest { + + private static final int ROWS = 500; + private static final int COLS = 4; + + @Test + public void removeEmptyRowsFallbackOLE() { + runFallback(CompressionType.OLE, true); + } + + @Test + public void removeEmptyRowsFallbackRLE() { + runFallback(CompressionType.RLE, true); + } + + @Test + public void removeEmptyColsFallbackRLE() { + runFallback(CompressionType.RLE, false); + } + + private void runFallback(CompressionType ct, boolean rows) { + MatrixBlock mb = CompressibleInputGenerator.getInput(ROWS, COLS, ct, 10, 0.6, 7); + + CompressionSettingsBuilder csb = new CompressionSettingsBuilder().setMinimumCompressionRatio(0.0) + .setValidCompressions(EnumSet.of(ct)); + MatrixBlock compressed = CompressedMatrixBlockFactory.compress(mb, 1, csb).getLeft(); + assertTrue("Expected the input to compress into a " + ct + " backed block", + compressed instanceof CompressedMatrixBlock); + CompressedMatrixBlock cmb = (CompressedMatrixBlock) compressed; + assertTrue("Expected at least one " + ct + " column group to exercise the fallback path", + containsType(cmb, ct)); + + // Use a strict subset selection so the column path reaches removeEmptyColsSubset (which throws + // NotImplementedException for OLE/RLE) rather than the copyAndSet all-selected shortcut. + final MatrixBlock select; + if(rows) { + select = new MatrixBlock(ROWS, 1, false); + for(int i = 0; i < ROWS; i += 2) + select.set(i, 0, 1); + } + else { + select = new MatrixBlock(1, COLS, false); + select.set(0, 0, 1); + } + + // Must not throw NotImplementedException; must match the uncompressed reference via decompression fallback. + MatrixBlock actual = cmb.removeEmptyOperations(null, rows, false, select); + MatrixBlock expected = mb.removeEmptyOperations(null, rows, false, select); + TestUtils.compareMatrices(expected, actual, 0.0, "removeEmpty fallback for " + ct + " rows=" + rows); + } + + private static boolean containsType(CompressedMatrixBlock cmb, CompressionType ct) { + for(AColGroup g : cmb.getColGroups()) + if(g.getCompType() == ct) + return true; + return false; + } +} diff --git a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java index e6e41755dd9..af21b14206a 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java +++ b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java @@ -49,6 +49,7 @@ import org.apache.sysds.runtime.compress.estim.CompressedSizeInfo; import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup; import org.apache.sysds.runtime.compress.lib.CLALibLeftMultBy; +import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.data.SparseBlockMCSR; @@ -468,6 +469,18 @@ public AColGroup[] splitReshapePushDown(int multiplier, int nRow, int nColOrg, E // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'splitReshapePushDown'"); } + + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'removeEmptyRows'"); + } + + @Override + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'removeEmptyColsSubset'"); + } } private class FakeDictBasedColGroup extends ADictBasedColGroup { @@ -777,5 +790,17 @@ public AColGroup[] splitReshapePushDown(int multiplier, int nRow, int nColOrg, E // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'splitReshapePushDown'"); } + + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'removeEmptyRows'"); + } + + @Override + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'removeEmptyColsSubset'"); + } } } diff --git a/src/test/java/org/apache/sysds/test/component/compress/offset/CustomOffsetTest.java b/src/test/java/org/apache/sysds/test/component/compress/offset/CustomOffsetTest.java index 2e901eeb14d..3755365c018 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/offset/CustomOffsetTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/offset/CustomOffsetTest.java @@ -28,13 +28,14 @@ import org.apache.sysds.runtime.compress.colgroup.offset.AIterator; import org.apache.sysds.runtime.compress.colgroup.offset.AOffset; import org.apache.sysds.runtime.compress.colgroup.offset.AOffset.OffsetSliceInfo; +import org.apache.sysds.runtime.compress.colgroup.offset.AOffset.RemoveEmptyOffsetsTmp; import org.apache.sysds.runtime.compress.colgroup.offset.OffsetFactory; import org.junit.Test; public class CustomOffsetTest { protected static final Log LOG = LogFactory.getLog(CustomOffsetTest.class.getName()); - static{ + static { CompressedMatrixBlock.debug = true; } @@ -96,4 +97,95 @@ public void printCache() { String s = off.toString(); assertTrue(s.contains("CacheRow")); } + + @Test + public void removeEmptyRows1() { + AOffset of = OffsetFactory.createOffset(new int[] {1, 2, 3, 4, 5}); + RemoveEmptyOffsetsTmp t = of.removeEmptyRows(new boolean[] {false, true, false, false, false, false}, 0); + assertEquals(1, t.select.size()); + assertEquals(0, t.select.get(0)); + assertEquals(1, t.retOffset.getSize()); + assertEquals(OffsetFactory.createOffset(new int[] {0}), t.retOffset); + } + + @Test + public void removeEmptyRows2() { + AOffset of = OffsetFactory.createOffset(new int[] {1, 2, 3, 4, 5}); + RemoveEmptyOffsetsTmp t = of.removeEmptyRows(new boolean[] {false, false, true, false, false, false}, 0); + assertEquals(1, t.select.size()); + assertEquals(1, t.select.get(0)); + assertEquals(1, t.retOffset.getSize()); + assertEquals(OffsetFactory.createOffset(new int[] {0}), t.retOffset); + } + + @Test + public void removeEmptyRows3() { + AOffset of = OffsetFactory.createOffset(new int[] {1, 2, 3, 4, 5}); + RemoveEmptyOffsetsTmp t = of.removeEmptyRows(new boolean[] {false, true, true, false, false, false}, 0); + assertEquals(2, t.select.size()); + assertEquals(0, t.select.get(0)); + assertEquals(1, t.select.get(1)); + assertEquals(2, t.retOffset.getSize()); + assertEquals(OffsetFactory.createOffset(new int[] {0, 1}), t.retOffset); + } + + @Test + public void removeEmptyRows4() { + AOffset of = OffsetFactory.createOffset(new int[] {1, 2, 3, 4, 5}); + RemoveEmptyOffsetsTmp t = of.removeEmptyRows(new boolean[] {false, true, true, false, false, true}, 0); + assertEquals(3, t.select.size()); + assertEquals(0, t.select.get(0)); + assertEquals(1, t.select.get(1)); + assertEquals(4, t.select.get(2)); + assertEquals(3, t.retOffset.getSize()); + assertEquals(OffsetFactory.createOffset(new int[] {0, 1, 2}), t.retOffset); + } + + @Test + public void removeEmptyRows5() { + AOffset of = OffsetFactory.createOffset(new int[] {1, 2, 3, 4, 5}); + RemoveEmptyOffsetsTmp t = of.removeEmptyRows(new boolean[] {false, false, false, false, false, true}, 0); + assertEquals(1, t.select.size()); + assertEquals(4, t.select.get(0)); + assertEquals(1, t.retOffset.getSize()); + assertEquals(OffsetFactory.createOffset(new int[] {0}), t.retOffset); + } + + @Test + public void removeEmptyRows6() { + AOffset of = OffsetFactory.createOffset(new int[] {1, 2, 5}); + RemoveEmptyOffsetsTmp t = of.removeEmptyRows(new boolean[] {false, false, false, true, true, true}, 0); + assertEquals(1, t.select.size()); + assertEquals(2, t.select.get(0)); + assertEquals(1, t.retOffset.getSize()); + assertEquals(OffsetFactory.createOffset(new int[] {2}), t.retOffset); + } + + @Test + public void removeEmptyRows7() { + AOffset of = OffsetFactory.createOffset(new int[] {1, 2, 5}); + RemoveEmptyOffsetsTmp t = of.removeEmptyRows(new boolean[] {true, false, false, true, true, true}, 0); + assertEquals(1, t.select.size()); + assertEquals(2, t.select.get(0)); + assertEquals(1, t.retOffset.getSize()); + assertEquals(OffsetFactory.createOffset(new int[] {3}), t.retOffset); + } + + @Test + public void removeEmptyRows8() { + AOffset of = OffsetFactory.createOffset(new int[] {1, 2, 3, 4, 5}); + RemoveEmptyOffsetsTmp t = of.removeEmptyRows(new boolean[] {true, false, false, false, false, true}, 0); + assertEquals(1, t.select.size()); + assertEquals(4, t.select.get(0)); + assertEquals(1, t.retOffset.getSize()); + assertEquals(OffsetFactory.createOffset(new int[] {1}), t.retOffset); + } + + @Test + public void removeEmptyRowsEmpty() { + AOffset of = OffsetFactory.createOffset(new int[] {1, 2, 3, 4, 5}); + RemoveEmptyOffsetsTmp t = of.removeEmptyRows(new boolean[] {false, false, false, false, false, false}, 0); + assertEquals(0, t.select.size()); + assertEquals(OffsetFactory.createOffset(new int[] {}), t.retOffset); + } } From b36b6c77b10b308dec862aac77ceaf9893f45a6c Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Wed, 24 Jun 2026 14:43:42 +0200 Subject: [PATCH 046/132] [MINOR][CI] Make federated client and monitoring threads daemon (#2508) Several federated Netty event-loop groups and a monitoring stats pool used non-daemon threads, which can keep a surefire test fork JVM alive after its tests complete and stall the job until the GitHub Actions timeout. This is the same class of leak addressed for the server-side worker and common thread pools, applied to the remaining sources. Graceful shutdown remains the normal path; daemon threads ensure a leaked or in-flight group can never block fork/JVM exit. --- .../controlprogram/federated/FederatedData.java | 7 ++++++- .../monitoring/FederatedMonitoringServer.java | 7 +++++-- .../monitoring/services/WorkerService.java | 14 +++++++++++++- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java index 98572e2ddd0..19277ba0843 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java @@ -66,6 +66,7 @@ import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.codec.serialization.ObjectEncoder; import io.netty.handler.timeout.ReadTimeoutHandler; +import io.netty.util.concurrent.DefaultThreadFactory; import io.netty.util.concurrent.Promise; @SuppressWarnings("deprecation") @@ -299,7 +300,11 @@ public static void clearWorkGroup() { public synchronized static void createWorkGroup() { if(workerGroup == null) - workerGroup = new NioEventLoopGroup(DMLConfig.DEFAULT_NUMBER_OF_FEDERATED_WORKER_THREADS); + // Daemon event loops so a leaked client-side group (e.g. in-JVM coordinator tests, a missed + // clearWorkGroup(), or an in-flight async shutdownGracefully) cannot block JVM exit. This mirrors + // the daemon factory used for the server-side worker in FederatedWorker. + workerGroup = new NioEventLoopGroup(DMLConfig.DEFAULT_NUMBER_OF_FEDERATED_WORKER_THREADS, + new DefaultThreadFactory("fed-client-worker", true)); } private static class DataRequestHandler extends ChannelInboundHandlerAdapter { diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/FederatedMonitoringServer.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/FederatedMonitoringServer.java index 6b3b180a260..d1a482a689a 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/FederatedMonitoringServer.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/FederatedMonitoringServer.java @@ -34,6 +34,7 @@ import io.netty.handler.codec.http.HttpServerCodec; import io.netty.handler.codec.http.cors.CorsConfigBuilder; import io.netty.handler.codec.http.cors.CorsHandler; +import io.netty.util.concurrent.DefaultThreadFactory; public class FederatedMonitoringServer { protected static Logger log = Logger.getLogger(FederatedMonitoringServer.class); @@ -51,8 +52,10 @@ public FederatedMonitoringServer(int port, boolean debug) { public void run() { log.info("Setting up Federated Monitoring Backend on port " + _port); - EventLoopGroup bossGroup = new NioEventLoopGroup(); - EventLoopGroup workerGroup = new NioEventLoopGroup(); + // Daemon event loops so a leaked in-JVM (test) monitoring server cannot block JVM exit. This mirrors + // the daemon factory used for the federated worker and client in FederatedWorker and FederatedData. + EventLoopGroup bossGroup = new NioEventLoopGroup(0, new DefaultThreadFactory("fed-monitoring-boss", true)); + EventLoopGroup workerGroup = new NioEventLoopGroup(0, new DefaultThreadFactory("fed-monitoring-pool", true)); try { var corsConfig = CorsConfigBuilder.forAnyOrigin() diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/WorkerService.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/WorkerService.java index a2ee2843405..f294fbd5c17 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/WorkerService.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/WorkerService.java @@ -25,6 +25,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.tuple.MutablePair; @@ -117,11 +118,22 @@ private static synchronized void updateCachedWorkers(List workers, private static synchronized void startStatsCollectionProcess(int threadCount, double frequencySeconds) { if (executorService == null) { - executorService = Executors.newScheduledThreadPool(threadCount); + // Daemon threads so this never-shut-down background stats collector cannot block JVM exit + // (e.g. keep a surefire test fork alive after the monitoring tests complete). + executorService = Executors.newScheduledThreadPool(threadCount, daemonThreadFactory()); executorService.scheduleAtFixedRate(syncWorkerStatisticsRunnable(), 0, Math.round(frequencySeconds * 1000), TimeUnit.MILLISECONDS); } } + private static ThreadFactory daemonThreadFactory() { + final ThreadFactory base = Executors.defaultThreadFactory(); + return r -> { + Thread t = base.newThread(r); + t.setDaemon(true); + return t; + }; + } + public static void syncWorkerStatisticsWithDB(StatisticsModel stats, Long id) { // NOTE: This part of the code is not directly connected to requests coming from the frontend From 51c57520a02ca03c01da28aac974d06a9f5816c3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:31:06 +0200 Subject: [PATCH 047/132] Bump actions/checkout from 6 to 7 (#2493) Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v6...v7) --- .github/workflows/build-cron.yml | 2 +- .github/workflows/build.yml | 2 +- .github/workflows/cleanup-transient-artifacts.yml | 2 +- .github/workflows/docker-cd.yml | 2 +- .github/workflows/docker-release.yml | 2 +- .github/workflows/docker-testImage.yml | 2 +- .github/workflows/documentation.yml | 4 ++-- .github/workflows/javaCodestyle.yml | 2 +- .github/workflows/javaTests.yml | 4 ++-- .github/workflows/license.yml | 2 +- .github/workflows/monitoringUITests.yml | 2 +- .github/workflows/python.yml | 2 +- .github/workflows/pythonFormatting.yml | 2 +- .github/workflows/release-scripts.yml | 2 +- 14 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-cron.yml b/.github/workflows/build-cron.yml index 730b276685e..6822e036045 100644 --- a/.github/workflows/build-cron.yml +++ b/.github/workflows/build-cron.yml @@ -54,7 +54,7 @@ jobs: ] steps: - name: Checkout Repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Setup Java ${{ matrix.java }} ${{ matrix.javadist }} uses: actions/setup-java@v5 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 77d492a2e22..ca822f1b8a7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -72,7 +72,7 @@ jobs: ] steps: - name: Checkout Repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Setup Java ${{ matrix.java }} ${{ matrix.javadist }} uses: actions/setup-java@v5 diff --git a/.github/workflows/cleanup-transient-artifacts.yml b/.github/workflows/cleanup-transient-artifacts.yml index d249064cf34..4a5b38e17e8 100644 --- a/.github/workflows/cleanup-transient-artifacts.yml +++ b/.github/workflows/cleanup-transient-artifacts.yml @@ -38,7 +38,7 @@ jobs: if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - name: Checkout Repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Delete Artifacts run: | diff --git a/.github/workflows/docker-cd.yml b/.github/workflows/docker-cd.yml index 809cab15c6a..7819c8daa75 100644 --- a/.github/workflows/docker-cd.yml +++ b/.github/workflows/docker-cd.yml @@ -38,7 +38,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 # https://github.com/docker/metadata-action - name: Configure Docker metadata diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 5b8057b2a85..fb7ceea1cbe 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -40,7 +40,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 - run: git checkout ${{ github.event.inputs.branch_or_tag }} # https://github.com/docker/metadata-action diff --git a/.github/workflows/docker-testImage.yml b/.github/workflows/docker-testImage.yml index 340d7e30297..44adf31415c 100644 --- a/.github/workflows/docker-testImage.yml +++ b/.github/workflows/docker-testImage.yml @@ -36,7 +36,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 # https://github.com/docker/metadata-action - name: Configure Docker metadata diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 78b5ce0458e..9b9fccfb812 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -51,7 +51,7 @@ jobs: name: Java steps: - name: Checkout Repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Setup Java ${{ matrix.java }} ${{ matrix.javadist }} uses: actions/setup-java@v5 @@ -68,7 +68,7 @@ jobs: name: Python steps: - name: Checkout Repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Setup Python uses: actions/setup-python@v6 diff --git a/.github/workflows/javaCodestyle.yml b/.github/workflows/javaCodestyle.yml index 2649edcbd0c..7dedc5f4865 100644 --- a/.github/workflows/javaCodestyle.yml +++ b/.github/workflows/javaCodestyle.yml @@ -51,7 +51,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Setup Java 17 adopt uses: actions/setup-java@v5 diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml index 0d4c71e946b..d181bd7acd5 100644 --- a/.github/workflows/javaTests.yml +++ b/.github/workflows/javaTests.yml @@ -96,7 +96,7 @@ jobs: name: ${{ matrix.tests }} steps: - name: Checkout Repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: ${{ matrix.tests }} uses: ./.github/action/ @@ -132,7 +132,7 @@ jobs: javadist: ['adopt'] steps: - name: Checkout Repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Cache Maven Dependencies uses: actions/cache@v5 diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml index d42e073c77b..b28a4ec0051 100644 --- a/.github/workflows/license.yml +++ b/.github/workflows/license.yml @@ -58,7 +58,7 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Setup Java ${{ matrix.java }} ${{ matrix.javadist }} uses: actions/setup-java@v5 diff --git a/.github/workflows/monitoringUITests.yml b/.github/workflows/monitoringUITests.yml index 2fcfc90651b..559cc8be3ab 100644 --- a/.github/workflows/monitoringUITests.yml +++ b/.github/workflows/monitoringUITests.yml @@ -56,7 +56,7 @@ jobs: node-version: ["lts/*"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Build the application, with Node.js ${{ matrix.node-version }} uses: actions/setup-node@v6 with: diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index ee4e771937d..e5f0d3bc2a0 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -64,7 +64,7 @@ jobs: name: ${{ matrix.os }} Java ${{ matrix.java }} ${{ matrix.javadist }} Python ${{ matrix.python-version }}/ ${{ matrix.test_mode}} steps: - name: Checkout Repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Setup Java ${{ matrix.java }} ${{ matrix.javadist }} uses: actions/setup-java@v5 diff --git a/.github/workflows/pythonFormatting.yml b/.github/workflows/pythonFormatting.yml index aca79c10f41..a8db5671116 100644 --- a/.github/workflows/pythonFormatting.yml +++ b/.github/workflows/pythonFormatting.yml @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Setup Python uses: actions/setup-python@v6 diff --git a/.github/workflows/release-scripts.yml b/.github/workflows/release-scripts.yml index b804a3db507..0dbe9c8899e 100644 --- a/.github/workflows/release-scripts.yml +++ b/.github/workflows/release-scripts.yml @@ -42,7 +42,7 @@ jobs: steps: # Java setup docs: # https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#installing-custom-java-package-type - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up JDK 17 uses: actions/setup-java@v5 with: From 5c72d53784f99f345944c04986a521a4929ece6d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:31:25 +0200 Subject: [PATCH 048/132] Bump actions/cache from 5 to 6 (#2506) Bumps [actions/cache](https://github.com/actions/cache) from 5 to 6. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v5...v6) --- .github/workflows/documentation.yml | 2 +- .github/workflows/javaTests.yml | 2 +- .github/workflows/python.yml | 8 ++++---- .github/workflows/release-scripts.yml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 9b9fccfb812..e2735eb1e2e 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -77,7 +77,7 @@ jobs: architecture: 'x64' - name: Cache Pip Dependencies - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-docs-${{ hashFiles('src/main/python/docs/requires-docs.txt') }} diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml index d181bd7acd5..747d8930cd3 100644 --- a/.github/workflows/javaTests.yml +++ b/.github/workflows/javaTests.yml @@ -135,7 +135,7 @@ jobs: uses: actions/checkout@v7 - name: Cache Maven Dependencies - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-test-${{ hashFiles('**/pom.xml') }} diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index e5f0d3bc2a0..e00cd6dfe1e 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -74,13 +74,13 @@ jobs: cache: 'maven' - name: Cache Pip Dependencies - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('src/main/python/setup.py') }} - name: Cache Datasets - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: | src/main/python/systemds/examples/tutorials/mnist @@ -88,7 +88,7 @@ jobs: key: ${{ runner.os }}-mnist-${{ hashFiles('src/main/python/systemds/examples/tutorials/mnist.py') }}-${{ hashFiles('src/main/python/systemds/examples/tutorials/adult.py') }} - name: Cache Deb Dependencies - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: /var/cache/apt/archives key: ${{ runner.os }}-${{ hashFiles('.github/workflows/python.yml') }} @@ -150,7 +150,7 @@ jobs: - name: Cache Torch Hub if: ${{ matrix.test_mode == 'scuro' }} id: torch-cache - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: .torch key: ${{ runner.os }}-torch-${{ hashFiles('requirements.txt') }} diff --git a/.github/workflows/release-scripts.yml b/.github/workflows/release-scripts.yml index 0dbe9c8899e..08b7750af89 100644 --- a/.github/workflows/release-scripts.yml +++ b/.github/workflows/release-scripts.yml @@ -54,7 +54,7 @@ jobs: - run: printf "JAVA_HOME = $JAVA_HOME \n" - name: Cache local Maven repository - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} From 8e235f4d187d11708919b4d50572b150ec4c2ace Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Thu, 25 Jun 2026 13:14:25 +0200 Subject: [PATCH 049/132] [BWARE] Add sort support to compressed column groups (#2507) Add sort support to compressed column groups --- .../compress/CompressedMatrixBlock.java | 6 +- .../runtime/compress/colgroup/AColGroup.java | 10 + .../compress/colgroup/ColGroupConst.java | 5 + .../compress/colgroup/ColGroupDDC.java | 19 ++ .../compress/colgroup/ColGroupDDCFOR.java | 19 ++ .../compress/colgroup/ColGroupDDCLZW.java | 6 + .../compress/colgroup/ColGroupEmpty.java | 5 + .../colgroup/ColGroupLinearFunctional.java | 5 + .../compress/colgroup/ColGroupOLE.java | 5 + .../compress/colgroup/ColGroupRLE.java | 5 + .../compress/colgroup/ColGroupSDC.java | 46 +++ .../compress/colgroup/ColGroupSDCFOR.java | 45 +++ .../compress/colgroup/ColGroupSDCSingle.java | 20 ++ .../colgroup/ColGroupSDCSingleZeros.java | 20 ++ .../compress/colgroup/ColGroupSDCZeros.java | 45 +++ .../colgroup/ColGroupUncompressed.java | 12 + .../colgroup/ColGroupUncompressedArray.java | 5 + .../dictionary/AIdentityDictionary.java | 6 + .../colgroup/dictionary/DeltaDictionary.java | 5 + .../colgroup/dictionary/Dictionary.java | 63 ++++ .../colgroup/dictionary/IDictionary.java | 9 + .../dictionary/MatrixBlockDictionary.java | 9 + .../colgroup/dictionary/PlaceHolderDict.java | 5 + .../colgroup/dictionary/QDictionary.java | 6 + .../runtime/compress/lib/CLALibReorg.java | 18 +- .../runtime/compress/lib/CLALibSort.java | 151 ++++++++++ .../compress/CompressedSortTest.java | 279 ++++++++++++++++++ .../compress/CompressedVectorTest.java | 22 ++ .../colgroup/ColGroupNegativeTests.java | 12 + .../compress/dictionary/DictionaryTests.java | 35 +++ 30 files changed, 890 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java create mode 100644 src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java index 58e33a616ca..dae13ed9f94 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java @@ -65,6 +65,7 @@ import org.apache.sysds.runtime.compress.lib.CLALibRexpand; import org.apache.sysds.runtime.compress.lib.CLALibScalar; import org.apache.sysds.runtime.compress.lib.CLALibSlice; +import org.apache.sysds.runtime.compress.lib.CLALibSort; import org.apache.sysds.runtime.compress.lib.CLALibSquash; import org.apache.sysds.runtime.compress.lib.CLALibTSMM; import org.apache.sysds.runtime.compress.lib.CLALibTernaryOp; @@ -847,9 +848,8 @@ public CmCovObject covOperations(COVOperator op, MatrixBlock that, MatrixBlock w } @Override - public MatrixBlock sortOperations(MatrixValue weights, MatrixBlock result) { - MatrixBlock right = getUncompressed(weights); - return getUncompressed("sortOperations").sortOperations(right, result); + public MatrixBlock sortOperations(MatrixValue weights, MatrixBlock result, int k) { + return CLALibSort.sort(this, weights, result, k); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java index f30cf8b17b2..354325e293b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java @@ -974,6 +974,16 @@ public AColGroup[] splitReshapePushDown(final int multiplier, final int nRow, fi return splitReshape(multiplier, nRow, nColOrg); } + /** + * Sort the values of the column group according to double comparison operations and return as another compressed + * group. + * + * This sorting assumes that the column group is sorted independently of everything else. + * + * @return The sorted group + */ + public abstract AColGroup sort(); + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java index 7d0b2469ec8..64f3f4fda07 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java @@ -769,4 +769,9 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { return ColGroupConst.create(newColumnIDs, _dict.sliceColumns(selectedColumns, getNumCols())); } + + @Override + public AColGroup sort() { + return this; + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java index 6ac1544e61e..b316e48474a 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java @@ -1178,4 +1178,23 @@ public AColGroup convertToDeltaDDC() { public AColGroup convertToDDCLZW() { return ColGroupDDCLZW.create(_colIndexes, _dict, _data, null); } + + @Override + public AColGroup sort() { + // TODO restore support for run length encoding to exploit the runs + + int[] counts = getCounts(); + // get the sort index + int[] r = _dict.sort(); + + AMapToData m = MapToFactory.create(_data.size(), counts.length); + int off = 0; + for(int i = 0; i < counts.length; i++) { + for(int j = 0; j < counts[r[i]]; j++) { + m.set(off++, r[i]); + } + } + + return ColGroupDDC.create(_colIndexes, _dict, m, counts); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java index 6a4a92469d2..d8a8ed1ade6 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java @@ -571,4 +571,23 @@ public String toString() { sb.append(Arrays.toString(_reference)); return sb.toString(); } + + @Override + public AColGroup sort() { + // TODO restore support for run length encoding. + + int[] counts = getCounts(); + // get the sort index + int[] r = _dict.sort(); + + AMapToData m = MapToFactory.create(_data.size(), counts.length); + int off = 0; + for(int i = 0; i < counts.length; i++) { + for(int j = 0; j < counts[r[i]]; j++) { + m.set(off++, r[i]); + } + } + + return ColGroupDDCFOR.create(_colIndexes, _dict, m, counts, _reference); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCLZW.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCLZW.java index c820f875a05..1f3e5934288 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCLZW.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCLZW.java @@ -1022,4 +1022,10 @@ protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, ColGroupDDC g = (ColGroupDDC) convertToDDC(); return g.removeEmptyColsSubset(newColumnIDs, selectedColumns); } + + @Override + public AColGroup sort() { + ColGroupDDC g = (ColGroupDDC) convertToDDC(); + return g.sort(); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java index 7c0a15e123b..64114a054ab 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java @@ -488,4 +488,9 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut){ protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ return new ColGroupEmpty(newColumnIDs); } + + @Override + public AColGroup sort() { + return this; + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java index 5ac168b9406..fa8aa104ffb 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java @@ -750,4 +750,9 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } + + @Override + public AColGroup sort() { + throw new NotImplementedException("Unimplemented method 'sort'"); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java index 5833729c378..a251d828b5f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java @@ -741,4 +741,9 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } + + @Override + public AColGroup sort() { + throw new NotImplementedException(); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java index c9fc920a845..347cea9c0da 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java @@ -1200,4 +1200,9 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } + + @Override + public AColGroup sort() { + throw new NotImplementedException(); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java index 5522a33e3e0..faa5ca7fa27 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java @@ -903,4 +903,50 @@ public String toString() { sb.append(_data.toString()); return sb.toString(); } + + @Override + public AColGroup sort() { + if(getNumCols() > 1) + throw new NotImplementedException(); + // TODO restore support for run length encoding. + + final int[] counts = getCounts(); + // get the sort index + final int[] r = _dict.sort(); + + // find default value position. + // todo use binary search for minor improvements. + final double def = _defaultTuple[0]; + int defIdx = counts.length; + for(int i = 0; i < r.length; i++) { + if(_dict.getValue(r[i], 0, 1) >= def) { + defIdx = i; + break; + } + } + + int nondefault = _data.size(); + int defaultLength = _numRows - nondefault; + AMapToData m = MapToFactory.create(nondefault, counts.length); + int[] offsets = new int[nondefault]; + + int off = 0; + for(int i = 0; i < counts.length; i++) { + if(i < defIdx) { + for(int j = 0; j < counts[r[i]]; j++) { + offsets[off] = off; + m.set(off++, r[i]); + } + } + else {// if( i >= defIdx){ + for(int j = 0; j < counts[r[i]]; j++) { + offsets[off] = off + defaultLength; + m.set(off++, r[i]); + } + } + } + + AOffset o = OffsetFactory.createOffset(offsets); + return ColGroupSDC.create(_colIndexes, _numRows, _dict, _defaultTuple, o, m, counts); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java index 2ef7f3012bc..815ecacf378 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java @@ -651,4 +651,49 @@ public String toString() { return sb.toString(); } + @Override + public AColGroup sort() { + if(getNumCols() > 1) + throw new NotImplementedException(); + // TODO restore support for run length encoding. + + final int[] counts = getCounts(); + // get the sort index + final int[] r = _dict.sort(); + + // find default value position. + // todo use binary search for minor improvements. + int defIdx = counts.length; + for(int i = 0; i < r.length; i++) { + if(_dict.getValue(r[i], 0, 1) >= 0) { + defIdx = i; + break; + } + } + + int nondefault = _data.size(); + int defaultLength = _numRows - nondefault; + AMapToData m = MapToFactory.create(nondefault, counts.length); + int[] offsets = new int[nondefault]; + + int off = 0; + for(int i = 0; i < counts.length; i++) { + if(i < defIdx) { + for(int j = 0; j < counts[r[i]]; j++) { + offsets[off] = off; + m.set(off++, r[i]); + } + } + else {// if( i >= defIdx){ + for(int j = 0; j < counts[r[i]]; j++) { + offsets[off] = off + defaultLength; + m.set(off++, r[i]); + } + } + } + + AOffset o = OffsetFactory.createOffset(offsets); + return ColGroupSDCFOR.create(_colIndexes, _numRows, _dict, o, m, counts, _reference); + } + } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java index 0f89e54d975..8a9f401c10c 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java @@ -746,4 +746,24 @@ public String toString() { sb.append(_indexes.toString()); return sb.toString(); } + + @Override + public AColGroup sort() { + if(getNumCols() > 1) + throw new NotImplementedException(); + + // Only a single non-default value exists, so sorting is a contiguous block of that value placed before the + // default values if it is smaller than the default, and after them otherwise. + final int[] counts = getCounts(); + final int nondefault = counts[0]; + final int defaultLength = _numRows - nondefault; + final int base = _dict.getValue(0, 0, 1) >= _defaultTuple[0] ? defaultLength : 0; + + final int[] offsets = new int[nondefault]; + for(int j = 0; j < nondefault; j++) + offsets[j] = base + j; + + AOffset o = OffsetFactory.createOffset(offsets); + return ColGroupSDCSingle.create(_colIndexes, _numRows, _dict, _defaultTuple, o, counts); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java index d9341bb9ea8..26b3cc4ee37 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java @@ -1071,4 +1071,24 @@ public String toString() { sb.append(_indexes.toString()); return sb.toString(); } + + @Override + public AColGroup sort() { + if(getNumCols() > 1) + throw new NotImplementedException(); + + // Only a single non-default value exists, so sorting is a contiguous block of that value placed before the + // zeros (default) if it is negative, and after the zeros otherwise. + final int[] counts = getCounts(); + final int nondefault = counts[0]; + final int defaultLength = _numRows - nondefault; + final int base = _dict.getValue(0, 0, 1) >= 0 ? defaultLength : 0; + + final int[] offsets = new int[nondefault]; + for(int j = 0; j < nondefault; j++) + offsets[j] = base + j; + + AOffset o = OffsetFactory.createOffset(offsets); + return ColGroupSDCSingleZeros.create(_colIndexes, _numRows, _dict, o, counts); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java index 86cd9866a75..09f222bfeee 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java @@ -1113,4 +1113,49 @@ public String toString() { sb.append(_data); return sb.toString(); } + + @Override + public AColGroup sort() { + if(getNumCols() > 1) + throw new NotImplementedException(); + // TODO restore support for run length encoding. + + final int[] counts = getCounts(); + // get the sort index + final int[] r = _dict.sort(); + + // find default value position. + // todo use binary search for minor improvements. + int defIdx = counts.length; + for(int i = 0; i < r.length; i++) { + if(_dict.getValue(r[i], 0, 1) >= 0) { + defIdx = i; + break; + } + } + + int nondefault = _data.size(); + int defaultLength = _numRows - nondefault; + AMapToData m = MapToFactory.create(nondefault, counts.length); + int[] offsets = new int[nondefault]; + + int off = 0; + for(int i = 0; i < counts.length; i++) { + if(i < defIdx) { + for(int j = 0; j < counts[r[i]]; j++) { + offsets[off] = off; + m.set(off++, r[i]); + } + } + else {// if( i >= defIdx){ + for(int j = 0; j < counts[r[i]]; j++) { + offsets[off] = off + defaultLength; + m.set(off++, r[i]); + } + } + } + + AOffset o = OffsetFactory.createOffset(offsets); + return ColGroupSDCZeros.create(_colIndexes, _numRows, _dict, o, m, counts); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java index e4e98da46f2..611add6480f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java @@ -56,6 +56,7 @@ import org.apache.sysds.runtime.functionobjects.Multiply; import org.apache.sysds.runtime.functionobjects.ReduceAll; import org.apache.sysds.runtime.functionobjects.ReduceRow; +import org.apache.sysds.runtime.functionobjects.SortIndex; import org.apache.sysds.runtime.functionobjects.ValueFunction; import org.apache.sysds.runtime.instructions.cp.CmCovObject; import org.apache.sysds.runtime.matrix.data.LibMatrixMult; @@ -65,6 +66,7 @@ import org.apache.sysds.runtime.matrix.operators.AggregateUnaryOperator; import org.apache.sysds.runtime.matrix.operators.BinaryOperator; import org.apache.sysds.runtime.matrix.operators.CMOperator; +import org.apache.sysds.runtime.matrix.operators.ReorgOperator; import org.apache.sysds.runtime.matrix.operators.ScalarOperator; import org.apache.sysds.runtime.matrix.operators.UnaryOperator; import org.apache.sysds.utils.stats.InfrastructureAnalyzer; @@ -1331,4 +1333,14 @@ public String toString() { return sb.toString(); } + + @Override + public AColGroup sort() { + if(getNumCols() > 1) + throw new NotImplementedException(); + // sortOperations builds a value/weight table for quantiles; for an ascending column sort we reorder the rows. + MatrixBlock sorted = _data.reorgOperations(new ReorgOperator(new SortIndex(1, false, false), 1), + new MatrixBlock(), 0, 0, 0); + return create(sorted, _colIndexes); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java index 0c8f07685b6..51e26a3f9d2 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java @@ -293,4 +293,9 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } + + @Override + public AColGroup sort() { + throw new NotImplementedException("Unimplemented method 'sort'"); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java index 17b382f06ad..a7e715b59b8 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java @@ -19,6 +19,7 @@ package org.apache.sysds.runtime.compress.colgroup.dictionary; +import org.apache.commons.lang3.NotImplementedException; import org.apache.sysds.runtime.compress.DMLCompressionException; public abstract class AIdentityDictionary extends ACachingMBDictionary { @@ -74,4 +75,9 @@ public double[] productAllRowsToDoubleWithDefault(double[] defaultTuple) { ret[ret.length - 1] *= defaultTuple[i]; return ret; } + + @Override + public int[] sort(){ + throw new NotImplementedException(); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java index c26de004373..9a0412145f0 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java @@ -142,4 +142,9 @@ public IDictionary clone() { public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ throw new NotImplementedException(); } + + @Override + public int[] sort() { + throw new NotImplementedException(); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java index 06bd811b50b..fd8dfd127db 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java @@ -1348,4 +1348,67 @@ public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { return getMBDict(nCol).sliceColumns(selectedColumns, nCol); } + @Override + public int[] sort() { + return sort(_values); + } + + protected static int[] sort(double[] values) { + int[] indices = new int[values.length]; + for(int i = 0; i < indices.length; i++) { + indices[i] = i; + } + + // quicksort with stack + int[] stack = new int[values.length]; + + int top = -1; + stack[++top] = 0; + stack[++top] = values.length - 1; + + while(top >= 0) { + int high = stack[top--]; + int low = stack[top--]; + + if(low < high) { + + int pivotIndex = partition(indices, values, low, high); + // Left side + if(pivotIndex - 1 > low) { + stack[++top] = low; + stack[++top] = pivotIndex - 1; + } + + // Right side + if(pivotIndex + 1 < high) { + stack[++top] = pivotIndex + 1; + stack[++top] = high; + } + } + } + + return indices; + } + + private static int partition(int[] indices, double[] values, int low, int high) { + double pivotValue = values[indices[high]]; + int i = low - 1; + + for(int j = low; j < high; j++) { + if(values[indices[j]] <= pivotValue) { + i++; + swap(indices, i, j); + } + } + + swap(indices, i + 1, high); + return i + 1; + } + + private static void swap(int[] arr, int i, int j) { + int tmp = arr[i]; + arr[i] = arr[j]; + arr[j] = tmp; + } + } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java index 726df96d5c8..c8ddfc4883a 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java @@ -1062,4 +1062,13 @@ public IDictionary rightMMPreAggSparse(int numVals, SparseBlock b, IColIndex thi */ public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol); + /** + * Sort the values of this dictionary via an index of how the values mapped previously. + * + * In practice this design means we can reuse the previous dictionary for the resulting column group + * + * @return The sorted index. + */ + public int[] sort(); + } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java index c1d2ecc5296..b77eacd2205 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java @@ -2845,4 +2845,13 @@ public static double[] sliceColumns(MatrixBlock mb, IntArrayList selectedColumns return ret; } + @Override + public int[] sort() { + if(_data.getNumColumns() > 1) + throw new RuntimeException("Not supported sort on multicolumn dictionaries"); + _data.sparseToDense(); + + return Dictionary.sort(_data.getDenseBlockValues()); + } + } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/PlaceHolderDict.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/PlaceHolderDict.java index 2d9075f73c9..c38af0be122 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/PlaceHolderDict.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/PlaceHolderDict.java @@ -107,4 +107,9 @@ public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { throw new RuntimeException("Invalid call"); } + @Override + public int[] sort() { + throw new RuntimeException("Invalid call"); + } + } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/QDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/QDictionary.java index 30b9d806c1f..6912ee12525 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/QDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/QDictionary.java @@ -23,6 +23,7 @@ import java.io.DataOutput; import java.io.IOException; +import org.apache.commons.lang3.NotImplementedException; import org.apache.sysds.runtime.compress.utils.IntArrayList; import org.apache.sysds.runtime.functionobjects.Builtin; import org.apache.sysds.runtime.matrix.data.MatrixBlock; @@ -282,4 +283,9 @@ public MatrixBlockDictionary createMBDict(int nCol) { public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { return getMBDict().sliceColumns(selectedColumns, nCol); } + + @Override + public int[] sort() { + throw new NotImplementedException(); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibReorg.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibReorg.java index d587d26c3cb..5cfcf223213 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibReorg.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibReorg.java @@ -32,6 +32,7 @@ import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.data.SparseBlockMCSR; +import org.apache.sysds.runtime.functionobjects.SortIndex; import org.apache.sysds.runtime.functionobjects.SwapIndex; import org.apache.sysds.runtime.matrix.data.LibMatrixReorg; import org.apache.sysds.runtime.matrix.data.MatrixBlock; @@ -65,12 +66,19 @@ else if(op.fn instanceof SwapIndex) { // the compressed matrix. https://issues.apache.org/jira/browse/SYSTEMDS-3025 return transpose(cmb, ret, op.getNumThreads()); } - else { - String message = !warned ? op.getClass().getSimpleName() + " -- " + op.fn.getClass().getSimpleName() : null; - MatrixBlock tmp = cmb.getUncompressed(message, op.getNumThreads()); - warned = true; - return tmp.reorgOperations(op, ret, startRow, startColumn, length); + else if(op.fn instanceof SortIndex) { + // order: keep the result compressed when a single column / single group is sorted ascending. + MatrixBlock res = CLALibSort.sort(cmb, (SortIndex) op.fn); + if(res != null) + return res; + // otherwise fall through to the decompression fallback below. } + + // Decompression fallback for reorg operations not supported directly on the compressed representation. + String message = !warned ? op.getClass().getSimpleName() + " -- " + op.fn.getClass().getSimpleName() : null; + MatrixBlock tmp = cmb.getUncompressed(message, op.getNumThreads()); + warned = true; + return tmp.reorgOperations(op, ret, startRow, startColumn, length); } private static MatrixBlock transpose(CompressedMatrixBlock cmb, MatrixBlock ret, int k) { diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java new file mode 100644 index 00000000000..b94f11ae723 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.compress.lib; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang3.NotImplementedException; +import org.apache.sysds.runtime.compress.CompressedMatrixBlock; +import org.apache.sysds.runtime.compress.colgroup.AColGroup; +import org.apache.sysds.runtime.functionobjects.SortIndex; +import org.apache.sysds.runtime.matrix.data.LibMatrixReorg; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.data.MatrixValue; +import org.apache.sysds.runtime.matrix.operators.ReorgOperator; + +public final class CLALibSort { + + private CLALibSort() { + // private constructor for utility class. + } + + /** + * Sort (order) a compressed matrix in place of the {@code order} built-in, while keeping the result compressed. + * + * The compressed fast-path only supports the case the user can benefit from: a single column held in a single column + * group, sorted ascending and returning the sorted values (not the index permutation). For everything else (multiple + * columns, multiple column groups, descending order, index return, or a column-group encoding without a sort + * implementation) this returns {@code null} so the caller can fall back to a decompressed reorg. + * + * @param mb the compressed matrix to sort + * @param fn the sort specification carried by the reorg operator + * @return the sorted compressed matrix, or {@code null} if the compressed fast-path does not apply + */ + public static MatrixBlock sort(CompressedMatrixBlock mb, SortIndex fn) { + final boolean singleColumn = mb.getNumColumns() == 1 && mb.getColGroups().size() == 1; + if(!singleColumn || fn.getDecreasing() || fn.getIndexReturn()) + return null; + + final AColGroup sorted = sortSingleColumn(mb); + if(sorted == null) + return null; + + final List rg = new ArrayList<>(1); + rg.add(sorted); + return new CompressedMatrixBlock(mb.getNumRows(), mb.getNumColumns(), mb.getNonZeros(), false, rg); + } + + /** + * Compute the sorted value/weight table used by the quantile/median/IQM operations (the {@code sort} / qsort lop), + * exploiting compression to sort the few distinct values instead of all rows. + * + * The compressed fast-path applies to an unweighted sort of a single column held in a single column group. The + * produced table is bit-for-bit identical to {@link MatrixBlock#sortOperations(MatrixValue, MatrixBlock, int)}: a + * {@code (1 + nnz) x 2} matrix holding one row per non-zero value (weight 1) plus a single collapsed row for the + * zeros (weight = number of zeros), sorted ascending by value. For every other case (weights present, multiple + * columns or groups, or an encoding without a sort implementation) it falls back to a decompressed sort. + * + * @param mb the compressed matrix to sort + * @param weights optional per-row weights, or {@code null} + * @param result the result matrix (reused by the fallback) + * @param k the parallelization degree + * @return the sorted value/weight table + */ + public static MatrixBlock sort(CompressedMatrixBlock mb, MatrixValue weights, MatrixBlock result, int k) { + final MatrixBlock w = CompressedMatrixBlock.getUncompressed(weights); + if(w == null && mb.getNumColumns() == 1 && mb.getColGroups().size() == 1) { + final MatrixBlock fast = sortTableSingleColumn(mb, result, k); + if(fast != null) + return fast; + } + + // fallback to uncompressed sort. + return CompressedMatrixBlock.getUncompressed(mb, "sortOperations", k).sortOperations(w, result, k); + } + + private static AColGroup sortSingleColumn(CompressedMatrixBlock mb) { + try { + return mb.getColGroups().get(0).sort(); + } + catch(NotImplementedException e) { + // the column-group encoding does not implement sort -> let the caller decompress. + return null; + } + } + + private static MatrixBlock sortTableSingleColumn(CompressedMatrixBlock mb, MatrixBlock result, int k) { + final long lnnz = mb.getNonZeros(); + if(lnnz < 0) // unknown number of non-zeros, cannot size the table. + return null; + + final AColGroup sorted = sortSingleColumn(mb); + if(sorted == null) + return null; + + final int nRows = mb.getNumRows(); + final int nnz = (int) lnnz; + final int zeroCount = nRows - nnz; + + // decompress the already-sorted single column once (ascending, zeros contiguous). + final List rg = new ArrayList<>(1); + rg.add(sorted); + final MatrixBlock sortedCol = new CompressedMatrixBlock(nRows, 1, lnnz, false, rg).decompress(k); + + // build the value/weight table: one row per non-zero value (weight 1) plus a single + // collapsed zero row (weight = number of zeros). The row order is irrelevant because the + // table is sorted by the reorg below, exactly as MatrixBlock.sortOperations does. + final MatrixBlock tdw = new MatrixBlock(1 + nnz, 2, false); + tdw.allocateDenseBlock(); + int w = 0; + for(int i = 0; i < nRows; i++) { + final double v = sortedCol.get(i, 0); + if(v != 0) { + tdw.set(w, 0, v); + tdw.set(w, 1, 1); + w++; + } + } + tdw.set(w, 0, 0); // collapsed zero row (weight 0 when the column is dense) + tdw.set(w, 1, zeroCount); + + // Emit through the same reorg used by MatrixBlock.sortOperations so the produced table is + // bit-for-bit identical to the uncompressed path, including its (intentionally unmaintained) + // non-zero metadata. This keeps downstream quantile/median consumers and result comparisons + // consistent regardless of whether the input was compressed. + if(result == null) + result = new MatrixBlock(1 + nnz, 2, false); + else + result.reset(1 + nnz, 2, false); + final ReorgOperator rop = new ReorgOperator(new SortIndex(1, false, false), k); + LibMatrixReorg.reorg(tdw, result, rop); + return result; + } +} diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java new file mode 100644 index 00000000000..7ab7187eb78 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java @@ -0,0 +1,279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.compress; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.Random; + +import org.apache.sysds.runtime.compress.CompressedMatrixBlock; +import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory; +import org.apache.sysds.runtime.compress.CompressionSettingsBuilder; +import org.apache.sysds.runtime.compress.colgroup.AColGroup; +import org.apache.sysds.runtime.compress.colgroup.AColGroup.CompressionType; +import org.apache.sysds.runtime.compress.colgroup.ColGroupUncompressed; +import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory; +import org.apache.sysds.runtime.functionobjects.SortIndex; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.operators.ReorgOperator; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; + +/** + * Tests the {@code order} (sort) reorg operation on compressed matrices. A single column held in a single column group + * is sorted ascending while staying compressed (via {@link org.apache.sysds.runtime.compress.lib.CLALibSort}); every + * other configuration falls back to a decompressed reorg. In all cases the result must match the uncompressed reference. + */ +public class CompressedSortTest { + + private static final int ROWS = 1000; + + private static final ReorgOperator ASC = new ReorgOperator(new SortIndex(1, false, false), 1); + private static final ReorgOperator DESC = new ReorgOperator(new SortIndex(1, true, false), 1); + private static final ReorgOperator INDEX = new ReorgOperator(new SortIndex(1, false, true), 1); + + @Test + public void sortDDC() { + runCompressed(generate(ROWS, 1, 8, 1.0, 1, 50, 7), CompressionType.DDC); + } + + @Test + public void sortDDCWithNegatives() { + runCompressed(generate(ROWS, 1, 10, 1.0, -25, 25, 13), CompressionType.DDC); + } + + @Test + public void sortSDCZeros() { + runCompressed(generate(ROWS, 1, 6, 0.2, 1, 40, 23), CompressionType.SDC); + } + + @Test + public void sortSDCWithNegatives() { + runCompressed(generate(ROWS, 1, 8, 0.3, -20, 20, 41), CompressionType.SDC); + } + + @Test + public void sortSDCSingleValueZeros() { + // sparse with a single distinct non-zero value -> SDCSingleZeros + runCompressed(generate(ROWS, 1, 1, 0.25, 5, 5, 99), CompressionType.SDC); + } + + @Test + public void sortSDCSingleNonZeroDefault() { + // two distinct non-zero values, one dominant default -> SDCSingle + runCompressed(twoValueColumn(3, 7), CompressionType.SDC); + } + + @Test + public void sortSDCSingleNonZeroDefaultNegative() { + // dominant non-zero default with a single smaller (negative) value -> SDCSingle + runCompressed(twoValueColumn(-4, 7), CompressionType.SDC); + } + + @Test + public void sortConst() { + MatrixBlock mb = new MatrixBlock(ROWS, 1, false); + for(int i = 0; i < ROWS; i++) + mb.set(i, 0, 3); + mb.recomputeNonZeros(); + runCompressed(mb, CompressionType.CONST); + } + + @Test + public void sortUncompressedColGroup() { + // a CompressedMatrixBlock holding a single uncompressed column group must also sort correctly + MatrixBlock raw = generate(ROWS, 1, ROWS, 1.0, -100000, 100000, 5); + List groups = new ArrayList<>(1); + groups.add(ColGroupUncompressed.create(raw, ColIndexFactory.create(1))); + CompressedMatrixBlock cmb = new CompressedMatrixBlock(ROWS, 1, raw.getNonZeros(), false, groups); + + MatrixBlock actual = cmb.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); + assertTrue("Expected the sorted result to stay compressed", actual instanceof CompressedMatrixBlock); + MatrixBlock expected = raw.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); + TestUtils.compareMatrices(expected, CompressedMatrixBlock.getUncompressed(actual, "sort"), 0.0, + "sort UNCOMPRESSED colgroup"); + } + + @Test + public void sortDescendingFallback() { + // descending order is not supported by the compressed fast-path -> decompress fallback + runFallback(generate(ROWS, 1, 8, 1.0, 1, 50, 7), CompressionType.DDC, DESC); + } + + @Test + public void sortMultiColumnFallback() { + // order on a multi-column matrix sorts rows by the first column -> decompress fallback + runFallback(generate(ROWS, 3, 6, 1.0, 1, 30, 31), CompressionType.DDC, ASC); + } + + @Test + public void sortIndexReturnFallback() { + // returning the sort permutation (index.return=TRUE) is not supported by the fast-path -> decompress fallback + runFallback(generate(ROWS, 1, 8, 1.0, 1, 50, 7), CompressionType.DDC, INDEX); + } + + @Test + public void sortUnsupportedEncodingFallback() { + // OLE does not implement colgroup sort -> the fast-path declines and the order falls back to decompression + runFallback(generate(ROWS, 1, 8, 0.3, 1, 40, 23), CompressionType.OLE, ASC); + } + + @Test + public void quantileTableDDC() { + runQuantile(generate(ROWS, 1, 8, 1.0, 1, 50, 7), CompressionType.DDC); + } + + @Test + public void quantileTableDDCWithNegatives() { + runQuantile(generate(ROWS, 1, 10, 1.0, -25, 25, 13), CompressionType.DDC); + } + + @Test + public void quantileTableSDCZeros() { + runQuantile(generate(ROWS, 1, 6, 0.2, 1, 40, 23), CompressionType.SDC); + } + + @Test + public void quantileTableSDCWithNegatives() { + runQuantile(generate(ROWS, 1, 8, 0.3, -20, 20, 41), CompressionType.SDC); + } + + @Test + public void quantileTableAllNegative() { + runQuantile(generate(ROWS, 1, 8, 0.4, -50, -1, 57), CompressionType.SDC); + } + + @Test + public void quantileTableAllNegativeDense() { + // dense column with no zeros -> the collapsed zero row carries weight 0 + runQuantile(generate(ROWS, 1, 8, 1.0, -50, -1, 57), CompressionType.DDC); + } + + @Test + public void quantileTableConst() { + MatrixBlock mb = new MatrixBlock(ROWS, 1, false); + for(int i = 0; i < ROWS; i++) + mb.set(i, 0, 3); + mb.recomputeNonZeros(); + runQuantile(mb, CompressionType.CONST); + } + + @Test + public void quantileTableUnsupportedEncodingFallback() { + // OLE does not implement colgroup sort -> the quantile table is built via the decompressed fallback + runQuantile(generate(ROWS, 1, 8, 0.3, 1, 40, 23), CompressionType.OLE); + } + + @Test + public void quantileWeightedFallback() { + MatrixBlock mb = generate(ROWS, 1, 8, 1.0, 1, 50, 7); + MatrixBlock weights = new MatrixBlock(ROWS, 1, false); + Random r = new Random(123); + for(int i = 0; i < ROWS; i++) + weights.set(i, 0, r.nextInt(4) + 1); + weights.recomputeNonZeros(); + MatrixBlock expected = new MatrixBlock(mb).sortOperations(weights, new MatrixBlock(), 1); + + CompressedMatrixBlock cmb = compress(mb, CompressionType.DDC); + MatrixBlock actual = cmb.sortOperations(weights, new MatrixBlock(), 1); + + expected.recomputeNonZeros(); + actual.recomputeNonZeros(); + TestUtils.compareMatrices(expected, actual, 0.0, "weighted sortOperations fallback"); + } + + private void runQuantile(MatrixBlock mb, CompressionType ct) { + // reference is computed on a copy because compression may consume the input. + MatrixBlock expected = new MatrixBlock(mb).sortOperations(null, new MatrixBlock(), 1); + + CompressedMatrixBlock cmb = compress(mb, ct); + assertEquals("Expected a single column group", 1, cmb.getColGroups().size()); + + MatrixBlock actual = cmb.sortOperations(null, new MatrixBlock(), 1); + + // sortOperations leaves the non-zero count unmaintained; recompute so the value comparison reads the data. + expected.recomputeNonZeros(); + actual.recomputeNonZeros(); + + // the value/weight table must match the uncompressed reference bit-for-bit ... + TestUtils.compareMatrices(expected, actual, 0.0, "sortOperations table " + ct); + // ... so the downstream median/quantile picks are identical. + assertEquals("median " + ct, expected.median(), actual.median(), 0.0); + assertEquals("q25 " + ct, expected.pickValue(0.25), actual.pickValue(0.25), 0.0); + assertEquals("q90 " + ct, expected.pickValue(0.90), actual.pickValue(0.90), 0.0); + } + + private void runCompressed(MatrixBlock mb, CompressionType ct) { + CompressedMatrixBlock cmb = compress(mb, ct); + assertEquals("Expected a single column group", 1, cmb.getColGroups().size()); + + MatrixBlock actual = cmb.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); + assertTrue("Expected the sorted result to stay compressed for " + ct, + actual instanceof CompressedMatrixBlock); + + MatrixBlock expected = mb.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); + TestUtils.compareMatrices(expected, CompressedMatrixBlock.getUncompressed(actual, "sort"), 0.0, "sort " + ct); + } + + private void runFallback(MatrixBlock mb, CompressionType ct, ReorgOperator op) { + CompressedMatrixBlock cmb = compress(mb, ct); + + MatrixBlock actual = cmb.reorgOperations(op, new MatrixBlock(), 0, 0, 0); + MatrixBlock expected = mb.reorgOperations(op, new MatrixBlock(), 0, 0, 0); + TestUtils.compareMatrices(expected, CompressedMatrixBlock.getUncompressed(actual, "sort"), 0.0, + "sort fallback " + ct); + } + + private static CompressedMatrixBlock compress(MatrixBlock mb, CompressionType ct) { + CompressionSettingsBuilder csb = new CompressionSettingsBuilder().setMinimumCompressionRatio(0.0) + .setValidCompressions(EnumSet.of(ct)); + MatrixBlock compressed = CompressedMatrixBlockFactory.compress(mb, 1, csb).getLeft(); + assertTrue("Expected the input to compress into a " + ct + " backed block", + compressed instanceof CompressedMatrixBlock); + return (CompressedMatrixBlock) compressed; + } + + private static MatrixBlock twoValueColumn(int rare, int dominant) { + MatrixBlock mb = new MatrixBlock(ROWS, 1, false); + for(int i = 0; i < ROWS; i++) + mb.set(i, 0, i % 10 < 3 ? rare : dominant); + mb.recomputeNonZeros(); + return mb; + } + + private static MatrixBlock generate(int rows, int cols, int unique, double sparsity, int min, int max, int seed) { + final MatrixBlock mb = new MatrixBlock(rows, cols, false); + final Random pos = new Random(seed); + final Random val = new Random(seed * 31 + 1); + final double[] values = new double[Math.max(unique, 1)]; + for(int i = 0; i < values.length; i++) + values[i] = min + (max > min ? val.nextInt(max - min + 1) : 0); + for(int i = 0; i < rows; i++) + for(int j = 0; j < cols; j++) + if(pos.nextDouble() < sparsity) + mb.set(i, j, values[pos.nextInt(values.length)]); + mb.recomputeNonZeros(); + return mb; + } +} diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedVectorTest.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedVectorTest.java index 8aea861d9ee..b2165201c52 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/CompressedVectorTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/CompressedVectorTest.java @@ -29,9 +29,11 @@ import org.apache.sysds.runtime.compress.CompressionSettingsBuilder; import org.apache.sysds.runtime.compress.colgroup.AColGroup.CompressionType; import org.apache.sysds.runtime.functionobjects.CM; +import org.apache.sysds.runtime.functionobjects.SortIndex; import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.runtime.matrix.operators.CMOperator; import org.apache.sysds.runtime.matrix.operators.CMOperator.AggregateOperationTypes; +import org.apache.sysds.runtime.matrix.operators.ReorgOperator; import org.apache.sysds.test.TestUtils; import org.apache.sysds.test.component.compress.TestConstants.MatrixTypology; import org.apache.sysds.test.component.compress.TestConstants.OverLapping; @@ -147,6 +149,26 @@ public void testSortOperations() { } } + @Test + public void testSort() { + try { + if(!(cmb instanceof CompressedMatrixBlock) || cols != 1) + return; // Input was not compressed then just pass test + + // order() builtin: sort the single column ascending (compressed fast-path or decompress fallback). + ReorgOperator op = new ReorgOperator(new SortIndex(1, false, false), _k); + MatrixBlock ret1 = mb.reorgOperations(op, new MatrixBlock(), 0, 0, 0); + MatrixBlock ret2 = cmb.reorgOperations(op, new MatrixBlock(), 0, 0, 0); + + compareResultMatrices(ret1, ret2, 1); + + } + catch(Exception e) { + e.printStackTrace(); + throw new RuntimeException(bufferedToString + "\n" + e.getMessage(), e); + } + } + @Test public void testReExpandRow() { // does not make much sense since it would entail the compression was on a matrix with one row. diff --git a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java index af21b14206a..81fdcb2a876 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java +++ b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java @@ -481,6 +481,12 @@ protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList s // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'removeEmptyColsSubset'"); } + + @Override + public AColGroup sort() { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'sort'"); + } } private class FakeDictBasedColGroup extends ADictBasedColGroup { @@ -802,5 +808,11 @@ protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList s // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'removeEmptyColsSubset'"); } + + @Override + public AColGroup sort() { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'sort'"); + } } } diff --git a/src/test/java/org/apache/sysds/test/component/compress/dictionary/DictionaryTests.java b/src/test/java/org/apache/sysds/test/component/compress/dictionary/DictionaryTests.java index 3dd48636ae4..58f8bb2df32 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/dictionary/DictionaryTests.java +++ b/src/test/java/org/apache/sysds/test/component/compress/dictionary/DictionaryTests.java @@ -365,6 +365,41 @@ public void sum3() { assertEquals(as, bs, 0.0000001); } + @Test + public void sort() { + if(nCol != 1) + return; // sort is only defined for single-column dictionaries. + + final double[] sa = sortedValues(a); + final double[] sb = sortedValues(b); + if(sa == null && sb == null) + return; // neither encoding implements sort -> nothing to compare. + + if(sa != null && sb != null) + TestUtils.compareMatricesBitAvgDistance(sa, sb, 10, 10, "Sorted values differ between dictionaries"); + } + + /** + * Reorders the dictionary values by {@link IDictionary#sort()} and asserts the permutation yields a non-decreasing + * sequence. Returns {@code null} when the encoding does not implement sort. + */ + private double[] sortedValues(IDictionary d) { + final int[] perm; + try { + perm = d.sort(); + } + catch(NotImplementedException e) { + return null; // encoding does not support sort. + } + assertEquals("sort must return one index per value", nRow, perm.length); + final double[] sorted = new double[perm.length]; + for(int i = 0; i < perm.length; i++) + sorted[i] = d.getValue(perm[i], 0, 1); + for(int i = 1; i < sorted.length; i++) + assertTrue("sort did not produce a non-decreasing sequence", sorted[i - 1] <= sorted[i]); + return sorted; + } + @Test public void getValues() { try { From a8f3462dac3cf4649bd8240b25605ba6b34ed64e Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Thu, 25 Jun 2026 13:49:27 +0200 Subject: [PATCH 050/132] [MINOR] Bump jackson to 2.15.4 (core/databind/annotations) (#2512) * Bump jackson core/databind/annotations to 2.15.4 Align the full jackson trio on 2.15.4, the latest patch in the 2.15.x line. Previously only jackson-databind was declared (2.15.2) while jackson-core and jackson-annotations were resolved transitively, now they are all three explicit. --- pom.xml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5762dc2289e..cfd3d8464fb 100644 --- a/pom.xml +++ b/pom.xml @@ -43,6 +43,7 @@ 4.8 3.23.4 3.5.7 + 2.15.4 2.12.18 2.12 yyyy-MM-dd HH:mm:ss z @@ -1433,10 +1434,22 @@ 1.4 + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + com.fasterxml.jackson.core jackson-databind - 2.15.2 + ${jackson.version} + + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} From ed849bb64a70b03d86e68d0877c251dbf87ce042 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Fri, 26 Jun 2026 15:41:42 +0200 Subject: [PATCH 051/132] Serialize federated monitoring and multitenant tests in CI (#2517) The **.functions.federated.monitoring.**,**.functions.federated.multitenant.** job was the only federated test group running with the default surefire parallelism (parallel=classes, threadCount=2). Run this group with -Dtest-threadCount=1 -Dtest-forkCount=1, matching every other federated group. --- .github/workflows/javaTests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml index 747d8930cd3..07762a17427 100644 --- a/.github/workflows/javaTests.yml +++ b/.github/workflows/javaTests.yml @@ -75,7 +75,7 @@ jobs: "**.functions.federated.primitives.part3.** -Dtest-threadCount=1 -Dtest-forkCount=1", "**.functions.federated.primitives.part4.** -Dtest-threadCount=1 -Dtest-forkCount=1", "**.functions.federated.primitives.part5.** -Dtest-threadCount=1 -Dtest-forkCount=1", - "**.functions.federated.monitoring.**,**.functions.federated.multitenant.**", + "**.functions.federated.monitoring.**,**.functions.federated.multitenant.** -Dtest-threadCount=1 -Dtest-forkCount=1", "**.functions.federated.codegen.**,**.functions.federated.FederatedTestObjectConstructor", "**.functions.codegenalg.partone.**", "**.functions.builtin.part1.**", From 6d1d218767f1fd863aeb83ce6b48405abb0ee22a Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Fri, 26 Jun 2026 17:15:32 +0200 Subject: [PATCH 052/132] [BWARE] Tighten MatrixBlock quantile/sort API surface (#2513) Tightens the MatrixBlock quantile/sort API surface so the entry points are sealed against accidental overrides and the weighted/unweighted quantile logic is clearly separated. --- .../compress/CompressedMatrixBlock.java | 15 +- .../runtime/matrix/data/MatrixBlock.java | 71 +++++++++- .../compress/CompressedSortTest.java | 34 +++++ .../component/matrix/QuantilePickTest.java | 129 ++++++++++++++++++ 4 files changed, 230 insertions(+), 19 deletions(-) create mode 100644 src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java index dae13ed9f94..3a79443157b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java @@ -1201,8 +1201,9 @@ public void examSparsity(boolean allowCSR, int k) { } @Override - public void sparseToDense(int k) { - // do nothing + public MatrixBlock sparseToDense(int k) { + // a compressed block has no sparse representation to convert; return unchanged + return this; } @Override @@ -1235,16 +1236,6 @@ public double interQuartileMean() { return getUncompressed("interQuartileMean").interQuartileMean(); } - @Override - public MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret) { - return getUncompressed("pickValues").pickValues(quantiles, ret); - } - - @Override - public double pickValue(double quantile, boolean average) { - return getUncompressed("pickValue").pickValue(quantile, average); - } - @Override public double sumWeightForQuantile() { return getUncompressed("sumWeightForQuantile").sumWeightForQuantile(); diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java index b1c06cdd51e..361d190bd02 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java @@ -1387,12 +1387,13 @@ public void denseToSparse(boolean allowCSR, int k){ LibMatrixDenseToSparse.denseToSparse(this, allowCSR, k); } - public final void sparseToDense() { - sparseToDense(1); + public final MatrixBlock sparseToDense() { + return sparseToDense(1); } - public void sparseToDense(int k) { + public MatrixBlock sparseToDense(int k) { LibMatrixSparseToDense.sparseToDense(this, k); + return this; } /** @@ -4650,7 +4651,7 @@ public final MatrixBlock sortOperations(MatrixValue weights){ return sortOperations(weights, null); } - public MatrixBlock sortOperations(MatrixValue weights, MatrixBlock result) { + public final MatrixBlock sortOperations(MatrixValue weights, MatrixBlock result) { return sortOperations(weights, result, 1); } @@ -4754,7 +4755,17 @@ public static double computeIQMCorrection(double sum, double sum_wt, return (sum + q25Part*q25Val - q75Part*q75Val) / (sum_wt*0.5); } - public MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret) { + /** + * Pick the quantiles out of this matrix. If this matrix contains two columns it is weighted quantile picking. + * If a single column it is unweighted. + * + * Note the values are assumed to be sorted. + * + * @param quantiles The quantiles to pick + * @param ret The result matrix + * @return The result matrix + */ + public final MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret) { return pickValues(quantiles, ret, false); } @@ -4779,16 +4790,62 @@ public MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret, boolean av return output; } + /** + * Pick the median value from this matrix. If this matrix has two columns it is weighted picking using the + * weight column, otherwise it is unweighted over the single column. + * + * Note the values are assumed to be sorted. + * + * @return The median value + */ public double median() { + if(getNumColumns() == 1) + return pickValue(0.5, getNumRows() % 2 == 0); double sum_wt = sumWeightForQuantile(); return pickValue(0.5, sum_wt%2==0); } - + + /** + * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is unweighted. + * + * Note the values are assumed to be sorted. + * + * @param quantile The quantile to pick + * @return The quantile + */ public final double pickValue(double quantile){ return pickValue(quantile, false); } - public double pickValue(double quantile, boolean average) { + /** + * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is unweighted. + * + * Note the values are assumed to be sorted. + * + * @param quantile The quantile to pick + * @param average If the quantile is averaged. + * @return The quantile + */ + public final double pickValue(double quantile, boolean average) { + if(this.getNumColumns() == 1) + return pickUnweightedValue(quantile, average); + return pickWeightedValue(quantile, average); + } + + private double pickUnweightedValue(double quantile, boolean average) { + // Mirror the weighted convention (pickWeightedValue) with an implicit weight of 1 per value, so a single + // column yields the same quantile as the equivalent two-column (value, weight) representation: take the + // ceil-based rank and only average adjacent order statistics when an even number of values straddles it. + final int rows = getNumRows(); + average = average && (rows % 2 == 0); + final int pos = (int) Math.ceil(quantile * rows); // 1-based rank + final int i = Math.min(Math.max(pos - 1, 0), rows - 1); + if(average && pos > 0 && pos < rows) + return (get(i, 0) + get(i + 1, 0)) / 2; + return get(i, 0); + } + + private double pickWeightedValue(double quantile, boolean average) { double sum_wt = sumWeightForQuantile(); // do averaging only if it is asked for; and sum_wt is even diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java index 7ab7187eb78..083a29f965b 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java @@ -203,6 +203,40 @@ public void quantileWeightedFallback() { TestUtils.compareMatrices(expected, actual, 0.0, "weighted sortOperations fallback"); } + @Test + public void pickDirectlyOnCompressedColumnDDC() { + runDirectPick(generate(ROWS, 1, 8, 1.0, 1, 50, 7), CompressionType.DDC); + } + + @Test + public void pickDirectlyOnCompressedColumnSDCZeros() { + runDirectPick(generate(ROWS, 1, 6, 0.2, 1, 40, 23), CompressionType.SDC); + } + + @Test + public void pickDirectlyOnCompressedColumnWithNegatives() { + runDirectPick(generate(ROWS, 1, 8, 0.3, -20, 20, 41), CompressionType.SDC); + } + + /** + * Quantile picking normally runs on the uncompressed value/weight table produced by sortOperations, so the + * inherited (no longer overridden) pickValue path is never reached on a compressed block through that flow. This + * exercises it directly: the single column is sorted while staying compressed, then pickValue is invoked on the + * CompressedMatrixBlock itself and must match the uncompressed sorted column element for element. median() is not + * used here because it requires the two-column weighted representation. + */ + private void runDirectPick(MatrixBlock mb, CompressionType ct) { + CompressedMatrixBlock cmb = compress(mb, ct); + MatrixBlock sortedC = cmb.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); + assertTrue("Expected the sorted result to stay compressed for " + ct, sortedC instanceof CompressedMatrixBlock); + MatrixBlock sortedU = mb.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); + + for(double q : new double[] {0.0, 0.25, 0.5, 0.75, 0.9, 1.0}) { + assertEquals("pick q=" + q + " " + ct, sortedU.pickValue(q, false), sortedC.pickValue(q, false), 0.0); + assertEquals("pick avg q=" + q + " " + ct, sortedU.pickValue(q, true), sortedC.pickValue(q, true), 0.0); + } + } + private void runQuantile(MatrixBlock mb, CompressionType ct) { // reference is computed on a copy because compression may consume the input. MatrixBlock expected = new MatrixBlock(mb).sortOperations(null, new MatrixBlock(), 1); diff --git a/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java b/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java new file mode 100644 index 00000000000..472b61d8cd6 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.matrix; + +import static org.junit.Assert.assertEquals; + +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.junit.Test; + +/** + * Tests the single-column (unweighted) branch of {@link MatrixBlock#pickValue(double, boolean)} and + * {@link MatrixBlock#median()}. The values are assumed to be sorted in ascending order, mirroring the contract used + * by the quantile pick instructions. The unweighted branch uses the same ceil-based rank as the two-column weighted + * branch (with an implicit weight of 1 per value), so a single column yields the same quantile as the equivalent + * (value, weight) representation. The two-column (weighted) branch is exercised separately through the compressed + * sort tests. + */ +public class QuantilePickTest { + + private static MatrixBlock singleColumn(double[] values, boolean sparse) { + MatrixBlock mb = new MatrixBlock(values.length, 1, sparse); + for(int i = 0; i < values.length; i++) + mb.set(i, 0, values[i]); + mb.recomputeNonZeros(); + return mb; + } + + @Test + public void pickOddLengthNoAverage() { + // rank = ceil(quantile * 5), value at (rank-1). + MatrixBlock mb = singleColumn(new double[] {10, 20, 30, 40, 50}, false); + assertEquals("q=0.0", 10, mb.pickValue(0.0, false), 0); // rank 0 -> idx 0 + assertEquals("q=0.2", 10, mb.pickValue(0.2, false), 0); // rank ceil(1.0)=1 -> idx 0 + assertEquals("q=0.5", 30, mb.pickValue(0.5, false), 0); // rank ceil(2.5)=3 -> idx 2 + assertEquals("q=0.75", 40, mb.pickValue(0.75, false), 0); // rank ceil(3.75)=4 -> idx 3 + assertEquals("q=1.0", 50, mb.pickValue(1.0, false), 0); // rank ceil(5.0)=5 -> idx 4 + } + + @Test + public void pickOddLengthAverageSuppressed() { + // Odd number of values -> averaging is suppressed, so average matches no-average. + MatrixBlock mb = singleColumn(new double[] {10, 20, 30, 40, 50}, false); + assertEquals("q=0.5 avg", 30, mb.pickValue(0.5, true), 0); + assertEquals("q=0.75 avg", 40, mb.pickValue(0.75, true), 0); + } + + @Test + public void pickEvenLengthAverage() { + // Even number of values -> averaging of adjacent order statistics applies. + MatrixBlock mb = singleColumn(new double[] {10, 20, 30, 40}, false); + assertEquals("q=0.25 avg", 15, mb.pickValue(0.25, true), 0); // rank 1 -> (idx0+idx1)/2 + assertEquals("q=0.375 avg", 25, mb.pickValue(0.375, true), 0); // rank ceil(1.5)=2 -> (idx1+idx2)/2 + assertEquals("q=0.5 avg", 25, mb.pickValue(0.5, true), 0); // rank 2 -> (idx1+idx2)/2 + assertEquals("q=0.75 avg", 35, mb.pickValue(0.75, true), 0); // rank 3 -> (idx2+idx3)/2 + } + + @Test + public void pickEvenLengthNoAverage() { + MatrixBlock mb = singleColumn(new double[] {10, 20, 30, 40}, false); + assertEquals("q=0.25", 10, mb.pickValue(0.25, false), 0); // rank 1 -> idx 0 + assertEquals("q=0.5", 20, mb.pickValue(0.5, false), 0); // rank 2 -> idx 1 + assertEquals("q=0.75", 30, mb.pickValue(0.75, false), 0); // rank 3 -> idx 2 + } + + @Test + public void pickAverageClampedAtTop() { + // Top quantile: rank reaches the last element so there is no successor to average with. + MatrixBlock even = singleColumn(new double[] {10, 20, 30, 40}, false); + assertEquals("even q=0.95 avg", 40, even.pickValue(0.95, true), 0); // rank ceil(3.8)=4 -> idx 3, no avg + assertEquals("even q=1.0 avg", 40, even.pickValue(1.0, true), 0); + MatrixBlock odd = singleColumn(new double[] {10, 20, 30, 40, 50}, false); + assertEquals("odd q=0.95 avg", 50, odd.pickValue(0.95, true), 0); // odd -> avg suppressed + } + + @Test + public void pickSingleElement() { + MatrixBlock mb = singleColumn(new double[] {42}, false); + assertEquals("q=0.0", 42, mb.pickValue(0.0, false), 0); + assertEquals("q=0.5", 42, mb.pickValue(0.5, false), 0); + assertEquals("q=1.0", 42, mb.pickValue(1.0, false), 0); + assertEquals("q=0.5 avg", 42, mb.pickValue(0.5, true), 0); + assertEquals("median", 42, mb.median(), 0); + } + + @Test + public void pickSparseSingleColumnWithZeros() { + // Sorted ascending including leading zeros, stored sparse. + MatrixBlock mb = singleColumn(new double[] {0, 0, 10, 20, 30}, true); + assertEquals("q=0.0", 0, mb.pickValue(0.0, false), 0); // rank 0 -> idx 0 (zero) + assertEquals("q=0.5", 10, mb.pickValue(0.5, false), 0); // rank ceil(2.5)=3 -> idx 2 + assertEquals("q=0.75", 20, mb.pickValue(0.75, false), 0); // rank ceil(3.75)=4 -> idx 3 + assertEquals("q=1.0", 30, mb.pickValue(1.0, false), 0); // rank 5 -> idx 4 + } + + @Test + public void medianSingleColumn() { + // Odd length -> middle element; even length -> average of the two middle elements. + assertEquals("odd median", 30, singleColumn(new double[] {10, 20, 30, 40, 50}, false).median(), 0); + assertEquals("even median", 25, singleColumn(new double[] {10, 20, 30, 40}, false).median(), 0); + assertEquals("sparse median", 10, singleColumn(new double[] {0, 0, 10, 20, 30}, true).median(), 0); + } + + @Test + public void pickSingleColumnMatchesDenseAndSparse() { + double[] v = {-5, -1, 0, 2, 7, 9}; + MatrixBlock dense = singleColumn(v, false); + MatrixBlock sparse = singleColumn(v, true); + for(double q : new double[] {0.0, 0.1, 0.25, 0.5, 0.75, 0.9, 1.0}) + for(boolean avg : new boolean[] {false, true}) + assertEquals("q=" + q + " avg=" + avg, dense.pickValue(q, avg), sparse.pickValue(q, avg), 0); + } +} From b11943541fe1bbef9ecc74715b229dd2efa13e07 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Fri, 26 Jun 2026 17:59:52 +0200 Subject: [PATCH 053/132] [MINOR][CI] Run component.c tests in parallel again (#2518) Remove -Dtest-threadCount=1 -Dtest-forkCount=1 from the component.c test matrix entry so these tests run with the default parallel fork/thread configuration instead of serially. --- .github/workflows/javaTests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml index 07762a17427..61089807820 100644 --- a/.github/workflows/javaTests.yml +++ b/.github/workflows/javaTests.yml @@ -59,7 +59,7 @@ jobs: tests: [ "org.apache.sysds.test.applications.**", "**.test.usertest.**", - "**.component.c**.** -Dtest-threadCount=1 -Dtest-forkCount=1", + "**.component.c**.**", "**.component.e**.**,**.component.f**.**,**.component.m**.**,**.component.o**.**", "**.component.p**.**,**.component.r**.**,**.component.s**.**,**.component.t**.**,**.component.u**.**", "**.functions.a**.**,**.functions.binary.matrix.**,**.functions.binary.scalar.**,**.functions.binary.tensor.**", From 8ab6b98f472b5be36994645cca63a55016345e84 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Sat, 27 Jun 2026 20:08:38 +0200 Subject: [PATCH 054/132] [MINOR] Reference-count shared Spark context to prevent concurrent shutdown (#2516) The singleton JVM-wide SparkContext in SparkExecutionContext was stopped unconditionally at the end of every DMLScript.execute() run. When two DML script executions run concurrently in the same JVM (e.g. surefire parallel tests with parallel=classes and threadCount=2), one execution finishing its script would call close() and stop the context while the other execution still had an in-flight spark job. The wedged job never receives a completion event from the now-stopped DAGScheduler. --- .../java/org/apache/sysds/api/DMLScript.java | 11 +- .../context/SparkExecutionContext.java | 49 ++++++- .../SparkContextReferenceCountTest.java | 126 ++++++++++++++++++ 3 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java b/src/main/java/org/apache/sysds/api/DMLScript.java index c286b8d3b52..a7a175bb7b6 100644 --- a/src/main/java/org/apache/sysds/api/DMLScript.java +++ b/src/main/java/org/apache/sysds/api/DMLScript.java @@ -508,6 +508,11 @@ private static void execute(String dmlScriptStr, String fnameOptConfig, Map 0) + _activeExecutions--; + } } public void close() { - synchronized( SparkExecutionContext.class) { + synchronized(SparkExecutionContext.class) { + //keep the shared context alive while a registered execution still uses + //it; close() never changes the count, so an unpaired close() (a caller + //that never entered) cannot stop a context another execution is using + if(_activeExecutions > 0) { + if(LOG.isDebugEnabled()) + LOG.debug("Keeping shared spark context alive; " + _activeExecutions + + " execution(s) still active"); + return; + } if(_spctx != null) { Logger spL = Logger.getLogger("org.apache.spark.network.client.TransportResponseHandler"); spL.setLevel(Level.FATAL); diff --git a/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java b/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java new file mode 100644 index 00000000000..3493da7d2b1 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.context; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; + +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.sysds.runtime.controlprogram.context.ExecutionContextFactory; +import org.apache.sysds.runtime.controlprogram.context.SparkExecutionContext; +import org.junit.Test; + +@net.jcip.annotations.NotThreadSafe +public class SparkContextReferenceCountTest { + + /** + * Two DML executions sharing the JVM-wide singleton spark context (as happens + * with surefire parallel tests, threadCount>1). When the first execution + * finishes and calls close(), the shared context must stay alive because the + * second execution still has in-flight work. Before reference counting, + * close() stopped the context unconditionally, which cancelled the second + * execution's spark job and wedged it until the test watchdog. + */ + @Test + public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { + SparkExecutionContext ecA = null; + SparkExecutionContext ecB = null; + try { + // execution A: create the context then register (as in DMLScript.execute) + ecA = ExecutionContextFactory.createSparkExecutionContext(); + SparkExecutionContext.enterSparkExecution(); + JavaSparkContext sc = ecA.getSparkContext(); + + // execution B is a second concurrent user with its own context instance, + // sharing the same JVM-wide singleton spark context + ecB = ExecutionContextFactory.createSparkExecutionContext(); + SparkExecutionContext.enterSparkExecution(); + + // B's in-flight work + JavaRDD rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4)); + + // A finishes first: release its registration; close() must NOT stop the + // context that B still uses + SparkExecutionContext.exitSparkExecution(); + ecA.close(); + assertFalse("shared context must stay alive while another execution is active", + sc.sc().isStopped()); + assertEquals("B's job must still run on the live context", + 10L, rdd.reduce(Integer::sum).longValue()); + + // B finishes last: releasing the final registration lets close() stop it + SparkExecutionContext.exitSparkExecution(); + ecB.close(); + assertTrue("shared context must be stopped once the last execution closes", + sc.sc().isStopped()); + } + finally { + // drain any remaining registrations and stop the context so a failed + // assertion cannot leak ref-count state into other tests in this JVM + // (exit/close are clamped and no-op once already drained/stopped) + SparkExecutionContext.exitSparkExecution(); + SparkExecutionContext.exitSparkExecution(); + if(ecA != null) + ecA.close(); + } + } + + /** + * An unpaired close() (a caller that borrows the shared context but never + * registered via enterSparkExecution()) must not stop a context another + * execution still uses. This fails on the old unconditional-stop code, which + * tore the context down out from under the active execution. + */ + @Test + public void unpairedCloseDoesNotStopAContextStillInUse() { + SparkExecutionContext active = null; + SparkExecutionContext unregistered = null; + try { + // a registered, in-flight execution holds the shared context + active = ExecutionContextFactory.createSparkExecutionContext(); + SparkExecutionContext.enterSparkExecution(); + JavaSparkContext sc = active.getSparkContext(); + + // a context that never registered closes (e.g. a caller that only + // borrows the shared context): close() must not stop a context in use + unregistered = ExecutionContextFactory.createSparkExecutionContext(); + unregistered.close(); + assertFalse("unpaired close() must not stop a context still in use", + sc.sc().isStopped()); + + // the registered execution finishing stops the context as the last user + SparkExecutionContext.exitSparkExecution(); + active.close(); + assertTrue("context must stop once the last registered execution closes", + sc.sc().isStopped()); + } + finally { + SparkExecutionContext.exitSparkExecution(); + if(active != null) + active.close(); + if(unregistered != null) + unregistered.close(); + } + } +} From 35af80f582eeeffca85e958f19ad861166db9808 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Sun, 28 Jun 2026 14:56:13 +0200 Subject: [PATCH 055/132] [BWARE] Restructure CLALibBinaryCellOp and decompression fallbacks (#2519) Restructure CLALibBinaryCellOp and decompression fallbacks for binary ops Reworks how the compressed library handles binary cell-wise operations, broadening the set of inputs it can keep compressed and tightening the "give up and decompress" path so it is consistent across operation shapes. --- .../compress/CompressedMatrixBlock.java | 21 +- .../CompressedMatrixBlockFactory.java | 25 +- .../compress/lib/CLALibBinaryCellOp.java | 348 +++++++++++++----- .../cp/BinaryMatrixMatrixCPInstruction.java | 9 +- .../lib/CLALibBinaryCellOpCustomTest.java | 41 +++ ...CompressedBinaryMatrixMatrixSolveTest.java | 127 +++++++ 6 files changed, 461 insertions(+), 110 deletions(-) create mode 100644 src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java index 3a79443157b..d0ba5363939 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java @@ -59,8 +59,8 @@ import org.apache.sysds.runtime.compress.lib.CLALibMatrixMult; import org.apache.sysds.runtime.compress.lib.CLALibMerge; import org.apache.sysds.runtime.compress.lib.CLALibRemoveEmpty; -import org.apache.sysds.runtime.compress.lib.CLALibReplace; import org.apache.sysds.runtime.compress.lib.CLALibReorg; +import org.apache.sysds.runtime.compress.lib.CLALibReplace; import org.apache.sysds.runtime.compress.lib.CLALibReshape; import org.apache.sysds.runtime.compress.lib.CLALibRexpand; import org.apache.sysds.runtime.compress.lib.CLALibScalar; @@ -103,6 +103,7 @@ import org.apache.sysds.runtime.util.IndexRange; import org.apache.sysds.utils.DMLCompressionStatistics; import org.apache.sysds.utils.stats.InfrastructureAnalyzer; +import org.apache.sysds.utils.stats.Timing; public class CompressedMatrixBlock extends MatrixBlock { private static final Log LOG = LogFactory.getLog(CompressedMatrixBlock.class.getName()); @@ -477,16 +478,20 @@ public void readFields(DataInput in) throws IOException { } public static CompressedMatrixBlock read(DataInput in) throws IOException { + Timing t = new Timing(); int rlen = in.readInt(); int clen = in.readInt(); long nonZeros = in.readLong(); boolean overlappingColGroups = in.readBoolean(); List groups = ColGroupIO.readGroups(in, rlen); - return new CompressedMatrixBlock(rlen, clen, nonZeros, overlappingColGroups, groups); + CompressedMatrixBlock ret = new CompressedMatrixBlock(rlen, clen, nonZeros, overlappingColGroups, groups); + LOG.debug("Compressed read serialization time: " + t.stop()); + return ret; } @Override public void write(DataOutput out) throws IOException { + Timing t = new Timing(); final long estimateUncompressed = nonZeros > 0 ? MatrixBlock.estimateSizeOnDisk(rlen, clen, nonZeros) : Long.MAX_VALUE; final long estDisk = nonZeros > 0 ? getExactSizeOnDisk() : Long.MAX_VALUE; @@ -514,6 +519,7 @@ public void write(DataOutput out) throws IOException { out.writeLong(nonZeros); out.writeBoolean(overlappingColGroups); ColGroupIO.writeGroups(out, _colGroups); + LOG.debug("Compressed write serialization time: " + t.stop()); } /** @@ -613,16 +619,7 @@ public MatrixBlock aggregateUnaryOperations(AggregateUnaryOperator op, MatrixVal public MatrixBlock transposeSelfMatrixMultOperations(MatrixBlock out, MMTSJType tstype, int k) { // check for transpose type if(tstype == MMTSJType.LEFT) { - if(isEmpty()) - return new MatrixBlock(clen, clen, true); - // create output matrix block - if(out == null) - out = new MatrixBlock(clen, clen, false); - else - out.reset(clen, clen, false); - out.allocateDenseBlock(); - CLALibTSMM.leftMultByTransposeSelf(this, out, k); - return out; + return CLALibTSMM.leftMultByTransposeSelf(this, out, k); } else { throw new DMLRuntimeException("Invalid MMTSJ type '" + tstype.toString() + "'."); diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java index 4c48effb4df..7eae9ca0a7e 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java +++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java @@ -64,6 +64,9 @@ public class CompressedMatrixBlockFactory { private static final Log LOG = LogFactory.getLog(CompressedMatrixBlockFactory.class.getName()); + /** Global lock serializing all async compressions to bound concurrent compression memory/CPU. */ + private static final Object asyncCompressLock = new Object(); + /** Timing object to measure the time of each phase in the compression */ private final Timing time = new Timing(true); /** Compression statistics gathered throughout the compression */ @@ -181,21 +184,23 @@ public static Future compressAsync(ExecutionContext ec, String varName) { } public static Future compressAsync(ExecutionContext ec, String varName, InstructionTypeCounter ins) { - LOG.debug("Compressing Async"); final ExecutorService pool = CommonThreadPool.get(); // We have to guarantee that a thread pool is allocated. return CompletableFuture.runAsync(() -> { // method call or code to be async try { CacheableData data = ec.getCacheableData(varName); - if(data instanceof MatrixObject) { - MatrixObject mo = (MatrixObject) data; - MatrixBlock mb = mo.acquireReadAndRelease(); - MatrixBlock mbc = CompressedMatrixBlockFactory.compress(mo.acquireReadAndRelease(), ins).getLeft(); - if(mbc instanceof CompressedMatrixBlock) { - ExecutionContext.createCacheableData(mb); - mo.acquireModify(mbc); - mo.release(); - mbc.sum(); // calculate sum to forcefully materialize counts + synchronized(asyncCompressLock) { // global lock: serialize all async compressions (not per-matrix) + if(data instanceof MatrixObject) { + LOG.debug("Compressing Async"); + MatrixObject mo = (MatrixObject) data; + MatrixBlock mb = mo.acquireReadAndRelease(); + MatrixBlock mbc = CompressedMatrixBlockFactory.compress(mb, ins).getLeft(); + if(mbc instanceof CompressedMatrixBlock) { + ExecutionContext.createCacheableData(mb); + mo.acquireModify(mbc); + mo.release(); + mbc.sum(); // calculate sum to forcefully materialize counts + } } } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java index ce52bcd23fd..d981ab87838 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java @@ -48,6 +48,7 @@ import org.apache.sysds.runtime.compress.colgroup.mapping.AMapToData; import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory; import org.apache.sysds.runtime.compress.colgroup.offset.AIterator; +import org.apache.sysds.runtime.compress.utils.HashMapIntToInt; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.DenseBlockFP64; import org.apache.sysds.runtime.data.SparseBlock; @@ -55,7 +56,6 @@ import org.apache.sysds.runtime.data.SparseRow; import org.apache.sysds.runtime.data.SparseRowScalar; import org.apache.sysds.runtime.data.SparseRowVector; -import org.apache.sysds.runtime.frame.data.columns.HashMapToInt; import org.apache.sysds.runtime.functionobjects.Divide; import org.apache.sysds.runtime.functionobjects.Minus; import org.apache.sysds.runtime.functionobjects.Multiply; @@ -77,7 +77,7 @@ public final class CLALibBinaryCellOp { private static final Log LOG = LogFactory.getLog(CLALibBinaryCellOp.class.getName()); - public static final int DECOMPRESSION_BLEN = 16384; + public static final int DECOMPRESSION_BLEN = 8192; private CLALibBinaryCellOp() { // empty private constructor. @@ -86,7 +86,7 @@ private CLALibBinaryCellOp() { public static MatrixBlock binaryOperationsRight(BinaryOperator op, CompressedMatrixBlock m1, MatrixBlock that) { try { - op = LibMatrixBincell.replaceOpWithSparseSafeIfApplicable(m1, that, op); + op = LibMatrixBincell.replaceOpWithSparseSafeIfApplicable(m1, that, op); if((that.getNumRows() == 1 && that.getNumColumns() == 1) || that.isEmpty()) { ScalarOperator sop = new RightScalarOperator(op.fn, that.get(0, 0), op.getNumThreads()); @@ -113,7 +113,7 @@ public static MatrixBlock binaryOperationsLeft(BinaryOperator op, CompressedMatr return selectProcessingBasedOnAccessType(op, m1, that, atype, true); } catch(Exception e) { - throw new DMLRuntimeException("Failed Left Binary Compressed Operation", e); + throw new DMLRuntimeException("Failed Left Binary Compressed Operation: " + op, e); } } @@ -122,8 +122,8 @@ private static MatrixBlock binaryOperationsRightFiltered(BinaryOperator op, Comp BinaryAccessType atype = LibMatrixBincell.getBinaryAccessTypeExtended(m1, that); if(isDoubleCompressedOpApplicable(m1, that)) return doubleCompressedBinaryOp(op, m1, (CompressedMatrixBlock) that); - if(that instanceof CompressedMatrixBlock && that.getNumColumns() == m1.getNumColumns() - && that.getInMemorySize() < m1.getInMemorySize() ) { + if(that instanceof CompressedMatrixBlock && that.getNumColumns() == m1.getNumColumns() && + that.getInMemorySize() < m1.getInMemorySize()) { MatrixBlock m1uc = CompressedMatrixBlock.getUncompressed(m1, "Decompressing left side in BinaryOps"); return selectProcessingBasedOnAccessType(op, (CompressedMatrixBlock) that, m1uc, atype, true); } @@ -135,16 +135,15 @@ private static MatrixBlock binaryOperationsRightFiltered(BinaryOperator op, Comp } private static boolean isDoubleCompressedOpApplicable(CompressedMatrixBlock m1, MatrixBlock that) { - return that instanceof CompressedMatrixBlock - && !m1.isOverlapping() - && m1.getColGroups().get(0) instanceof ColGroupDDC - && !((CompressedMatrixBlock) that).isOverlapping() - && ((CompressedMatrixBlock) that).getColGroups().get(0) instanceof ColGroupDDC - && ((IMapToDataGroup) m1.getColGroups().get(0)).getMapToData() == - ((IMapToDataGroup) ((CompressedMatrixBlock) that).getColGroups().get(0)).getMapToData(); + return that instanceof CompressedMatrixBlock && !m1.isOverlapping() && + m1.getColGroups().get(0) instanceof ColGroupDDC && !((CompressedMatrixBlock) that).isOverlapping() && + ((CompressedMatrixBlock) that).getColGroups().get(0) instanceof ColGroupDDC && + ((IMapToDataGroup) m1.getColGroups().get(0)) + .getMapToData() == ((IMapToDataGroup) ((CompressedMatrixBlock) that).getColGroups().get(0)).getMapToData(); } - private static CompressedMatrixBlock doubleCompressedBinaryOp(BinaryOperator op, CompressedMatrixBlock m1, CompressedMatrixBlock m2) { + private static CompressedMatrixBlock doubleCompressedBinaryOp(BinaryOperator op, CompressedMatrixBlock m1, + CompressedMatrixBlock m2) { LOG.debug("Double Compressed BinaryOp"); AColGroup left = m1.getColGroups().get(0); AColGroup right = m2.getColGroups().get(0); @@ -201,6 +200,7 @@ private static MatrixBlock mvCol(BinaryOperator op, CompressedMatrixBlock m1, Ma // Column vector access MatrixBlock d_compressed = m1.getCachedDecompressed(); if(d_compressed != null) { + LOG.debug("Using cached decompressed for Matrix column vector compressed operation"); if(left) throw new NotImplementedException("Binary row op left is not supported for Uncompressed Matrix, " + "Implement support for VMr in MatrixBlock Binary Cell operations"); @@ -416,17 +416,29 @@ private static MatrixBlock mvColCompressed(CompressedMatrixBlock m1, MatrixBlock Pair tuple = evaluateSparsityMVCol(m1, m2, op, left); double estSparsity = tuple.getKey(); double estNnzPerRow = tuple.getValue(); - boolean shouldBeSparseOut = MatrixBlock.evalSparseFormatInMemory(nRows, nCols, (long) (estSparsity * nRows * nCols)); + boolean shouldBeSparseOut = MatrixBlock.evalSparseFormatInMemory(nRows, nCols, + (long) (estSparsity * nRows * nCols)); // currently also jump into that case if estNnzPerRow == 0 - if(estNnzPerRow <= 2 && nCols <= 31 && op.fn instanceof ValueComparisonFunction){ - return k <= 1 ? binaryMVComparisonColSingleThreadCompressed(m1, m2, op, left) : - binaryMVComparisonColMultiCompressed(m1, m2, op, left); + if(estNnzPerRow <= 2 && nCols <= 31 && op.fn instanceof ValueComparisonFunction) { + return k <= 1 ? binaryMVComparisonColSingleThreadCompressed(m1, m2, op, + left) : binaryMVComparisonColMultiCompressed(m1, m2, op, left); } MatrixBlock ret = new MatrixBlock(nRows, nCols, shouldBeSparseOut, -1).allocateBlock(); if(shouldBeSparseOut) { - if(k <= 1) + // The sparse-sparse path only visits the stored non-zeros of m1, so it is correct only when the operation + // maps a zero in m1 to a zero output for every value of the vector m2 (i.e. it does not introduce non-zeros + // from m1's zeros). Otherwise fall back to the dense-scan sparse path that evaluates every cell. + final boolean sparseSafeOnM1Zeros = left ? op.isRowSafeLeft(m2) : op.isRowSafeRight(m2); + if(sparseSafeOnM1Zeros && !m1.isOverlapping() && + MatrixBlock.evalSparseFormatInMemory(nRows, nCols, m1.getNonZeros())) { + if(k <= 1) + nnz = binaryMVColSingleThreadSparseSparse(m1, m2, op, left, ret); + else + nnz = binaryMVColMultiThreadSparseSparse(m1, m2, op, left, ret); + } + else if(k <= 1) nnz = binaryMVColSingleThreadSparse(m1, m2, op, left, ret); else nnz = binaryMVColMultiThreadSparse(m1, m2, op, left, ret); @@ -438,7 +450,7 @@ private static MatrixBlock mvColCompressed(CompressedMatrixBlock m1, MatrixBlock nnz = binaryMVColMultiThreadDense(m1, m2, op, left, ret); } - if(op.fn instanceof ValueComparisonFunction) { + if(op.fn instanceof ValueComparisonFunction) { // potentially empty or filled. if(nnz == (long) nRows * nCols)// all was 1 return CompressedMatrixBlockFactory.createConstant(nRows, nCols, 1.0); else if(nnz == 0) // all was 0 -> return empty. @@ -452,19 +464,19 @@ else if(nnz == 0) // all was 0 -> return empty. } private static MatrixBlock binaryMVComparisonColSingleThreadCompressed(CompressedMatrixBlock m1, MatrixBlock m2, - BinaryOperator op, boolean left) { + BinaryOperator op, boolean left) { final int nRows = m1.getNumRows(); final int nCols = m1.getNumColumns(); // get indicators (one-hot-encoded comparison results) - BinaryMVColTaskCompressed task = new BinaryMVColTaskCompressed(m1, m2, 0, nRows, op, left); + BinaryMVColTaskCompressed task = new BinaryMVColTaskCompressed(m1, m2, 0, nRows, op, left); long nnz = task.call(); int[] indicators = task._ret; // map each unique indicator to an index - HashMapToInt hm = new HashMapToInt<>(nCols*3); + HashMapIntToInt hm = new HashMapIntToInt(nCols * 3); int[] colMap = new int[nRows]; - for(int i = 0; i < m1.getNumRows(); i++){ + for(int i = 0; i < m1.getNumRows(); i++) { int nextId = hm.size(); int id = hm.putIfAbsentI(indicators[i], nextId); colMap[i] = id == -1 ? nextId : id; @@ -477,37 +489,39 @@ private static MatrixBlock binaryMVComparisonColSingleThreadCompressed(Compresse return getCompressedMatrixBlock(m1, colMap, hm.size(), outMb, nRows, nCols, nnz); } - private static void fillSparseBlockFromIndicatorFromIndicatorInt(int numCol, Integer indicator, Integer rix, SparseBlockMCSR out) { + private static void fillSparseBlockFromIndicatorFromIndicatorInt(int numCol, Integer indicator, Integer rix, + SparseBlockMCSR out) { ArrayList colIndices = new ArrayList<>(8); - for (int c = numCol - 1; c >= 0; c--) { + for(int c = numCol - 1; c >= 0; c--) { if(indicator <= 0) break; - if(indicator % 2 == 1){ + if(indicator % 2 == 1) { colIndices.add(c); } indicator = indicator >> 1; } SparseRow row = null; - if(colIndices.size() > 1){ + if(colIndices.size() > 1) { double[] vals = new double[colIndices.size()]; Arrays.fill(vals, 1); int[] indices = new int[colIndices.size()]; - for (int i = 0, j = colIndices.size() - 1; i < colIndices.size(); i++, j--) + for(int i = 0, j = colIndices.size() - 1; i < colIndices.size(); i++, j--) indices[i] = colIndices.get(j); row = new SparseRowVector(vals, indices); - } else if(colIndices.size() == 1){ + } + else if(colIndices.size() == 1) { row = new SparseRowScalar(colIndices.get(0), 1.0); } out.set(rix, row, false); } private static MatrixBlock binaryMVComparisonColMultiCompressed(CompressedMatrixBlock m1, MatrixBlock m2, - BinaryOperator op, boolean left) throws Exception { + BinaryOperator op, boolean left) throws Exception { final int nRows = m1.getNumRows(); final int nCols = m1.getNumColumns(); final int k = op.getNumThreads(); - final int blkz = nRows / k; + final int blkz = Math.max((nRows + k) / k, 1000); // get indicators (one-hot-encoded comparison results) long nnz = 0; @@ -518,14 +532,11 @@ private static MatrixBlock binaryMVComparisonColMultiCompressed(CompressedMatrix tasks.add(new BinaryMVColTaskCompressed(m1, m2, i, Math.min(nRows, i + blkz), op, left)); } List> futures = pool.invokeAll(tasks); - HashMapToInt hm = new HashMapToInt<>(nCols*2); + HashMapIntToInt hm = new HashMapIntToInt(nCols * 2); int[] colMap = new int[nRows]; - for(Future f : futures) - nnz += f.get(); - // map each unique indicator to an index - mergeMVColTaskResults(tasks, blkz, hm, colMap); + nnz = mergeMVColTaskResults(futures, tasks, blkz, hm, colMap); // decode the unique indicator ints to SparseVectors MatrixBlock outMb = getMCSRMatrixBlock(hm, nCols); @@ -539,48 +550,53 @@ private static MatrixBlock binaryMVComparisonColMultiCompressed(CompressedMatrix } - private static void mergeMVColTaskResults(ArrayList tasks, int blkz, HashMapToInt hm, int[] colMap) { - + private static long mergeMVColTaskResults(List> futures, ArrayList tasks, + int blkz, HashMapIntToInt hm, int[] colMap) throws InterruptedException, ExecutionException { + long nnz = 0; for(int j = 0; j < tasks.size(); j++) { + nnz += futures.get(j).get(); // ensure task was finished. int[] indicators = tasks.get(j)._ret; - int offset = j* blkz; - - final int remainders = indicators.length % 8; - final int endVecLen = indicators.length - remainders; - for (int i = 0; i < endVecLen; i+= 8) { - colMap[offset + i] = hm.putIfAbsentReturnVal(indicators[i], hm.size()); - colMap[offset + i + 1] = hm.putIfAbsentReturnVal(indicators[i + 1], hm.size()); - colMap[offset + i + 2] = hm.putIfAbsentReturnVal(indicators[i + 2], hm.size()); - colMap[offset + i + 3] = hm.putIfAbsentReturnVal(indicators[i + 3], hm.size()); - colMap[offset + i + 4] = hm.putIfAbsentReturnVal(indicators[i + 4], hm.size()); - colMap[offset + i + 5] = hm.putIfAbsentReturnVal(indicators[i + 5], hm.size()); - colMap[offset + i + 6] = hm.putIfAbsentReturnVal(indicators[i + 6], hm.size()); - colMap[offset + i + 7] = hm.putIfAbsentReturnVal(indicators[i + 7], hm.size()); + int offset = j * blkz; - } - for (int i = 0; i < remainders; i++) { - colMap[offset + endVecLen + i] = hm.putIfAbsentReturnVal(indicators[endVecLen + i], hm.size()); - } + mergeMVColUnrolled(hm, colMap, indicators, offset); } + return nnz; } + private static void mergeMVColUnrolled(HashMapIntToInt hm, int[] colMap, int[] indicators, int offset) { + final int remainders = indicators.length % 8; + final int endVecLen = indicators.length - remainders; + for(int i = 0; i < endVecLen; i += 8) { + colMap[offset + i] = hm.putIfAbsentReturnVal(indicators[i], hm.size()); + colMap[offset + i + 1] = hm.putIfAbsentReturnVal(indicators[i + 1], hm.size()); + colMap[offset + i + 2] = hm.putIfAbsentReturnVal(indicators[i + 2], hm.size()); + colMap[offset + i + 3] = hm.putIfAbsentReturnVal(indicators[i + 3], hm.size()); + colMap[offset + i + 4] = hm.putIfAbsentReturnVal(indicators[i + 4], hm.size()); + colMap[offset + i + 5] = hm.putIfAbsentReturnVal(indicators[i + 5], hm.size()); + colMap[offset + i + 6] = hm.putIfAbsentReturnVal(indicators[i + 6], hm.size()); + colMap[offset + i + 7] = hm.putIfAbsentReturnVal(indicators[i + 7], hm.size()); + + } + for(int i = 0; i < remainders; i++) { + colMap[offset + endVecLen + i] = hm.putIfAbsentReturnVal(indicators[endVecLen + i], hm.size()); + } + } - private static CompressedMatrixBlock getCompressedMatrixBlock(CompressedMatrixBlock m1, int[] colMap, - int mapSize, MatrixBlock outMb, int nRows, int nCols, long nnz) { + private static CompressedMatrixBlock getCompressedMatrixBlock(CompressedMatrixBlock m1, int[] colMap, int mapSize, + MatrixBlock outMb, int nRows, int nCols, long nnz) { final IColIndex i = ColIndexFactory.create(0, m1.getNumColumns()); final AMapToData map = MapToFactory.create(m1.getNumRows(), colMap, mapSize); final AColGroup rgroup = ColGroupDDC.create(i, MatrixBlockDictionary.create(outMb), map, null); final ArrayList groups = new ArrayList<>(1); groups.add(rgroup); - return new CompressedMatrixBlock(nRows, nCols, nnz, false, groups); + return new CompressedMatrixBlock(nRows, nCols, nnz, false, groups); } - private static MatrixBlock getMCSRMatrixBlock(HashMapToInt hm, int nCols) { + private static MatrixBlock getMCSRMatrixBlock(HashMapIntToInt hm, int nCols) { // decode the unique indicator ints to SparseVectors SparseBlockMCSR out = new SparseBlockMCSR(hm.size()); - hm.forEach((indicator, rix) -> - fillSparseBlockFromIndicatorFromIndicatorInt(nCols, indicator, rix, out)); - return new MatrixBlock(hm.size(), nCols, -1, out); + hm.forEach((indicator, rix) -> fillSparseBlockFromIndicatorFromIndicatorInt(nCols, indicator, rix, out)); + return new MatrixBlock(hm.size(), nCols, -1, out); } private static long binaryMVColSingleThreadDense(CompressedMatrixBlock m1, MatrixBlock m2, BinaryOperator op, @@ -599,6 +615,14 @@ private static long binaryMVColSingleThreadSparse(CompressedMatrixBlock m1, Matr return nnz; } + private static long binaryMVColSingleThreadSparseSparse(CompressedMatrixBlock m1, MatrixBlock m2, BinaryOperator op, + boolean left, MatrixBlock ret) { + final int nRows = m1.getNumRows(); + long nnz = 0; + nnz += new BinaryMVColTaskSparseSparse(m1, m2, ret, 0, nRows, op, left).call(); + return nnz; + } + private static long binaryMVColMultiThreadDense(CompressedMatrixBlock m1, MatrixBlock m2, BinaryOperator op, boolean left, MatrixBlock ret) throws Exception { final int nRows = m1.getNumRows(); @@ -641,6 +665,27 @@ private static long binaryMVColMultiThreadSparse(CompressedMatrixBlock m1, Matri return nnz; } + private static long binaryMVColMultiThreadSparseSparse(CompressedMatrixBlock m1, MatrixBlock m2, BinaryOperator op, + boolean left, MatrixBlock ret) throws Exception { + final int nRows = m1.getNumRows(); + final int k = op.getNumThreads(); + final int blkz = Math.max(nRows / k, 64); + long nnz = 0; + final ExecutorService pool = CommonThreadPool.get(op.getNumThreads()); + try { + final ArrayList> tasks = new ArrayList<>(); + for(int i = 0; i < nRows; i += blkz) { + tasks.add(new BinaryMVColTaskSparseSparse(m1, m2, ret, i, Math.min(nRows, i + blkz), op, left)); + } + for(Future f : pool.invokeAll(tasks)) + nnz += f.get(); + } + finally { + pool.shutdown(); + } + return nnz; + } + private static MatrixBlock mmCompressed(CompressedMatrixBlock m1, MatrixBlock m2, BinaryOperator op, boolean left) throws Exception { final int nCols = m1.getNumColumns(); @@ -724,8 +769,8 @@ private static class BinaryMVColTaskCompressed implements Callable { private MatrixBlock tmp; - protected BinaryMVColTaskCompressed(CompressedMatrixBlock m1, MatrixBlock m2, int rl, int ru, - BinaryOperator op, boolean left) { + protected BinaryMVColTaskCompressed(CompressedMatrixBlock m1, MatrixBlock m2, int rl, int ru, BinaryOperator op, + boolean left) { _m1 = m1; _m2 = m2; _op = op; @@ -738,21 +783,21 @@ protected BinaryMVColTaskCompressed(CompressedMatrixBlock m1, MatrixBlock m2, in @Override public Long call() { - tmp = allocateTempUncompressedBlock(_m1.getNumColumns()); - final int _blklen = tmp.getNumRows(); + final int _blklen = Math.max(DECOMPRESSION_BLEN / _m1.getNumColumns(), 64); + tmp = allocateTempUncompressedBlock(_blklen, _m1.getNumColumns()); final List groups = _m1.getColGroups(); final AIterator[] its = getIterators(groups, _rl); long nnz = 0; if(!_left) - for (int rl = _rl, retIxOff = 0; rl < _ru; rl += _blklen, retIxOff += _blklen){ + for(int rl = _rl, retIxOff = 0; rl < _ru; rl += _blklen, retIxOff += _blklen) { int ru = Math.min(rl + _blklen, _ru); decompressToTmpBlock(rl, ru, tmp.getDenseBlock(), groups, its); nnz += processDense(rl, ru, retIxOff); tmp.reset(); } else - for (int rl = _rl, retIxOff = 0; rl < _ru; rl += _blklen, retIxOff += _blklen){ + for(int rl = _rl, retIxOff = 0; rl < _ru; rl += _blklen, retIxOff += _blklen) { int ru = Math.min(rl + _blklen, _ru); decompressToTmpBlock(rl, ru, tmp.getDenseBlock(), groups, its); nnz += processDenseLeft(rl, ru, retIxOff); @@ -770,15 +815,21 @@ private final long processDense(final int rl, final int ru, final int retIxOffse for(int row = rl, retIx = retIxOffset; row < ru; row++, retIx++) { final double vr = _m2Dense[row]; final int tmpOff = (row - rl) * nCol; - int indicatorVector = 0; - for(int col = 0; col < nCol; col++) { - indicatorVector = indicatorVector << 1; - int indicator = _compFn.compare(_tmpDense[tmpOff + col], vr) ? 1 : 0; - indicatorVector += indicator; - nnz += indicator; - } - _ret[retIx] = indicatorVector; + nnz = processRow(nCol, _tmpDense, nnz, retIx, vr, tmpOff); + } + return nnz; + } + + private final long processRow(final int nCol, final double[] _tmpDense, long nnz, int retIx, final double vr, + final int tmpOff) { + int indicatorVector = 0; + for(int col = tmpOff; col < nCol + tmpOff; col++) { + indicatorVector = indicatorVector << 1; + int indicator = _compFn.compare(_tmpDense[col], vr) ? 1 : 0; + indicatorVector += indicator; + nnz += indicator; } + _ret[retIx] = indicatorVector; return nnz; } @@ -847,7 +898,8 @@ private final void processBlock(final int rl, final int ru, final List groups, final AIterator[] its) { + private final void processBlockLeft(final int rl, final int ru, final List groups, + final AIterator[] its) { // unsafe decompress, since we count nonzeros afterwards. final DenseBlock db = _ret.getDenseBlock(); decompressToSubBlock(rl, ru, db, groups, its); @@ -887,7 +939,7 @@ private void processRow(final int ncol, final double[] ret, final int posR, fina private void processRowLeft(final int ncol, final double[] ret, final int posR, final double vr) { for(int col = 0; col < ncol; col++) - ret[posR + col] = _op.fn.execute(vr,ret[posR + col]); + ret[posR + col] = _op.fn.execute(vr, ret[posR + col]); } } @@ -917,8 +969,8 @@ protected BinaryMVColTaskSparse(CompressedMatrixBlock m1, MatrixBlock m2, Matrix @Override public Long call() { - tmp = allocateTempUncompressedBlock(_m1.getNumColumns()); - final int _blklen = tmp.getNumRows(); + final int _blklen = Math.max(DECOMPRESSION_BLEN / _m1.getNumColumns(), 64); + tmp = allocateTempUncompressedBlock(_blklen, _m1.getNumColumns()); final List groups = _m1.getColGroups(); final AIterator[] its = getIterators(groups, _rl); if(!_left) @@ -936,7 +988,8 @@ private final void processBlock(final int rl, final int ru, final List groups, final AIterator[] its) { + private final void processBlockLeft(final int rl, final int ru, final List groups, + final AIterator[] its) { decompressToTmpBlock(rl, ru, tmp.getDenseBlock(), groups, its); processDenseLeft(rl, ru); tmp.reset(); @@ -971,8 +1024,110 @@ private final void processDenseLeft(final int rl, final int ru) { } } - private static MatrixBlock allocateTempUncompressedBlock(int cols) { - MatrixBlock out = new MatrixBlock(Math.max(DECOMPRESSION_BLEN / cols, 64), cols, false); + private static class BinaryMVColTaskSparseSparse implements Callable { + private final int _rl; + private final int _ru; + private final CompressedMatrixBlock _m1; + private final MatrixBlock _m2; + private final MatrixBlock _ret; + private final BinaryOperator _op; + + private MatrixBlock tmp; + + private boolean _left; + + protected BinaryMVColTaskSparseSparse(CompressedMatrixBlock m1, MatrixBlock m2, MatrixBlock ret, int rl, int ru, + BinaryOperator op, boolean left) { + _m1 = m1; + _m2 = m2; + _ret = ret; + _op = op; + _rl = rl; + _ru = ru; + _left = left; + } + + @Override + public Long call() { + final int _blklen = Math.max(DECOMPRESSION_BLEN / _m1.getNumColumns(), 64); + tmp = allocateTempUncompressedBlockSparse(_blklen, _m1.getNumColumns()); + final List groups = _m1.getColGroups(); + final AIterator[] its = getIterators(groups, _rl); + if(!_left) + for(int r = _rl; r < _ru; r += _blklen) + processBlock(r, Math.min(r + _blklen, _ru), groups, its); + else + for(int r = _rl; r < _ru; r += _blklen) + processBlockLeft(r, Math.min(r + _blklen, _ru), groups, its); + return _ret.recomputeNonZeros(_rl, _ru - 1); + } + + private final void processBlock(final int rl, final int ru, final List groups, final AIterator[] its) { + decompressToTmpBlock(rl, ru, tmp.getSparseBlock(), groups, its); + // decompressing multiple column groups can leave the temp rows with unsorted column indices, so sort + // before reading them in stored order into the (column-sorted) output sparse block. + tmp.sortSparseRows(0, ru - rl); + processSparse(rl, ru); + tmp.reset(); + } + + private final void processBlockLeft(final int rl, final int ru, final List groups, + final AIterator[] its) { + decompressToTmpBlock(rl, ru, tmp.getSparseBlock(), groups, its); + tmp.sortSparseRows(0, ru - rl); + processSparseLeft(rl, ru); + tmp.reset(); + } + + private final void processSparse(final int rl, final int ru) { + final SparseBlock sb = _ret.getSparseBlock(); + final SparseBlock _tmpSparse = tmp.getSparseBlock(); + final double[] _m2Dense = _m2.getDenseBlockValues(); + for(int row = rl; row < ru; row++) { + final double vr = _m2Dense[row]; + final int tmpOff = (row - rl); + if(!_tmpSparse.isEmpty(tmpOff)) { + int[] aoff = _tmpSparse.indexes(tmpOff); + double[] aval = _tmpSparse.values(tmpOff); + int apos = _tmpSparse.pos(tmpOff); + int alen = apos + _tmpSparse.size(tmpOff); + + for(int j = apos; j < alen; j++) { + sb.append(row, aoff[j], _op.fn.execute(aval[j], vr)); + } + } + + } + } + + private final void processSparseLeft(final int rl, final int ru) { + final SparseBlock sb = _ret.getSparseBlock(); + final SparseBlock _tmpSparse = tmp.getSparseBlock(); + final double[] _m2Dense = _m2.getDenseBlockValues(); + for(int row = rl; row < ru; row++) { + final double vr = _m2Dense[row]; + final int tmpOff = (row - rl); + if(!_tmpSparse.isEmpty(tmpOff)) { + int[] aoff = _tmpSparse.indexes(tmpOff); + double[] aval = _tmpSparse.values(tmpOff); + int apos = _tmpSparse.pos(tmpOff); + int alen = apos + _tmpSparse.size(tmpOff); + for(int j = apos; j < alen; j++) { + sb.append(row, aoff[j], _op.fn.execute(vr, aval[j])); + } + } + } + } + } + + private static MatrixBlock allocateTempUncompressedBlock(int blklen, int cols) { + MatrixBlock out = new MatrixBlock(blklen, cols, false); + out.allocateBlock(); + return out; + } + + private static MatrixBlock allocateTempUncompressedBlockSparse(int blklen, int cols) { + MatrixBlock out = new MatrixBlock(blklen, cols, true); out.allocateBlock(); return out; } @@ -1199,6 +1354,25 @@ protected static void decompressToTmpBlock(final int rl, final int ru, final Den } } + protected static void decompressToTmpBlock(final int rl, final int ru, final SparseBlock db, + final List groups, final AIterator[] its) { + Timing time = new Timing(true); + for(int i = 0; i < groups.size(); i++) { + final AColGroup g = groups.get(i); + if(g.getCompType() == CompressionType.SDC) + ((ASDCZero) g).decompressToSparseBlock(db, rl, ru, -rl, 0, its[i]); + else + g.decompressToSparseBlock(db, rl, ru, -rl, 0); + } + + if(DMLScript.STATISTICS) { + final double t = time.stop(); + DMLCompressionStatistics.addDecompressToBlockTime(t, 1); + if(LOG.isTraceEnabled()) + LOG.trace("decompressed block w/ k=" + 1 + " in " + t + "ms."); + } + } + protected static AIterator[] getIterators(final List groups, final int rl) { final AIterator[] its = new AIterator[groups.size()]; for(int i = 0; i < groups.size(); i++) { @@ -1210,8 +1384,8 @@ protected static AIterator[] getIterators(final List groups, final in return its; } - private static Pair evaluateSparsityMVCol(CompressedMatrixBlock m1, MatrixBlock m2, BinaryOperator op, - boolean left) { + private static Pair evaluateSparsityMVCol(CompressedMatrixBlock m1, MatrixBlock m2, + BinaryOperator op, boolean left) { final List groups = m1.getColGroups(); final int nCol = m1.getNumColumns(); final int nRow = m1.getNumRows(); @@ -1247,7 +1421,7 @@ private static Pair evaluateSparsityMVCol(CompressedMatrixBlock for(int r = 0; r < sampleRow; r++) { final double m = m2v[r]; final int off = r * sampleCol; - for(int c = 0; c < sampleCol; c++){ + for(int c = 0; c < sampleCol; c++) { int outVal = op.fn.execute(dv[off + c], m) != 0 ? 1 : 0; nnz += outVal; nnzPerRow[r] += outVal; diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java index 2ec23037385..d76dbe0d45e 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java @@ -80,8 +80,15 @@ public void processInstruction(ExecutionContext ec) { retBlock = inBlock1; } else { - if(LibCommonsMath.isSupportedMatrixMatrixOperation(getOpcode()) && !compressedLeft && !compressedRight) + if(LibCommonsMath.isSupportedMatrixMatrixOperation(getOpcode()) ){ + if(compressedLeft) + inBlock1 = CompressedMatrixBlock.getUncompressed(inBlock1, getOpcode()); + + if(compressedRight) + inBlock2 = CompressedMatrixBlock.getUncompressed(inBlock2, getOpcode()); + retBlock = LibCommonsMath.matrixMatrixOperations(inBlock1, inBlock2, getOpcode()); + } else { // Perform computation using input matrices, and produce the result matrix BinaryOperator bop = (BinaryOperator) _optr; diff --git a/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibBinaryCellOpCustomTest.java b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibBinaryCellOpCustomTest.java index 1ce05fab616..48bac7a7920 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibBinaryCellOpCustomTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibBinaryCellOpCustomTest.java @@ -22,14 +22,20 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; +import static org.junit.Assert.assertTrue; + import org.apache.commons.lang3.tuple.Pair; import org.apache.sysds.runtime.compress.CompressedMatrixBlock; import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory; import org.apache.sysds.runtime.compress.CompressionStatistics; +import org.apache.sysds.runtime.compress.colgroup.AColGroup; +import org.apache.sysds.runtime.compress.colgroup.AColGroup.CompressionType; import org.apache.sysds.runtime.compress.lib.CLALibBinaryCellOp; import org.apache.sysds.runtime.functionobjects.GreaterThanEquals; import org.apache.sysds.runtime.functionobjects.LessThanEquals; import org.apache.sysds.runtime.functionobjects.Minus; +import org.apache.sysds.runtime.functionobjects.Multiply; +import org.apache.sysds.runtime.matrix.data.LibMatrixBincell; import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.runtime.matrix.operators.BinaryOperator; import org.apache.sysds.test.TestUtils; @@ -132,6 +138,41 @@ public void OVV2() { TestUtils.compareMatricesBitAvgDistance(new MatrixBlock(10, 10, 2.5 - 324.0), cRet, 0, 0, op.toString()); } + @Test + public void sparseSparseColVectorNonSDCGroup() { + // Drive the sparse-sparse column-vector path of CLALibBinaryCellOp through a compressed input whose + // column groups are not all SDC. The constant non-zero first column compresses to a non-SDC (Const/DDC) + // group, while the remaining columns stay sparse so the overall matrix is sparse enough to pick the + // sparse-sparse path. This exercises the non-SDC branch of decompressToTmpBlock(SparseBlock), which the + // all-SDC sparse inputs of CLALibBinaryCellOpTest never reach. + final int nRow = 300; + final int nCol = 12; + MatrixBlock mb = new MatrixBlock(nRow, nCol, false); + mb.allocateDenseBlock(); + for(int i = 0; i < nRow; i++) + mb.set(i, 0, 3.0); // constant non-zero column -> non-SDC group + for(int i = 0; i < nRow; i += 7) + mb.set(i, 3, 1.0 + (i % 4)); // a few sparse non-zeros + for(int i = 0; i < nRow; i += 11) + mb.set(i, 8, 4.0); + mb.recomputeNonZeros(); + + CompressedMatrixBlock cmb = (CompressedMatrixBlock) CompressedMatrixBlockFactory.compress(mb, 1).getLeft(); + assertTrue("input must compress to exercise the compressed path", cmb instanceof CompressedMatrixBlock); + boolean hasNonSDC = false; + for(AColGroup g : cmb.getColGroups()) + hasNonSDC |= g.getCompType() != CompressionType.SDC; + assertTrue("need a non-SDC column group to cover the non-SDC decompress branch", hasNonSDC); + + // Multiply is sparse-safe (f(0,v)==0), so the sparse-safe gate routes it through the sparse-sparse path. + BinaryOperator op = new BinaryOperator(Multiply.getMultiplyFnObject(), 1); + MatrixBlock cv = TestUtils.round(TestUtils.generateTestMatrixBlock(nRow, 1, -5, 5, 1.0, 7)); + + MatrixBlock cRet = CLALibBinaryCellOp.binaryOperationsRight(op, cmb, cv); + MatrixBlock uRet = LibMatrixBincell.bincellOp(mb, cv, null, op); + TestUtils.compareMatricesBitAvgDistance(uRet, cRet, 0, 0, op.toString()); + } + @Test public void overwriteToCompressedOnSecondCompressed() { BinaryOperator op = new BinaryOperator(Minus.getMinusFnObject(), 2); diff --git a/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java b/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java new file mode 100644 index 00000000000..549010a78cb --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.compress.lib; + +import static org.junit.Assert.assertTrue; + +import org.apache.sysds.common.Types.DataType; +import org.apache.sysds.common.Types.FileFormat; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.compress.CompressedMatrixBlock; +import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory; +import org.apache.sysds.runtime.controlprogram.LocalVariableMap; +import org.apache.sysds.runtime.controlprogram.caching.CacheableData; +import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; +import org.apache.sysds.runtime.controlprogram.context.ExecutionContext; +import org.apache.sysds.runtime.instructions.InstructionUtils; +import org.apache.sysds.runtime.instructions.cp.BinaryCPInstruction; +import org.apache.sysds.runtime.instructions.cp.BinaryMatrixMatrixCPInstruction; +import org.apache.sysds.runtime.matrix.data.LibCommonsMath; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.meta.MatrixCharacteristics; +import org.apache.sysds.runtime.meta.MetaDataFormat; +import org.apache.sysds.test.TestUtils; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Drive the solve opcode through {@link BinaryMatrixMatrixCPInstruction} with compressed inputs to cover the + * commons-math matrix-matrix branch that decompresses compressed left/right operands before solving. The + * script-level solve tests only ever see uncompressed inputs, so this branch is otherwise unreached. + */ +public class CompressedBinaryMatrixMatrixSolveTest { + + private static final String SOLVE = "solve"; + + @BeforeClass + public static void init() throws java.io.IOException { + CacheableData.initCaching("compressed_solve_instruction_test"); + } + + @Test + public void solveCompressedLeftCompressedRight() { + // A is a compressible, invertible matrix (constant off-diagonal with a larger diagonal), b is a + // compressed constant right-hand side: both inputs are CompressedMatrixBlock so the instruction must + // decompress both before delegating to commons-math solve. + final int n = 200; + MatrixBlock aUC = new MatrixBlock(n, n, false); + aUC.allocateDenseBlock(); + for(int i = 0; i < n; i++) + for(int j = 0; j < n; j++) + aUC.set(i, j, i == j ? 5.0 : 1.0); + aUC.recomputeNonZeros(); + + CompressedMatrixBlock aC = (CompressedMatrixBlock) CompressedMatrixBlockFactory.compress(aUC, 1).getLeft(); + assertTrue("A must compress to exercise the compressed-left path", aC instanceof CompressedMatrixBlock); + CompressedMatrixBlock bC = CompressedMatrixBlockFactory.createConstant(n, 2, 1.0); + + MatrixBlock expected = LibCommonsMath.matrixMatrixOperations( + CompressedMatrixBlock.getUncompressed(aC), CompressedMatrixBlock.getUncompressed(bC), SOLVE); + + MatrixBlock actual = runSolve(aC, bC); + TestUtils.compareMatricesBitAvgDistance(expected, actual, 0, 0, SOLVE); + } + + @Test + public void solveCompressedLeftDenseRight() { + // Only the left operand is compressed; the right-hand side stays dense (a single column). + final int n = 200; + MatrixBlock aUC = new MatrixBlock(n, n, false); + aUC.allocateDenseBlock(); + for(int i = 0; i < n; i++) + for(int j = 0; j < n; j++) + aUC.set(i, j, i == j ? 5.0 : 1.0); + aUC.recomputeNonZeros(); + + CompressedMatrixBlock aC = (CompressedMatrixBlock) CompressedMatrixBlockFactory.compress(aUC, 1).getLeft(); + assertTrue("A must compress to exercise the compressed-left path", aC instanceof CompressedMatrixBlock); + MatrixBlock b = TestUtils.round(TestUtils.generateTestMatrixBlock(n, 1, -5, 5, 1.0, 7)); + + MatrixBlock expected = LibCommonsMath.matrixMatrixOperations( + CompressedMatrixBlock.getUncompressed(aC), b, SOLVE); + + MatrixBlock actual = runSolve(aC, b); + TestUtils.compareMatricesBitAvgDistance(expected, actual, 0, 0, SOLVE); + } + + private static MatrixBlock runSolve(MatrixBlock a, MatrixBlock b) { + ExecutionContext ec = new ExecutionContext(new LocalVariableMap()); + ec.setAutoCreateVars(true); + ec.setVariable("A", matrixObject("A", a)); + ec.setVariable("b", matrixObject("b", b)); + solveInstruction().processInstruction(ec); + return ec.getMatrixObject("out").acquireReadAndRelease(); + } + + private static BinaryMatrixMatrixCPInstruction solveInstruction() { + String in1 = InstructionUtils.concatOperandParts("A", DataType.MATRIX.name(), ValueType.FP64.name(), "false"); + String in2 = InstructionUtils.concatOperandParts("b", DataType.MATRIX.name(), ValueType.FP64.name(), "false"); + String out = InstructionUtils.concatOperandParts("out", DataType.MATRIX.name(), ValueType.FP64.name(), "false"); + String str = InstructionUtils.concatOperands("CP", SOLVE, in1, in2, out); + return (BinaryMatrixMatrixCPInstruction) BinaryCPInstruction.parseInstruction(str); + } + + private static MatrixObject matrixObject(String name, MatrixBlock mb) { + MatrixCharacteristics mc = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), 1000, mb.getNonZeros()); + MatrixObject mo = new MatrixObject(ValueType.FP64, "/dev/null/" + name, + new MetaDataFormat(mc, FileFormat.BINARY), mb); + return mo; + } +} From eb6928ecb1934384486b53a64942a7e13769376e Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Sun, 28 Jun 2026 23:17:52 +0200 Subject: [PATCH 056/132] [SYSTEMDS-3949] Add native Delta Lake matrix read/write via Delta Kernel (#2511) Introduce a DELTA file format that reads and writes Delta Lake tables natively through the Spark-free Delta Kernel library, for matrices on the single-node CP path. DML read/write with format="delta". --- .github/workflows/javaTests.yml | 2 +- pom.xml | 44 ++ .../java/org/apache/sysds/common/Types.java | 3 +- .../sysds/conf/ConfigurationManager.java | 15 + .../java/org/apache/sysds/conf/DMLConfig.java | 6 + .../apache/sysds/parser/DMLTranslator.java | 3 +- .../apache/sysds/parser/DataExpression.java | 13 +- .../controlprogram/caching/MatrixObject.java | 3 +- .../sysds/runtime/io/DeltaKernelUtils.java | 425 +++++++++++++ .../sysds/runtime/io/MatrixReaderFactory.java | 11 +- .../sysds/runtime/io/MatrixWriterFactory.java | 3 + .../apache/sysds/runtime/io/ReaderDelta.java | 179 ++++++ .../sysds/runtime/io/ReaderDeltaParallel.java | 217 +++++++ .../apache/sysds/runtime/io/WriterDelta.java | 219 +++++++ .../component/io/DeltaMatrixCoverageTest.java | 301 +++++++++ .../io/DeltaMatrixReadWriteTest.java | 595 ++++++++++++++++++ .../io/DeltaMatrixSparkInteropTest.java | 271 ++++++++ .../io/delta/DeltaReadWriteTest.java | 130 ++++ .../functions/io/delta/DeltaReadCompare.dml | 34 + .../scripts/functions/io/delta/DeltaWrite.dml | 30 + 20 files changed, 2495 insertions(+), 9 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java create mode 100644 src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java create mode 100644 src/main/java/org/apache/sysds/runtime/io/ReaderDeltaParallel.java create mode 100644 src/main/java/org/apache/sysds/runtime/io/WriterDelta.java create mode 100644 src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java create mode 100644 src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java create mode 100644 src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java create mode 100644 src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java create mode 100644 src/test/scripts/functions/io/delta/DeltaReadCompare.dml create mode 100644 src/test/scripts/functions/io/delta/DeltaWrite.dml diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml index 61089807820..16e925d155b 100644 --- a/.github/workflows/javaTests.yml +++ b/.github/workflows/javaTests.yml @@ -60,7 +60,7 @@ jobs: "org.apache.sysds.test.applications.**", "**.test.usertest.**", "**.component.c**.**", - "**.component.e**.**,**.component.f**.**,**.component.m**.**,**.component.o**.**", + "**.component.e**.**,**.component.f**.**,**.component.i**.**,**.component.m**.**,**.component.o**.**", "**.component.p**.**,**.component.r**.**,**.component.s**.**,**.component.t**.**,**.component.u**.**", "**.functions.a**.**,**.functions.binary.matrix.**,**.functions.binary.scalar.**,**.functions.binary.tensor.**", "**.functions.blocks.**,**.functions.data.rand.**,", diff --git a/pom.xml b/pom.xml index cfd3d8464fb..068bed2e8ea 100644 --- a/pom.xml +++ b/pom.xml @@ -46,6 +46,8 @@ 2.15.4 2.12.18 2.12 + 3.3.2 + 1.13.1 yyyy-MM-dd HH:mm:ss z 1 false @@ -968,6 +970,48 @@ + + + io.delta + delta-kernel-api + ${delta-kernel.version} + + + io.delta + delta-kernel-defaults + ${delta-kernel.version} + + + + org.apache.parquet + parquet-hadoop + ${parquet.version} + + + org.apache.parquet + parquet-column + ${parquet.version} + + + org.apache.parquet + parquet-common + ${parquet.version} + + + + io.delta + delta-spark_${scala.binary.version} + ${delta-kernel.version} + test + org.jcuda jcuda diff --git a/src/main/java/org/apache/sysds/common/Types.java b/src/main/java/org/apache/sysds/common/Types.java index c2832aeb8cd..624c9eed3c6 100644 --- a/src/main/java/org/apache/sysds/common/Types.java +++ b/src/main/java/org/apache/sysds/common/Types.java @@ -878,6 +878,7 @@ public enum FileFormat { HDF5, // Hierarchical Data Format (HDF) COG, // Cloud-optimized GeoTIFF PARQUET, // parquet format for columnar data storage + DELTA, // Delta Lake table (transaction log + parquet), read/written via Delta Kernel UNKNOWN; public boolean isIJV() { @@ -885,7 +886,7 @@ public boolean isIJV() { } public boolean isTextFormat() { - return this != BINARY && this != COMPRESSED; + return this != BINARY && this != COMPRESSED && this != DELTA; } public static boolean isTextFormat(String fmt) { diff --git a/src/main/java/org/apache/sysds/conf/ConfigurationManager.java b/src/main/java/org/apache/sysds/conf/ConfigurationManager.java index 0d5ee888d86..83676da47a7 100644 --- a/src/main/java/org/apache/sysds/conf/ConfigurationManager.java +++ b/src/main/java/org/apache/sysds/conf/ConfigurationManager.java @@ -258,6 +258,21 @@ public static int getFederatedTimeout(){ return getDMLConfig().getIntValue(DMLConfig.FEDERATED_TIMEOUT); } + /** @return rows per parquet read batch for the native Delta reader */ + public static int getDeltaReaderBatchSize() { + return getDMLConfig().getIntValue(DMLConfig.DELTA_READER_BATCH_SIZE); + } + + /** @return matrix rows materialized per columnar batch for the native Delta writer */ + public static int getDeltaWriterBatchSize() { + return getDMLConfig().getIntValue(DMLConfig.DELTA_WRITER_BATCH_SIZE); + } + + /** @return target data-file size (bytes) for the native Delta writer */ + public static long getDeltaWriterTargetFileSize() { + return Long.parseLong(getDMLConfig().getTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE)); + } + public static boolean isFederatedSSL(){ return getDMLConfig().getBooleanValue(DMLConfig.USE_SSL_FEDERATED_COMMUNICATION); } diff --git a/src/main/java/org/apache/sysds/conf/DMLConfig.java b/src/main/java/org/apache/sysds/conf/DMLConfig.java index a6339656fb0..e06b58b07c8 100644 --- a/src/main/java/org/apache/sysds/conf/DMLConfig.java +++ b/src/main/java/org/apache/sysds/conf/DMLConfig.java @@ -71,6 +71,9 @@ public class DMLConfig public static final String CP_PARALLEL_OPS = "sysds.cp.parallel.ops"; public static final String CP_PARALLEL_IO = "sysds.cp.parallel.io"; public static final String IO_COMPRESSION_CODEC = "sysds.io.compression.encoding"; + public static final String DELTA_READER_BATCH_SIZE = "sysds.io.delta.reader.batchsize"; // int: rows per parquet read batch + public static final String DELTA_WRITER_BATCH_SIZE = "sysds.io.delta.writer.batchsize"; // int: matrix rows materialized per columnar batch handed to the engine + public static final String DELTA_WRITER_TARGET_FILE_SIZE = "sysds.io.delta.writer.targetfilesize"; // long: target data-file size in bytes (smaller -> more files -> more parallel-read throughput) public static final String PARALLEL_ENCODE = "sysds.parallel.encode"; // boolean: enable multi-threaded transformencode and apply public static final String PARALLEL_ENCODE_STAGED = "sysds.parallel.encode.staged"; public static final String PARALLEL_ENCODE_APPLY_BLOCKS = "sysds.parallel.encode.applyBlocks"; @@ -158,6 +161,9 @@ public class DMLConfig _defaultVals.put(CP_PARALLEL_OPS, "true" ); _defaultVals.put(CP_PARALLEL_IO, "true" ); _defaultVals.put(IO_COMPRESSION_CODEC, "none"); + _defaultVals.put(DELTA_READER_BATCH_SIZE, "4096"); // rows per parquet read batch (Delta Kernel default 1024) + _defaultVals.put(DELTA_WRITER_BATCH_SIZE, "4096"); // matrix rows materialized per columnar batch handed to the engine + _defaultVals.put(DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(64L * 1024 * 1024)); // 64MB target data-file size (Delta Kernel default 128MB) -> more files -> more parallel-read throughput _defaultVals.put(PARALLEL_TOKENIZE, "false"); _defaultVals.put(PARALLEL_TOKENIZE_NUM_BLOCKS, "64"); _defaultVals.put(FRAME_TO_MATRIX_WARN_CAST, "false"); diff --git a/src/main/java/org/apache/sysds/parser/DMLTranslator.java b/src/main/java/org/apache/sysds/parser/DMLTranslator.java index e14cfd31388..a8e1667d049 100644 --- a/src/main/java/org/apache/sysds/parser/DMLTranslator.java +++ b/src/main/java/org/apache/sysds/parser/DMLTranslator.java @@ -1057,7 +1057,8 @@ public void constructHops(StatementBlock sb) { case CSV: case LIBSVM: case HDF5: - // write output in textcell format + case DELTA: + // columnar/text formats: no block layout (blocksize -1) ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), -1); break; case BINARY: diff --git a/src/main/java/org/apache/sysds/parser/DataExpression.java b/src/main/java/org/apache/sysds/parser/DataExpression.java index 22dbe21c187..68a3d1b7ffe 100644 --- a/src/main/java/org/apache/sysds/parser/DataExpression.java +++ b/src/main/java/org/apache/sysds/parser/DataExpression.java @@ -1178,6 +1178,10 @@ else if( getVarParam(READNNZPARAM) != null ) { boolean isCOG = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.COG.toString())); + // Delta tables are self-describing (schema + dimensions discovered from the + // transaction log at read time), so dimensions are optional like CSV. + boolean isDelta = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.DELTA.toString())); + dataTypeString = (getVarParam(DATATYPEPARAM) == null) ? null : getVarParam(DATATYPEPARAM).toString(); if ( dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE) @@ -1202,8 +1206,8 @@ else if( getVarParam(READNNZPARAM) != null ) { // initialize size of target data identifier to UNKNOWN getOutput().setDimensions(-1, -1); - if (!isCSV && !isLIBSVM && !isHDF5 && !isCOG && ConfigurationManager.getCompilerConfig() - .getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) //skip check for csv/libsvm format / jmlc api + if (!isCSV && !isLIBSVM && !isHDF5 && !isCOG && !isDelta && ConfigurationManager.getCompilerConfig() + .getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) //skip check for csv/libsvm/delta format / jmlc api && (getVarParam(READROWPARAM) == null || getVarParam(READCOLPARAM) == null) ) { raiseValidateError("Missing or incomplete dimension information in read statement: " + mtdFileName, conditional, LanguageErrorCodes.INVALID_PARAMETERS); @@ -1215,7 +1219,7 @@ && getVarParam(READCOLPARAM) instanceof ConstIdentifier) // these are strings that are long values Long dim1 = (getVarParam(READROWPARAM) == null) ? null : Long.valueOf( getVarParam(READROWPARAM).toString()); Long dim2 = (getVarParam(READCOLPARAM) == null) ? null : Long.valueOf( getVarParam(READCOLPARAM).toString()); - if ( !isCSV && (dim1 < 0 || dim2 < 0) && ConfigurationManager + if ( !isCSV && !isDelta && (dim1 < 0 || dim2 < 0) && ConfigurationManager .getCompilerConfig().getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) ) { raiseValidateError("Invalid dimension information in read statement", conditional, LanguageErrorCodes.INVALID_PARAMETERS); } @@ -1333,7 +1337,8 @@ else if (valueTypeString.equalsIgnoreCase(ValueType.UNKNOWN.name())) } //validate read filename - if (getVarParam(FORMAT_TYPE) == null || FileFormat.isTextFormat(getVarParam(FORMAT_TYPE).toString())) + if (getVarParam(FORMAT_TYPE) == null || FileFormat.isTextFormat(getVarParam(FORMAT_TYPE).toString()) + || checkFormatType(FileFormat.DELTA)) //delta: columnar, no block layout getOutput().setBlocksize(-1); else if (checkFormatType(FileFormat.BINARY, FileFormat.COMPRESSED, FileFormat.UNKNOWN)) { if( getVarParam(ROWBLOCKCOUNTPARAM)!=null ) diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java index 0b1a1ee27cb..5430a7a6c32 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java @@ -453,7 +453,8 @@ protected MatrixBlock readBlobFromHDFS(String fname, long[] dims) throws IOExcep DataConverter.readMatrixFromHDFS(fname, iimd.getFileFormat(), rlen, clen, blen, mc.getNonZeros(), getFileFormatProperties()); - if(iimd.getFileFormat() == FileFormat.CSV) { + if(iimd.getFileFormat() == FileFormat.CSV || iimd.getFileFormat() == FileFormat.DELTA) { + //dimensions/nnz are discovered at read time for these self-describing formats _metaData = _metaData instanceof MetaDataFormat ? new MetaDataFormat(newData.getDataCharacteristics(), iimd.getFileFormat()) : new MetaData(newData.getDataCharacteristics()); } diff --git a/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java new file mode 100644 index 00000000000..1e06f9acb56 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java @@ -0,0 +1,425 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.io; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.util.HDFSTool; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.delta.kernel.DataWriteContext; +import io.delta.kernel.Operation; +import io.delta.kernel.Scan; +import io.delta.kernel.Snapshot; +import io.delta.kernel.Table; +import io.delta.kernel.Transaction; +import io.delta.kernel.TransactionBuilder; +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.data.ColumnarBatch; +import io.delta.kernel.data.FilteredColumnarBatch; +import io.delta.kernel.data.Row; +import io.delta.kernel.defaults.engine.DefaultEngine; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.internal.InternalScanFileUtils; +import io.delta.kernel.internal.ScanImpl; +import io.delta.kernel.internal.data.ScanStateRow; +import io.delta.kernel.internal.util.Utils; +import io.delta.kernel.types.BooleanType; +import io.delta.kernel.types.ByteType; +import io.delta.kernel.types.DataType; +import io.delta.kernel.types.DoubleType; +import io.delta.kernel.types.FloatType; +import io.delta.kernel.types.IntegerType; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.ShortType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructType; +import io.delta.kernel.utils.CloseableIterable; +import io.delta.kernel.utils.CloseableIterator; +import io.delta.kernel.utils.DataFileStatus; +import io.delta.kernel.utils.FileStatus; + +/** + * Shared helpers for the native (Spark-free) Delta Lake read/write paths used + * by both the matrix and frame readers/writers. Centralizes engine creation, + * path qualification, the scan loop (snapshot -> data files -> logical + * columnar batches, honoring deletion vectors), and the write transaction + * (logical data -> parquet -> commit). + */ +public class DeltaKernelUtils { + + private static final String ENGINE_INFO = "Apache SystemDS"; + + /** Reused thread-safe JSON reader for the per-file Delta stats (numRecords). */ + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + + /** Delta Kernel config key: number of rows per parquet read batch, overridable via + * {@link org.apache.sysds.conf.DMLConfig#DELTA_READER_BATCH_SIZE}. */ + private static final String CONF_READER_BATCH_SIZE = "delta.kernel.default.parquet.reader.batch-size"; + /** Delta Kernel config key: target size (bytes) at which the writer rolls a new data file, overridable via + * {@link org.apache.sysds.conf.DMLConfig#DELTA_WRITER_TARGET_FILE_SIZE}. */ + private static final String CONF_WRITER_TARGET_FILE_SIZE = "delta.kernel.default.parquet.writer.targetMaxFileSize"; + + /** Internal Delta column type codes shared by the matrix and frame readers to + * dispatch boxing-free primitive column access. */ + public static final int T_DOUBLE = 0; + public static final int T_FLOAT = 1; + public static final int T_LONG = 2; + public static final int T_INT = 3; + public static final int T_SHORT = 4; + public static final int T_BYTE = 5; + public static final int T_BOOLEAN = 6; + public static final int T_STRING = 7; + + //derived configuration cached to avoid copying the (large) base conf on every + //engine creation (createEngine is called once per data file in parallel reads); + //rebuilt whenever the base conf or the relevant SystemDS settings change. + private static Configuration cachedConf; + private static Configuration cachedConfBase; + private static int cachedBatchSize; + private static long cachedTargetFileSize; + + private DeltaKernelUtils() {} + + /** + * Consumes a whole columnar batch. {@code selected} is {@code null} when all + * {@code size} rows are live; otherwise {@code selected[r]} indicates whether + * row {@code r} survived the deletion/selection vector. Batch-level consumption + * lets callers extract data column-at-a-time (cache friendly, boxing free) + * instead of paying a per-row callback. + */ + @FunctionalInterface + public interface BatchConsumer { + void accept(ColumnVector[] cols, int size, boolean[] selected); + } + + /** + * Map a Delta Kernel {@link DataType} to an internal type code (see the + * {@code T_*} constants). Returned once per column so the per-cell read loop + * can switch on a primitive int instead of repeating {@code instanceof} checks. + * + * @param dt the Delta column data type + * @return the matching {@code T_*} code, or {@code -1} if the type is not supported + */ + public static int typeCode(DataType dt) { + if( dt instanceof DoubleType ) return T_DOUBLE; + if( dt instanceof FloatType ) return T_FLOAT; + if( dt instanceof LongType ) return T_LONG; + if( dt instanceof IntegerType ) return T_INT; + if( dt instanceof ShortType ) return T_SHORT; + if( dt instanceof ByteType ) return T_BYTE; + if( dt instanceof BooleanType ) return T_BOOLEAN; + if( dt instanceof StringType ) return T_STRING; + return -1; + } + + /** + * @param size number of rows in the batch + * @param selected per-row selection mask, or {@code null} if all rows are live + * @return the number of live rows in the batch + */ + public static int countSelected(int size, boolean[] selected) { + if(selected == null) + return size; + int n = 0; + for(int r = 0; r < size; r++) + if(selected[r]) + n++; + return n; + } + + private static synchronized Configuration deltaConf() { + Configuration base = ConfigurationManager.getCachedJobConf(); + int batchSize = ConfigurationManager.getDeltaReaderBatchSize(); + long targetFileSize = ConfigurationManager.getDeltaWriterTargetFileSize(); + if(cachedConf == null || cachedConfBase != base + || cachedBatchSize != batchSize || cachedTargetFileSize != targetFileSize) + { + Configuration c = new Configuration(base); + c.setInt(CONF_READER_BATCH_SIZE, batchSize); + c.setLong(CONF_WRITER_TARGET_FILE_SIZE, targetFileSize); + cachedConf = c; + cachedConfBase = base; + cachedBatchSize = batchSize; + cachedTargetFileSize = targetFileSize; + } + return cachedConf; + } + + public static Engine createEngine() { + return DefaultEngine.create(deltaConf()); + } + + /** + * Resolve a (possibly relative) path to a fully-qualified URI so the + * kernel's default engine can locate the table on the right filesystem. + * + * @param fname input path + * @return fully-qualified table path + */ + public static String qualify(String fname) { + try { + Configuration conf = ConfigurationManager.getCachedJobConf(); + Path path = new Path(fname); + return path.getFileSystem(conf).makeQualified(path).toString(); + } + catch(IOException ex) { + throw new DMLRuntimeException("Failed to resolve Delta table path: " + fname, ex); + } + } + + /** + * Opened latest snapshot of a Delta table: the logical schema plus everything + * needed to (re)read its data files, including the list of per-data-file scan + * rows. Delta Kernel scan-file rows are self-contained (the kernel's + * distributed design serializes them to workers), so they can be retained and + * read independently / in parallel. + */ + public static final class ScanHandle { + public final StructType schema; + public final Row scanState; + public final StructType physicalReadSchema; + public final List scanFiles; + /** + * Per-file record counts taken from the Delta {@code numRecords} statistic, + * aligned with {@link #scanFiles}; {@code -1} where the statistic is absent. + */ + public final long[] numRecords; + /** + * Per-file flag indicating a deletion vector is present (so the live row + * count differs from {@link #numRecords}), aligned with {@link #scanFiles}. + */ + public final boolean[] hasDeletionVector; + + private ScanHandle(StructType schema, Row scanState, StructType physicalReadSchema, + List scanFiles, long[] numRecords, boolean[] hasDeletionVector) + { + this.schema = schema; + this.scanState = scanState; + this.physicalReadSchema = physicalReadSchema; + this.scanFiles = scanFiles; + this.numRecords = numRecords; + this.hasDeletionVector = hasDeletionVector; + } + + /** + * @return true iff every data file carries a {@code numRecords} statistic + * and none has a deletion vector, i.e. exact per-file row offsets + * can be derived from metadata without reading the data. + */ + public boolean hasExactRowCounts() { + for( int i=0; i scanFileIter = (scan instanceof ScanImpl) + ? ((ScanImpl) scan).getScanFiles(engine, true) + : scan.getScanFiles(engine); + + List files = new ArrayList<>(); + List recs = new ArrayList<>(); + List dvs = new ArrayList<>(); + try( CloseableIterator scanFiles = scanFileIter ) { + while( scanFiles.hasNext() ) { + FilteredColumnarBatch scanFileBatch = scanFiles.next(); + try( CloseableIterator scanFileRows = scanFileBatch.getRows() ) { + while( scanFileRows.hasNext() ) { + Row scanFileRow = scanFileRows.next(); + files.add(scanFileRow); + recs.add(numRecords(scanFileRow)); + dvs.add(InternalScanFileUtils.getDeletionVectorDescriptorFromRow(scanFileRow) != null); + } + } + } + } + long[] numRecords = new long[recs.size()]; + boolean[] hasDv = new boolean[dvs.size()]; + for( int i=0; i physicalData = engine.getParquetHandler() + .readParquetFiles(Utils.singletonCloseableIterator(dataFile), physicalReadSchema, Optional.empty()); + try( CloseableIterator logicalData = + Scan.transformPhysicalData(engine, scanState, scanFileRow, physicalData) ) + { + while( logicalData.hasNext() ) + consumeBatch(logicalData.next(), consumer); + } + } + + /** + * Scan the latest snapshot of a Delta table sequentially, invoking the batch + * consumer for every data batch. The consumer is created lazily from the table + * schema (so callers can size buffers / derive per-column types up front). + * + * @param engine delta kernel engine + * @param tablePath fully-qualified table path + * @param consumerFactory builds the batch consumer from the table schema + * @return the logical table schema + * @throws IOException on read failure + */ + public static StructType scan(Engine engine, String tablePath, Function consumerFactory) + throws IOException + { + ScanHandle h = openScan(engine, tablePath); + BatchConsumer consumer = consumerFactory.apply(h.schema); + for( Row scanFileRow : h.scanFiles ) + readScanFile(engine, h.scanState, h.physicalReadSchema, scanFileRow, consumer); + return h.schema; + } + + private static void consumeBatch(FilteredColumnarBatch fcb, BatchConsumer consumer) { + ColumnarBatch batch = fcb.getData(); + int ncol = batch.getSchema().length(); + ColumnVector[] cols = new ColumnVector[ncol]; + for( int c=0; c all rows live) + Optional selVector = fcb.getSelectionVector(); + boolean[] selected = null; + if( selVector.isPresent() ) { + ColumnVector sv = selVector.get(); + selected = new boolean[size]; + for( int r=0; r logicalData) throws IOException + { + //replace any existing table at the path (the other SystemDS writers delete + //the output first; the caching layer does not do it on our behalf) + HDFSTool.deleteFileIfExistOnHDFS(tablePath); + + Table table = Table.forPath(engine, tablePath); + TransactionBuilder txnBuilder = table + .createTransactionBuilder(engine, ENGINE_INFO, Operation.CREATE_TABLE) + .withSchema(engine, schema); + Transaction txn = txnBuilder.build(engine); + Row txnState = txn.getTransactionState(engine); + + CloseableIterator physicalData = + Transaction.transformLogicalData(engine, txnState, logicalData, Collections.emptyMap()); + DataWriteContext writeContext = + Transaction.getWriteContext(engine, txnState, Collections.emptyMap()); + CloseableIterator dataFiles = engine.getParquetHandler() + .writeParquetFiles(writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns()); + CloseableIterator appendActions = + Transaction.generateAppendActions(engine, txnState, dataFiles, writeContext); + txn.commit(engine, CloseableIterable.inMemoryIterable(appendActions)); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/io/MatrixReaderFactory.java b/src/main/java/org/apache/sysds/runtime/io/MatrixReaderFactory.java index dc1c7da230c..e10d358d629 100644 --- a/src/main/java/org/apache/sysds/runtime/io/MatrixReaderFactory.java +++ b/src/main/java/org/apache/sysds/runtime/io/MatrixReaderFactory.java @@ -78,7 +78,11 @@ public static MatrixReader createMatrixReader(FileFormat fmt) { case COMPRESSED: reader = ReaderCompressed.create(); break; - + + case DELTA: + reader = par ? new ReaderDeltaParallel() : new ReaderDelta(); + break; + default: throw new DMLRuntimeException("Failed to create matrix reader for unknown format: " + fmt.toString()); } @@ -140,6 +144,11 @@ public static MatrixReader createMatrixReader( ReadProperties props ) { case COMPRESSED: reader = new ReaderCompressed(); break; + + case DELTA: + reader = par ? new ReaderDeltaParallel() : new ReaderDelta(); + break; + default: throw new DMLRuntimeException("Failed to create matrix reader for unknown format: " + fmt.toString()); } diff --git a/src/main/java/org/apache/sysds/runtime/io/MatrixWriterFactory.java b/src/main/java/org/apache/sysds/runtime/io/MatrixWriterFactory.java index bb0b0c940f7..091194edc81 100644 --- a/src/main/java/org/apache/sysds/runtime/io/MatrixWriterFactory.java +++ b/src/main/java/org/apache/sysds/runtime/io/MatrixWriterFactory.java @@ -94,6 +94,9 @@ else if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_WRITE case COMPRESSED: return WriterCompressed.create(props); + case DELTA: + return new WriterDelta(); + default: throw new DMLRuntimeException("Failed to create matrix writer for unknown format: " + fmt.toString()); } diff --git a/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java b/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java new file mode 100644 index 00000000000..58a98741975 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.io; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; + +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.data.DenseBlock; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.types.DataType; +import io.delta.kernel.types.StructType; + +/** + * Single-threaded native Delta Lake reader for matrices, built on the + * Spark-free Delta Kernel library. It opens the latest snapshot of a Delta + * table directory, reads its parquet data files through the kernel's default + * engine (honoring deletion vectors), and materializes the numeric columns + * into a dense {@link MatrixBlock}. + * + *

Only numeric columns (double/float/long/int/short/byte/boolean) are + * supported, matching the all-double nature of a SystemDS matrix. Dimensions + * do not need to be known up front: the row count is discovered while scanning + * and the column count is taken from the table schema.

+ */ +public class ReaderDelta extends MatrixReader { + + @Override + public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int blen, long estnnz) + throws IOException, DMLRuntimeException + { + Engine engine = DeltaKernelUtils.createEngine(); + String tablePath = DeltaKernelUtils.qualify(fname); + + //Scan column-at-a-time into one row-major buffer per batch (no per-row + //allocation, no boxing, no per-cell set()). Buffers are concatenated into + //the dense output via bulk array copies below. + ArrayList batches = new ArrayList<>(); + int[] nrowH = new int[1]; + StructType schema = DeltaKernelUtils.scan(engine, tablePath, sch -> { + int[] types = columnTypes(sch); + int ncol = sch.length(); + return (cols, size, selected) -> { + batches.add(extractBatch(cols, size, selected, types, ncol)); + nrowH[0] += DeltaKernelUtils.countSelected(size, selected); + }; + }); + + int ncol = schema.length(); + int nrow = nrowH[0]; + long lestnnz = (estnnz >= 0) ? estnnz : (long) nrow * ncol; + MatrixBlock ret = createOutputMatrixBlock(nrow, ncol, Math.max(nrow, 1), lestnnz, true, false); + + if( nrow > 0 && ncol > 0 ) + fillDense(ret, batches); + ret.recomputeNonZeros(); + ret.examSparsity(); + return ret; + } + + /** Derive the per-column internal type codes from the table schema. */ + static int[] columnTypes(StructType schema) { + int ncol = schema.length(); + int[] types = new int[ncol]; + for( int c=0; c batches) { + DenseBlock db = ret.getDenseBlock(); + if( db.isContiguous() ) { + double[] dv = db.valuesAt(0); + int off = 0; + for( double[] buf : batches ) { + System.arraycopy(buf, 0, dv, off, buf.length); + off += buf.length; + } + } + else { + //rare large multi-block fallback: route each row through the block API + int ncol = ret.getNumColumns(); + int r = 0; + for( double[] buf : batches ) { + int rowsInBuf = buf.length / ncol; + for( int i=0; iThe expensive part of a Delta read is the parquet decode, which the kernel + * performs per data file; parallelizing across files is therefore the natural + * way to bridge the gap to the (near-raw) binary reader. A table backed by a + * single data file (the default for tables <= the parquet target file size) + * cannot be split this way, so the reader transparently falls back to the + * sequential {@link ReaderDelta} path in that case.

+ */ +public class ReaderDeltaParallel extends ReaderDelta { + + private final int _numThreads; + + public ReaderDeltaParallel() { + _numThreads = OptimizerUtils.getParallelBinaryReadParallelism(); + } + + @Override + public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int blen, long estnnz) + throws IOException, DMLRuntimeException + { + Engine engine = DeltaKernelUtils.createEngine(); + String tablePath = DeltaKernelUtils.qualify(fname); + DeltaKernelUtils.ScanHandle handle = DeltaKernelUtils.openScan(engine, tablePath); + + final int nfiles = handle.scanFiles.size(); + //nothing to gain from parallelism for single-file (or empty) tables + if( _numThreads <= 1 || nfiles <= 1 ) + return super.readMatrixFromHDFS(fname, rlen, clen, blen, estnnz); + + final int ncol = handle.schema.length(); + final int[] types = columnTypes(handle.schema); + + //fast path: exact per-file row counts are known from metadata and the dense + //output fits a single contiguous array -> pre-size once and let each thread + //decode directly into its slice (no intermediate buffers, no serial copy). + if( useDirectPath(handle) ) { + long total = 0; + for( long r : handle.numRecords ) + total += r; + if( total > 0 && (long) total * ncol <= Integer.MAX_VALUE ) + return readDirect(fname, handle, ncol, types, (int) total, estnnz); + } + + return readBuffered(fname, handle, ncol, types, estnnz); + } + + /** + * Whether the metadata-driven direct-write fast path can be used for this + * table (exact per-file row counts and no deletion vectors). Visible for + * testing: the buffered fallback is otherwise only reachable for tables + * lacking row statistics or carrying deletion vectors, which the SystemDS + * Delta writer never produces. + * + * @param handle the opened scan handle + * @return true if the direct path is applicable + */ + protected boolean useDirectPath(DeltaKernelUtils.ScanHandle handle) { + return handle.hasExactRowCounts(); + } + + /** + * Fast path: each thread decodes one data file straight into the final dense + * array at a metadata-derived row offset. Single allocation, fully parallel. + */ + private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, + int ncol, int[] types, int nrow, long estnnz) throws IOException + { + final int nfiles = handle.scanFiles.size(); + final int[] rowOffset = new int[nfiles]; + int acc = 0; + for( int i=0; i> tasks = new ArrayList<>(nfiles); + for( int i=0; i { + int[] cur = new int[] {base}; + Engine eng = DeltaKernelUtils.createEngine(); + DeltaKernelUtils.readScanFile(eng, handle.scanState, handle.physicalReadSchema, scanFileRow, + (cols, size, selected) -> { + if( cur[0] + DeltaKernelUtils.countSelected(size, selected) > limit ) + throw new DMLRuntimeException("Delta file produced more rows than its " + + "numRecords statistic; refusing parallel direct read of " + fname); + cur[0] += extractBatchInto(cols, size, selected, types, ncol, dv, cur[0]); + }); + return null; + }); + } + awaitFileTasks(tasks, fname); + + ret.recomputeNonZeros(_numThreads); + ret.examSparsity(); + return ret; + } + + /** + * Fallback path: decode each file in parallel into per-file buffers (used when + * row counts are unknown, deletion vectors are present, or the matrix exceeds a + * single contiguous array), then concatenate in file order. + */ + private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handle, + int ncol, int[] types, long estnnz) throws IOException + { + final int nfiles = handle.scanFiles.size(); + @SuppressWarnings("unchecked") + final ArrayList[] fileBufs = new ArrayList[nfiles]; + final int[] fileRows = new int[nfiles]; + ArrayList> tasks = new ArrayList<>(nfiles); + for( int i=0; i { + ArrayList bufs = new ArrayList<>(); + int[] rows = new int[1]; + Engine eng = DeltaKernelUtils.createEngine(); + DeltaKernelUtils.readScanFile(eng, handle.scanState, handle.physicalReadSchema, scanFileRow, + (cols, size, selected) -> { + bufs.add(extractBatch(cols, size, selected, types, ncol)); + rows[0] += DeltaKernelUtils.countSelected(size, selected); + }); + fileBufs[fi] = bufs; + fileRows[fi] = rows[0]; + return null; + }); + } + awaitFileTasks(tasks, fname); + + int nrow = 0; + for( int i=0; i ordered = new ArrayList<>(); + for( int i=0; i= 0) ? estnnz : (long) nrow * ncol; + MatrixBlock ret = createOutputMatrixBlock(nrow, ncol, Math.max(nrow, 1), lestnnz, true, false); + if( nrow > 0 && ncol > 0 ) + fillDense(ret, ordered); + ret.recomputeNonZeros(_numThreads); + ret.examSparsity(); + return ret; + } + + /** + * Run one decode task per data file on the shared common thread pool and await + * completion. Full parallelism is requested (the task count, one per data file, + * naturally caps concurrency); this avoids the per-thread pool-size caching in + * {@code CommonThreadPool.get(k)} that could otherwise throttle this reader to a + * smaller pool created earlier on the same thread. + */ + private void awaitFileTasks(List> tasks, String fname) throws IOException { + ExecutorService pool = CommonThreadPool.get(_numThreads); + try { + for( Future f : pool.invokeAll(tasks) ) + f.get(); + } + catch(Exception ex) { + throw new IOException("Failed parallel read of Delta table: " + fname, ex); + } + finally { + pool.shutdown(); + } + } +} diff --git a/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java b/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java new file mode 100644 index 00000000000..0f08bf5517d --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.io; + +import java.io.IOException; +import java.util.NoSuchElementException; +import java.util.Optional; + +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.data.ColumnarBatch; +import io.delta.kernel.data.FilteredColumnarBatch; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.types.DataType; +import io.delta.kernel.types.DoubleType; +import io.delta.kernel.types.StructType; +import io.delta.kernel.utils.CloseableIterable; +import io.delta.kernel.utils.CloseableIterator; + +/** + * Single-threaded native Delta Lake writer for matrices, built on the + * Spark-free Delta Kernel library. It creates a Delta table at the target + * directory with an all-double schema {@code c0..c(n-1)} (replacing any existing + * table at that path), streams the {@link MatrixBlock} rows as columnar batches + * into parquet data files via the kernel's default engine, and commits the + * corresponding add-file actions to the transaction log. + */ +public class WriterDelta extends MatrixWriter { + + @Override + public void writeMatrixToHDFS(MatrixBlock src, String fname, long rlen, long clen, int blen, long nnz, boolean diag) + throws IOException + { + if( src.getNumRows() != rlen || src.getNumColumns() != clen ) + throw new IOException("Matrix dimensions mismatch with metadata: (" + + src.getNumRows() + "x" + src.getNumColumns() + ") vs (" + rlen + "x" + clen + ")."); + int ncol = (int) clen; + int nrow = (int) rlen; + int batchRows = ConfigurationManager.getDeltaWriterBatchSize(); + //fast path: a contiguous dense block lets the column views read straight + //from the backing double[] (avoids per-cell MatrixBlock.get dispatch). + double[] dense = (!src.isInSparseFormat() && src.getDenseBlock() != null + && src.getDenseBlock().isContiguous()) ? src.getDenseBlockValues() : null; + Engine engine = DeltaKernelUtils.createEngine(); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), + buildSchema(ncol), new MatrixBatchIterator(src, dense, nrow, ncol, batchRows)); + } + + @Override + public void writeEmptyMatrixToHDFS(String fname, long rlen, long clen, int blen) + throws IOException + { + //empty table: create with schema but no data files + Engine engine = DeltaKernelUtils.createEngine(); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), + buildSchema((int) clen), CloseableIterable.emptyIterable().iterator()); + } + + private static StructType buildSchema(int ncol) { + StructType schema = new StructType(); + for( int c=0; c stream, long rlen, long clen, int blen) + throws IOException + { + throw new UnsupportedOperationException("Out-of-core stream write is not supported for the Delta format."); + } + + /** Chunks a MatrixBlock into fixed-size columnar batches for the kernel write path. */ + private static class MatrixBatchIterator implements CloseableIterator { + private final MatrixBlock _mb; + private final double[] _dense; + private final int _nrow; + private final int _ncol; + private final int _batchRows; + private final StructType _schema; + private int _pos = 0; + + MatrixBatchIterator(MatrixBlock mb, double[] dense, int nrow, int ncol, int batchRows) { + _mb = mb; + _dense = dense; + _nrow = nrow; + _ncol = ncol; + _batchRows = batchRows; + _schema = buildSchema(ncol); + } + + @Override + public boolean hasNext() { + return _pos < _nrow; + } + + @Override + public FilteredColumnarBatch next() { + if( !hasNext() ) + throw new NoSuchElementException(); + int size = Math.min(_batchRows, _nrow - _pos); + ColumnarBatch batch = new MatrixColumnarBatch(_mb, _dense, _schema, _pos, size, _ncol); + _pos += size; + //no selection vector: all rows in the batch are written + return new FilteredColumnarBatch(batch, Optional.empty()); + } + + @Override + public void close() { + //nothing to release + } + } + + /** Read-only view of a row range of a MatrixBlock as a Delta Kernel columnar batch. */ + private static class MatrixColumnarBatch implements ColumnarBatch { + private final MatrixBlock _mb; + private final double[] _dense; + private final StructType _schema; + private final int _rowStart; + private final int _size; + private final int _ncol; + + MatrixColumnarBatch(MatrixBlock mb, double[] dense, StructType schema, int rowStart, int size, int ncol) { + _mb = mb; + _dense = dense; + _schema = schema; + _rowStart = rowStart; + _size = size; + _ncol = ncol; + } + + @Override + public StructType getSchema() { + return _schema; + } + + @Override + public ColumnVector getColumnVector(int ordinal) { + if( ordinal < 0 || ordinal >= _ncol ) + throw new IndexOutOfBoundsException("column ordinal " + ordinal); + return new MatrixColumnVector(_mb, _dense, _rowStart, _size, _ncol, ordinal); + } + + @Override + public int getSize() { + return _size; + } + } + + /** Read-only double column view over one column of a MatrixBlock row range. */ + private static class MatrixColumnVector implements ColumnVector { + private final MatrixBlock _mb; + private final double[] _dense; // contiguous dense backing array, or null + private final int _rowStart; + private final int _size; + private final int _ncol; + private final int _col; + + MatrixColumnVector(MatrixBlock mb, double[] dense, int rowStart, int size, int ncol, int col) { + _mb = mb; + _dense = dense; + _rowStart = rowStart; + _size = size; + _ncol = ncol; + _col = col; + } + + @Override + public DataType getDataType() { + return DoubleType.DOUBLE; + } + + @Override + public int getSize() { + return _size; + } + + @Override + public boolean isNullAt(int rowId) { + return false; + } + + @Override + public double getDouble(int rowId) { + //dense contiguous single block => index fits in int (getDenseBlockValues + //is only handed over for single-block dense matrices) + return (_dense != null) + ? _dense[(_rowStart + rowId) * _ncol + _col] + : _mb.get(_rowStart + rowId, _col); + } + + @Override + public void close() { + //nothing to release + } + } +} diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java new file mode 100644 index 00000000000..8d7ad14539a --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.io; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.commons.io.FileUtils; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.data.DenseBlock; +import org.apache.sysds.runtime.data.DenseBlockLDRB; +import org.apache.sysds.runtime.data.DenseBlockLFP64; +import org.apache.sysds.runtime.io.DeltaKernelUtils; +import org.apache.sysds.runtime.io.ReaderDelta; +import org.apache.sysds.runtime.io.ReaderDeltaParallel; +import org.apache.sysds.runtime.io.WriterDelta; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.data.Row; +import io.delta.kernel.internal.InternalScanFileUtils; +import io.delta.kernel.types.BinaryType; +import io.delta.kernel.types.DataType; +import io.delta.kernel.types.DateType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructType; +import io.delta.kernel.types.TimestampType; + +/** + * Targeted tests for the error/defensive branches of the native Delta matrix + * read/write code that the round-trip and interop tests do not reach: malformed + * per-file statistics, unsupported column types, unsupported stream operations, + * bad table paths, and the non-dense writer input path. + * + *

A few of these branches guard against inputs that the SystemDS writer and + * the Delta Kernel scan API never produce in a normal round trip (e.g. a + * statistics JSON without {@code numRecords}, or a column type code outside the + * supported set). They are exercised here by mocking the Delta Kernel data + * objects and invoking the (package-private) helpers reflectively, rather than + * widening their production visibility purely for testing. + */ +public class DeltaMatrixCoverageTest { + + // --------------------------------------------------------------------- + // public defensive paths + // --------------------------------------------------------------------- + + @Test + public void qualifyRejectsUnknownFilesystemScheme() { + try { + DeltaKernelUtils.qualify("nosuchfs://host/path/to/table"); + fail("expected a DMLRuntimeException for an unresolvable table path"); + } + catch(DMLRuntimeException ex) { + assertTrue("message should reference the bad path, got: " + ex.getMessage(), + ex.getMessage() != null && ex.getMessage().contains("Delta table path")); + } + } + + @Test + public void typeCodeReturnsNegativeForUnsupportedTypes() { + //non-numeric / unsupported Delta types must map to the sentinel -1 so the + //reader can reject them with a clear message rather than mis-decoding. + assertEquals(-1, DeltaKernelUtils.typeCode(DateType.DATE)); + assertEquals(-1, DeltaKernelUtils.typeCode(TimestampType.TIMESTAMP)); + assertEquals(-1, DeltaKernelUtils.typeCode(BinaryType.BINARY)); + } + + @Test(expected = UnsupportedOperationException.class) + public void readerRejectsInputStream() throws Exception { + new ReaderDelta().readMatrixFromInputStream(null, 1, 1, -1, -1); + } + + @Test(expected = UnsupportedOperationException.class) + public void writerRejectsStreamWrite() throws Exception { + new WriterDelta().writeMatrixFromStream("dummy", null, 1, 1, -1); + } + + // --------------------------------------------------------------------- + // mocked / reflective coverage of internal defensive branches + // --------------------------------------------------------------------- + + @Test + public void numRecordsHandlesAbsentNullAndMalformedStats() throws Exception { + //no "stats" field at all -> -1 + assertEquals(-1, numRecords(addFileRow(new StructType().add("path", StringType.STRING), false, null))); + //stats column present but null-at -> -1 + assertEquals(-1, numRecords(addFileRow(statsSchema(), true, null))); + //stats string explicitly null -> -1 + assertEquals(-1, numRecords(addFileRow(statsSchema(), false, null))); + //malformed JSON -> JsonProcessingException -> -1 + assertEquals(-1, numRecords(addFileRow(statsSchema(), false, "{not valid json"))); + //valid JSON but no numRecords field -> -1 + assertEquals(-1, numRecords(addFileRow(statsSchema(), false, "{\"minValues\":{}}"))); + //well-formed stats -> the parsed count + assertEquals(1234L, numRecords(addFileRow(statsSchema(), false, "{\"numRecords\":1234}"))); + } + + @Test + public void getDoubleValueRejectsUnknownTypeCode() throws Exception { + Method m = ReaderDelta.class.getDeclaredMethod("getDoubleValue", ColumnVector.class, int.class, int.class); + m.setAccessible(true); + try { + //type code outside the supported T_* set; the switch default must throw + //before touching the (null) vector. + m.invoke(null, (ColumnVector) null, 0, 999); + fail("expected a DMLRuntimeException for an unsupported type code"); + } + catch(InvocationTargetException ite) { + assertTrue(ite.getCause() instanceof DMLRuntimeException); + } + } + + @Test + public void numericTypeCodeRejectsNonNumericType() throws Exception { + Method m = ReaderDelta.class.getDeclaredMethod("numericTypeCode", DataType.class, String.class); + m.setAccessible(true); + try { + m.invoke(null, DateType.DATE, "d"); + fail("expected a DMLRuntimeException for a non-numeric column type"); + } + catch(InvocationTargetException ite) { + assertTrue(ite.getCause() instanceof DMLRuntimeException); + } + } + + @Test + public void parallelReadWrapsFileFailure() throws Exception { + //a per-file decode failure in the parallel reader must surface as a single + //clear IOException (the awaitFileTasks catch), not a raw executor error. + //Provoke it by deleting one data file after the table (and its log) exist. + MatrixBlock in = TestUtils.generateTestMatrixBlock(100_000, 8, -10, 10, 1.0, 13); + in.recomputeNonZeros(); + DMLConfig conf = new DMLConfig(); + conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(256L * 1024)); + ConfigurationManager.setLocalConfig(conf); + Path dir = Files.createTempDirectory("sysds_delta_fail_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + + //delete one parquet data file; the transaction log still references it, + //so the scan enumerates it but the decode task fails. + File victim; + try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + victim = s.filter(p -> p.toString().endsWith(".parquet")) + .findFirst().map(Path::toFile).orElse(null); + } + assertTrue("expected at least one data file to delete", victim != null && victim.delete()); + + try { + new ReaderDeltaParallel().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + fail("expected an IOException when a Delta data file is missing"); + } + catch(java.io.IOException ex) { + assertTrue("message should describe the failed parallel read, got: " + ex.getMessage(), + ex.getMessage() != null && ex.getMessage().contains("parallel read of Delta table")); + } + } + finally { + ConfigurationManager.clearLocalConfigs(); + FileUtils.deleteQuietly(dir.toFile()); + } + } + + // --------------------------------------------------------------------- + // non-dense writer input path + // --------------------------------------------------------------------- + + @Test + public void sparseFormatMatrixRoundTrips() throws Exception { + //a sparse-backed MatrixBlock takes the writer's non-contiguous path (no + //direct double[] view), exercising MatrixColumnVector.get via MatrixBlock. + MatrixBlock in = TestUtils.generateTestMatrixBlock(2000, 7, -5, 5, 0.05, 13); + in.recomputeNonZeros(); + in.examSparsity(); + assertTrue("input should be in sparse format to exercise the non-dense path", in.isInSparseFormat()); + + Path dir = Files.createTempDirectory("sysds_delta_sparse_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + assertEquals("rows", in.getNumRows(), out.getNumRows()); + assertEquals("cols", in.getNumColumns(), out.getNumColumns()); + TestUtils.compareMatrices(in, out, 1e-12, "sparse-format-roundtrip"); + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void fillDenseHandlesNonContiguousBlock() throws Exception { + //the dense fill normally hits the contiguous fast path; force a multi-block + //(non-contiguous) dense block so the row-by-row fallback is exercised. Such + //blocks only arise for matrices beyond a single contiguous array, so we + //shrink the per-block allocation cap to provoke it on a tiny matrix. + int rows = 5, cols = 4; + int savedMaxAlloc = DenseBlockLDRB.MAX_ALLOC; + DenseBlock db; + try { + DenseBlockLDRB.MAX_ALLOC = 2 * cols; //~2 rows per block -> multiple blocks + db = new DenseBlockLFP64(new int[] {rows, cols}); + } + finally { + DenseBlockLDRB.MAX_ALLOC = savedMaxAlloc; + } + assertTrue("expected a non-contiguous (multi-block) dense block", !db.isContiguous()); + + //two row-major batches (3 rows + 2 rows) covering all 5 rows + double[] b0 = new double[3 * cols]; + double[] b1 = new double[2 * cols]; + for( int r = 0; r < 3; r++ ) + for( int c = 0; c < cols; c++ ) + b0[r * cols + c] = cell(r, c); + for( int r = 0; r < 2; r++ ) + for( int c = 0; c < cols; c++ ) + b1[r * cols + c] = cell(3 + r, c); + java.util.ArrayList batches = new java.util.ArrayList<>(); + batches.add(b0); + batches.add(b1); + + MatrixBlock ret = new MatrixBlock(rows, cols, db); + Method m = ReaderDelta.class.getDeclaredMethod("fillDense", MatrixBlock.class, java.util.ArrayList.class); + m.setAccessible(true); + m.invoke(null, ret, batches); + + for( int r = 0; r < rows; r++ ) + for( int c = 0; c < cols; c++ ) + assertEquals("r" + r + " c" + c, cell(r, c), ret.getDenseBlock().get(r, c), 0.0); + } + + private static double cell(int r, int c) { + return r * 10 + c; + } + + // --------------------------------------------------------------------- + // helpers + // --------------------------------------------------------------------- + + private static StructType statsSchema() { + return new StructType().add("stats", StringType.STRING); + } + + /** + * Build a mocked scan-file row whose AddFile child has the given schema, null + * flag and (when not null) stats string, matching what {@code numRecords} reads. + */ + private static Row addFileRow(StructType addSchema, boolean statsNull, String statsValue) { + Row outer = mock(Row.class); + Row add = mock(Row.class); + when(outer.getStruct(InternalScanFileUtils.ADD_FILE_ORDINAL)).thenReturn(add); + when(add.getSchema()).thenReturn(addSchema); + int statsOrd = addSchema.fieldNames().indexOf("stats"); + if( statsOrd >= 0 ) { + when(add.isNullAt(statsOrd)).thenReturn(statsNull); + if( !statsNull ) + when(add.getString(statsOrd)).thenReturn(statsValue); + } + return outer; //the scan-file row numRecords consumes (its AddFile child is 'add') + } + + private static long numRecords(Row scanFileRow) throws Exception { + Method m = DeltaKernelUtils.class.getDeclaredMethod("numRecords", Row.class); + m.setAccessible(true); + return (Long) m.invoke(null, scanFileRow); + } +} diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java new file mode 100644 index 00000000000..54a3bcf6334 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java @@ -0,0 +1,595 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.io; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.NoSuchElementException; +import java.util.Optional; + +import org.apache.commons.io.FileUtils; +import org.apache.sysds.common.Types.FileFormat; +import org.apache.sysds.conf.CompilerConfig; +import org.apache.sysds.conf.CompilerConfig.ConfigType; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.io.DeltaKernelUtils; +import org.apache.sysds.runtime.io.MatrixReader; +import org.apache.sysds.runtime.io.MatrixReaderFactory; +import org.apache.sysds.runtime.io.ReaderDelta; +import org.apache.sysds.runtime.io.ReaderDeltaParallel; +import org.apache.sysds.runtime.io.WriterDelta; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.data.ColumnarBatch; +import io.delta.kernel.data.FilteredColumnarBatch; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.types.BooleanType; +import io.delta.kernel.types.ByteType; +import io.delta.kernel.types.DataType; +import io.delta.kernel.types.DoubleType; +import io.delta.kernel.types.FloatType; +import io.delta.kernel.types.IntegerType; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.ShortType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructType; +import io.delta.kernel.utils.CloseableIterator; + +/** + * Direct (no DML) round-trip tests for the native Delta Kernel based matrix + * reader/writer. Each test writes a MatrixBlock to a fresh local Delta table + * directory and reads it back, asserting dimensions and values match. + */ +public class DeltaMatrixReadWriteTest { + + //small writer target file size (bytes) used to force a multi-file table + //layout cheaply, instead of brute-forcing huge row counts. + private static final long SMALL_TARGET_FILE_SIZE = 256L * 1024; + private static final int ROWS_MULTI_FILE = 100_000; + + private static MatrixBlock writeThenRead(MatrixBlock in) throws Exception { + Path dir = Files.createTempDirectory("sysds_delta_"); + //WriterDelta creates the table at the given (empty) directory + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + WriterDelta writer = new WriterDelta(); + writer.writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + + MatrixReader reader = new ReaderDelta(); + return reader.readMatrixFromHDFS(tablePath, in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void parallelReadMatchesSerialMultiFile() throws Exception { + //force a multi-file table cheaply via a small writer target file size + //(rather than a huge row count), so the parallel per-file path is + //actually exercised rather than falling back to serial. + MatrixBlock in = TestUtils.generateTestMatrixBlock(ROWS_MULTI_FILE, 8, -10, 10, 1.0, 13); + in.recomputeNonZeros(); + + DMLConfig conf = new DMLConfig(); + conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(SMALL_TARGET_FILE_SIZE)); + ConfigurationManager.setLocalConfig(conf); + Path dir = Files.createTempDirectory("sysds_delta_par_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + + //sanity: confirm the table really is split across multiple files + long files; + try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + files = s.filter(p -> p.toString().endsWith(".parquet")).count(); + } + assertTrue("expected a multi-file Delta table to exercise the parallel path, got " + files, + files > 1); + + MatrixBlock serial = new ReaderDelta() + .readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + MatrixBlock parallel = new ReaderDeltaParallel() + .readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + + assertEquals("rows", serial.getNumRows(), parallel.getNumRows()); + assertEquals("cols", serial.getNumColumns(), parallel.getNumColumns()); + assertEquals("nnz", serial.getNonZeros(), parallel.getNonZeros()); + TestUtils.compareMatrices(serial, parallel, 0, "serial-vs-parallel"); + } + finally { + ConfigurationManager.clearLocalConfigs(); + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void roundTripDenseSmall() throws Exception { + MatrixBlock in = new MatrixBlock(3, 4, false); + double v = 1.0; + for( int i=0; i<3; i++ ) + for( int j=0; j<4; j++ ) + in.set(i, j, v++); + in.recomputeNonZeros(); + + MatrixBlock out = writeThenRead(in); + assertEquals("rows", 3, out.getNumRows()); + assertEquals("cols", 4, out.getNumColumns()); + TestUtils.compareMatrices(in, out, 1e-12, "dense-small"); + } + + @Test + public void roundTripDenseRandom() throws Exception { + MatrixBlock in = TestUtils.generateTestMatrixBlock(500, 17, -10, 10, 1.0, 7); + MatrixBlock out = writeThenRead(in); + assertEquals("rows", in.getNumRows(), out.getNumRows()); + assertEquals("cols", in.getNumColumns(), out.getNumColumns()); + TestUtils.compareMatrices(in, out, 1e-12, "dense-random"); + } + + @Test + public void roundTripSparseRandom() throws Exception { + //values written are dense parquet, but exercise a sparse-ish input + MatrixBlock in = TestUtils.generateTestMatrixBlock(1200, 9, -5, 5, 0.1, 13); + in.recomputeNonZeros(); + MatrixBlock out = writeThenRead(in); + assertEquals("rows", in.getNumRows(), out.getNumRows()); + assertEquals("cols", in.getNumColumns(), out.getNumColumns()); + TestUtils.compareMatrices(in, out, 1e-12, "sparse-random"); + } + + @Test + public void roundTripMultiBatch() throws Exception { + //more rows than the writer batch size (4096) to exercise chunking + MatrixBlock in = TestUtils.generateTestMatrixBlock(10000, 5, 0, 100, 1.0, 1); + MatrixBlock out = writeThenRead(in); + assertEquals("rows", 10000, out.getNumRows()); + assertEquals("cols", 5, out.getNumColumns()); + TestUtils.compareMatrices(in, out, 1e-12, "multi-batch"); + } + + @Test + public void readDiscoversUnknownDimensions() throws Exception { + MatrixBlock in = TestUtils.generateTestMatrixBlock(123, 6, -1, 1, 1.0, 3); + Path dir = Files.createTempDirectory("sysds_delta_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + //pass -1 dimensions: the reader must discover them from the table + MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + assertEquals("rows", 123, out.getNumRows()); + assertEquals("cols", 6, out.getNumColumns()); + TestUtils.compareMatrices(in, out, 1e-12, "unknown-dims"); + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void emptyMatrixRoundTrip() throws Exception { + Path dir = Files.createTempDirectory("sysds_delta_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + new WriterDelta().writeEmptyMatrixToHDFS(tablePath, 0, 4, -1); + MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + assertEquals("rows", 0, out.getNumRows()); + assertEquals("cols", 4, out.getNumColumns()); + assertEquals("nnz", 0, out.getNonZeros()); + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void readNonDoubleNumericColumns() throws Exception { + //tables produced by external tools (or the frame writer) can carry + //long/int/boolean columns; the matrix reader must coerce them to double + Path dir = Files.createTempDirectory("sysds_delta_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + double[] longVals = {1, -2, 1_000_000_000L, 0}; + double[] intVals = {7, -8, 123456, 0}; + double[] boolVals = {1, 0, 1, 0}; + writeTypedColumns(tablePath, + new DataType[] {LongType.LONG, IntegerType.INTEGER, BooleanType.BOOLEAN}, + new double[][] {longVals, intVals, boolVals}); + + MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + assertEquals("rows", 4, out.getNumRows()); + assertEquals("cols", 3, out.getNumColumns()); + for( int r=0; r<4; r++ ) { + assertEquals("long col r" + r, longVals[r], out.get(r, 0), 0.0); + assertEquals("int col r" + r, intVals[r], out.get(r, 1), 0.0); + assertEquals("bool col r" + r, boolVals[r], out.get(r, 2), 0.0); + } + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void rewriteSamePathReplacesData() throws Exception { + //writing to a path that already holds a Delta table must fully replace it + Path dir = Files.createTempDirectory("sysds_delta_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + MatrixBlock first = TestUtils.generateTestMatrixBlock(50, 8, 0, 100, 1.0, 1); + new WriterDelta().writeMatrixToHDFS(first, tablePath, 50, 8, -1, first.getNonZeros()); + + //second write has different dimensions and values + MatrixBlock second = TestUtils.generateTestMatrixBlock(20, 3, -5, 5, 1.0, 2); + new WriterDelta().writeMatrixToHDFS(second, tablePath, 20, 3, -1, second.getNonZeros()); + + MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + assertEquals("rows", 20, out.getNumRows()); + assertEquals("cols", 3, out.getNumColumns()); + TestUtils.compareMatrices(second, out, 1e-12, "rewrite-replace"); + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void parallelBufferedPathMatchesSerial() throws Exception { + //the direct fast path is always taken for SystemDS-written tables (exact + //row stats, no deletion vectors); force the buffered fallback to exercise + //its per-file decode + serial concatenation and assert it matches serial. + //force a multi-file table cheaply via a small writer target file size. + MatrixBlock in = TestUtils.generateTestMatrixBlock(ROWS_MULTI_FILE, 8, -10, 10, 1.0, 23); + in.recomputeNonZeros(); + + DMLConfig conf = new DMLConfig(); + conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(SMALL_TARGET_FILE_SIZE)); + ConfigurationManager.setLocalConfig(conf); + Path dir = Files.createTempDirectory("sysds_delta_buf_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + + long files; + try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + files = s.filter(p -> p.toString().endsWith(".parquet")).count(); + } + assertTrue("expected a multi-file Delta table, got " + files, files > 1); + + MatrixBlock serial = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + //subclass that always declines the direct path -> readBuffered() + MatrixBlock buffered = new ReaderDeltaParallel() { + @Override protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { return false; } + }.readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + + assertEquals("rows", serial.getNumRows(), buffered.getNumRows()); + assertEquals("cols", serial.getNumColumns(), buffered.getNumColumns()); + assertEquals("nnz", serial.getNonZeros(), buffered.getNonZeros()); + TestUtils.compareMatrices(serial, buffered, 0, "serial-vs-buffered"); + } + finally { + ConfigurationManager.clearLocalConfigs(); + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void writerTargetFileSizeConfigProducesMoreFiles() throws Exception { + //a smaller configured target file size must make the writer roll more + //data files for the same matrix (the lever the parallel reader relies on). + MatrixBlock in = TestUtils.generateTestMatrixBlock(400_000, 16, -10, 10, 1.0, 7); + in.recomputeNonZeros(); + + //isolate the override in a fresh thread-local config (restored in finally) + DMLConfig conf = new DMLConfig(); + conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(1L * 1024 * 1024)); + ConfigurationManager.setLocalConfig(conf); + Path dir = Files.createTempDirectory("sysds_delta_cfg_"); + try { + assertEquals("config getter reflects the override", + 1L * 1024 * 1024, ConfigurationManager.getDeltaWriterTargetFileSize()); + + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + long files; + try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + files = s.filter(p -> p.toString().endsWith(".parquet")).count(); + } + assertTrue("expected >1 data file with a 1MB target, got " + files, files > 1); + + //data still round-trips correctly with the custom layout + MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + TestUtils.compareMatrices(in, out, 1e-12, "small-target-roundtrip"); + } + finally { + ConfigurationManager.clearLocalConfigs(); + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void readerBatchSizeConfigRoundTrips() throws Exception { + //a non-default reader batch size must not change the result (more, smaller + //batches exercise the per-batch extract/concatenate loop more often). + MatrixBlock in = TestUtils.generateTestMatrixBlock(5000, 7, -10, 10, 1.0, 11); + //isolate the override in a fresh thread-local config (restored in finally) + DMLConfig conf = new DMLConfig(); + conf.setTextValue(DMLConfig.DELTA_READER_BATCH_SIZE, "128"); + ConfigurationManager.setLocalConfig(conf); + Path dir = Files.createTempDirectory("sysds_delta_bs_"); + try { + assertEquals("config getter reflects the override", + 128, ConfigurationManager.getDeltaReaderBatchSize()); + + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + TestUtils.compareMatrices(in, out, 1e-12, "small-batch-roundtrip"); + } + finally { + ConfigurationManager.clearLocalConfigs(); + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void factoryRoutesDeltaToParallelWhenEnabled() { + //the factory must pick the parallel reader iff parallel CP read is enabled + CompilerConfig cc = ConfigurationManager.getCompilerConfig(); + try { + cc.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, true); + ConfigurationManager.setLocalConfig(cc); + MatrixReader par = MatrixReaderFactory.createMatrixReader(FileFormat.DELTA); + assertTrue("expected ReaderDeltaParallel when parallel read enabled", + par instanceof ReaderDeltaParallel); + + cc.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, false); + ConfigurationManager.setLocalConfig(cc); + MatrixReader ser = MatrixReaderFactory.createMatrixReader(FileFormat.DELTA); + assertTrue("expected serial ReaderDelta when parallel read disabled", + ser instanceof ReaderDelta && !(ser instanceof ReaderDeltaParallel)); + } + finally { + ConfigurationManager.clearLocalConfigs(); + } + } + + @Test + public void readFloatColumnsCoercedToDouble() throws Exception { + //float columns must be widened to double on read (exact-representable values) + Path dir = Files.createTempDirectory("sysds_delta_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + double[] f0 = {1.5, -2.25, 0.0, 1024.5}; + double[] f1 = {-0.5, 3.75, 100.125, -7.0}; + writeTypedColumns(tablePath, + new DataType[] {FloatType.FLOAT, FloatType.FLOAT}, + new double[][] {f0, f1}); + + MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + assertEquals("rows", 4, out.getNumRows()); + assertEquals("cols", 2, out.getNumColumns()); + for( int r=0; r<4; r++ ) { + assertEquals("f0 r" + r, f0[r], out.get(r, 0), 0.0); + assertEquals("f1 r" + r, f1[r], out.get(r, 1), 0.0); + } + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void readShortByteColumnsCoercedToDouble() throws Exception { + //short/byte columns must be coerced to double on read, exercising the + //T_SHORT / T_BYTE branches of ReaderDelta.getDoubleValue. + Path dir = Files.createTempDirectory("sysds_delta_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + double[] shortVals = {1, -2, 30000, 0}; + double[] byteVals = {7, -8, 120, 0}; + writeTypedColumns(tablePath, + new DataType[] {ShortType.SHORT, ByteType.BYTE}, + new double[][] {shortVals, byteVals}); + + MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + assertEquals("rows", 4, out.getNumRows()); + assertEquals("cols", 2, out.getNumColumns()); + for( int r=0; r<4; r++ ) { + assertEquals("short col r" + r, shortVals[r], out.get(r, 0), 0.0); + assertEquals("byte col r" + r, byteVals[r], out.get(r, 1), 0.0); + } + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void writerRejectsDimensionMismatch() throws Exception { + //WriterDelta validates that the passed rlen/clen match the MatrixBlock + //and rejects a mismatch with an IOException. + MatrixBlock in = TestUtils.generateTestMatrixBlock(10, 4, -1, 1, 1.0, 5); + in.recomputeNonZeros(); + Path dir = Files.createTempDirectory("sysds_delta_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + new WriterDelta().writeMatrixToHDFS(in, tablePath, 11, 4, -1, in.getNonZeros()); + fail("expected an IOException for mismatched matrix dimensions"); + } + catch(java.io.IOException ex) { + assertTrue("message should mention the dimension mismatch, got: " + ex.getMessage(), + ex.getMessage() != null && ex.getMessage().contains("dimensions mismatch")); + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void readNullCellsBecomeZero() throws Exception { + //nullable numeric columns with null cells must read back as 0.0 + Path dir = Files.createTempDirectory("sysds_delta_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + double[] vals = {3.0, 7.0, 9.0, 11.0}; + boolean[] nulls = {false, true, false, true}; + writeNullableDoubleColumn(tablePath, vals, nulls); + + MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + assertEquals("rows", 4, out.getNumRows()); + assertEquals("cols", 1, out.getNumColumns()); + for( int r=0; r<4; r++ ) + assertEquals("r" + r, nulls[r] ? 0.0 : vals[r], out.get(r, 0), 0.0); + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void readStringColumnRejected() throws Exception { + //string columns cannot back an all-double matrix -> reader must reject them + Path dir = Files.createTempDirectory("sysds_delta_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + writeStringColumn(tablePath, new String[] {"a", "b", "c"}); + try { + new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + fail("expected a DMLRuntimeException for a non-numeric (string) Delta column"); + } + catch(DMLRuntimeException ex) { + assertTrue("message should mention the non-numeric column, got: " + ex.getMessage(), + ex.getMessage() != null && ex.getMessage().contains("non-numeric")); + } + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + /** Writes a single-batch Delta table with one column per given (type, values) pair. */ + private static void writeTypedColumns(String tablePath, DataType[] types, double[][] vals) throws Exception { + Engine engine = DeltaKernelUtils.createEngine(); + StructType schema = new StructType(); + for( int c=0; c singleton(FilteredColumnarBatch fcb) { + return new CloseableIterator() { + private boolean _done = false; + @Override public boolean hasNext() { return !_done; } + @Override public FilteredColumnarBatch next() { + if( _done ) throw new NoSuchElementException(); + _done = true; + return fcb; + } + @Override public void close() {} + }; + } + + /** Minimal in-memory columnar batch backed by per-column double[] values, with + * an optional per-column null mask ({@code nulls==null} => no nulls). */ + private static class TypedBatch implements ColumnarBatch { + private final StructType _schema; + private final DataType[] _types; + private final double[][] _vals; + private final boolean[][] _nulls; + TypedBatch(StructType schema, DataType[] types, double[][] vals, boolean[][] nulls) { + _schema = schema; _types = types; _vals = vals; _nulls = nulls; + } + @Override public StructType getSchema() { return _schema; } + @Override public int getSize() { return _vals[0].length; } + @Override public ColumnVector getColumnVector(int ordinal) { + return new TypedVector(_types[ordinal], _vals[ordinal], + _nulls == null ? null : _nulls[ordinal]); + } + } + + /** Column view exposing a double[] as the requested Delta primitive type. */ + private static class TypedVector implements ColumnVector { + private final DataType _type; + private final double[] _vals; + private final boolean[] _nulls; + TypedVector(DataType type, double[] vals, boolean[] nulls) { _type = type; _vals = vals; _nulls = nulls; } + @Override public DataType getDataType() { return _type; } + @Override public int getSize() { return _vals.length; } + @Override public boolean isNullAt(int rowId) { return _nulls != null && _nulls[rowId]; } + @Override public double getDouble(int rowId) { return _vals[rowId]; } + @Override public float getFloat(int rowId) { return (float) _vals[rowId]; } + @Override public long getLong(int rowId) { return (long) _vals[rowId]; } + @Override public int getInt(int rowId) { return (int) _vals[rowId]; } + @Override public short getShort(int rowId) { return (short) _vals[rowId]; } + @Override public byte getByte(int rowId) { return (byte) _vals[rowId]; } + @Override public boolean getBoolean(int rowId) { return _vals[rowId] != 0; } + @Override public void close() {} + } + + /** Column view exposing a String[] as a Delta string column. */ + private static class StringVector implements ColumnVector { + private final String[] _vals; + StringVector(String[] vals) { _vals = vals; } + @Override public DataType getDataType() { return StringType.STRING; } + @Override public int getSize() { return _vals.length; } + @Override public boolean isNullAt(int rowId) { return _vals[rowId] == null; } + @Override public String getString(int rowId) { return _vals[rowId]; } + @Override public void close() {} + } +} diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java new file mode 100644 index 00000000000..2d79b79f2dd --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.io; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; +import org.apache.sysds.runtime.io.ReaderDelta; +import org.apache.sysds.runtime.io.ReaderDeltaParallel; +import org.apache.sysds.runtime.io.WriterDelta; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Cross-engine interoperability tests for the native (Delta Kernel based) matrix + * reader/writer against the reference Delta implementation (Delta's Spark + * connector, {@code delta-spark}, pulled in test-only). + * + *

The other Delta matrix tests round-trip exclusively through SystemDS' own + * Kernel-based read/write paths, so they cannot catch a table that SystemDS + * writes in a way other Delta engines reject (or vice versa). These tests close + * that gap by routing data through two independent engines: + *

    + *
  • SystemDS writes -> Spark/Delta reads (our output is spec-compliant), and
    • + *
  • Spark/Delta writes -> SystemDS reads, including a multi-file layout and a + * table with deletion vectors / a second commit that the SystemDS writer + * never produces itself.
    • + *
+ * + *

Row order is never assumed: every table carries a unique id in column 0 and + * comparisons are keyed by that id, since neither engine guarantees row order + * across files. + */ +@net.jcip.annotations.NotThreadSafe +public class DeltaMatrixSparkInteropTest { + + private static SparkSession spark; + + @BeforeClass + public static void startSpark() { + //each test class runs in its own fork (surefire reuseForks=false), so this + //is the only SparkSession in the JVM and gets the Delta extensions injected. + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + spark = SparkSession.builder() + .appName("sysds-delta-interop") + .master("local[2]") + .config("spark.ui.enabled", "false") + .config("spark.sql.shuffle.partitions", "2") + .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") + .getOrCreate(); + } + + @AfterClass + public static void stopSpark() { + if( spark != null ) + spark.stop(); + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + spark = null; + } + + @Test + public void systemdsWriteSparkReadMultiFile() throws Exception { + //SystemDS writes a (forced) multi-file Delta table; the reference Delta + //engine (Spark) must read every data file back with matching values. + int rows = 500, cols = 5; + MatrixBlock in = indexedMatrix(rows, cols); + + //small target file size -> multiple parquet data files (exercise that an + //external reader stitches all of our data files, not just the first). + DMLConfig conf = new DMLConfig(); + conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(16L * 1024)); + ConfigurationManager.setLocalConfig(conf); + Path dir = Files.createTempDirectory("sysds_delta_s2s_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + new WriterDelta().writeMatrixToHDFS(in, tablePath, rows, cols, -1, in.getNonZeros()); + assertTrue("writer should have produced a multi-file table", countParquet(tablePath) > 1); + + Dataset df = spark.read().format("delta").load(tablePath); + assertEquals("rows", rows, df.count()); + assertEquals("cols", cols, df.schema().fields().length); + + List read = df.collectAsList(); + assertEquals(rows, read.size()); + for( Row r : read ) { + int id = (int) Math.round(r.getDouble(0)); + assertTrue("id in range: " + id, id >= 0 && id < rows); + for( int c = 0; c < cols; c++ ) + assertEquals("r" + id + " c" + c, in.get(id, c), r.getDouble(c), 1e-9); + } + } + finally { + ConfigurationManager.clearLocalConfigs(); + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void sparkWriteSystemdsReadMultiFile() throws Exception { + //the reference Delta engine writes a multi-file table; both the serial and + //parallel SystemDS readers must reconstruct it (coercing long ids to double). + int rows = 600, cols = 4; + Dataset df = indexedDataFrame(rows, cols).repartition(3); //-> multiple data files + Path dir = Files.createTempDirectory("sysds_delta_p2s_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + df.write().format("delta").save(tablePath); + assertTrue("spark should have written a multi-file table", countParquet(tablePath) > 1); + + Map expected = expectedById(rows, cols); + assertMatchesById(new ReaderDelta() + .readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, "serial"); + assertMatchesById(new ReaderDeltaParallel() + .readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, "parallel"); + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void sparkDeletionVectorsSystemdsRead() throws Exception { + //a Delta table with deletion vectors + a second commit (the DELETE) is a + //layout the SystemDS writer never emits; the readers must honor the DV and + //return only the surviving rows. This exercises the hasDeletionVector path. + int rows = 400, cols = 3, deleteBelow = 50; + Path dir = Files.createTempDirectory("sysds_delta_dv_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + //enable deletion vectors for tables created in this block, then delete a + //row range so Delta records a DV rather than rewriting the data files. + spark.conf().set(DV_DEFAULT, "true"); + indexedDataFrame(rows, cols).write().format("delta").save(tablePath); + spark.sql("DELETE FROM delta.`" + tablePath + "` WHERE c0 < " + deleteBelow); + + Map expected = expectedById(rows, cols); + expected.keySet().removeIf(id -> id < deleteBelow); + + MatrixBlock serial = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + assertEquals("surviving rows (serial)", rows - deleteBelow, serial.getNumRows()); + assertMatchesById(serial, expected, cols, "serial-dv"); + + MatrixBlock parallel = new ReaderDeltaParallel().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + assertEquals("surviving rows (parallel)", rows - deleteBelow, parallel.getNumRows()); + assertMatchesById(parallel, expected, cols, "parallel-dv"); + } + finally { + //fresh fork per test class, so simply clearing the override is enough + spark.conf().unset(DV_DEFAULT); + FileUtils.deleteQuietly(dir.toFile()); + } + } + + private static final String DV_DEFAULT = + "spark.databricks.delta.properties.defaults.enableDeletionVectors"; + + /** Matrix whose column 0 is the row index and remaining columns are exact doubles. */ + private static MatrixBlock indexedMatrix(int rows, int cols) { + MatrixBlock mb = new MatrixBlock(rows, cols, false); + for( int r = 0; r < rows; r++ ) { + mb.set(r, 0, r); + for( int c = 1; c < cols; c++ ) + mb.set(r, c, value(r, c)); + } + mb.recomputeNonZeros(); + return mb; + } + + /** Spark DataFrame mirroring {@link #indexedMatrix} with columns c0..c(cols-1) as doubles. */ + private static Dataset indexedDataFrame(int rows, int cols) { + StructField[] fields = new StructField[cols]; + for( int c = 0; c < cols; c++ ) + fields[c] = DataTypes.createStructField("c" + c, DataTypes.DoubleType, false); + StructType schema = DataTypes.createStructType(fields); + + List data = new ArrayList<>(rows); + for( int r = 0; r < rows; r++ ) { + Object[] vals = new Object[cols]; + vals[0] = (double) r; + for( int c = 1; c < cols; c++ ) + vals[c] = value(r, c); + data.add(RowFactory.create(vals)); + } + return spark.createDataFrame(data, schema); + } + + /** Deterministic, exactly-representable cell value for (row,col), col>=1. */ + private static double value(int row, int col) { + return row * 0.5 - col; + } + + private static Map expectedById(int rows, int cols) { + Map exp = new HashMap<>(rows); + for( int r = 0; r < rows; r++ ) { + double[] row = new double[cols]; + row[0] = r; + for( int c = 1; c < cols; c++ ) + row[c] = value(r, c); + exp.put(r, row); + } + return exp; + } + + /** Asserts every row of {@code out} (keyed by its column-0 id) matches {@code expected}. */ + private static void assertMatchesById(MatrixBlock out, Map expected, int cols, String tag) { + assertEquals(tag + " rows", expected.size(), out.getNumRows()); + assertEquals(tag + " cols", cols, out.getNumColumns()); + boolean[] seen = new boolean[expected.size() == 0 ? 0 : maxId(expected) + 1]; + for( int r = 0; r < out.getNumRows(); r++ ) { + int id = (int) Math.round(out.get(r, 0)); + double[] exp = expected.get(id); + assertTrue(tag + ": unexpected/duplicate id " + id, exp != null && id < seen.length && !seen[id]); + seen[id] = true; + for( int c = 0; c < cols; c++ ) + assertEquals(tag + " id" + id + " c" + c, exp[c], out.get(r, c), 1e-9); + } + } + + private static int maxId(Map expected) { + int m = 0; + for( int id : expected.keySet() ) + m = Math.max(m, id); + return m; + } + + private static long countParquet(String tablePath) throws Exception { + try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + return s.filter(p -> p.toString().endsWith(".parquet")).count(); + } + } +} diff --git a/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java b/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java new file mode 100644 index 00000000000..a4013c3672d --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.io.delta; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.util.HashMap; + +import org.apache.sysds.runtime.controlprogram.caching.CacheStatistics; +import org.apache.sysds.runtime.matrix.data.MatrixValue.CellIndex; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; + +/** + * End-to-end DML test of the native Delta read/write path. + * + *

The write and the read are run as two separate SystemDS executions + * on purpose. If they shared a single script/process, SystemDS would reuse the + * still-materialized in-memory matrix for the subsequent read and never invoke + * {@link org.apache.sysds.runtime.io.ReaderDelta} at all (verified: the cache + * reports 0 HDFS hits in that case). Splitting the executions forces a genuine + * read from disk, and we additionally assert via {@link CacheStatistics} that + * the read run actually performed HDFS reads (the Delta table + the text + * reference) rather than serving the matrix from cache.

+ */ +public class DeltaReadWriteTest extends AutomatedTestBase { + + private final static String TEST_DIR = "functions/io/delta/"; + private final static String TEST_CLASS_DIR = TEST_DIR + DeltaReadWriteTest.class.getSimpleName() + "/"; + private final static String WRITE_NAME = "DeltaWrite"; + private final static String READ_NAME = "DeltaReadCompare"; + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration(WRITE_NAME, + new TestConfiguration(TEST_CLASS_DIR, WRITE_NAME, new String[] { "ref" })); + addTestConfiguration(READ_NAME, + new TestConfiguration(TEST_CLASS_DIR, READ_NAME, new String[] { "R" })); + } + + @Test + public void testDenseRoundTrip() { + runDeltaRoundTrip(200, 12, 1.0); + } + + @Test + public void testSparseRoundTrip() { + runDeltaRoundTrip(640, 8, 0.2); + } + + @Test + public void testMultiBatchRoundTrip() { + runDeltaRoundTrip(9000, 4, 1.0); + } + + private void runDeltaRoundTrip(int rows, int cols, double sparsity) { + try { + String HOME = SCRIPT_DIR + TEST_DIR; + + // ---- phase 1: write the matrix as a Delta table + text reference ---- + getAndLoadTestConfiguration(WRITE_NAME); + String deltaPath = output("deltaTable"); + String refPath = output("ref"); + fullDMLScriptName = HOME + WRITE_NAME + ".dml"; + programArgs = new String[] { "-stats", "-args", + String.valueOf(rows), String.valueOf(cols), String.valueOf(sparsity), + deltaPath, refPath }; + runTest(true, false, null, -1); + + // the write run must have materialized two matrices to disk (the Delta + // table under test + the text reference); WriterDelta genuinely hitting + // HDFS is what produces these write-side cache statistics. + long hdfsWrites = CacheStatistics.getHDFSWrites(); + assertTrue("expected >= 2 HDFS writes in the write run (delta + reference), got " + + hdfsWrites, hdfsWrites >= 2); + // and a real Delta table (transaction log) must have been created + assertTrue("missing Delta transaction log under " + deltaPath, + new File(deltaPath, "_delta_log").isDirectory()); + + // ---- phase 2: fresh execution reads the Delta table and compares ---- + getAndLoadTestConfiguration(READ_NAME); + fullDMLScriptName = HOME + READ_NAME + ".dml"; + programArgs = new String[] { "-stats", "-args", + deltaPath, refPath, output("R") }; + runTest(true, false, null, -1); + + // the read run must have materialized two matrices from disk (the Delta + // table under test + the text reference); a cached/short-circuited read + // would report fewer HDFS hits and fail here. + long hdfsReads = CacheStatistics.getHDFSHits(); + assertTrue("expected >= 2 HDFS reads in the read run (delta + reference), got " + + hdfsReads, hdfsReads >= 2); + + HashMap R = readDMLMatrixFromOutputDir("R"); + //text-cell output omits exact zeros, so a missing cell means 0.0 + double diff = R.getOrDefault(new CellIndex(1, 1), 0.0); + double nrow = R.getOrDefault(new CellIndex(1, 2), 0.0); + double ncol = R.getOrDefault(new CellIndex(1, 3), 0.0); + + assertEquals("reconstruction error", 0.0, diff, 1e-12); + assertEquals("discovered rows", rows, (int) nrow); + assertEquals("discovered cols", cols, (int) ncol); + } + catch(Exception ex) { + throw new RuntimeException(ex); + } + } +} diff --git a/src/test/scripts/functions/io/delta/DeltaReadCompare.dml b/src/test/scripts/functions/io/delta/DeltaReadCompare.dml new file mode 100644 index 00000000000..5caf992f39c --- /dev/null +++ b/src/test/scripts/functions/io/delta/DeltaReadCompare.dml @@ -0,0 +1,34 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# Reader side of the native Delta round-trip test. Reads the Delta table +# (dimensions discovered from the transaction log) and the text reference, +# both genuine HDFS reads in a fresh process, and reports the elementwise +# reconstruction error together with the discovered dimensions. + +Y = read($1, format="delta") +Xref = read($2, format="text") + +R = matrix(0, rows=1, cols=3) +R[1,1] = sum(abs(Xref - Y)) # 0 if ReaderDelta reconstructed the matrix exactly +R[1,2] = nrow(Y) # discovered row count +R[1,3] = ncol(Y) # discovered column count +write(R, $3) diff --git a/src/test/scripts/functions/io/delta/DeltaWrite.dml b/src/test/scripts/functions/io/delta/DeltaWrite.dml new file mode 100644 index 00000000000..41c5b0899a7 --- /dev/null +++ b/src/test/scripts/functions/io/delta/DeltaWrite.dml @@ -0,0 +1,30 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# Writer side of the native Delta round-trip test. Generates a matrix and +# materializes it twice: once as a Delta table (under test) and once as a +# plain text reference. Running the read/compare in a SEPARATE process is +# intentional: it prevents SystemDS from short-circuiting the subsequent +# read against the still-in-memory matrix, so ReaderDelta is actually used. + +X = rand(rows=$1, cols=$2, min=-5, max=5, seed=7, sparsity=$3) +write(X, $4, format="delta") +write(X, $5, format="text") From 5e476bb645436bf2408a76a4b745ae69ab5bb978 Mon Sep 17 00:00:00 2001 From: ywcb00 <52667438+ywcb00@users.noreply.github.com> Date: Mon, 29 Jun 2026 13:07:01 +0200 Subject: [PATCH 057/132] [MINOR][TEST] Add Asserts to ScalarIOTest for Result Validation (#2522) This PR fixes the ScalarIOTest by adding respective asserts to validate the output and uncommenting checks about the variables written to disk. Additionally, some unnecessary indents in the test class are removed. --- .../sysds/test/functions/io/ScalarIOTest.java | 90 ++++++++++--------- 1 file changed, 47 insertions(+), 43 deletions(-) diff --git a/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java b/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java index d04cfe3125c..a1b51ee9d81 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java @@ -31,18 +31,18 @@ public class ScalarIOTest extends AutomatedTestBase { - + private final static String TEST_NAME = "scalarIOTest"; private final static String TEST_DIR = "functions/io/"; private final static String OUT_FILE = "a.scalar"; private final static String TEST_CLASS_DIR = TEST_DIR + ScalarIOTest.class.getSimpleName() + "/"; private final static String HOME = SCRIPT_DIR + TEST_DIR; - + @Override public void setUp() { addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] { "a.scalar" }) ); - + getAndLoadTestConfiguration(TEST_NAME); } @@ -50,40 +50,39 @@ public void setUp() { public void testIntScalarWrite() { int int_scalar = 464; - + fullDMLScriptName = HOME + "ScalarWrite.dml"; programArgs = new String[]{ "-args", String.valueOf(int_scalar), output("a.scalar") }; runTest(true, false, null, -1); - + int int_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1,1)).intValue(); Assert.assertEquals("Values not equal: " + int_scalar + Opcodes.NOTEQUAL.toString() + int_out_scalar, int_scalar, int_out_scalar); - + // Invoke the DML script that does computations and then writes scalar to HDFS fullDMLScriptName = HOME + "ScalarComputeWrite.dml"; runTest(true, false, null, -1); - + int_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1,1)).intValue(); Assert.assertEquals("Computation test for Integers failed: Values not equal: " + int_scalar + Opcodes.NOTEQUAL.toString() + int_out_scalar, int_scalar, int_out_scalar); } @Test - public void testDoubleScalarWrite() - { + public void testDoubleScalarWrite() { Double double_scalar = 464.55; fullDMLScriptName = HOME + "ScalarWrite.dml"; programArgs = new String[]{ "-args", String.valueOf(double_scalar), output("a.scalar") }; runTest(true, false, null, -1); - + Double double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1,1)).doubleValue(); Assert.assertEquals("Values not equal: " + double_scalar + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar); // Invoke the DML script that does computations and then writes scalar to HDFS fullDMLScriptName = HOME + "ScalarComputeWrite.dml"; runTest(true, false, null, -1); - + double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1,1)).doubleValue(); - Assert.assertEquals("Computation test for Integers failed: Values not equal: " + double_scalar + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar); + Assert.assertEquals("Computation test for Doubles failed: Values not equal: " + double_scalar + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar); } @Test @@ -96,7 +95,7 @@ public void testBooleanScalarWrite() { runTest(true, false, null, -1); boolean boolean_out_scalar = TestUtils.readDMLBoolean(output(OUT_FILE)); - + Assert.assertEquals("Values not equal: " + boolean_scalar + Opcodes.NOTEQUAL.toString() + boolean_out_scalar, boolean_scalar, boolean_out_scalar); } @@ -110,86 +109,91 @@ public void testStringScalarWrite() { runTest(true, false, null, -1); String string_out_scalar = TestUtils.readDMLString(output(OUT_FILE)); - + Assert.assertEquals("Values not equal: " + string_scalar + Opcodes.NOTEQUAL.toString() + string_out_scalar, string_scalar, string_out_scalar); } - + @Test public void testIntScalarRead() { - int int_scalar = 464; - + setOutputBuffering(true); fullDMLScriptName = HOME + "ScalarWrite.dml"; programArgs = new String[]{"-args", String.valueOf(int_scalar), output("a.scalar")}; runTest(true, false, null, -1); - - //int int_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1,1)).intValue(); - //assertEquals("Values not equal: " + int_scalar + "!=" + int_out_scalar, int_scalar, int_out_scalar); - + + int int_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)) + .get(new CellIndex(1,1)).intValue(); + Assert.assertEquals("Values not equal: " + int_scalar + Opcodes.NOTEQUAL.toString() + int_out_scalar, + int_scalar, int_out_scalar); + // Invoke the DML script that reads the scalar and prints to stdout fullDMLScriptName = HOME + "ScalarRead.dml"; programArgs = new String[] { "-args", output("a.scalar"), "int" }; - - ByteArrayOutputStream stdout = runTest(true, false, null, -1); - bufferContainsString(stdout, String.valueOf(int_scalar)); + + ByteArrayOutputStream stdout = runTest(true, false, null, -1); + Assert.assertTrue(bufferContainsString(stdout, String.valueOf(int_scalar))); } @Test public void testDoubleScalarRead() { - double double_scalar = 464.5; - + fullDMLScriptName = HOME + "ScalarWrite.dml"; programArgs = new String[]{ "-args", String.valueOf(double_scalar), output("a.scalar") }; runTest(true, false, null, -1); - - //double double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1,1)).doubleValue(); - //assertEquals("Values not equal: " + double_scalar + "!=" + double_out_scalar, double_scalar, double_out_scalar); - + + double double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)) + .get(new CellIndex(1,1)).doubleValue(); + Assert.assertEquals("Values not equal: " + double_scalar + Opcodes.NOTEQUAL.toString() + double_out_scalar, + double_scalar, double_out_scalar, 0); + // Invoke the DML script that reads the scalar and prints to stdout fullDMLScriptName = HOME + "ScalarRead.dml"; programArgs = new String[] { "-args", output("a.scalar"), "double" }; - + ByteArrayOutputStream stdout = runTest(true, false, null, -1); - bufferContainsString(stdout, String.valueOf(double_scalar)); + Assert.assertTrue(bufferContainsString(stdout, String.valueOf(double_scalar))); } @Test public void testBooleanScalarRead() { - boolean boolean_scalar = true; - + fullDMLScriptName = HOME + "ScalarWrite.dml"; programArgs = new String[]{ "-args", String.valueOf(boolean_scalar).toUpperCase(), output("a.scalar") }; runTest(true, false, null, -1); + boolean boolean_out_scalar = TestUtils.readDMLBoolean(output(OUT_FILE)); + Assert.assertEquals("Values not equal: " + boolean_scalar + Opcodes.NOTEQUAL.toString() + boolean_out_scalar, + boolean_scalar, boolean_out_scalar); + // Invoke the DML script that reads the scalar and prints to stdout fullDMLScriptName = HOME + "ScalarRead.dml"; programArgs = new String[] { "-args", output("a.scalar"), "boolean" }; - - // setExpectedStdOut(String.valueOf(boolean_scalar).toUpperCase()); + ByteArrayOutputStream stdout = runTest(true, false, null, -1); - bufferContainsString(stdout, String.valueOf(boolean_scalar).toUpperCase()); + Assert.assertTrue(bufferContainsString(stdout, String.valueOf(boolean_scalar).toUpperCase())); } @Test public void testStringScalarRead() { - String string_scalar = "String Test.!"; - + fullDMLScriptName = HOME + "ScalarWrite.dml"; programArgs = new String[]{ "-args", String.valueOf(string_scalar), output("a.scalar") }; runTest(true, false, null, -1); + String string_out_scalar = TestUtils.readDMLString(output(OUT_FILE)); + Assert.assertEquals("Values not equal: " + string_scalar + Opcodes.NOTEQUAL.toString() + string_out_scalar, + string_scalar, string_out_scalar); + // Invoke the DML script that reads the scalar and prints to stdout fullDMLScriptName = HOME + "ScalarRead.dml"; programArgs = new String[] { "-args", output("a.scalar"), "string" }; - - ByteArrayOutputStream stdout = runTest(true, false, null, -1); - bufferContainsString(stdout, string_scalar); + ByteArrayOutputStream stdout = runTest(true, false, null, -1); + Assert.assertTrue(bufferContainsString(stdout, string_scalar)); } - } From ef54020b019aacb233028af8beae45d462acc57f Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Wed, 1 Jul 2026 14:38:54 +0200 Subject: [PATCH 058/132] [SYSTEMDS-3835] Fine grained window size parameters This patch adds a mechanism to evaluate more fine grained context windows in the unimodal optimizer. --- .../scuro/dataloader/timeseries_loader.py | 21 +++-- .../systemds/scuro/dataloader/video_loader.py | 4 + .../scuro/drsearch/hyperparameter_tuner.py | 2 + .../systemds/scuro/drsearch/node_executor.py | 53 +++++++++++-- .../scuro/drsearch/operator_registry.py | 78 ++++++++++++++++++- .../scuro/drsearch/representation_dag.py | 1 + .../python/systemds/scuro/drsearch/task.py | 4 +- .../scuro/drsearch/unimodal_optimizer.py | 67 ++++++++++++---- .../python/systemds/scuro/modality/type.py | 22 ++++++ .../systemds/scuro/representations/lstm.py | 2 +- .../scuro/representations/mel_spectrogram.py | 9 +++ .../systemds/scuro/representations/mfcc.py | 41 ++++++---- .../scuro/representations/mlp_averaging.py | 21 +---- .../multimodal_attention_fusion.py | 7 ++ .../scuro/representations/spectrogram.py | 8 ++ .../timeseries_representations.py | 59 +++++++++----- .../systemds/scuro/representations/wav2vec.py | 5 +- .../representations/window_aggregation.py | 27 ++++++- src/main/python/tests/scuro/data_generator.py | 3 + 19 files changed, 352 insertions(+), 82 deletions(-) diff --git a/src/main/python/systemds/scuro/dataloader/timeseries_loader.py b/src/main/python/systemds/scuro/dataloader/timeseries_loader.py index 6b697e6a7ad..8e6c11316b0 100644 --- a/src/main/python/systemds/scuro/dataloader/timeseries_loader.py +++ b/src/main/python/systemds/scuro/dataloader/timeseries_loader.py @@ -36,6 +36,8 @@ class TimeseriesStats: num_signals: int output_shape: tuple output_shape_is_known: bool + avg_length: float + sampling_rate: int class TimeseriesLoader(BaseLoader): @@ -49,15 +51,14 @@ def __init__( sampling_rate: Optional[int] = None, normalize: bool = True, file_format: str = "npy", + modality_type: Optional[ModalityType] = ModalityType.TIMESERIES, ): - super().__init__( - source_path, indices, data_type, chunk_size, ModalityType.TIMESERIES - ) + super().__init__(source_path, indices, data_type, chunk_size, modality_type) self.signal_names = signal_names self.sampling_rate = sampling_rate self.normalize = normalize self.file_format = file_format.lower() - self.stats = self.get_stats(source_path) + self.stats = self.get_stats(source_path, sampling_rate) if self.file_format not in ["npy", "mat", "hdf5", "txt"]: raise ValueError(f"Unsupported file format: {self.file_format}") @@ -148,17 +149,25 @@ def _load_csv_with_header(self, file: str, delimiter: str = None) -> np.ndarray: data = df[selected].to_numpy(dtype=self._data_type) return data - def get_stats(self, source_path: str): + def get_stats(self, source_path: str, sampling_rate: int): self.file_sanity_check(source_path) max_length = 0 num_instances = 0 num_signals = 0 + avg_length = 0 for file_name in self.indices: file = source_path + file_name + "." + self.file_format data = self._load_data(file) max_length = max(max_length, data.shape[0]) + avg_length += data.shape[0] num_instances += 1 num_signals = max(num_signals, data.shape[1]) return TimeseriesStats( - max_length, num_instances, num_signals, (max_length,), True + max_length, + num_instances, + num_signals, + (max_length,), + True, + avg_length / num_instances, + sampling_rate, ) diff --git a/src/main/python/systemds/scuro/dataloader/video_loader.py b/src/main/python/systemds/scuro/dataloader/video_loader.py index b35a22a8b66..bf7bdd846c7 100644 --- a/src/main/python/systemds/scuro/dataloader/video_loader.py +++ b/src/main/python/systemds/scuro/dataloader/video_loader.py @@ -33,6 +33,7 @@ class VideoStats: fps: int max_length: int + avg_length: float max_width: int max_height: int max_channels: int @@ -117,6 +118,7 @@ def get_stats(self, source_path: str): max_height = 0 max_num_channels = 0 num_instances = 0 + avg_length = 0 for file in os.listdir(source_path): file_name = file.split(".")[0] if file_name not in self.indices: @@ -129,6 +131,7 @@ def get_stats(self, source_path: str): height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) num_channels = 3 max_length = max(max_length, length) + avg_length += length max_width = max(max_width, width) max_height = max(max_height, height) max_num_channels = max(max_num_channels, num_channels) @@ -142,6 +145,7 @@ def get_stats(self, source_path: str): return VideoStats( fps, max_length, + avg_length / num_instances, max_width, max_height, max_num_channels, diff --git a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py index 0305f613b63..4f04bffcbe5 100644 --- a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py +++ b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py @@ -233,6 +233,8 @@ def __init__(self, tasks, modalities): def add_result(self, results): # TODO: Check if order of best results matters (deterministic) for result in results: + if result is None: + continue if result.mm_opt: self.results[result.task_name]["mm_results"].append(result) else: diff --git a/src/main/python/systemds/scuro/drsearch/node_executor.py b/src/main/python/systemds/scuro/drsearch/node_executor.py index 4b9b2acc658..a6a7ffe2ca4 100644 --- a/src/main/python/systemds/scuro/drsearch/node_executor.py +++ b/src/main/python/systemds/scuro/drsearch/node_executor.py @@ -234,6 +234,7 @@ def _execute_multiple_reps_for_leaf_dependencies( def _execute_node_worker(node, input_mods, task, rep_cache, gpu_id): + start_time = time.perf_counter() if gpu_id is not None: device = torch.device(f"cuda:{gpu_id}") torch.cuda.set_device(device) @@ -287,23 +288,28 @@ def _run_node_op(): ) else: result = _run_node_op() - + end_time = time.perf_counter() + pid = os.getpid() return { "result": result, "peak_bytes": peak_delta_bytes, "peak_abs_rss_bytes": peak_abs_rss, "gpu_peak_bytes": gpu_peak_bytes, "operation_name": operation_name, + "start_time": start_time, + "end_time": end_time, + "pid": pid, } def _execute_task_worker( task_node_id: str, task: Any, - data: Any, + modality: Any, gpu_id: Optional[int], + aggregation: AggregatedRepresentation = None, ) -> Dict[str, Any]: - + start_time = time.perf_counter() if DEBUG: print(f"Executing task {task_node_id} on GPU {gpu_id}") if gpu_id is not None: @@ -316,6 +322,14 @@ def _execute_task_worker( def _run_task(): start = time.perf_counter() + if aggregation is not None: + data = ( + aggregation.operation(params=aggregation.parameters) + .transform(modality) + .data + ) + else: + data = modality.data scores = task.run(data) end = time.perf_counter() return scores, end - start @@ -336,12 +350,16 @@ def _run_task(): ) else: result = _run_task() - + end_time = time.perf_counter() + pid = os.getpid() return { "scores": result[0], "task_time": result[1], "peak_bytes": peak_delta_bytes, "gpu_peak_bytes": gpu_peak_bytes, + "start_time": start_time, + "end_time": end_time, + "pid": pid, } @@ -385,6 +403,9 @@ def __init__( checkpoint_every=1, resume=False, ) + self.statistics = {} + self.statistics["worker_stats"] = {} + self.statistics["node_stats"] = {} def _shm_names_for_submit( self, parent_node_ids: List[str], payload_data: Any @@ -451,6 +472,10 @@ def submit_node(node_id: str): ] if self._is_task_node(node): + # potentially batch task nodes and then execute them together + # by the the same task type (index, gpu vs cpu) + # either enough nodes to batch or enough time to batch whatever happens first + task_result = ResultEntry( dag=self._get_dag_from_node_ids(node_id), representation_time=parent_results[0].transform_time, @@ -458,17 +483,20 @@ def submit_node(node_id: str): task_results[node_id] = task_result task_idx = int(node.parameters.get("_task_idx", 0)) payload_data = ( - self.modalities[0].data + self.modalities[0] if parent_results is None - else parent_results[0].data + else parent_results[0] ) retained = self._retain_for_submit(parent_node_ids, payload_data) + aggregation = node.aggregation + future = executor.submit( _execute_task_worker, node_id, self.tasks[task_idx], payload_data, gpu_id, + aggregation, ) else: payload_data = ( @@ -538,6 +566,14 @@ def submit_new_ready_nodes(): peak_bytes = result["peak_bytes"] gpu_peak_bytes = result["gpu_peak_bytes"] node = self.scheduler.mapping[node_id] + self.statistics["worker_stats"][result["pid"]] = { + "start_time": result["start_time"], + "end_time": result["end_time"], + } + self.statistics["node_stats"][node_id] = { + "start_time": result["start_time"], + "end_time": result["end_time"], + } if self._is_task_node(node): task_results[node_id].task_time = result["task_time"] task_results[node_id].train_score = result["scores"][ @@ -594,7 +630,10 @@ def submit_new_ready_nodes(): self.result_cache.cleanup_all() self._cleanup_leaf_shared_memory() - return {"task_results": list(task_results.values())} + return { + "task_results": list(task_results.values()), + "statistics": self.statistics, + } def _handle_modality_result( self, diff --git a/src/main/python/systemds/scuro/drsearch/operator_registry.py b/src/main/python/systemds/scuro/drsearch/operator_registry.py index bc9bd406a8f..4c5641fdd91 100644 --- a/src/main/python/systemds/scuro/drsearch/operator_registry.py +++ b/src/main/python/systemds/scuro/drsearch/operator_registry.py @@ -19,7 +19,7 @@ # # ------------------------------------------------------------- from typing import Union, List - +import math from systemds.scuro.modality.type import ModalityType from systemds.scuro.representations.representation import Representation @@ -57,6 +57,24 @@ def set_representations(self, modality_type, representations): else: self._representations[modality_type] = [representations] + def set_context_operators(self, modality_type, context_operators): + if isinstance(context_operators, list): + self._context_operators[modality_type] = context_operators + else: + self._context_operators[modality_type] = [context_operators] + + def set_context_representation_operators( + self, modality_type, context_representation_operators + ): + if isinstance(context_representation_operators, list): + self._context_representation_operators[modality_type] = ( + context_representation_operators + ) + else: + self._context_representation_operators[modality_type] = [ + context_representation_operators + ] + def add_representation( self, representation: Representation, modality: ModalityType ): @@ -139,6 +157,64 @@ def get_representation_by_name(self, representation_name, modality_type): def get_context_representations(self, modality_type): return self._context_representation_operators[modality_type] + def get_context_lenghts_for_modality(self, modality_type, statistics): + if modality_type == ModalityType.AUDIO: + window_lengths = [ + 0.010, + 0.020, + 0.050, + 0.075, + 0.100, + 0.5, + 1, + 2, + 5, + 10, + ] # seconds + + if ( + modality_type == ModalityType.TIMESERIES + or modality_type == ModalityType.PHYSIOLOGICAL + ): + window_lengths = [0.05, 0.1, 0.5, 0.75, 1, 2, 5, 10, 30, 60] # seconds + + if modality_type == ModalityType.VIDEO: + window_lengths = [0.5, 1, 2, 5, 10] # seconds + + if ( + modality_type == ModalityType.AUDIO + or modality_type == ModalityType.TIMESERIES + or modality_type == ModalityType.PHYSIOLOGICAL + ): + max_length_in_seconds = statistics.max_length / statistics.sampling_rate + window_lengths = [ + length for length in window_lengths if length <= max_length_in_seconds + ] + + effective_window_lenghts = [ + statistics.sampling_rate * length for length in window_lengths + ] + num_windows = [ + math.ceil(statistics.avg_length / length) + for length in effective_window_lenghts + ] + return effective_window_lenghts, num_windows + + if modality_type == ModalityType.VIDEO: + max_length_in_seconds = statistics.max_length / statistics.fps + window_lengths = [ + length for length in window_lengths if length <= max_length_in_seconds + ] + + effective_window_lenghts = [ + statistics.fps * length for length in window_lengths + ] + num_windows = [ + math.ceil(statistics.avg_length / length) + for length in effective_window_lenghts + ] + return effective_window_lenghts, num_windows + def register_representation(modalities: Union[ModalityType, List[ModalityType]]): """ diff --git a/src/main/python/systemds/scuro/drsearch/representation_dag.py b/src/main/python/systemds/scuro/drsearch/representation_dag.py index b1d5835ad3f..a19c44396fd 100644 --- a/src/main/python/systemds/scuro/drsearch/representation_dag.py +++ b/src/main/python/systemds/scuro/drsearch/representation_dag.py @@ -80,6 +80,7 @@ class RepresentationNode: representation_index: int = None parameters: Dict[str, Any] = field(default_factory=dict) gpu_id: int = None + aggregation: AggregatedRepresentation = None @dataclass diff --git a/src/main/python/systemds/scuro/drsearch/task.py b/src/main/python/systemds/scuro/drsearch/task.py index 404624ec3a5..e74e791794c 100644 --- a/src/main/python/systemds/scuro/drsearch/task.py +++ b/src/main/python/systemds/scuro/drsearch/task.py @@ -259,7 +259,9 @@ def run(self, data): val_y = self._gather_by_indices(self.labels, fold_val_indices) self._run_fold(model, train_X, train_y, val_X, val_y, test_X, test_y) - + if hasattr(model, "clean_up"): + model.clean_up() + del model return [ self.train_scores.compute_averages(), self.val_scores.compute_averages(), diff --git a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py index 8dc2e1a082f..a67cbe12029 100644 --- a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py +++ b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py @@ -20,6 +20,8 @@ # ------------------------------------------------------------- import copy import pickle +import csv +from pathlib import Path import time from concurrent.futures import ProcessPoolExecutor, as_completed from dataclasses import dataclass @@ -39,6 +41,7 @@ ) from systemds.scuro.modality.modality import Modality from systemds.scuro.drsearch.operator_registry import Registry +from systemds.scuro.representations.window_aggregation import WindowAggregation from systemds.scuro.utils.checkpointing import CheckpointManager from systemds.scuro.drsearch.representation_dag import ( RepresentationDag, @@ -63,6 +66,8 @@ def __init__( resume: bool = False, max_num_workers: int = -1, enable_checkpointing: bool = True, + enable_execution_profile: bool = False, + execution_profile_path: Optional[str] = None, ): self.enable_checkpointing = enable_checkpointing self.modalities = modalities @@ -92,7 +97,8 @@ def __init__( self.operator_performance = UnimodalResults( modalities, tasks, debug, True, k, self.metric_name ) - + self.enable_execution_profile = enable_execution_profile + self.execution_profile_path = execution_profile_path self._tasks_require_same_dims = True self.expected_dimensions = tasks[0].expected_dim @@ -242,7 +248,7 @@ def optimize(self): for modality in self.modalities: try: - local_result = self._process_modality( + local_result, execution_time = self._process_modality( modality, ( int( @@ -274,6 +280,7 @@ def optimize(self): self.operator_performance.results, {} ) raise + return execution_time def _expand_dags_with_task_roots( self, dags: List[RepresentationDag] @@ -281,7 +288,16 @@ def _expand_dags_with_task_roots( expanded_dags: List[RepresentationDag] = [] for dag in dags: - root_id = dag.root_node_id + dag = copy.deepcopy(dag) + root_node = dag.get_node_by_id(dag.root_node_id) + if root_node and root_node.operation == AggregatedRepresentation: + aggregation = root_node + dag.nodes = [n for n in dag.nodes if n.node_id != root_node.node_id] + root_id = aggregation.inputs[0] + dag.root_node_id = root_id + else: + aggregation = None + root_id = dag.root_node_id for task_idx, _ in enumerate(self.tasks): task_node_id = f"task_{root_id}_{task_idx}" @@ -294,6 +310,7 @@ def _expand_dags_with_task_roots( "_task_idx": task_idx, "_dag_root_id": root_id, }, + aggregation=aggregation, ) task_root_dag = RepresentationDag( @@ -326,12 +343,18 @@ def _process_modality(self, modality, skip_remaining: int = 0, scheduler=None): self.result_path, enable_checkpointing=self.enable_checkpointing, ) - + start_time = time.perf_counter() exec_out = node_executor.run() + end_time = time.perf_counter() task_results = exec_out["task_results"] for task_result in task_results: local_results.add_task_result(task_result, dags) + statistics = exec_out["statistics"] + for worker_stat in statistics["worker_stats"]: + local_results.add_worker_stat(worker_stat, modality.modality_id) + for node_stat in statistics["node_stats"]: + local_results.add_node_stat(node_stat, modality.modality_id) if self.save_all_results: timestr = time.strftime("%Y%m%d-%H%M%S") @@ -339,7 +362,7 @@ def _process_modality(self, modality, skip_remaining: int = 0, scheduler=None): with open(file_name, "wb") as f: pickle.dump(local_results.results, f) - return local_results + return local_results, end_time - start_time def _build_execution_dags_for_modality( self, modality: Modality, skip_remaining: int = 0 @@ -354,6 +377,7 @@ def _build_execution_dags_for_modality( if ( modality.modality_type == ModalityType.TIMESERIES or modality.modality_type == ModalityType.AUDIO + or modality.modality_type == ModalityType.PHYSIOLOGICAL ): dags.extend( self.temporal_context_operators( @@ -509,7 +533,6 @@ def _build_modality_dag( def _aggregation_needed(self, dag: RepresentationDag) -> bool: input_stats = {} - # TODO: adapt this to the fusion of multiple modalities, list of input stats needed for modality in self.modalities: if modality.modality_id == dag.nodes[0].modality_id: input_stats[dag.nodes[0].node_id] = modality.stats @@ -584,16 +607,26 @@ def temporal_context_operators(self, modality, builder, leaf_id): modality.modality_type ) context_operators = self._get_context_operators(modality.modality_type) - + window_lengths, num_windows = ( + self.operator_registry.get_context_lenghts_for_modality( + modality.modality_type, modality.stats + ) + ) dags = [] for agg in aggregators: for context_operator in context_operators: - context_node_id = builder.create_operation_node( - context_operator, - [leaf_id], - context_operator(agg()).get_current_parameters(), - ) - dags.append(builder.build(context_node_id)) + for window_size, num_window in zip(window_lengths, num_windows): + context_operator_instance = context_operator(agg()) + if hasattr(context_operator, "num_windows"): + context_operator_instance.num_windows = num_window + elif hasattr(context_operator_instance, "window_size"): + context_operator_instance.window_size = window_size + context_node_id = builder.create_operation_node( + context_operator, + [leaf_id], + context_operator_instance.get_current_parameters(), + ) + dags.append(builder.build(context_node_id)) return dags @@ -619,6 +652,8 @@ def __init__( for modality in self.modality_ids: self.results[modality] = {task_name: [] for task_name in self.task_names} self.cache[modality] = {task_name: [] for task_name in self.task_names} + self.worker_stats = {} + self.node_stats = {} def add_task_result(self, task_result: ResultEntry, dags: List[RepresentationDag]): dag_id = task_result.dag.dag_id @@ -750,6 +785,12 @@ def get_k_best_results( return results, cache + def add_worker_stat(self, worker_stats, modality_id): + self.worker_stats[modality_id] = worker_stats + + def add_node_stat(self, node_stats, modality_id): + self.node_stats[modality_id] = node_stats + def get_dag_by_id(dags: List[RepresentationDag], dag_id: int) -> RepresentationDag: for dag in dags: diff --git a/src/main/python/systemds/scuro/modality/type.py b/src/main/python/systemds/scuro/modality/type.py index 0493edf5bdd..a648044562a 100644 --- a/src/main/python/systemds/scuro/modality/type.py +++ b/src/main/python/systemds/scuro/modality/type.py @@ -66,6 +66,11 @@ class ModalitySchemas: "num_columns": "integer", } + PHYSIOLOGICAL_SCHEMA = { + **TEMPORAL_BASE_SCHEMA, + "num_columns": "integer", + } + _metadata_handlers = {} @classmethod @@ -184,6 +189,22 @@ def handle_timeseries_metadata(md, data): return md +@ModalitySchemas.register_metadata_handler("PHYSIOLOGICAL") +def handle_physiological_metadata(md, data): + new_frequency = calculate_new_frequency(len(data), md["length"], md["frequency"]) + md.update( + { + "length": len(data), + "num_columns": ( + data.shape[1] if isinstance(data, np.ndarray) and data.ndim > 1 else 1 + ), + "frequency": new_frequency, + "timestamp": create_timestamps(new_frequency, len(data)), + } + ) + return md + + @ModalitySchemas.register_metadata_handler("TEXT") def handle_text_metadata(md, data): md.update({"length": len(data)}) @@ -205,6 +226,7 @@ class ModalityType(Flag): "VIDEO": "create_video_metadata", "IMAGE": "create_image_metadata", "TIMESERIES": "create_ts_metadata", + "PHYSIOLOGICAL": "create_ts_metadata", "EMBEDDING": "create_embedding_metadata", } diff --git a/src/main/python/systemds/scuro/representations/lstm.py b/src/main/python/systemds/scuro/representations/lstm.py index 7243b65966a..c15776284ce 100644 --- a/src/main/python/systemds/scuro/representations/lstm.py +++ b/src/main/python/systemds/scuro/representations/lstm.py @@ -25,7 +25,7 @@ from torch import nn from torch.utils.data import DataLoader, TensorDataset from typing import List, Dict, Any -from systemds.scuro.utils.static_variables import get_device, get_device_for_model +from systemds.scuro.utils.static_variables import get_device import numpy as np from systemds.scuro.modality.modality import Modality diff --git a/src/main/python/systemds/scuro/representations/mel_spectrogram.py b/src/main/python/systemds/scuro/representations/mel_spectrogram.py index 6d378806475..3cf25b44c4a 100644 --- a/src/main/python/systemds/scuro/representations/mel_spectrogram.py +++ b/src/main/python/systemds/scuro/representations/mel_spectrogram.py @@ -36,6 +36,14 @@ PY_LIST_SLOT_BYTES, ) +import warnings + +warnings.filterwarnings( + "ignore", + message=r"n_fft=\d+ is too (?:small|large) for input signal", + module=r"librosa", +) + @register_representation(ModalityType.AUDIO) @register_context_representation_operator(ModalityType.AUDIO) @@ -56,6 +64,7 @@ def __init__(self, n_mels=128, hop_length=512, n_fft=2048, params=None): self.n_mels = int(n_mels) self.hop_length = int(hop_length) self.n_fft = int(n_fft) + self.window_size = self.n_fft def transform(self, modality, aggregation=None): transformed_modality = TransformedModality( diff --git a/src/main/python/systemds/scuro/representations/mfcc.py b/src/main/python/systemds/scuro/representations/mfcc.py index 406fc6616c1..483ae3eef81 100644 --- a/src/main/python/systemds/scuro/representations/mfcc.py +++ b/src/main/python/systemds/scuro/representations/mfcc.py @@ -35,43 +35,54 @@ PY_LIST_HEADER_BYTES, PY_LIST_SLOT_BYTES, ) +import warnings + +warnings.filterwarnings( + "ignore", + message=r"n_fft=\d+ is too (?:small|large) for input signal", + module=r"librosa", +) @register_representation(ModalityType.AUDIO) @register_context_representation_operator(ModalityType.AUDIO) class MFCC(UnimodalRepresentation): - def __init__(self, n_mfcc=12, dct_type=2, n_mels=128, hop_length=512, params=None): + def __init__( + self, n_mfcc=12, dct_type=2, n_mels=128, hop_length=512, n_fft=2048, params=None + ): parameters = { "n_mfcc": [x for x in range(10, 26)], "dct_type": [1, 2, 3], "hop_length": [256, 512, 1024, 2048], "n_mels": [20, 32, 64, 128], - } # TODO + "n_fft": [1024, 2048, 4096], + } + super().__init__("MFCC", ModalityType.TIMESERIES, parameters, False) - # Allow construction from a parameter dict (used by optimizer) if params is not None: n_mfcc = params.get("n_mfcc", n_mfcc) dct_type = params.get("dct_type", dct_type) n_mels = params.get("n_mels", n_mels) hop_length = params.get("hop_length", hop_length) + n_fft = params.get("n_fft", n_fft) self.n_mfcc = int(n_mfcc) self.dct_type = int(dct_type) self.n_mels = int(n_mels) self.hop_length = int(hop_length) + self.n_fft = int(n_fft) + self.window_size = self.n_fft def transform(self, modality, aggregation=None): transformed_modality = TransformedModality( modality, self, self.output_modality_type ) result = [] - - for i, sample in enumerate(modality.data): - sr = modality.metadata[i]["frequency"] - computed_feature = self.compute_feature(sample, sr) + sr = modality.metadata[0]["frequency"] if modality.metadata else 22050 + for sample in modality.data: + computed_feature = self.compute_feature(sample, sr=sr) result.append(computed_feature) - transformed_modality.data = result return transformed_modality @@ -79,20 +90,22 @@ def compute_feature(self, instance, sr=None): if sr is None: sr = 22050 mfcc = librosa.feature.mfcc( - y=np.array(instance), + y=np.asarray(instance, dtype=np.float32), sr=sr, n_mfcc=self.n_mfcc, dct_type=self.dct_type, hop_length=self.hop_length, n_mels=self.n_mels, + n_fft=self.n_fft, ) if mfcc.ndim == 2: - mean = np.mean(mfcc, keepdims=True) - std = np.std(mfcc, keepdims=True) + mean = mfcc.mean(keepdims=True) + std = mfcc.std(keepdims=True) else: - mean = np.mean(mfcc, axis=(1, 2), keepdims=True) - std = np.std(mfcc, axis=(1, 2), keepdims=True) - mfcc = (mfcc - mean) / np.maximum(std, 1e-8) + mean = mfcc.mean(axis=(1, 2), keepdims=True) + std = mfcc.std(axis=(1, 2), keepdims=True) + mfcc -= mean + mfcc /= np.maximum(std, 1e-8) if instance.ndim == 1: return mfcc.T diff --git a/src/main/python/systemds/scuro/representations/mlp_averaging.py b/src/main/python/systemds/scuro/representations/mlp_averaging.py index 46fe04899aa..8c8d67a06ec 100644 --- a/src/main/python/systemds/scuro/representations/mlp_averaging.py +++ b/src/main/python/systemds/scuro/representations/mlp_averaging.py @@ -54,12 +54,8 @@ def __init__(self, output_dim=512, batch_size=32, params=None): "batch_size": [8, 16, 32, 64, 128], } super().__init__("MLPAveraging", parameters) - if params is not None: - self.output_dim = params.get("output_dim", output_dim) - self.batch_size = params.get("batch_size", batch_size) - else: - self.output_dim = output_dim - self.batch_size = batch_size + self.output_dim = output_dim + self.batch_size = batch_size self.device = None self.data_type = np.float32 self.gpu_id = None @@ -74,10 +70,7 @@ def gpu_id(self, gpu_id): self.device = get_device(gpu_id) def get_output_stats(self, input_stats: RepresentationStats) -> RepresentationStats: - if ( - len(input_stats.output_shape) > 1 - and np.prod(input_stats.output_shape) > self.output_dim - ): + if len(input_stats.output_shape) > 1: return RepresentationStats( input_stats.num_instances, (self.output_dim,), @@ -94,13 +87,7 @@ def get_output_stats(self, input_stats: RepresentationStats) -> RepresentationSt ) return RepresentationStats( input_stats.num_instances, - ( - ( - np.prod(input_stats.output_shape) - if np.prod(input_stats.output_shape) < self.output_dim - else self.output_dim - ), - ), + (self.output_dim,), output_shape_is_known=input_stats.output_shape_is_known, ) diff --git a/src/main/python/systemds/scuro/representations/multimodal_attention_fusion.py b/src/main/python/systemds/scuro/representations/multimodal_attention_fusion.py index 066f3432159..af5a83c7e50 100644 --- a/src/main/python/systemds/scuro/representations/multimodal_attention_fusion.py +++ b/src/main/python/systemds/scuro/representations/multimodal_attention_fusion.py @@ -60,6 +60,13 @@ def __init__( self.batch_size = int(batch_size) self.num_epochs = int(num_epochs) self.learning_rate = float(learning_rate) + if params is not None: + self.hidden_dim = int(params.get("hidden_dim", self.hidden_dim)) + self.num_heads = int(params.get("num_heads", self.num_heads)) + self.dropout = float(params.get("dropout", self.dropout)) + self.batch_size = int(params.get("batch_size", self.batch_size)) + self.num_epochs = int(params.get("num_epochs", self.num_epochs)) + self.learning_rate = float(params.get("learning_rate", self.learning_rate)) self.needs_training = True self.needs_alignment = True diff --git a/src/main/python/systemds/scuro/representations/spectrogram.py b/src/main/python/systemds/scuro/representations/spectrogram.py index ab0c8a6c649..9d14931ef0e 100644 --- a/src/main/python/systemds/scuro/representations/spectrogram.py +++ b/src/main/python/systemds/scuro/representations/spectrogram.py @@ -35,6 +35,13 @@ PY_LIST_HEADER_BYTES, PY_LIST_SLOT_BYTES, ) +import warnings + +warnings.filterwarnings( + "ignore", + message=r"n_fft=\d+ is too (?:small|large) for input signal", + module=r"librosa", +) @register_representation(ModalityType.AUDIO) @@ -45,6 +52,7 @@ def __init__(self, hop_length=512, n_fft=2048, params=None): super().__init__("Spectrogram", ModalityType.TIMESERIES, parameters, False) self.hop_length = int(hop_length) self.n_fft = int(n_fft) + self.window_size = self.n_fft def transform(self, modality, aggregation=None): transformed_modality = TransformedModality( diff --git a/src/main/python/systemds/scuro/representations/timeseries_representations.py b/src/main/python/systemds/scuro/representations/timeseries_representations.py index 26c8c1a8d98..14fcacf724f 100644 --- a/src/main/python/systemds/scuro/representations/timeseries_representations.py +++ b/src/main/python/systemds/scuro/representations/timeseries_representations.py @@ -51,9 +51,13 @@ def transform(self, modality, aggregation=None): feature = self.compute_feature(signal) result.append(feature) - transformed_modality.data = np.vstack(np.array(result)).astype( - modality.metadata[0]["data_layout"]["type"] - ) + maxlen = max(r.size for r in result) + padded_result = [ + np.pad(r, (0, maxlen - r.size), mode="constant", constant_values=0.0) + for r in result + ] + dtype = modality.metadata[0]["data_layout"]["type"] + transformed_modality.data = np.vstack(np.asarray(padded_result)).astype(dtype) return transformed_modality def get_output_stats(self, input_stats): @@ -71,8 +75,10 @@ def estimate_peak_memory_bytes(self, input_stats): } -@register_representation([ModalityType.TIMESERIES]) +@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) @register_context_representation_operator(ModalityType.TIMESERIES) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +@register_context_representation_operator(ModalityType.AUDIO) class Mean(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("Mean") @@ -81,8 +87,9 @@ def compute_feature(self, signal, axis=-1): return np.array(np.mean(signal, axis=axis)) -@register_representation([ModalityType.TIMESERIES]) +@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) @register_context_representation_operator(ModalityType.TIMESERIES) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) class Min(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("Min") @@ -91,8 +98,9 @@ def compute_feature(self, signal, axis=-1): return np.array(np.min(signal, axis=axis)) -@register_representation([ModalityType.TIMESERIES]) +@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) @register_context_representation_operator(ModalityType.TIMESERIES) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) class Max(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("Max") @@ -101,8 +109,9 @@ def compute_feature(self, signal, axis=-1): return np.array(np.max(signal, axis=axis)) -@register_representation([ModalityType.TIMESERIES]) +@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) @register_context_representation_operator(ModalityType.TIMESERIES) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) class Sum(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("Sum") @@ -111,8 +120,10 @@ def compute_feature(self, signal, axis=-1): return np.array(np.sum(signal, axis=axis)) -@register_representation([ModalityType.TIMESERIES]) +@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) @register_context_representation_operator(ModalityType.TIMESERIES) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +@register_context_representation_operator(ModalityType.AUDIO) class Std(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("Std") @@ -121,8 +132,10 @@ def compute_feature(self, signal, axis=-1): return np.array(np.std(signal, axis=axis)) -@register_representation([ModalityType.TIMESERIES]) +@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) @register_context_representation_operator(ModalityType.TIMESERIES) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +@register_context_representation_operator(ModalityType.AUDIO) class Skew(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("Skew") @@ -131,12 +144,13 @@ def compute_feature(self, signal, axis=-1): return np.array(stats.skew(signal, axis=axis)) -@register_representation([ModalityType.TIMESERIES]) +@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) @register_context_representation_operator(ModalityType.TIMESERIES) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) class Quantile(TimeSeriesRepresentation): def __init__(self, quantile=0.9, params=None): super().__init__( - "Qunatile", {"quantile": [0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99]} + "Qunatile", {"quantile": [0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99]} ) self.quantile = quantile @@ -144,8 +158,10 @@ def compute_feature(self, signal, axis=-1): return np.array(np.quantile(signal, self.quantile, axis=axis)) -@register_representation([ModalityType.TIMESERIES]) +@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) @register_context_representation_operator(ModalityType.TIMESERIES) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +@register_context_representation_operator(ModalityType.AUDIO) class Kurtosis(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("Kurtosis") @@ -154,8 +170,10 @@ def compute_feature(self, signal, axis=-1): return np.array(stats.kurtosis(signal, fisher=True, bias=True, axis=axis)) -@register_representation([ModalityType.TIMESERIES]) +@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) @register_context_representation_operator(ModalityType.TIMESERIES) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +@register_context_representation_operator(ModalityType.AUDIO) class RMS(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("RMS") @@ -164,8 +182,9 @@ def compute_feature(self, signal, axis=-1): return np.array(np.sqrt(np.mean(np.square(signal), axis=axis))) -@register_representation([ModalityType.TIMESERIES]) +@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) @register_context_representation_operator(ModalityType.TIMESERIES) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) class ZeroCrossingRate(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("ZeroCrossingRate") @@ -174,8 +193,9 @@ def compute_feature(self, signal, axis=-1): return np.array(np.sum(np.diff(np.signbit(signal), axis=axis) != 0, axis=axis)) -@register_representation([ModalityType.TIMESERIES]) +@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) @register_context_representation_operator(ModalityType.TIMESERIES) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) class ACF(TimeSeriesRepresentation): def __init__(self, k=1, params=None): super().__init__("ACF", {"k": [1, 2, 5, 10, 20, 25, 50, 100, 200, 500]}) @@ -209,8 +229,9 @@ def get_k_values(self, max_length, percent=0.2, num=10, log=False): return k_vals.tolist() -@register_representation([ModalityType.TIMESERIES]) +@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) @register_context_representation_operator(ModalityType.TIMESERIES) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) class FrequencyMagnitude(TimeSeriesRepresentation): def __init__(self, params=None): super().__init__("FrequencyMagnitude") @@ -219,7 +240,8 @@ def compute_feature(self, signal, axis=-1): return np.array(np.abs(np.fft.rfft(signal, axis=axis))) -@register_representation([ModalityType.TIMESERIES]) +@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) @register_context_representation_operator(ModalityType.TIMESERIES) class SpectralCentroid(TimeSeriesRepresentation): def __init__(self, fs=1.0, params=None): @@ -240,7 +262,8 @@ def compute_feature(self, signal, axis=-1): return np.array(num / den) -@register_representation([ModalityType.TIMESERIES]) +@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) @register_context_representation_operator(ModalityType.TIMESERIES) class BandpowerFFT(TimeSeriesRepresentation): def __init__(self, fs=1.0, f1=0.0, f2=0.5, params=None): diff --git a/src/main/python/systemds/scuro/representations/wav2vec.py b/src/main/python/systemds/scuro/representations/wav2vec.py index ece034e099b..5e03baf8bc4 100644 --- a/src/main/python/systemds/scuro/representations/wav2vec.py +++ b/src/main/python/systemds/scuro/representations/wav2vec.py @@ -28,10 +28,11 @@ from systemds.scuro.representations.representation import RepresentationStats from systemds.scuro.representations.unimodal import UnimodalRepresentation from systemds.scuro.drsearch.operator_registry import register_representation +from systemds.scuro.utils.memory_utility import get_device -import warnings +from transformers.utils import logging as transformers_logging -warnings.filterwarnings("ignore", message="Some weights of") +transformers_logging.set_verbosity_error() @register_representation(ModalityType.AUDIO) diff --git a/src/main/python/systemds/scuro/representations/window_aggregation.py b/src/main/python/systemds/scuro/representations/window_aggregation.py index 36e12c0cd5c..52a17401959 100644 --- a/src/main/python/systemds/scuro/representations/window_aggregation.py +++ b/src/main/python/systemds/scuro/representations/window_aggregation.py @@ -155,11 +155,21 @@ def __init__( pad=True, params=None, ): + window_size_set = False + if isinstance(aggregation_function, Representation) and hasattr( + aggregation_function, "window_size" + ): + window_size = aggregation_function.window_size + window_size_set = True + if params is not None: if isinstance( params.get("aggregation_function"), (Aggregation, Representation) ): aggregation_function = params["aggregation_function"] + if hasattr(aggregation_function, "window_size"): + window_size = aggregation_function.window_size + window_size_set = True else: nested_agg = { key[len("aggregation_function_") :]: value @@ -177,7 +187,8 @@ def __init__( aggregation_function = params.get( "aggregation_function", aggregation_function ) - window_size = params["window_size"] + + window_size = params["window_size"] if not window_size_set else window_size pad = params.get("pad", True) super().__init__("WindowAggregation", aggregation_function) self.parameters["window_size"] = (4, 128) @@ -234,6 +245,7 @@ def estimate_peak_memory_bytes(self, input_stats: RepresentationStats) -> dict: effective_seq_len = in_shape[0] in_numel = effective_seq_len * self._rest_numel(in_shape) + output_bytes = self.estimate_output_memory_bytes(input_stats) one_instance_bytes = in_numel * np.dtype(self.data_type).itemsize input_bytes = one_instance_bytes * input_stats.num_instances @@ -269,6 +281,7 @@ def execute(self, modality): ) original_lengths.append(windowed_instance.shape[0]) windowed_data.append(windowed_instance) + if self.pad and not isinstance(windowed_data, np.ndarray): target_length = max(original_lengths) @@ -322,7 +335,17 @@ def window_aggregate_single_level(self, instance, new_length): full_result = self.aggregation_function.compute_feature(full_batches) if tail.size: tail_result = self.aggregation_function.compute_feature(tail) - full_result = np.concatenate([full_result, tail_result[None, :]]) + if tail_result.shape == full_result.shape[1:]: + tail_row = tail_result + else: + tail_row = np.zeros_like(full_result[0]) + slices = tuple( + slice(0, min(d, s)) + for d, s in zip(tail_row.shape, tail_result.shape) + ) + tail_row[slices] = tail_result[slices] + full_result = np.concatenate([full_result, tail_row[None, :]]) + return full_result def window_aggregate_nested_level(self, instance, new_length): diff --git a/src/main/python/tests/scuro/data_generator.py b/src/main/python/tests/scuro/data_generator.py index a51ea510ea3..937fd622d85 100644 --- a/src/main/python/tests/scuro/data_generator.py +++ b/src/main/python/tests/scuro/data_generator.py @@ -75,6 +75,7 @@ def __init__(self, indices, chunk_size, modality_type, data, data_type, metadata self.stats = VideoStats( 30, max(d.shape[0] for d in data), + sum(d.shape[0] for d in data) / len(data), max(d.shape[1] for d in data), max(d.shape[2] for d in data), max(d.shape[3] for d in data), @@ -88,6 +89,8 @@ def __init__(self, indices, chunk_size, modality_type, data, data_type, metadata sum(len(d) for d in data) / len(data), (max(len(d) for d in data),), True, + sum(len(d) for d in data) / len(data), + 16000, ) elif modality_type == ModalityType.IMAGE: self.stats = ImageStats( From aca41a492964a1ed7984e65812a2e4678b3b6ddf Mon Sep 17 00:00:00 2001 From: Jessica Priebe Date: Wed, 1 Jul 2026 17:00:10 +0200 Subject: [PATCH 059/132] [SYSTEMDS-3891] Add OOC reshape Closes #2521. --- .../sysds/runtime/data/DenseBlockFP64.java | 20 + .../instructions/OOCInstructionParser.java | 3 +- .../instructions/ooc/CachingStream.java | 1 + .../instructions/ooc/OOCInstruction.java | 2 +- .../instructions/ooc/ReorgOOCInstruction.java | 42 +- .../ooc/ReshapeOOCInstruction.java | 437 ++++++++++++++++++ .../sysds/test/functions/ooc/ReshapeTest.java | 160 +++++++ .../functions/ooc/MatrixReshapeColWise.dml | 26 ++ .../functions/ooc/MatrixReshapeRowWise.dml | 26 ++ 9 files changed, 676 insertions(+), 41 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java create mode 100644 src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java create mode 100644 src/test/scripts/functions/ooc/MatrixReshapeColWise.dml create mode 100644 src/test/scripts/functions/ooc/MatrixReshapeRowWise.dml diff --git a/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64.java b/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64.java index 94909444198..0a734261b9f 100644 --- a/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64.java +++ b/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64.java @@ -181,6 +181,26 @@ public DenseBlock set(int rl, int ru, int ol, int ou, DenseBlock db) { return this; } + public DenseBlock setPartialRow(DenseBlock row, int rIdx, int srcOffset, int destOffset, int length) { + if(destOffset + length > _odims[0]) + throw new RuntimeException( + "Partial row assignment exceeds row length: " + (destOffset + length) + " > " + _odims[0]); + System.arraycopy(row.valuesAt(0), srcOffset, _data, this.pos(rIdx, destOffset), length); + return this; + } + + public DenseBlock setPartialCol(DenseBlock col, int cIdx, int srcOffset, int destOffset, int length) { + if(destOffset + length > _rlen) + throw new RuntimeException( + "Partial column assignment exceeds column length: " + (destOffset + length) + " > " + _rlen); + int destPos = this.pos(destOffset, cIdx); + double[] src = col.valuesAt(0); + for(int i = 0; i < length; i++) { + _data[destPos + i * _odims[0]] = src[srcOffset + i]; + } + return this; + } + @Override public DenseBlock set(int r, double[] v) { System.arraycopy(v, 0, _data, pos(r), _odims[0]); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/OOCInstructionParser.java b/src/main/java/org/apache/sysds/runtime/instructions/OOCInstructionParser.java index ae41639687b..98a454283e2 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/OOCInstructionParser.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/OOCInstructionParser.java @@ -43,6 +43,7 @@ import org.apache.sysds.runtime.instructions.ooc.ReorgOOCInstruction; import org.apache.sysds.runtime.instructions.ooc.TeeOOCInstruction; import org.apache.sysds.runtime.instructions.ooc.AppendOOCInstruction; +import org.apache.sysds.runtime.instructions.ooc.ReshapeOOCInstruction; import org.apache.sysds.runtime.instructions.ooc.QuaternaryOOCInstruction; public class OOCInstructionParser extends InstructionParser { @@ -97,7 +98,7 @@ else if(parts.length == 4) case Reorg: return ReorgOOCInstruction.parseInstruction(str); case Reshape: - return ReorgOOCInstruction.parseInstruction(str); + return ReshapeOOCInstruction.parseInstruction(str); case Tee: return TeeOOCInstruction.parseInstruction(str); case CentralMoment: diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java index 56c265fe5e6..38929dcafcc 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java @@ -99,6 +99,7 @@ public CachingStream(OOCStream source, long streamId) { // Capture a short context to help identify origin OOCWatchdog.registerOpen(_watchdogId, toString(), getCtxMsg(), this); } + activateIndexing(); _downstreamRelays = null; source.setSubscriber(tmp -> { try(tmp) { diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java index 859bca42dfe..80d71231646 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java @@ -80,7 +80,7 @@ public abstract class OOCInstruction extends Instruction { public enum OOCType { Reblock, Tee, Binary, Ternary, Unary, AggregateUnary, AggregateBinary, AggregateTernary, MAPMM, MMTSJ, - MAPMMCHAIN, Reorg, CM, Ctable, MatrixIndexing, ParameterizedBuiltin, Rand, Append, Quaternary + MAPMMCHAIN, Reorg, CM, Ctable, MatrixIndexing, ParameterizedBuiltin, Rand, Append, Quaternary, Reshape } protected final OOCInstruction.OOCType _ooctype; diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java index cf77d559727..273d33341ab 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java @@ -41,31 +41,17 @@ public class ReorgOOCInstruction extends ComputationOOCInstruction { private final CPOperand _col; private final CPOperand _desc; private final CPOperand _ixret; - // reshape-specific attributes - private final CPOperand _opRows; - private final CPOperand _opCols; - //private final CPOperand _opDims; - private final CPOperand _opByRow; protected ReorgOOCInstruction(ReorgOperator op, CPOperand in1, CPOperand out, String opcode, String istr) { - this(op, in1, out, null, null, null, null, null, null, null, opcode, istr); - } - - private ReorgOOCInstruction(Operator op, CPOperand in, CPOperand out, CPOperand opRows, CPOperand opCols, - CPOperand opDims, CPOperand opByRow, String opcode, String istr) { - this(op, in, out, null, null, null, opRows, opCols, opDims, opByRow, opcode, istr); + this(op, in1, out, null, null, null, opcode, istr); } private ReorgOOCInstruction(Operator op, CPOperand in, CPOperand out, CPOperand col, CPOperand desc, CPOperand ixret, - CPOperand opRows, CPOperand opCols, CPOperand opDims, CPOperand opByRow, String opcode, String istr) { + String opcode, String istr) { super(OOCType.Reorg, op, in, out, opcode, istr); _col = col; _desc = desc; _ixret = ixret; - _opRows = opRows; - _opCols = opCols; - //_opDims = opDims; - _opByRow = opByRow; } public static ReorgOOCInstruction parseInstruction(String str) { @@ -92,35 +78,13 @@ else if(opcode.equalsIgnoreCase(Opcodes.SORT.toString())) { CPOperand ixret = new CPOperand(parts[4]); int k = Integer.parseInt(parts[6]); return new ReorgOOCInstruction(new ReorgOperator(new SortIndex(1, false, false), k), - in, out, col, desc, ixret, null, null, null, null, opcode, str); - } - else if(opcode.equalsIgnoreCase(Opcodes.RESHAPE.toString())) { - InstructionUtils.checkNumFields(parts, 6); - in.split(parts[1]); - CPOperand rows = new CPOperand(parts[2]); - CPOperand cols = new CPOperand(parts[3]); - CPOperand dims = new CPOperand(parts[4]); - CPOperand byRow = new CPOperand(parts[5]); - out.split(parts[6]); - return new ReorgOOCInstruction(new Operator(true), in, out, rows, cols, dims, byRow, opcode, str); + in, out, col, desc, ixret, opcode, str); } else throw new NotImplementedException(); } public void processInstruction( ExecutionContext ec ) { - if(getOpcode().equalsIgnoreCase(Opcodes.RESHAPE.toString())) { - // TODO Make reshape truly out-of-core - int rows = (int) ec.getScalarInput(_opRows).getLongValue(); - int cols = (int) ec.getScalarInput(_opCols).getLongValue(); - boolean byRow = ec.getScalarInput(_opByRow).getBooleanValue(); - MatrixBlock in = ec.getMatrixInput(input1.getName()); - MatrixBlock out = in.reshape(rows, cols, byRow); - ec.releaseMatrixInput(input1.getName()); - ec.setMatrixOutput(output.getName(), out); - return; - } - // Create thread and process the transpose/sort operation MatrixObject min = ec.getMatrixObject(input1); ReorgOperator r_op = (ReorgOperator) _optr; diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java new file mode 100644 index 00000000000..7590438b949 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java @@ -0,0 +1,437 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.instructions.ooc; + +import org.apache.sysds.common.Opcodes; +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; +import org.apache.sysds.runtime.controlprogram.context.ExecutionContext; +import org.apache.sysds.runtime.data.DenseBlockFP64; +import org.apache.sysds.runtime.instructions.InstructionUtils; +import org.apache.sysds.runtime.instructions.cp.CPOperand; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.matrix.operators.Operator; + +import java.util.ArrayList; +import java.util.concurrent.CompletableFuture; + +public class ReshapeOOCInstruction extends ComputationOOCInstruction { + private final CPOperand _opRows; + private final CPOperand _opCols; + // private final CPOperand _opDims; + private final CPOperand _opByRow; + + private ReshapeOOCInstruction(Operator op, CPOperand in, CPOperand out, CPOperand rows, CPOperand cols, + CPOperand dims, CPOperand byRow, String opcode, String istr) { + super(OOCType.Reshape, op, in, out, opcode, istr); + _opRows = rows; + _opCols = cols; + // _opDims = dims; + _opByRow = byRow; + } + + public static ReshapeOOCInstruction parseInstruction(String str) { + String[] parts = InstructionUtils.getInstructionPartsWithValueType(str); + InstructionUtils.checkNumFields(parts, 6); + String opcode = parts[0]; + + if(!opcode.equalsIgnoreCase(Opcodes.RESHAPE.toString())) + throw new DMLRuntimeException("Unknown opcode while parsing ReshapeInstruction: " + str); + + CPOperand in = new CPOperand(parts[1]); + CPOperand rows = new CPOperand(parts[2]); + CPOperand cols = new CPOperand(parts[3]); + CPOperand dims = new CPOperand(parts[4]); + CPOperand byRow = new CPOperand(parts[5]); + CPOperand out = new CPOperand(parts[6]); + + return new ReshapeOOCInstruction(new Operator(true), in, out, rows, cols, dims, byRow, opcode, str); + } + + public void processInstruction(ExecutionContext ec) { + long rows = ec.getScalarInput(_opRows).getLongValue(); + long cols = ec.getScalarInput(_opCols).getLongValue(); + boolean byRow = ec.getScalarInput(_opByRow).getBooleanValue(); + + OOCStream qOut = createWritableStream(); + ec.getMatrixObject(output).setStreamHandle(qOut); + + MatrixObject in = ec.getMatrixObject(input1); + OOCStream qIn = in.getStreamHandle(); + int blen = in.getBlocksize(); + long rlen = in.getNumRows(); + long clen = in.getNumColumns(); + + if(rlen * clen != rows * cols) + throw new DMLRuntimeException("Reshape matrix requires consistent numbers of input/output cells (" + rlen + + ":" + clen + ", " + rows + ":" + cols + ")."); + + if(rlen == rows) { + mapOOC(qIn, qOut, tmp -> tmp); + return; + } + + if(clen <= blen && rlen <= blen && cols <= blen && rows <= blen) { + mapOOC(qIn, qOut, tmp -> { + MatrixBlock res = ((MatrixBlock) tmp.getValue()).reshape((int) rows, (int) cols, byRow); + return new IndexedMatrixValue(tmp.getIndexes(), res); + }); + return; + } + + int numBlocksPerRowIn = (int) Math.ceil((double) clen / blen); + int numBlocksPerColIn = (int) Math.ceil((double) rlen / blen); + int numBlocksPerRowOut = (int) Math.ceil((double) cols / blen); + int numBlocksPerColOut = (int) Math.ceil((double) rows / blen); + + if(byRow) { + OOCStream singleRowBlocks = new SubscribableTaskQueue<>(); + // split blocks into single rows and adapt index + CompletableFuture f = expandOOC(qIn, singleRowBlocks, tmp -> { + ArrayList out = new ArrayList<>(); + MatrixBlock blk = (MatrixBlock) tmp.getValue(); + for(int i = 0; i < blk.getNumRows(); i++) { + MatrixBlock slice = blk.slice(i, i); + long r = tmp.getIndexes().getRowIndex(); + long c = tmp.getIndexes().getColumnIndex(); + r = (r - 1) * blen + i + 1; + MatrixIndexes idx = new MatrixIndexes(r, c); + out.add(new IndexedMatrixValue(idx, slice)); + } + return out; + }); + + if(clen % blen == 0 && cols % blen == 0) { + // singleRowBlocks do not need to be split + if(rows == 1) { + // result is one single row + mapOOC(singleRowBlocks.getReadStream(), qOut, tmp -> { + long r = tmp.getIndexes().getRowIndex(); + long c = tmp.getIndexes().getColumnIndex(); + // adapt index to new position in row + return new IndexedMatrixValue(new MatrixIndexes(1, (r - 1) * numBlocksPerRowIn + c), tmp.getValue()); + }); + } + else { + f.join(); + reshapeFullColBlocks(rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, singleRowBlocks, qOut); + } + } + else { + f.join(); + reshapePartialColBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, singleRowBlocks, qOut); + } + } + else { + OOCStream singleColBlocks = new SubscribableTaskQueue<>(); + // split blocks into single cols and adapt index + CompletableFuture f = expandOOC(qIn, singleColBlocks, tmp -> { + ArrayList out = new ArrayList<>(); + MatrixBlock blk = (MatrixBlock) tmp.getValue(); + for(int i = 0; i < blk.getNumColumns(); i++) { + MatrixBlock slice = blk.slice(0, blk.getNumRows() - 1, i, i); + long r = tmp.getIndexes().getRowIndex(); + long c = tmp.getIndexes().getColumnIndex(); + c = (c - 1) * blen + i + 1; + MatrixIndexes idx = new MatrixIndexes(r, c); + out.add(new IndexedMatrixValue(idx, slice)); + } + return out; + }); + + if(rlen % blen == 0 && rows % blen == 0) { + // cols do not need to be split + if(cols == 1) { + // result is one single col + mapOOC(singleColBlocks.getReadStream(), qOut, tmp -> { + long r = tmp.getIndexes().getRowIndex(); + long c = tmp.getIndexes().getColumnIndex(); + // adapt index to new position in col + return new IndexedMatrixValue(new MatrixIndexes((c - 1) * numBlocksPerColIn + r, 1), tmp.getValue()); + }); + } + else { + f.join(); + reshapeFullRowBlocks(rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, singleColBlocks, qOut); + } + } + else { + f.join(); + reshapePartialRowBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, singleColBlocks, qOut); + } + } + } + + private void reshapeFullColBlocks(long rows, long cols, int blen, int numBlocksPerRowIn, int numBlocksPerRowOut, + int numBlocksPerColOut, OOCStream singleRowBlocks, OOCStream qOut) { + // use cache for accessing input rows by index + CachingStream singleRowBlockCache = new CachingStream(singleRowBlocks); + singleRowBlockCache.incrSubscriberCount(1); + singleRowBlockCache.scheduleDeletion(); + + // totalRowIdx corresponds to index of row block when all aligned in one row + // br * numBlocksPerRowOut * blen + b + r * numBlocksPerRowOut; + // with numBlocksPerRowOut * blen = cols + long totalIdx = -cols - 1 - numBlocksPerRowOut; + + // iterate through rows of output blocks + for(int br = 0; br < numBlocksPerColOut; br++) { + totalIdx += cols; + long tmp = totalIdx; + // for each block in row + for(int b = 0; b < numBlocksPerRowOut; b++) { + totalIdx += 1; + int localRows = (br == numBlocksPerColOut - 1 && rows % blen != 0) ? (int) rows % blen : blen; + MatrixBlock res = new MatrixBlock(localRows, blen, false); + long tmp2 = totalIdx; + // for each row in block + for(int r = 0; r < blen && r < localRows; r++) { + totalIdx += numBlocksPerRowOut; + // calc col idx for input + long colBlockIn = totalIdx % numBlocksPerRowIn + 1; + // calc row idx for input + long rowBlockIn = totalIdx / numBlocksPerRowIn + 1; + + try(OOCStream.QueueCallback cb = singleRowBlockCache + .findCached(new MatrixIndexes(rowBlockIn, colBlockIn))) { + MatrixBlock blk = (MatrixBlock) cb.get().getValue(); + res.setRow(r, blk.getDenseBlockValues()); + } + } + totalIdx = tmp2; + qOut.enqueue(new IndexedMatrixValue(new MatrixIndexes(br + 1, b + 1), res)); + } + totalIdx = tmp; + } + qOut.closeInput(); + } + + private void reshapePartialColBlocks(long rlen, long clen, long rows, long cols, int blen, int numBlocksPerRowIn, + int numBlocksPerRowOut, int numBlocksPerColOut, OOCStream singleRowBlocks, OOCStream qOut) { + // use cache for accessing input rows by index + CachingStream singleRowBlockCache = new CachingStream(singleRowBlocks); + singleRowBlockCache.incrSubscriberCount(1); + singleRowBlockCache.scheduleDeletion(); + + int br = 0; + int bc = 0; + int r = 0; + + // allocate row of output blocks + MatrixBlock[] outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut,true); + + int offsetOut = 0; + int localColsOut = (cols > blen) ? blen : (int) cols; + // iterate through input rows and add to row of output blocks + for(int i = 1; i <= rlen; i++) { + for(int j = 1; j <= numBlocksPerRowIn; j++) { + try(OOCStream.QueueCallback qcb = singleRowBlockCache.findCached(new MatrixIndexes(i, j))) { + MatrixBlock blk = (MatrixBlock) qcb.get().getValue(); + + int offsetIn = 0; + int localColsIn = (j == numBlocksPerRowIn && clen % blen != 0) ? (int) clen % blen : blen; + while(offsetIn < localColsIn) { + // until input row fully processed + int remIn = localColsIn - offsetIn; + int remOut = localColsOut - offsetOut; + if(remIn < remOut) { + // next input + setOutputEntries(blk, outputBlockRow[bc], r, offsetIn, offsetOut, remIn, true); + offsetIn += remIn; + offsetOut += remIn; + continue; + } + else if(remIn == remOut) { + // next input and next row + setOutputEntries(blk, outputBlockRow[bc], r, offsetIn, offsetOut, remIn, true); + offsetIn += remIn; + } + else { + // next row + setOutputEntries(blk, outputBlockRow[bc], r, offsetIn, offsetOut, remOut, true); + offsetIn += remOut; + } + bc++; + offsetOut = 0; + if(bc == numBlocksPerRowOut) { + // next row + r++; + if(r == outputBlockRow[0].getNumRows()) { + // enqueue filled output blocks and allocate new ones + for(int b = 0; b < outputBlockRow.length; b++) + qOut.enqueue(new IndexedMatrixValue(new MatrixIndexes(br + 1, b + 1), outputBlockRow[b])); + br++; + // allocate new block row + outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, true); + r = 0; + } + bc = 0; + } + localColsOut = (bc == numBlocksPerRowOut - 1 && cols % blen != 0) ? (int) cols % blen : blen; + } + } + } + } + qOut.closeInput(); + } + + private void reshapeFullRowBlocks(long rows, long cols, int blen, int numBlocksPerRowOut, int numBlocksPerColIn, + int numBlocksPerColOut, OOCStream singleColBlocks, OOCStream qOut) { + // use cache for accessing input cols by index + CachingStream singleColBlockCache = new CachingStream(singleColBlocks); + singleColBlockCache.incrSubscriberCount(1); + singleColBlockCache.scheduleDeletion(); + + // totalColIdx corresponds to index of col block when all aligned in one col + // bc * numBlocksPerColOut * blen + b + c * numBlocksPerColOut; + // with numBlocksPerColOut * blen = rows + long totalIdx = -rows - 1 - numBlocksPerColOut; + + // iterate through cols of output blocks + for(int bc = 0; bc < numBlocksPerRowOut; bc++) { + totalIdx += rows; + long tmp = totalIdx; + // for each block in col + for(int b = 0; b < numBlocksPerColOut; b++) { + totalIdx += 1; + int localCols = (bc == numBlocksPerRowOut - 1 && cols % blen != 0) ? (int) cols % blen : blen; + MatrixBlock res = new MatrixBlock(blen, localCols, false); + res.allocateDenseBlock(); + long tmp2 = totalIdx; + // for each col in block + for(int c = 0; c < blen && c < localCols; c++) { + totalIdx += numBlocksPerColOut; + // calc col idx for input + long colBlockIn = totalIdx / numBlocksPerColIn + 1; + // calc row idx for input + long rowBlockIn = totalIdx % numBlocksPerColIn + 1; + + try(OOCStream.QueueCallback cb = singleColBlockCache + .findCached(new MatrixIndexes(rowBlockIn, colBlockIn))) { + MatrixBlock blk = (MatrixBlock) cb.get().getValue(); + res.getDenseBlock().set(0, blen, c, c + 1, blk.getDenseBlock()); + } + } + totalIdx = tmp2; + res.recomputeNonZeros(); + qOut.enqueue(new IndexedMatrixValue(new MatrixIndexes(b + 1, bc + 1), res)); + } + totalIdx = tmp; + } + qOut.closeInput(); + } + + private void reshapePartialRowBlocks(long rlen, long clen, long rows, long cols, int blen, int numBlocksPerRowOut, + int numBlocksPerColIn, int numBlocksPerColOut, OOCStream singleColBlocks, OOCStream qOut) { + // use cache for accessing input cols by index + CachingStream singleRowBlockCache = new CachingStream(singleColBlocks); + singleRowBlockCache.incrSubscriberCount(1); + singleRowBlockCache.scheduleDeletion(); + + int br = 0; + int bc = 0; + int c = 0; + + // allocate col of output blocks + MatrixBlock[] outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, false); + + int offsetOut = 0; + int localRowsOut = (rows > blen) ? blen : (int) rows; + // iterate through input cols and add to col of output blocks + for(int j = 1; j <= clen; j++) { + for(int i = 1; i <= numBlocksPerColIn; i++) { + try(OOCStream.QueueCallback qcb = singleRowBlockCache.findCached(new MatrixIndexes(i, j))) { + MatrixBlock blk = (MatrixBlock) qcb.get().getValue(); + + int offsetIn = 0; + int localRowsIn = (i == numBlocksPerColIn && rlen % blen != 0) ? (int) rlen % blen : blen; + while(offsetIn < localRowsIn) { + // until input col fully processed + int remIn = localRowsIn - offsetIn; + int remOut = localRowsOut - offsetOut; + if(remIn < remOut) { + // next input + setOutputEntries(blk, outputBlockCol[br], c, offsetIn, offsetOut, remIn, false); + offsetIn += remIn; + offsetOut += remIn; + continue; + } + else if(remIn == remOut) { + // next input and next col + setOutputEntries(blk, outputBlockCol[br], c, offsetIn, offsetOut, remIn, false); + offsetIn += remIn; + } + else { + // next col + setOutputEntries(blk, outputBlockCol[br], c, offsetIn, offsetOut, remOut, false); + offsetIn += remOut; + } + br++; + offsetOut = 0; + if(br == numBlocksPerColOut) { + // next col + c++; + if(c == outputBlockCol[0].getNumColumns()) { + // enqueue filled output blocks and allocate new ones + for(int b = 0; b < outputBlockCol.length; b++) { + outputBlockCol[b].recomputeNonZeros(); + qOut.enqueue(new IndexedMatrixValue(new MatrixIndexes(b + 1, bc + 1), outputBlockCol[b])); + } + bc++; + // allocate new block col + outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, false); + c = 0; + } + br = 0; + } + localRowsOut = (br == numBlocksPerColOut - 1 && rows % blen != 0) ? (int) rows % blen : blen; + } + } + } + } + qOut.closeInput(); + } + + private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int blen, int numBlocksPerRow, int numBlocksPerCol, boolean isBlockRowSlice) { + int num = isBlockRowSlice ? numBlocksPerRow : numBlocksPerCol; + MatrixBlock[] res = new MatrixBlock[num]; + + // full inner blocks, adjust for outer blocks + int localRows = ((!isBlockRowSlice || idx == numBlocksPerCol - 1) && rows % blen != 0) ? (int) rows % blen : blen; + int localCols = ((isBlockRowSlice || idx == numBlocksPerRow - 1) && cols % blen != 0) ? (int) cols % blen : blen; + + for(int k = 0; k < num - 1; k++) { + res[k] = isBlockRowSlice ? new MatrixBlock(localRows, blen, false) : new MatrixBlock(blen, localCols, false); + res[k].allocateDenseBlock(); + } + res[num - 1] = new MatrixBlock(localRows, localCols, false); + res[num - 1].allocateDenseBlock(); + return res; + } + + private void setOutputEntries(MatrixBlock src, MatrixBlock dest, int idx, int srcOffset, int destOffset, int length, boolean rowWise) { + if(rowWise) + ((DenseBlockFP64) dest.getDenseBlock()).setPartialRow(src.getDenseBlock(), idx, srcOffset, destOffset, length); + else + ((DenseBlockFP64) dest.getDenseBlock()).setPartialCol(src.getDenseBlock(), idx, srcOffset, destOffset, length); + } +} diff --git a/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java new file mode 100644 index 00000000000..770c5b7c5bf --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.ooc; + +import org.apache.sysds.common.Opcodes; +import org.apache.sysds.common.Types; +import org.apache.sysds.common.Types.ExecMode; +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.instructions.Instruction; +import org.apache.sysds.runtime.io.MatrixWriter; +import org.apache.sysds.runtime.io.MatrixWriterFactory; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.meta.MatrixCharacteristics; +import org.apache.sysds.runtime.util.DataConverter; +import org.apache.sysds.runtime.util.HDFSTool; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.ArrayList; + +@RunWith(Parameterized.class) +@net.jcip.annotations.NotThreadSafe +public class ReshapeTest extends AutomatedTestBase { + private final static String TEST_NAME1 = "MatrixReshapeRowWise"; + private final static String TEST_NAME2 = "MatrixReshapeColWise"; + private final static String TEST_DIR = "functions/ooc/"; + private static final String TEST_CLASS_DIR = TEST_DIR + ReshapeTest.class.getSimpleName() + "/"; + private static final String INPUT_NAME = "X"; + private static final String OUTPUT_NAME = "Y"; + private static final double eps = 1e-8; + private static final int blen = 1000; + + private final int rlen; + private final int clen; + private final int rows; + private final int cols; + private final boolean rowWise; + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1)); + addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2)); + } + + public ReshapeTest(int rlen, int clen, int rows, int cols, boolean rowWise) { + this.rlen = rlen; + this.clen = clen; + this.rows = rows; + this.cols = cols; + this.rowWise = rowWise; + } + + @Parameterized.Parameters(name = "{0}x{1} {2}x{3} rowWise {4}") + public static Iterable getParams() { + + int[][][] dims = { + {{1000, 1000}, {1, 1000000}}, // single row/col + {{3000, 4000}, {1500, 8000}}, // partialBlocks + {{2400, 1400}, {800, 4200}} // fullBlocks + }; + + ArrayList params = new ArrayList<>(); + + for(int[][] d : dims) { + params.add(new Object[] {d[0][0], d[0][1], d[1][0], d[1][1], true}); + params.add(new Object[] {d[1][0], d[1][1], d[0][0], d[0][1], true}); + + params.add(new Object[] {d[0][1], d[0][0], d[1][1], d[1][0], false}); + params.add(new Object[] {d[1][1], d[1][0], d[0][1], d[0][0], false}); + } + + for(boolean rowWise : new boolean[] {true, false}) { + // single block + params.add(new Object[] {400, 300, 300, 400, rowWise}); + // non matching dims + params.add(new Object[] {1400, 1000, 5000, 1, rowWise}); + // no change + params.add(new Object[] {300, 400, 300, 400, rowWise}); + } + + return params; + } + + @Test + public void runTestMatrixReshapeOOC() { + ExecMode platformOld = setExecMode(ExecMode.SINGLE_NODE); + + try { + String TEST_NAME = (rowWise) ? TEST_NAME1 : TEST_NAME2; + getAndLoadTestConfiguration(TEST_NAME); + + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + + double[][] X = getRandomMatrix(rlen, clen, 0, 1, 1, 7); + MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(Types.FileFormat.BINARY); + writer.writeMatrixToHDFS(DataConverter.convertToMatrixBlock(X), input(INPUT_NAME), rlen, clen, 1000, rlen * clen); + HDFSTool.writeMetaDataFile(input(INPUT_NAME + ".mtd"), Types.ValueType.FP64, + new MatrixCharacteristics(rlen, clen, blen, rlen * clen), Types.FileFormat.BINARY); + + programArgs = new String[] {"-explain", "-stats", "-ooc", "-args", input(INPUT_NAME), String.valueOf(rlen), + String.valueOf(clen), String.valueOf(rows), String.valueOf(cols), output(OUTPUT_NAME)}; + + if(rlen * clen != rows * cols) { + runTest(true, true, DMLRuntimeException.class, -1); + return; + } + + runTest(true, false, null, -1); + if(rlen != rows) + Assert.assertTrue("OOC wasn't used for reshape", + heavyHittersContainsString(Instruction.OOC_INST_PREFIX + Opcodes.RESHAPE)); + else + Assert.assertTrue("OOC RBLK wasn't used for unchanged dimensions", + heavyHittersContainsString(Instruction.OOC_INST_PREFIX + Opcodes.RBLK)); + + // rerun without ooc flag + programArgs = new String[] {"-explain", "-stats", "-args", input(INPUT_NAME), String.valueOf(rlen), + String.valueOf(clen), String.valueOf(rows), String.valueOf(cols), output(OUTPUT_NAME + "_target")}; + runTest(true, false, null, -1); + + // compare results + MatrixBlock actual = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME), + Types.FileFormat.BINARY, rows, cols, blen); + MatrixBlock expected = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME + "_target"), + Types.FileFormat.BINARY, rows, cols, blen); + + TestUtils.compareMatrices(expected, actual, eps); + } + catch(Exception ex) { + Assert.fail(ex.getMessage()); + } + finally { + rtplatform = platformOld; + } + } +} diff --git a/src/test/scripts/functions/ooc/MatrixReshapeColWise.dml b/src/test/scripts/functions/ooc/MatrixReshapeColWise.dml new file mode 100644 index 00000000000..3af5463b65f --- /dev/null +++ b/src/test/scripts/functions/ooc/MatrixReshapeColWise.dml @@ -0,0 +1,26 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +X = read($1, rows=$2, cols=$3); + +Y = matrix(X, rows=$4, cols=$5, byrow=FALSE); +write(Y, $6, format="binary"); diff --git a/src/test/scripts/functions/ooc/MatrixReshapeRowWise.dml b/src/test/scripts/functions/ooc/MatrixReshapeRowWise.dml new file mode 100644 index 00000000000..edaaf258d90 --- /dev/null +++ b/src/test/scripts/functions/ooc/MatrixReshapeRowWise.dml @@ -0,0 +1,26 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +X = read($1, rows=$2, cols=$3); + +Y = matrix(X, rows=$4, cols=$5, byrow=TRUE); +write(Y, $6, format="binary"); From 119a7e1c910e89811f959e987af3063b099bee0c Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Fri, 3 Jul 2026 12:24:40 +0200 Subject: [PATCH 060/132] [OOC] Move Old Files to Legacy Package (#2530) This commit moves old files that are intended to be replaced by a new cache into cache.legacy --- .../runtime/instructions/ooc/OOCWatchdog.java | 2 +- .../sysds/runtime/ooc/cache/BlockEntry.java | 22 +++++++++---------- .../runtime/ooc/cache/OOCCacheManager.java | 2 ++ .../cache/{ => legacy}/DeferredReadQueue.java | 5 ++++- .../{ => legacy}/DeferredReadRequest.java | 4 +++- .../cache/{ => legacy}/OOCCacheScheduler.java | 5 ++++- .../{ => legacy}/OOCLRUCacheScheduler.java | 6 ++++- .../runtime/ooc/memory/CachedAllowance.java | 2 +- .../ooc/cache/OOCLRUCacheSchedulerTest.java | 2 +- .../cache/SourceBackedCacheSchedulerTest.java | 2 +- .../sysds/test/functions/ooc/LmCGTest.java | 2 +- 11 files changed, 34 insertions(+), 20 deletions(-) rename src/main/java/org/apache/sysds/runtime/ooc/cache/{ => legacy}/DeferredReadQueue.java (96%) rename src/main/java/org/apache/sysds/runtime/ooc/cache/{ => legacy}/DeferredReadRequest.java (96%) rename src/main/java/org/apache/sysds/runtime/ooc/cache/{ => legacy}/OOCCacheScheduler.java (96%) rename src/main/java/org/apache/sysds/runtime/ooc/cache/{ => legacy}/OOCLRUCacheScheduler.java (99%) diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCWatchdog.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCWatchdog.java index 289ef8e6b87..19eac2c0da2 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCWatchdog.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCWatchdog.java @@ -21,7 +21,7 @@ import org.apache.sysds.runtime.ooc.cache.BlockEntry; import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; -import org.apache.sysds.runtime.ooc.cache.OOCCacheScheduler; +import org.apache.sysds.runtime.ooc.cache.legacy.OOCCacheScheduler; import java.util.ArrayList; import java.util.Collection; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/BlockEntry.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/BlockEntry.java index c0604d017d8..3e040ef805e 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/BlockEntry.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/BlockEntry.java @@ -32,7 +32,7 @@ public final class BlockEntry { private int _retainHintCount; private int _referenceCount; // The number of references from different managing instances (e.g. CachingStream) - BlockEntry(BlockKey key, long size, Object data) { + public BlockEntry(BlockKey key, long size, Object data) { this._key = key; this._size = size; this._pinCount = 0; @@ -68,11 +68,11 @@ public boolean isGrouped() { throw new IllegalStateException("Cannot get the data of an unpinned entry"); } - Object getDataUnsafe() { + public Object getDataUnsafe() { return _data; } - void setDataUnsafe(Object data) { + public void setDataUnsafe(Object data) { if(data != null && _data != null) throw new IllegalStateException("Cannot overwrite data"); _data = data; @@ -86,15 +86,15 @@ public boolean isPinned() { return _pinCount > 0; } - synchronized int addReference() { + public synchronized int addReference() { return ++_referenceCount; } - synchronized int forget() { + public synchronized int forget() { return --_referenceCount; } - synchronized void setState(BlockState state) { + public synchronized void setState(BlockState state) { _state = state; } @@ -126,7 +126,7 @@ public synchronized int getRetainHintCount() { * Tries to clear the underlying data if it is not pinned * @return the number of cleared bytes (or 0 if could not clear or data was already cleared) */ - synchronized long clear() { + public synchronized long clear() { if (_pinCount != 0 || _data == null) return 0; if (_data instanceof IndexedMatrixValue) @@ -140,7 +140,7 @@ synchronized long clear() { * Pins the underlying data in memory * @return the new number of pins (0 if pin was unsuccessful) */ - synchronized int pin() { + public synchronized int pin() { if (_data == null) return 0; _pinCount++; @@ -151,7 +151,7 @@ synchronized int pin() { * Tries to increment pin-count if already pinned. Unpinned entries are not affected * by this operation. This allows bypassing the global cache lock. */ - synchronized boolean fastPin() { + public synchronized boolean fastPin() { if(_pinCount == 0) return false; _pinCount++; @@ -162,7 +162,7 @@ synchronized boolean fastPin() { * Unpins the underlying data * @return true if the data is now unpinned */ - synchronized boolean unpin() { + public synchronized boolean unpin() { if (_pinCount <= 0) throw new IllegalStateException("Cannot unpin data if it was not pinned"); _pinCount--; @@ -173,7 +173,7 @@ synchronized boolean unpin() { * Tries to unpin but guarantees that it will not * remove the last pin. This allows bypassing the global cache lock. */ - synchronized boolean fastUnpin() { + public synchronized boolean fastUnpin() { if(_pinCount <= 1) return false; _pinCount--; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java index 9f0f8c15b49..f1bb0cbf7f4 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java @@ -26,6 +26,8 @@ import org.apache.sysds.runtime.instructions.ooc.TeeOOCInstruction; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.ooc.cache.legacy.OOCCacheScheduler; +import org.apache.sysds.runtime.ooc.cache.legacy.OOCLRUCacheScheduler; import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; import org.apache.sysds.runtime.ooc.stats.OOCEventLog; import org.apache.sysds.utils.Statistics; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/DeferredReadQueue.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/DeferredReadQueue.java similarity index 96% rename from src/main/java/org/apache/sysds/runtime/ooc/cache/DeferredReadQueue.java rename to src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/DeferredReadQueue.java index b5564430fe2..fb56a50c1b9 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/DeferredReadQueue.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/DeferredReadQueue.java @@ -17,7 +17,10 @@ * under the License. */ -package org.apache.sysds.runtime.ooc.cache; +package org.apache.sysds.runtime.ooc.cache.legacy; + +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.BlockKey; import java.util.ArrayList; import java.util.Collections; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/DeferredReadRequest.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/DeferredReadRequest.java similarity index 96% rename from src/main/java/org/apache/sysds/runtime/ooc/cache/DeferredReadRequest.java rename to src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/DeferredReadRequest.java index 0ca6cbd2eab..6e4f225c48c 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/DeferredReadRequest.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/DeferredReadRequest.java @@ -17,7 +17,9 @@ * under the License. */ -package org.apache.sysds.runtime.ooc.cache; +package org.apache.sysds.runtime.ooc.cache.legacy; + +import org.apache.sysds.runtime.ooc.cache.BlockEntry; import java.util.List; import java.util.concurrent.CompletableFuture; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java similarity index 96% rename from src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheScheduler.java rename to src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java index f78327160fa..1820f15e8ae 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java @@ -17,10 +17,13 @@ * under the License. */ -package org.apache.sysds.runtime.ooc.cache; +package org.apache.sysds.runtime.ooc.cache.legacy; import org.apache.sysds.runtime.instructions.ooc.OOCStream; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.BlockKey; +import org.apache.sysds.runtime.ooc.cache.OOCIOHandler; import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; import java.util.Collection; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCLRUCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java similarity index 99% rename from src/main/java/org/apache/sysds/runtime/ooc/cache/OOCLRUCacheScheduler.java rename to src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java index e8af837d670..305a7419166 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCLRUCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.sysds.runtime.ooc.cache; +package org.apache.sysds.runtime.ooc.cache.legacy; import org.apache.commons.lang3.mutable.MutableObject; import org.apache.commons.logging.Log; @@ -25,6 +25,10 @@ import org.apache.sysds.api.DMLScript; import org.apache.sysds.runtime.instructions.ooc.OOCStream; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.BlockKey; +import org.apache.sysds.runtime.ooc.cache.BlockState; +import org.apache.sysds.runtime.ooc.cache.OOCIOHandler; import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; import org.apache.sysds.runtime.ooc.stats.OOCEventLog; import org.apache.sysds.utils.Statistics; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/memory/CachedAllowance.java b/src/main/java/org/apache/sysds/runtime/ooc/memory/CachedAllowance.java index 4649e47f81e..ffba3910b26 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/memory/CachedAllowance.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/memory/CachedAllowance.java @@ -25,7 +25,7 @@ import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.ooc.cache.BlockKey; import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; -import org.apache.sysds.runtime.ooc.cache.OOCCacheScheduler; +import org.apache.sysds.runtime.ooc.cache.legacy.OOCCacheScheduler; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; diff --git a/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCLRUCacheSchedulerTest.java b/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCLRUCacheSchedulerTest.java index 2741c10bdbb..527d89ec05f 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCLRUCacheSchedulerTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCLRUCacheSchedulerTest.java @@ -25,7 +25,7 @@ import org.apache.sysds.runtime.ooc.cache.BlockKey; import org.apache.sysds.runtime.ooc.cache.BlockState; import org.apache.sysds.runtime.ooc.cache.OOCIOHandler; -import org.apache.sysds.runtime.ooc.cache.OOCLRUCacheScheduler; +import org.apache.sysds.runtime.ooc.cache.legacy.OOCLRUCacheScheduler; import org.junit.After; import org.junit.Assert; import org.junit.Before; diff --git a/src/test/java/org/apache/sysds/test/component/ooc/cache/SourceBackedCacheSchedulerTest.java b/src/test/java/org/apache/sysds/test/component/ooc/cache/SourceBackedCacheSchedulerTest.java index 83c2fd59669..4d345601982 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/cache/SourceBackedCacheSchedulerTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/cache/SourceBackedCacheSchedulerTest.java @@ -29,7 +29,7 @@ import org.apache.sysds.runtime.ooc.cache.BlockKey; import org.apache.sysds.runtime.ooc.cache.BlockState; import org.apache.sysds.runtime.ooc.cache.OOCIOHandler; -import org.apache.sysds.runtime.ooc.cache.OOCLRUCacheScheduler; +import org.apache.sysds.runtime.ooc.cache.legacy.OOCLRUCacheScheduler; import org.apache.sysds.runtime.ooc.cache.OOCMatrixIOHandler; import org.apache.sysds.test.AutomatedTestBase; import org.apache.sysds.test.TestConfiguration; diff --git a/src/test/java/org/apache/sysds/test/functions/ooc/LmCGTest.java b/src/test/java/org/apache/sysds/test/functions/ooc/LmCGTest.java index f4c4d364e83..7e92d824985 100644 --- a/src/test/java/org/apache/sysds/test/functions/ooc/LmCGTest.java +++ b/src/test/java/org/apache/sysds/test/functions/ooc/LmCGTest.java @@ -37,7 +37,7 @@ public class LmCGTest extends AutomatedTestBase { private final static String TEST_NAME1 = "lmCG"; private final static String TEST_DIR = "functions/ooc/"; private final static String TEST_CLASS_DIR = TEST_DIR + LmCGTest.class.getSimpleName() + "/"; - private final static double eps = 1e-8; + private final static double eps = 1e-7; private static final String INPUT_NAME_1 = "X"; private static final String INPUT_NAME_2 = "y"; private static final String OUTPUT_NAME = "res"; From 9a4415f70fdde111b56a41efca09e5f294a84f8b Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Fri, 3 Jul 2026 13:38:32 +0200 Subject: [PATCH 061/132] [OOC] Move I/O Related Classes to cache.io Package (#2531) --- .../apache/sysds/runtime/instructions/ooc/CachingStream.java | 2 +- .../sysds/runtime/instructions/ooc/ReblockOOCInstruction.java | 2 +- .../org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java | 2 ++ .../sysds/runtime/ooc/cache/{ => io}/CloseableQueue.java | 2 +- .../apache/sysds/runtime/ooc/cache/{ => io}/OOCIOHandler.java | 4 +++- .../sysds/runtime/ooc/cache/{ => io}/OOCMatrixIOHandler.java | 4 +++- .../sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java | 2 +- .../sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java | 2 +- .../org/apache/sysds/runtime/ooc/stream/SourceOOCStream.java | 2 +- .../test/component/ooc/cache/OOCLRUCacheSchedulerTest.java | 2 +- .../component/ooc/cache/SourceBackedCacheSchedulerTest.java | 4 ++-- .../component/ooc/cache/SourceBackedReadOOCIOHandlerTest.java | 4 ++-- .../sysds/test/functions/ooc/SourceReadOOCIOHandlerTest.java | 4 ++-- 13 files changed, 21 insertions(+), 15 deletions(-) rename src/main/java/org/apache/sysds/runtime/ooc/cache/{ => io}/CloseableQueue.java (98%) rename src/main/java/org/apache/sysds/runtime/ooc/cache/{ => io}/OOCIOHandler.java (96%) rename src/main/java/org/apache/sysds/runtime/ooc/cache/{ => io}/OOCMatrixIOHandler.java (99%) diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java index 38929dcafcc..abf1efde9c2 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java @@ -28,7 +28,7 @@ import org.apache.sysds.runtime.meta.DataCharacteristics; import org.apache.sysds.runtime.ooc.cache.BlockKey; import org.apache.sysds.runtime.ooc.cache.GroupedBlockKey; -import org.apache.sysds.runtime.ooc.cache.OOCIOHandler; +import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; import org.apache.sysds.runtime.ooc.stream.SourceOOCStream; import org.apache.sysds.runtime.ooc.stream.message.OOCGetStreamTypeMessage; diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReblockOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReblockOOCInstruction.java index 4270836b755..75407f8cf05 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReblockOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReblockOOCInstruction.java @@ -30,7 +30,7 @@ import org.apache.sysds.runtime.matrix.operators.Operator; import org.apache.sysds.runtime.meta.DataCharacteristics; import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; -import org.apache.sysds.runtime.ooc.cache.OOCIOHandler; +import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; import org.apache.sysds.runtime.ooc.stream.SourceOOCStream; public class ReblockOOCInstruction extends ComputationOOCInstruction { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java index f1bb0cbf7f4..78d12348fe5 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java @@ -26,6 +26,8 @@ import org.apache.sysds.runtime.instructions.ooc.TeeOOCInstruction; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; +import org.apache.sysds.runtime.ooc.cache.io.OOCMatrixIOHandler; import org.apache.sysds.runtime.ooc.cache.legacy.OOCCacheScheduler; import org.apache.sysds.runtime.ooc.cache.legacy.OOCLRUCacheScheduler; import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/CloseableQueue.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java similarity index 98% rename from src/main/java/org/apache/sysds/runtime/ooc/cache/CloseableQueue.java rename to src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java index 4f1c5799736..94411242df1 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/CloseableQueue.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.sysds.runtime.ooc.cache; +package org.apache.sysds.runtime.ooc.cache.io; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java similarity index 96% rename from src/main/java/org/apache/sysds/runtime/ooc/cache/OOCIOHandler.java rename to src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java index 0bc5ace1274..21085626a71 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java @@ -17,12 +17,14 @@ * under the License. */ -package org.apache.sysds.runtime.ooc.cache; +package org.apache.sysds.runtime.ooc.cache.io; import org.apache.sysds.common.Types; import org.apache.sysds.runtime.instructions.ooc.OOCStream; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.BlockKey; import java.util.concurrent.CompletableFuture; import java.util.List; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCMatrixIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java similarity index 99% rename from src/main/java/org/apache/sysds/runtime/ooc/cache/OOCMatrixIOHandler.java rename to src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java index 7509b669701..029c9e8060f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCMatrixIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.sysds.runtime.ooc.cache; +package org.apache.sysds.runtime.ooc.cache.io; import org.apache.sysds.api.DMLScript; import org.apache.hadoop.fs.FileSystem; @@ -32,6 +32,8 @@ import org.apache.sysds.runtime.io.MatrixReader; import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.BlockKey; import org.apache.sysds.runtime.ooc.stats.OOCEventLog; import org.apache.sysds.runtime.ooc.stream.SourceOOCStream; import org.apache.sysds.runtime.util.FastBufferedDataInputStream; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java index 1820f15e8ae..ad161e95303 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java @@ -23,7 +23,7 @@ import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.ooc.cache.BlockEntry; import org.apache.sysds.runtime.ooc.cache.BlockKey; -import org.apache.sysds.runtime.ooc.cache.OOCIOHandler; +import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; import java.util.Collection; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java index 305a7419166..c1f7058b5dc 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java @@ -28,7 +28,7 @@ import org.apache.sysds.runtime.ooc.cache.BlockEntry; import org.apache.sysds.runtime.ooc.cache.BlockKey; import org.apache.sysds.runtime.ooc.cache.BlockState; -import org.apache.sysds.runtime.ooc.cache.OOCIOHandler; +import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; import org.apache.sysds.runtime.ooc.stats.OOCEventLog; import org.apache.sysds.utils.Statistics; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/SourceOOCStream.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/SourceOOCStream.java index 553767ef8ce..0941cf0ea5c 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/SourceOOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/SourceOOCStream.java @@ -25,7 +25,7 @@ import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; -import org.apache.sysds.runtime.ooc.cache.OOCIOHandler; +import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; import org.apache.sysds.runtime.ooc.stream.message.OOCStreamMessage; import org.apache.sysds.runtime.matrix.data.MatrixIndexes; diff --git a/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCLRUCacheSchedulerTest.java b/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCLRUCacheSchedulerTest.java index 527d89ec05f..002b19e57be 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCLRUCacheSchedulerTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCLRUCacheSchedulerTest.java @@ -24,7 +24,7 @@ import org.apache.sysds.runtime.ooc.cache.BlockEntry; import org.apache.sysds.runtime.ooc.cache.BlockKey; import org.apache.sysds.runtime.ooc.cache.BlockState; -import org.apache.sysds.runtime.ooc.cache.OOCIOHandler; +import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; import org.apache.sysds.runtime.ooc.cache.legacy.OOCLRUCacheScheduler; import org.junit.After; import org.junit.Assert; diff --git a/src/test/java/org/apache/sysds/test/component/ooc/cache/SourceBackedCacheSchedulerTest.java b/src/test/java/org/apache/sysds/test/component/ooc/cache/SourceBackedCacheSchedulerTest.java index 4d345601982..cc206bdff40 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/cache/SourceBackedCacheSchedulerTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/cache/SourceBackedCacheSchedulerTest.java @@ -28,9 +28,9 @@ import org.apache.sysds.runtime.ooc.cache.BlockEntry; import org.apache.sysds.runtime.ooc.cache.BlockKey; import org.apache.sysds.runtime.ooc.cache.BlockState; -import org.apache.sysds.runtime.ooc.cache.OOCIOHandler; +import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; import org.apache.sysds.runtime.ooc.cache.legacy.OOCLRUCacheScheduler; -import org.apache.sysds.runtime.ooc.cache.OOCMatrixIOHandler; +import org.apache.sysds.runtime.ooc.cache.io.OOCMatrixIOHandler; import org.apache.sysds.test.AutomatedTestBase; import org.apache.sysds.test.TestConfiguration; import org.apache.sysds.test.TestUtils; diff --git a/src/test/java/org/apache/sysds/test/component/ooc/cache/SourceBackedReadOOCIOHandlerTest.java b/src/test/java/org/apache/sysds/test/component/ooc/cache/SourceBackedReadOOCIOHandlerTest.java index 7c93af0ba09..a8de3cb7951 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/cache/SourceBackedReadOOCIOHandlerTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/cache/SourceBackedReadOOCIOHandlerTest.java @@ -28,8 +28,8 @@ import org.apache.sysds.runtime.ooc.cache.BlockEntry; import org.apache.sysds.runtime.ooc.cache.BlockKey; import org.apache.sysds.runtime.ooc.cache.BlockState; -import org.apache.sysds.runtime.ooc.cache.OOCIOHandler; -import org.apache.sysds.runtime.ooc.cache.OOCMatrixIOHandler; +import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; +import org.apache.sysds.runtime.ooc.cache.io.OOCMatrixIOHandler; import org.apache.sysds.test.AutomatedTestBase; import org.apache.sysds.test.TestConfiguration; import org.apache.sysds.test.TestUtils; diff --git a/src/test/java/org/apache/sysds/test/functions/ooc/SourceReadOOCIOHandlerTest.java b/src/test/java/org/apache/sysds/test/functions/ooc/SourceReadOOCIOHandlerTest.java index 34dd01d6620..e1ec384a310 100644 --- a/src/test/java/org/apache/sysds/test/functions/ooc/SourceReadOOCIOHandlerTest.java +++ b/src/test/java/org/apache/sysds/test/functions/ooc/SourceReadOOCIOHandlerTest.java @@ -23,8 +23,8 @@ import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.matrix.data.MatrixBlock; -import org.apache.sysds.runtime.ooc.cache.OOCIOHandler; -import org.apache.sysds.runtime.ooc.cache.OOCMatrixIOHandler; +import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; +import org.apache.sysds.runtime.ooc.cache.io.OOCMatrixIOHandler; import org.apache.sysds.runtime.controlprogram.parfor.LocalTaskQueue; import org.apache.sysds.runtime.io.MatrixWriter; import org.apache.sysds.runtime.io.MatrixWriterFactory; From 9b419c88999b0e396df14eac5ef2c943e294626d Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Fri, 3 Jul 2026 23:02:00 +0200 Subject: [PATCH 062/132] [SYSTEMDS-3949] Add native Delta Lake frame read/write via Delta Kernel (#2515) * [SYSTEMDS-3949] Add native Delta Lake frame read/write via Delta Kernel Extend the native Delta Lake support from matrices to frames, reading and writing Delta Lake tables through the Spark-free Delta Kernel library on the single-node CP path. DML read/write with format="delta" now works for frames, discovering schema, column names, and dimensions directly from the table. --- .../sysds/conf/ConfigurationManager.java | 7 +- .../java/org/apache/sysds/conf/DMLConfig.java | 6 +- .../controlprogram/caching/FrameObject.java | 18 +- .../frame/data/columns/ArrayFactory.java | 102 ++- .../sysds/runtime/io/DeltaKernelUtils.java | 62 +- .../sysds/runtime/io/FrameReaderDelta.java | 403 ++++++++++ .../runtime/io/FrameReaderDeltaParallel.java | 238 ++++++ .../sysds/runtime/io/FrameReaderFactory.java | 2 + .../sysds/runtime/io/FrameWriterDelta.java | 259 ++++++ .../sysds/runtime/io/FrameWriterFactory.java | 2 + .../apache/sysds/runtime/io/WriterDelta.java | 6 +- .../org/apache/sysds/performance/Main.java | 19 + .../performance/frame/DeltaFrameRead.java | 173 ++++ .../component/io/DeltaFrameReadWriteTest.java | 737 ++++++++++++++++++ .../io/DeltaFrameSparkInteropTest.java | 273 +++++++ .../component/io/DeltaFrameTestUtils.java | 40 + .../io/delta/FrameDeltaReadWriteTest.java | 120 +++ .../io/delta/FrameDeltaReadCompare.dml | 35 + .../functions/io/delta/FrameDeltaWrite.dml | 32 + 19 files changed, 2502 insertions(+), 32 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/io/FrameReaderDelta.java create mode 100644 src/main/java/org/apache/sysds/runtime/io/FrameReaderDeltaParallel.java create mode 100644 src/main/java/org/apache/sysds/runtime/io/FrameWriterDelta.java create mode 100644 src/test/java/org/apache/sysds/performance/frame/DeltaFrameRead.java create mode 100644 src/test/java/org/apache/sysds/test/component/io/DeltaFrameReadWriteTest.java create mode 100644 src/test/java/org/apache/sysds/test/component/io/DeltaFrameSparkInteropTest.java create mode 100644 src/test/java/org/apache/sysds/test/component/io/DeltaFrameTestUtils.java create mode 100644 src/test/java/org/apache/sysds/test/functions/io/delta/FrameDeltaReadWriteTest.java create mode 100644 src/test/scripts/functions/io/delta/FrameDeltaReadCompare.dml create mode 100644 src/test/scripts/functions/io/delta/FrameDeltaWrite.dml diff --git a/src/main/java/org/apache/sysds/conf/ConfigurationManager.java b/src/main/java/org/apache/sysds/conf/ConfigurationManager.java index 83676da47a7..8b0f5fe06b9 100644 --- a/src/main/java/org/apache/sysds/conf/ConfigurationManager.java +++ b/src/main/java/org/apache/sysds/conf/ConfigurationManager.java @@ -268,11 +268,16 @@ public static int getDeltaWriterBatchSize() { return getDMLConfig().getIntValue(DMLConfig.DELTA_WRITER_BATCH_SIZE); } - /** @return target data-file size (bytes) for the native Delta writer */ + /** @return upper bound (bytes) on the native Delta writer's target data-file size */ public static long getDeltaWriterTargetFileSize() { return Long.parseLong(getDMLConfig().getTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE)); } + /** @return whether the native Delta writer adaptively sizes data files for parallel reads */ + public static boolean isDeltaWriterAdaptiveFileSize() { + return getDMLConfig().getBooleanValue(DMLConfig.DELTA_WRITER_ADAPTIVE_FILE_SIZE); + } + public static boolean isFederatedSSL(){ return getDMLConfig().getBooleanValue(DMLConfig.USE_SSL_FEDERATED_COMMUNICATION); } diff --git a/src/main/java/org/apache/sysds/conf/DMLConfig.java b/src/main/java/org/apache/sysds/conf/DMLConfig.java index e06b58b07c8..d114ccf69b9 100644 --- a/src/main/java/org/apache/sysds/conf/DMLConfig.java +++ b/src/main/java/org/apache/sysds/conf/DMLConfig.java @@ -73,7 +73,8 @@ public class DMLConfig public static final String IO_COMPRESSION_CODEC = "sysds.io.compression.encoding"; public static final String DELTA_READER_BATCH_SIZE = "sysds.io.delta.reader.batchsize"; // int: rows per parquet read batch public static final String DELTA_WRITER_BATCH_SIZE = "sysds.io.delta.writer.batchsize"; // int: matrix rows materialized per columnar batch handed to the engine - public static final String DELTA_WRITER_TARGET_FILE_SIZE = "sysds.io.delta.writer.targetfilesize"; // long: target data-file size in bytes (smaller -> more files -> more parallel-read throughput) + public static final String DELTA_WRITER_TARGET_FILE_SIZE = "sysds.io.delta.writer.targetfilesize"; // long: upper bound on target data-file size in bytes; adaptive sizing may pick smaller -> more files -> more parallel-read throughput + public static final String DELTA_WRITER_ADAPTIVE_FILE_SIZE = "sysds.io.delta.writer.adaptivefilesize"; // boolean: size data files toward one per parallel reader (capped by targetfilesize) public static final String PARALLEL_ENCODE = "sysds.parallel.encode"; // boolean: enable multi-threaded transformencode and apply public static final String PARALLEL_ENCODE_STAGED = "sysds.parallel.encode.staged"; public static final String PARALLEL_ENCODE_APPLY_BLOCKS = "sysds.parallel.encode.applyBlocks"; @@ -163,7 +164,8 @@ public class DMLConfig _defaultVals.put(IO_COMPRESSION_CODEC, "none"); _defaultVals.put(DELTA_READER_BATCH_SIZE, "4096"); // rows per parquet read batch (Delta Kernel default 1024) _defaultVals.put(DELTA_WRITER_BATCH_SIZE, "4096"); // matrix rows materialized per columnar batch handed to the engine - _defaultVals.put(DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(64L * 1024 * 1024)); // 64MB target data-file size (Delta Kernel default 128MB) -> more files -> more parallel-read throughput + _defaultVals.put(DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(64L * 1024 * 1024)); // 64MB cap on target data-file size; adaptive sizing may pick smaller -> more files -> more parallel-read throughput + _defaultVals.put(DELTA_WRITER_ADAPTIVE_FILE_SIZE, "true"); // size data files toward one per parallel reader _defaultVals.put(PARALLEL_TOKENIZE, "false"); _defaultVals.put(PARALLEL_TOKENIZE_NUM_BLOCKS, "64"); _defaultVals.put(FRAME_TO_MATRIX_WARN_CAST, "false"); diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java index 7151d87211c..87d14dbf87e 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java @@ -23,6 +23,7 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.tuple.Pair; +import org.apache.sysds.api.DMLScript; import org.apache.sysds.common.Types.DataType; import org.apache.sysds.common.Types.FileFormat; import org.apache.sysds.common.Types.ValueType; @@ -203,13 +204,19 @@ protected FrameBlock readBlobFromHDFS(String fname, long[] dims) throws IOExcept .createFrameReader(iimd.getFileFormat(), getFileFormatProperties()) .readFrameFromHDFS(fname, lschema, dc.getRows(), dc.getCols()); - if(iimd.getFileFormat() == FileFormat.CSV) + // sanity check correct output (before dereferencing data below) + if(data == null) + throw new IOException("Unable to load frame from file: " + fname); + + //Delta and CSV discover dimensions (and Delta also schema) at read time, so + //refresh the cached metadata to reflect the materialized frame block. + if(iimd.getFileFormat() == FileFormat.CSV || iimd.getFileFormat() == FileFormat.DELTA) { _metaData = _metaData instanceof MetaDataFormat ? new MetaDataFormat(data.getDataCharacteristics(), iimd.getFileFormat()) : new MetaData(data.getDataCharacteristics()); + if(iimd.getFileFormat() == FileFormat.DELTA) + _schema = data.getSchema(); + } - // sanity check correct output - if(data == null) - throw new IOException("Unable to load frame from file: " + fname); return data; } @@ -293,6 +300,9 @@ protected void writeBlobToHDFS(String fname, String ofmt, int rep, FileFormatPro FrameWriter writer = FrameWriterFactory.createFrameWriter(fmt, fprop); writer.writeFrameToHDFS(_data, fname, getNumRows(), getNumColumns()); + + if(DMLScript.STATISTICS) + CacheStatistics.incrementHDFSWrites(); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java index 5f2d08a122f..80a5d699dfa 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java @@ -123,6 +123,87 @@ public static RaggedArray create(T[] col, int m) { return new RaggedArray<>(col, m); } + /** + * Wrap a fully populated raw typed column array into an {@link Array} of the given value type. The runtime type of + * {@code col} must match the primitive backing type of {@code vt} (e.g. {@code double[]} for {@link ValueType#FP64}, + * {@code String[]} for {@link ValueType#STRING}). + * + *

For {@link ValueType#BOOLEAN} this mirrors {@link #allocateBoolean(int)}: a {@code boolean[]} longer than + * {@link #bitSetSwitchPoint} is bit-packed into a compact {@link BitSetArray} (so a bulk decoder that fills a + * plain {@code boolean[]} still ends up with the same representation as every other frame allocation path), + * while shorter columns stay a plain {@link BooleanArray}.

+ * + * @param vt the value type of the column + * @param col the backing array to wrap + * @return an {@link Array} view over {@code col} (boolean columns may be bit-packed rather than wrapped in place) + */ + public static Array create(ValueType vt, Object col) { + switch(vt) { + case FP64: + return create((double[]) col); + case FP32: + return create((float[]) col); + case INT64: + return create((long[]) col); + case UINT4: + case UINT8: + case INT32: + return create((int[]) col); + case BOOLEAN: { + boolean[] b = (boolean[]) col; + return b.length > bitSetSwitchPoint ? new BitSetArray(b) : create(b); + } + case CHARACTER: + return create((char[]) col); + case HASH64: + return createHash64((long[]) col); + case HASH32: + return createHash32((int[]) col); + case UNKNOWN: + case STRING: + default: + return create((String[]) col); + } + } + + /** + * Allocate the raw backing array for a column of the given value type: the inverse of + * {@link #create(ValueType, Object)}. Returns {@code double[]} for {@link ValueType#FP64}, + * {@code int[]} for INT32/UINT/HASH32, {@code long[]} for INT64/HASH64, {@code String[]} for STRING, etc. The + * runtime array type matches what {@link #create(ValueType, Object)} expects, so a bulk decoder can fill this + * primitive array directly and then wrap it via {@code create(vt, backing)}. + * + * @param vt the value type of the column + * @param nRow the number of rows to allocate + * @return a freshly allocated raw backing array of the matching primitive/object type + */ + public static Object allocateBacking(ValueType vt, int nRow) { + switch(vt) { + case FP64: + return new double[nRow]; + case FP32: + return new float[nRow]; + case INT64: + case HASH64: + return new long[nRow]; + case UINT4: + case UINT8: + LOG.warn("Not supported allocation of UInt 4 or 8 array: defaulting to Int32"); + // fall through: UINT4/UINT8 are backed by int[] (wrapped as Int32) + case INT32: + case HASH32: + return new int[nRow]; + case BOOLEAN: + return new boolean[nRow]; + case CHARACTER: + return new char[nRow]; + case UNKNOWN: + case STRING: + default: + return new String[nRow]; + } + } + public static long getInMemorySize(ValueType type, int _numRows, boolean containsNull) { if(containsNull) { switch(type) { @@ -221,27 +302,8 @@ public static Array allocate(ValueType v, int nRow) { switch(v) { case BOOLEAN: return allocateBoolean(nRow); - case UINT4: - case UINT8: - LOG.warn("Not supported allocation of UInt 4 or 8 array: defaulting to Int32"); - case INT32: - return new IntegerArray(new int[nRow]); - case INT64: - return new LongArray(new long[nRow]); - case FP32: - return new FloatArray(new float[nRow]); - case FP64: - return new DoubleArray(new double[nRow]); - case CHARACTER: - return new CharArray(new char[nRow]); - case HASH64: - return new HashLongArray(new long[nRow]); - case HASH32: - return new HashIntegerArray(new int[nRow]); - case UNKNOWN: - case STRING: default: - return new StringArray(new String[nRow]); + return create(v, allocateBacking(v, nRow)); } } diff --git a/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java index 1e06f9acb56..bbca857a1cd 100644 --- a/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java +++ b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java @@ -26,9 +26,12 @@ import java.util.Optional; import java.util.function.Function; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.util.HDFSTool; @@ -77,6 +80,8 @@ */ public class DeltaKernelUtils { + private static final Log LOG = LogFactory.getLog(DeltaKernelUtils.class.getName()); + private static final String ENGINE_INFO = "Apache SystemDS"; /** Reused thread-safe JSON reader for the per-file Delta stats (numRecords). */ @@ -157,6 +162,17 @@ public static int countSelected(int size, boolean[] selected) { return n; } + /** Floor on the adaptive writer target file size. Below this the per-file metadata/open + * overhead (and tiny-file proliferation) outweighs the extra read parallelism. */ + public static final long ADAPTIVE_WRITER_MIN_FILE_SIZE = 4L * 1024 * 1024; + + private static Configuration buildConf(Configuration base, int batchSize, long targetFileSize) { + Configuration c = new Configuration(base); + c.setInt(CONF_READER_BATCH_SIZE, batchSize); + c.setLong(CONF_WRITER_TARGET_FILE_SIZE, targetFileSize); + return c; + } + private static synchronized Configuration deltaConf() { Configuration base = ConfigurationManager.getCachedJobConf(); int batchSize = ConfigurationManager.getDeltaReaderBatchSize(); @@ -164,10 +180,7 @@ private static synchronized Configuration deltaConf() { if(cachedConf == null || cachedConfBase != base || cachedBatchSize != batchSize || cachedTargetFileSize != targetFileSize) { - Configuration c = new Configuration(base); - c.setInt(CONF_READER_BATCH_SIZE, batchSize); - c.setLong(CONF_WRITER_TARGET_FILE_SIZE, targetFileSize); - cachedConf = c; + cachedConf = buildConf(base, batchSize, targetFileSize); cachedConfBase = base; cachedBatchSize = batchSize; cachedTargetFileSize = targetFileSize; @@ -179,6 +192,47 @@ public static Engine createEngine() { return DefaultEngine.create(deltaConf()); } + /** + * Compute the parquet target data-file size (bytes) for writing a table of the given + * estimated size. With adaptive sizing enabled the writer aims for roughly one data + * file per expected parallel reader (so the native per-file parallel read can use all + * threads): never above the configured target, and never below + * {@code ADAPTIVE_WRITER_MIN_FILE_SIZE} unless the configured target is itself smaller + * than that floor (in which case the configured target wins). + * + * @param estimatedBytes estimate of the table's size (the block in-memory size is a fine proxy) + * @return the target max parquet data-file size in bytes + */ + public static long adaptiveWriterTargetFileSize(long estimatedBytes) { + long configured = ConfigurationManager.getDeltaWriterTargetFileSize(); + if(!ConfigurationManager.isDeltaWriterAdaptiveFileSize() || estimatedBytes <= 0) + return configured; + int par = Math.max(1, OptimizerUtils.getParallelBinaryReadParallelism()); + long perReader = Math.max(1, estimatedBytes / par); + //never above the configured cap, never below the floor (unless the cap itself is lower) + long target = Math.min(configured, Math.max(ADAPTIVE_WRITER_MIN_FILE_SIZE, perReader)); + if(LOG.isDebugEnabled()) + LOG.debug("Delta adaptive file size: est=" + estimatedBytes + "B par=" + par + " -> target=" + target + + "B (cap=" + configured + "B, floor=" + ADAPTIVE_WRITER_MIN_FILE_SIZE + "B)"); + return target; + } + + /** + * Create an engine for writing a table of the given estimated size, configured with an + * adaptive target data-file size (see {@link #adaptiveWriterTargetFileSize(long)}). A fresh + * (uncached) configuration is built since writes happen once per table, not per data file. + * + * @param estimatedBytes estimate of the table's size (the block in-memory size is a fine proxy) + * @return a Delta Kernel engine for the write + */ + public static Engine createWriteEngine(long estimatedBytes) { + //the reader batch size is irrelevant on the write path but is set to keep the + //conf shape identical to deltaConf(); only the target file size matters here. + Configuration c = buildConf(ConfigurationManager.getCachedJobConf(), + ConfigurationManager.getDeltaReaderBatchSize(), adaptiveWriterTargetFileSize(estimatedBytes)); + return DefaultEngine.create(c); + } + /** * Resolve a (possibly relative) path to a fully-qualified URI so the * kernel's default engine can locate the table on the right filesystem. diff --git a/src/main/java/org/apache/sysds/runtime/io/FrameReaderDelta.java b/src/main/java/org/apache/sysds/runtime/io/FrameReaderDelta.java new file mode 100644 index 00000000000..9e8823f7ecf --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/io/FrameReaderDelta.java @@ -0,0 +1,403 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.io; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; + +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.frame.data.columns.Array; +import org.apache.sysds.runtime.frame.data.columns.ArrayFactory; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.data.Row; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.types.DataType; + +/** + * Single-threaded native Delta Lake reader for frames, built on the Spark-free Delta Kernel library. It opens the + * latest snapshot of a Delta table, reads its parquet data files through the kernel's default engine (honoring deletion + * vectors), and materializes the columns into a {@link FrameBlock} whose schema and column names are derived from the + * Delta table schema. + * + *

+ * Data is extracted column-at-a-time into primitive arrays (no per-cell boxing or {@code FrameBlock.set} dispatch) and + * the frame is constructed directly from typed column {@link Array}s. Supported column types map to SystemDS value + * types: double, float, long, int, short, byte, boolean, and string. Neither the schema nor the dimensions need to be + * supplied; they are discovered from the table. + *

+ */ +public class FrameReaderDelta extends FrameReader { + + // per-column read codes (how to pull a value out of the Delta column vector); + // aliases of the shared codes in DeltaKernelUtils so the frame read dispatch stays + // in lockstep with the matrix reader's type mapping. Package visible so the parallel + // reader can reuse the same dispatch. + static final int R_DOUBLE = DeltaKernelUtils.T_DOUBLE, R_FLOAT = DeltaKernelUtils.T_FLOAT, + R_LONG = DeltaKernelUtils.T_LONG, R_INT = DeltaKernelUtils.T_INT, R_SHORT = DeltaKernelUtils.T_SHORT, + R_BYTE = DeltaKernelUtils.T_BYTE, R_BOOLEAN = DeltaKernelUtils.T_BOOLEAN, R_STRING = DeltaKernelUtils.T_STRING; + + @Override + public FrameBlock readFrameFromHDFS(String fname, ValueType[] schema, String[] names, long rlen, long clen) + throws IOException, DMLRuntimeException { + Engine engine = DeltaKernelUtils.createEngine(); + String tablePath = DeltaKernelUtils.qualify(fname); + DeltaKernelUtils.ScanHandle handle = DeltaKernelUtils.openScan(engine, tablePath); + return readWithHandle(fname, engine, handle); + } + + /** + * Materialize the frame from an already-opened engine and scan handle. Factored out so the parallel reader can + * reuse a handle it already opened for its single-file/single-thread fallback instead of re-opening the (expensive) + * Delta snapshot a second time. + * + * @param fname the table path (for error messages) + * @param engine the Delta Kernel engine + * @param handle the opened scan handle + * @return the materialized frame block + */ + protected FrameBlock readWithHandle(String fname, Engine engine, DeltaKernelUtils.ScanHandle handle) + throws IOException { + final ReadPlan plan = planColumns(handle); + + // fast path: exact per-file row counts are known from metadata (no deletion + // vectors) -> pre-size one typed array per column and decode each file + // straight into its row offset, avoiding the per-batch extract + concatenate. + if(useDirectPath(handle)) { + long total = 0; + for(long r : handle.numRecords) + total += r; + // empty table: the typed column arrays cannot be zero-length, so return a + // schema-only frame with the discovered schema/names and zero rows. + if(total == 0) + return new FrameBlock(plan.vt, plan.cnames, 0); + if(total <= Integer.MAX_VALUE) + return readDirect(fname, engine, handle, plan, (int) total); + } + + // fallback: row counts unknown or deletion vectors present -> decode into + // per-batch arrays and concatenate per column in file order. + return readBuffered(engine, handle, plan); + } + + /** + * Immutable per-column read plan derived once from the Delta table schema: how to pull each column out of the + * kernel column vector ({@code readCodes}), the resulting SystemDS value types, and the column names. Shared by the + * serial and parallel readers so the schema-to-column mapping lives in exactly one place. + */ + protected static final class ReadPlan { + final int ncol; + final int[] readCodes; + final ValueType[] vt; + final String[] cnames; + + private ReadPlan(int ncol, int[] readCodes, ValueType[] vt, String[] cnames) { + this.ncol = ncol; + this.readCodes = readCodes; + this.vt = vt; + this.cnames = cnames; + } + } + + /** Derive the {@link ReadPlan} (read codes, value types, names) from the opened scan handle's schema. */ + protected static ReadPlan planColumns(DeltaKernelUtils.ScanHandle handle) { + final int ncol = handle.schema.length(); + final int[] readCodes = new int[ncol]; + final ValueType[] vt = new ValueType[ncol]; + final String[] cnames = new String[ncol]; + for(int c = 0; c < ncol; c++) { + DataType dt = handle.schema.at(c).getDataType(); + readCodes[c] = readCode(dt, handle.schema.at(c).getName()); + vt[c] = valueType(readCodes[c]); + cnames[c] = handle.schema.at(c).getName(); + } + return new ReadPlan(ncol, readCodes, vt, cnames); + } + + /** + * Whether the metadata-driven direct read fast path can be used for this table (exact per-file row counts and no + * deletion vectors, so the output can be pre-sized and each file decoded straight into its row offset). Visible for + * testing: the buffered fallback is otherwise only reachable for tables lacking row statistics or carrying deletion + * vectors, which the SystemDS Delta writer never produces. + * + * @param handle the opened scan handle + * @return true if the direct path is applicable + */ + protected boolean useDirectPath(DeltaKernelUtils.ScanHandle handle) { + return handle.hasExactRowCounts(); + } + + /** + * Fast path: decode each data file straight into pre-sized typed column arrays at a metadata-derived row offset. + * One allocation per column, single pass, no intermediate per-batch buffers or serial concatenation. + */ + private FrameBlock readDirect(String fname, Engine engine, DeltaKernelUtils.ScanHandle handle, ReadPlan plan, + int nrow) throws IOException { + final int ncol = plan.ncol; + final int[] readCodes = plan.readCodes; + final Object[] dest = new Object[ncol]; + for(int c = 0; c < ncol; c++) + dest[c] = ArrayFactory.allocateBacking(plan.vt[c], nrow); + + int base = 0; + for(int i = 0; i < handle.scanFiles.size(); i++) { + // exclusive upper row bound for this file's slice; a file decoding more + // rows than its numRecords statistic would otherwise overflow into the + // next file's region or off the array + final int limit = base + (int) handle.numRecords[i]; + final int[] cur = new int[] {base}; + DeltaKernelUtils.readScanFile(engine, handle.scanState, handle.physicalReadSchema, handle.scanFiles.get(i), + (cols, size, selected) -> { + int n = DeltaKernelUtils.countSelected(size, selected); + if(cur[0] + n > limit) + throw new DMLRuntimeException("Delta file produced more rows than its " + + "numRecords statistic; refusing direct read of " + fname); + for(int c = 0; c < ncol; c++) + extractColumnInto(cols[c], size, selected, readCodes[c], dest[c], cur[0]); + cur[0] += n; + }); + // also fail loud on underflow: a file decoding fewer rows than its + // numRecords statistic would leave the tail of the slice at the array + // default (0/null) while nrow still reports the (inflated) statistic. + if(cur[0] != limit) + throw new DMLRuntimeException("Delta file produced " + (cur[0] - base) + " rows, expected " + + (limit - base) + " from its numRecords statistic; refusing direct read of " + fname); + base = limit; + } + + Array[] columns = new Array[ncol]; + for(int c = 0; c < ncol; c++) + columns[c] = ArrayFactory.create(plan.vt[c], dest[c]); + FrameBlock ret = new FrameBlock(columns); + ret.setColumnNames(plan.cnames); + return ret; + } + + /** + * Fallback path: decode each batch into per-batch typed arrays and concatenate them per column in file order. Used + * when exact per-file row counts are not available (missing statistics or deletion vectors present), so the output + * cannot be pre-sized up front. + */ + private FrameBlock readBuffered(Engine engine, DeltaKernelUtils.ScanHandle handle, ReadPlan plan) + throws IOException { + final int ncol = plan.ncol; + final int[] readCodes = plan.readCodes; + final ArrayList batchCols = new ArrayList<>(); + final ArrayList batchSizes = new ArrayList<>(); + final int[] nrowHolder = new int[1]; + for(Row scanFileRow : handle.scanFiles) { + DeltaKernelUtils.readScanFile(engine, handle.scanState, handle.physicalReadSchema, scanFileRow, + (cols, size, selected) -> { + int n = DeltaKernelUtils.countSelected(size, selected); + Object[] extracted = new Object[ncol]; + for(int c = 0; c < ncol; c++) { + // decode into a fresh per-batch array via the shared alloc + + // decode primitives (the same ones the direct path uses) + Object col = ArrayFactory.allocateBacking(plan.vt[c], n); + extractColumnInto(cols[c], size, selected, readCodes[c], col, 0); + extracted[c] = col; + } + batchCols.add(extracted); + batchSizes.add(n); + nrowHolder[0] += n; + }); + } + + int nrow = nrowHolder[0]; + // empty table: return a schema-only frame with the discovered schema/names. + if(nrow == 0) + return new FrameBlock(plan.vt, plan.cnames, 0); + Array[] columns = new Array[ncol]; + for(int c = 0; c < ncol; c++) + columns[c] = concatColumn(plan.vt[c], nrow, batchCols, batchSizes, c); + FrameBlock ret = new FrameBlock(columns); + ret.setColumnNames(plan.cnames); + return ret; + } + + /** + * Concatenate the per-batch typed arrays of one column (in file/batch order) into a single pre-sized array and wrap + * it as a frame {@link Array}. The copy is type-agnostic ({@link System#arraycopy} works on the boxed primitive or + * object arrays), so there is no per-type dispatch here: allocation and wrapping reuse + * {@link ArrayFactory#allocateBacking(ValueType, int)} and {@link ArrayFactory#create(ValueType, Object)}, the same + * primitives the single-pass direct path uses. + * + *

+ * Only the buffered fallback needs this concatenation; the default direct path decodes straight into one pre-sized + * array per column with no intermediate per-batch arrays. + *

+ */ + static Array concatColumn(ValueType vt, int nrow, ArrayList batchCols, ArrayList batchSizes, + int c) { + Object full = ArrayFactory.allocateBacking(vt, nrow); + int off = 0; + for(int b = 0; b < batchCols.size(); b++) { + int n = batchSizes.get(b); + System.arraycopy(batchCols.get(b)[c], 0, full, off, n); + off += n; + } + return ArrayFactory.create(vt, full); + } + + static int readCode(DataType dt, String name) { + // reuse the shared Delta type -> code mapping; frames additionally reject the + // types the matrix reader also cannot map (typeCode returns -1) + int code = DeltaKernelUtils.typeCode(dt); + if(code < 0) + throw new DMLRuntimeException( + "Unsupported non-mappable Delta column '" + name + "' of type " + dt + " for frame read."); + return code; + } + + static ValueType valueType(int readCode) { + switch(readCode) { + case R_DOUBLE: + return ValueType.FP64; + case R_FLOAT: + return ValueType.FP32; + case R_LONG: + return ValueType.INT64; + case R_INT: + case R_SHORT: + case R_BYTE: + return ValueType.INT32; + case R_BOOLEAN: + return ValueType.BOOLEAN; + default: + return ValueType.STRING; + } + } + + /** + * Decode the live (selected, after deletion vector) rows of one column batch directly into a pre-sized typed array + * starting at absolute row {@code destOff}. Null numeric cells keep the array default (0); string nulls are stored + * as null. + */ + static void extractColumnInto(ColumnVector col, int size, boolean[] selected, int readCode, Object dest, + int destOff) { + switch(readCode) { + case R_DOUBLE: { + double[] a = (double[]) dest; + int lr = destOff; + for(int r = 0; r < size; r++) { + if(selected != null && !selected[r]) + continue; + if(!col.isNullAt(r)) + a[lr] = col.getDouble(r); + lr++; + } + break; + } + case R_FLOAT: { + float[] a = (float[]) dest; + int lr = destOff; + for(int r = 0; r < size; r++) { + if(selected != null && !selected[r]) + continue; + if(!col.isNullAt(r)) + a[lr] = col.getFloat(r); + lr++; + } + break; + } + case R_LONG: { + long[] a = (long[]) dest; + int lr = destOff; + for(int r = 0; r < size; r++) { + if(selected != null && !selected[r]) + continue; + if(!col.isNullAt(r)) + a[lr] = col.getLong(r); + lr++; + } + break; + } + case R_INT: { + int[] a = (int[]) dest; + int lr = destOff; + for(int r = 0; r < size; r++) { + if(selected != null && !selected[r]) + continue; + if(!col.isNullAt(r)) + a[lr] = col.getInt(r); + lr++; + } + break; + } + case R_SHORT: { + int[] a = (int[]) dest; + int lr = destOff; + for(int r = 0; r < size; r++) { + if(selected != null && !selected[r]) + continue; + if(!col.isNullAt(r)) + a[lr] = col.getShort(r); + lr++; + } + break; + } + case R_BYTE: { + int[] a = (int[]) dest; + int lr = destOff; + for(int r = 0; r < size; r++) { + if(selected != null && !selected[r]) + continue; + if(!col.isNullAt(r)) + a[lr] = col.getByte(r); + lr++; + } + break; + } + case R_BOOLEAN: { + boolean[] a = (boolean[]) dest; + int lr = destOff; + for(int r = 0; r < size; r++) { + if(selected != null && !selected[r]) + continue; + if(!col.isNullAt(r)) + a[lr] = col.getBoolean(r); + lr++; + } + break; + } + default: { // R_STRING + String[] a = (String[]) dest; + int lr = destOff; + for(int r = 0; r < size; r++) { + if(selected != null && !selected[r]) + continue; + a[lr] = col.isNullAt(r) ? null : col.getString(r); + lr++; + } + break; + } + } + } + + @Override + public FrameBlock readFrameFromInputStream(InputStream is, ValueType[] schema, String[] names, long rlen, long clen) + throws IOException, DMLRuntimeException { + throw new UnsupportedOperationException( + "Reading a Delta table from an input stream is not supported; Delta is a directory-based table format."); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/io/FrameReaderDeltaParallel.java b/src/main/java/org/apache/sysds/runtime/io/FrameReaderDeltaParallel.java new file mode 100644 index 00000000000..106264afe6c --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/io/FrameReaderDeltaParallel.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.io; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.hops.OptimizerUtils; +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.frame.data.columns.Array; +import org.apache.sysds.runtime.frame.data.columns.ArrayFactory; +import org.apache.sysds.runtime.util.CommonThreadPool; + +import io.delta.kernel.data.Row; +import io.delta.kernel.engine.Engine; + +/** + * Parallel native Delta Lake frame reader. Delta tables are stored as one or more parquet data files; this reader + * decodes those files concurrently (one task per data file) and assembles them into a column-major {@link FrameBlock} + * in the original file order. + * + *

+ * It mirrors {@link ReaderDeltaParallel} (the matrix variant) but produces typed column {@link Array}s instead of a + * dense {@code double[]}. As with the matrix reader, the expensive part of a Delta read is the per-file parquet decode, + * so parallelizing across data files is the natural speedup. A table backed by a single data file cannot be split this + * way, so the reader transparently falls back to the sequential {@link FrameReaderDelta}. + *

+ */ +public class FrameReaderDeltaParallel extends FrameReaderDelta { + + private final int _numThreads; + + public FrameReaderDeltaParallel() { + _numThreads = OptimizerUtils.getParallelBinaryReadParallelism(); + } + + @Override + public FrameBlock readFrameFromHDFS(String fname, ValueType[] schema, String[] names, long rlen, long clen) + throws IOException, DMLRuntimeException { + Engine engine = DeltaKernelUtils.createEngine(); + String tablePath = DeltaKernelUtils.qualify(fname); + DeltaKernelUtils.ScanHandle handle = DeltaKernelUtils.openScan(engine, tablePath); + + final int nfiles = handle.scanFiles.size(); + // nothing to gain from parallelism for single-file (or empty) tables: reuse + // the already-opened engine + scan handle instead of re-opening the snapshot. + if(_numThreads <= 1 || nfiles <= 1) + return readWithHandle(fname, engine, handle); + + // derive per-column read codes, value types and names once from the schema + final ReadPlan plan = planColumns(handle); + + // fast path: exact per-file row counts are known from metadata -> pre-size + // one typed array per column and let each thread decode directly into its + // row offset (no intermediate buffers, no serial concatenation). + if(useDirectPath(handle)) { + long total = 0; + for(long r : handle.numRecords) + total += r; + if(total > 0 && total <= Integer.MAX_VALUE) + return readDirect(fname, handle, plan, (int) total); + } + + return readBuffered(fname, handle, plan); + } + + /** + * Fast path: each thread decodes one data file straight into the final typed column arrays at a metadata-derived + * row offset. Single allocation per column, fully parallel. + */ + private FrameBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, ReadPlan plan, int nrow) + throws IOException { + final int ncol = plan.ncol; + final int[] readCodes = plan.readCodes; + final int nfiles = handle.scanFiles.size(); + final int[] rowOffset = new int[nfiles]; + int acc = 0; + for(int i = 0; i < nfiles; i++) { + rowOffset[i] = acc; + acc += (int) handle.numRecords[i]; + } + + // pre-size one typed array per column for the whole table + final Object[] dest = new Object[ncol]; + for(int c = 0; c < ncol; c++) + dest[c] = ArrayFactory.allocateBacking(plan.vt[c], nrow); + + ArrayList> tasks = new ArrayList<>(nfiles); + for(int i = 0; i < nfiles; i++) { + final Row scanFileRow = handle.scanFiles.get(i); + final int base = rowOffset[i]; + // exclusive upper row bound for this file's slice; a file decoding more + // rows than its numRecords statistic would otherwise overflow into the + // next file's region (concurrent overlapping writes) or off the array + final int limit = base + (int) handle.numRecords[i]; + tasks.add(() -> { + int[] cur = new int[] {base}; + Engine eng = DeltaKernelUtils.createEngine(); + DeltaKernelUtils.readScanFile(eng, handle.scanState, handle.physicalReadSchema, scanFileRow, + (cols, size, selected) -> { + int n = DeltaKernelUtils.countSelected(size, selected); + if(cur[0] + n > limit) + throw new DMLRuntimeException("Delta file produced more rows than its " + + "numRecords statistic; refusing parallel direct read of " + fname); + for(int c = 0; c < ncol; c++) + extractColumnInto(cols[c], size, selected, readCodes[c], dest[c], cur[0]); + cur[0] += n; + }); + // fail loud on underflow too: fewer decoded rows than the statistic + // would leave this slice's tail at the array default (0/null). + if(cur[0] != limit) + throw new DMLRuntimeException("Delta file produced " + (cur[0] - base) + " rows, expected " + + (limit - base) + " from its numRecords statistic; refusing parallel direct read of " + fname); + return null; + }); + } + awaitFileTasks(tasks, fname); + + Array[] columns = new Array[ncol]; + for(int c = 0; c < ncol; c++) + columns[c] = ArrayFactory.create(plan.vt[c], dest[c]); + + FrameBlock ret = new FrameBlock(columns); + ret.setColumnNames(plan.cnames); + return ret; + } + + /** + * Fallback path: decode each file in parallel into per-file per-column batch arrays (used when row counts are + * unknown or deletion vectors are present), then concatenate per column in file order via the shared + * {@link FrameReaderDelta#concatColumn} helper. + */ + private FrameBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handle, ReadPlan plan) + throws IOException { + final int ncol = plan.ncol; + final int[] readCodes = plan.readCodes; + final int nfiles = handle.scanFiles.size(); + @SuppressWarnings("unchecked") + final ArrayList[] fileCols = new ArrayList[nfiles]; + @SuppressWarnings("unchecked") + final ArrayList[] fileSizes = new ArrayList[nfiles]; + ArrayList> tasks = new ArrayList<>(nfiles); + for(int i = 0; i < nfiles; i++) { + final int fi = i; + final Row scanFileRow = handle.scanFiles.get(i); + tasks.add(() -> { + ArrayList fileBatchCols = new ArrayList<>(); + ArrayList fileBatchSizes = new ArrayList<>(); + Engine eng = DeltaKernelUtils.createEngine(); + DeltaKernelUtils.readScanFile(eng, handle.scanState, handle.physicalReadSchema, scanFileRow, + (cols, size, selected) -> { + int n = DeltaKernelUtils.countSelected(size, selected); + Object[] extracted = new Object[ncol]; + for(int c = 0; c < ncol; c++) { + // decode into a fresh per-batch array via the shared alloc + + // decode primitives (the same ones the direct path uses) + Object col = ArrayFactory.allocateBacking(plan.vt[c], n); + extractColumnInto(cols[c], size, selected, readCodes[c], col, 0); + extracted[c] = col; + } + fileBatchCols.add(extracted); + fileBatchSizes.add(n); + }); + fileCols[fi] = fileBatchCols; + fileSizes[fi] = fileBatchSizes; + return null; + }); + } + awaitFileTasks(tasks, fname); + + // flatten the per-file batches in file order and concatenate per column + ArrayList batchCols = new ArrayList<>(); + ArrayList batchSizes = new ArrayList<>(); + int nrow = 0; + for(int i = 0; i < nfiles; i++) { + batchCols.addAll(fileCols[i]); + batchSizes.addAll(fileSizes[i]); + for(int n : fileSizes[i]) + nrow += n; + } + + Array[] columns = new Array[ncol]; + for(int c = 0; c < ncol; c++) + columns[c] = concatColumn(plan.vt[c], nrow, batchCols, batchSizes, c); + + FrameBlock ret = new FrameBlock(columns); + ret.setColumnNames(plan.cnames); + return ret; + } + + /** + * Run one decode task per data file on the shared common thread pool and await completion. Full parallelism is + * requested (the task count, one per data file, naturally caps concurrency); this avoids the per-thread pool-size + * caching in {@code CommonThreadPool.get(k)} that could otherwise throttle this reader to a smaller pool created + * earlier on the same thread. + */ + private void awaitFileTasks(List> tasks, String fname) throws IOException { + ExecutorService pool = CommonThreadPool.get(_numThreads); + try { + for(Future f : pool.invokeAll(tasks)) + f.get(); + } + catch(InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted during parallel read of Delta table: " + fname, ex); + } + catch(Exception ex) { + throw new IOException("Failed parallel read of Delta table: " + fname, ex); + } + finally { + pool.shutdown(); + } + } + +} diff --git a/src/main/java/org/apache/sysds/runtime/io/FrameReaderFactory.java b/src/main/java/org/apache/sysds/runtime/io/FrameReaderFactory.java index 4e21d2c3f60..5efbf80b83e 100644 --- a/src/main/java/org/apache/sysds/runtime/io/FrameReaderFactory.java +++ b/src/main/java/org/apache/sysds/runtime/io/FrameReaderFactory.java @@ -51,6 +51,8 @@ public static FrameReader createFrameReader(FileFormat fmt, FileFormatProperties case PROTO: // TODO performance improvement: add parallel reader return new FrameReaderProto(); + case DELTA: + return textParallel ? new FrameReaderDeltaParallel() : new FrameReaderDelta(); default: throw new DMLRuntimeException("Failed to create frame reader for unknown format: " + fmt.toString()); } diff --git a/src/main/java/org/apache/sysds/runtime/io/FrameWriterDelta.java b/src/main/java/org/apache/sysds/runtime/io/FrameWriterDelta.java new file mode 100644 index 00000000000..fe66a7d195f --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/io/FrameWriterDelta.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.io; + +import java.io.IOException; +import java.util.NoSuchElementException; +import java.util.Optional; + +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.frame.data.columns.Array; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.data.ColumnarBatch; +import io.delta.kernel.data.FilteredColumnarBatch; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.types.BooleanType; +import io.delta.kernel.types.DataType; +import io.delta.kernel.types.DoubleType; +import io.delta.kernel.types.FloatType; +import io.delta.kernel.types.IntegerType; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructType; +import io.delta.kernel.utils.CloseableIterator; + +/** + * Single-threaded native Delta Lake writer for frames, built on the Spark-free Delta Kernel library. It creates (or + * recreates) a Delta table whose schema mirrors the frame schema (per-column {@link ValueType} mapped to a Delta type + * and the frame column names), streams the {@link FrameBlock} rows as columnar batches into parquet data files, and + * commits the add-file actions. + */ +public class FrameWriterDelta extends FrameWriter { + + @Override + public void writeFrameToHDFS(FrameBlock src, String fname, long rlen, long clen) + throws IOException, DMLRuntimeException { + if(src.getNumRows() != rlen || src.getNumColumns() != clen) + throw new IOException("Frame dimensions mismatch with metadata: (" + src.getNumRows() + "x" + + src.getNumColumns() + ") vs (" + rlen + "x" + clen + ")."); + int ncol = (int) clen; + int nrow = (int) rlen; + StructType schema = buildSchema(src.getSchema(), src.getColumnNames(), ncol); + + // snapshot the typed column arrays + per-column nullability once, so the + // hot per-cell path can read primitives directly (no boxing) and skip + // null-checks on non-nullable columns. + Array[] cols = new Array[ncol]; + boolean[] nullable = new boolean[ncol]; + for(int c = 0; c < ncol; c++) { + cols[c] = src.getColumn(c); + nullable[c] = cols[c].containsNull(); + } + + int batchRows = ConfigurationManager.getDeltaWriterBatchSize(); + // size data files adaptively (toward one file per parallel reader) for faster parallel reads + Engine engine = DeltaKernelUtils.createWriteEngine(src.getInMemorySize()); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), schema, + new FrameBatchIterator(cols, nullable, schema, nrow, ncol, batchRows)); + } + + private static StructType buildSchema(ValueType[] vtSchema, String[] names, int ncol) { + StructType schema = new StructType(); + for(int c = 0; c < ncol; c++) + schema = schema.add(names[c], toDeltaType(vtSchema[c]), true); + return schema; + } + + static DataType toDeltaType(ValueType vt) { + switch(vt) { + case FP64: + return DoubleType.DOUBLE; + case FP32: + return FloatType.FLOAT; + case INT64: + return LongType.LONG; + case INT32: + case UINT8: + case UINT4: + return IntegerType.INTEGER; + case BOOLEAN: + return BooleanType.BOOLEAN; + default: + return StringType.STRING; // STRING/CHARACTER/HASH*/UNKNOWN + } + } + + /** Chunks the frame columns into fixed-size columnar batches for the kernel write path. */ + private static class FrameBatchIterator implements CloseableIterator { + private final Array[] _cols; + private final boolean[] _nullable; + private final StructType _schema; + private final int _nrow; + private final int _ncol; + private final int _batchRows; + private int _pos = 0; + + FrameBatchIterator(Array[] cols, boolean[] nullable, StructType schema, int nrow, int ncol, int batchRows) { + _cols = cols; + _nullable = nullable; + _schema = schema; + _nrow = nrow; + _ncol = ncol; + _batchRows = batchRows; + } + + @Override + public boolean hasNext() { + return _pos < _nrow; + } + + @Override + public FilteredColumnarBatch next() { + if(!hasNext()) + throw new NoSuchElementException(); + int size = Math.min(_batchRows, _nrow - _pos); + ColumnarBatch batch = new FrameColumnarBatch(_cols, _nullable, _schema, _pos, size, _ncol); + _pos += size; + return new FilteredColumnarBatch(batch, Optional.empty()); + } + + @Override + public void close() { + // nothing to release + } + } + + /** Read-only view of a row range of the frame columns as a Delta Kernel columnar batch. */ + private static class FrameColumnarBatch implements ColumnarBatch { + private final Array[] _cols; + private final boolean[] _nullable; + private final StructType _schema; + private final int _rowStart; + private final int _size; + private final int _ncol; + + FrameColumnarBatch(Array[] cols, boolean[] nullable, StructType schema, int rowStart, int size, int ncol) { + _cols = cols; + _nullable = nullable; + _schema = schema; + _rowStart = rowStart; + _size = size; + _ncol = ncol; + } + + @Override + public StructType getSchema() { + return _schema; + } + + @Override + public ColumnVector getColumnVector(int ordinal) { + if(ordinal < 0 || ordinal >= _ncol) + throw new IndexOutOfBoundsException("column ordinal " + ordinal); + return new FrameColumnVector(_cols[ordinal], _nullable[ordinal], _schema.at(ordinal).getDataType(), + _rowStart, _size); + } + + @Override + public int getSize() { + return _size; + } + } + + /** + * Read-only typed column view over one column {@link Array} row range. Numeric values are read through + * {@link Array#getAsDouble(int)} to avoid boxing, and non-nullable columns short-circuit {@code isNullAt} so the + * kernel never pays for a redundant boxed fetch. + */ + private static class FrameColumnVector implements ColumnVector { + private final Array _col; + private final boolean _nullable; + private final DataType _type; + private final int _rowStart; + private final int _size; + + FrameColumnVector(Array col, boolean nullable, DataType type, int rowStart, int size) { + _col = col; + _nullable = nullable; + _type = type; + _rowStart = rowStart; + _size = size; + } + + @Override + public DataType getDataType() { + return _type; + } + + @Override + public int getSize() { + return _size; + } + + @Override + public boolean isNullAt(int rowId) { + return _nullable && _col.get(_rowStart + rowId) == null; + } + + @Override + public String getString(int rowId) { + Object v = _col.get(_rowStart + rowId); + return (v == null) ? null : v.toString(); + } + + @Override + public boolean getBoolean(int rowId) { + return _col.getAsDouble(_rowStart + rowId) != 0; + } + + @Override + public double getDouble(int rowId) { + return _col.getAsDouble(_rowStart + rowId); + } + + @Override + public float getFloat(int rowId) { + return (float) _col.getAsDouble(_rowStart + rowId); + } + + @Override + public long getLong(int rowId) { + // exact for INT64 (getAsDouble would lose precision beyond 2^53). This boxes one + // Number per cell because Array exposes no primitive getAsLong; a boxing-free + // getAsLong on Array would remove this write-path allocation (follow-up). The + // kernel only calls this after isNullAt() is false, so the cell is never null here. + return ((Number) _col.get(_rowStart + rowId)).longValue(); + } + + @Override + public int getInt(int rowId) { + return (int) _col.getAsDouble(_rowStart + rowId); + } + + @Override + public void close() { + // nothing to release + } + } +} diff --git a/src/main/java/org/apache/sysds/runtime/io/FrameWriterFactory.java b/src/main/java/org/apache/sysds/runtime/io/FrameWriterFactory.java index 3fb3968c96f..ff38eb395dd 100644 --- a/src/main/java/org/apache/sysds/runtime/io/FrameWriterFactory.java +++ b/src/main/java/org/apache/sysds/runtime/io/FrameWriterFactory.java @@ -50,6 +50,8 @@ public static FrameWriter createFrameWriter(FileFormat fmt, FileFormatProperties return binaryParallel ? new FrameWriterBinaryBlockParallel() : new FrameWriterBinaryBlock(); case PROTO: return new FrameWriterProto(); + case DELTA: + return new FrameWriterDelta(); default: throw new DMLRuntimeException("Failed to create frame writer for unknown format: " + fmt.toString()); } diff --git a/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java b/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java index 0f08bf5517d..55ea8a54297 100644 --- a/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java +++ b/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java @@ -62,7 +62,11 @@ public void writeMatrixToHDFS(MatrixBlock src, String fname, long rlen, long cle //from the backing double[] (avoids per-cell MatrixBlock.get dispatch). double[] dense = (!src.isInSparseFormat() && src.getDenseBlock() != null && src.getDenseBlock().isContiguous()) ? src.getDenseBlockValues() : null; - Engine engine = DeltaKernelUtils.createEngine(); + //size data files adaptively (toward one file per parallel reader) for faster parallel reads. + //Delta writes every cell as a double, so size by the dense footprint rather than the (possibly + //sparse) in-memory size, which would understate the on-disk table for sparse inputs. + long estimatedBytes = (long) nrow * ncol * 8L; + Engine engine = DeltaKernelUtils.createWriteEngine(estimatedBytes); DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), buildSchema(ncol), new MatrixBatchIterator(src, dense, nrow, ncol, batchRows)); } diff --git a/src/test/java/org/apache/sysds/performance/Main.java b/src/test/java/org/apache/sysds/performance/Main.java index f8d0bbea852..0622e789baa 100644 --- a/src/test/java/org/apache/sysds/performance/Main.java +++ b/src/test/java/org/apache/sysds/performance/Main.java @@ -24,6 +24,7 @@ import org.apache.sysds.performance.compression.Serialize; import org.apache.sysds.performance.compression.StreamCompress; import org.apache.sysds.performance.compression.TransformPerf; +import org.apache.sysds.performance.frame.DeltaFrameRead; import org.apache.sysds.performance.frame.Transform; import org.apache.sysds.performance.generators.ConstMatrix; import org.apache.sysds.performance.generators.FrameFile; @@ -113,6 +114,9 @@ private static void exec(int prog, String[] args) throws Exception { case 17: run17(args); break; + case 18: + run18(args); + break; case 1000: run1000(args); break; @@ -238,6 +242,21 @@ private static void run17(String[] args) throws Exception { new MatrixReplacePerf(100, g, k).run(); } + /** + * Repeatedly read the same on-disk Delta frame table (written once as setup). + * Args: {@code 18 [mode] [targetFileSizeMB]} + * where mode is one of serial|parallel|both (default parallel) and an omitted + * target file size uses the adaptive default sizing. + */ + private static void run18(String[] args) throws Exception { + int rows = Integer.parseInt(args[1]); + int k = Integer.parseInt(args[2]); + int n = Integer.parseInt(args[3]); + String mode = (args.length > 4) ? args[4] : "parallel"; + long targetFileSize = (args.length > 5) ? Long.parseLong(args[5]) * 1024 * 1024 : -1; + new DeltaFrameRead(n, DeltaFrameRead.mixedFrame(rows, 7), k, mode, targetFileSize).run(); + } + private static void run1000(String[] args) { MMSparsityPerformance perf; if (args.length < 3) { diff --git a/src/test/java/org/apache/sysds/performance/frame/DeltaFrameRead.java b/src/test/java/org/apache/sysds/performance/frame/DeltaFrameRead.java new file mode 100644 index 00000000000..ea76fef51b1 --- /dev/null +++ b/src/test/java/org/apache/sysds/performance/frame/DeltaFrameRead.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.performance.frame; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.commons.io.FileUtils; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; +import org.apache.sysds.performance.compression.APerfTest; +import org.apache.sysds.performance.generators.ConstFrame; +import org.apache.sysds.performance.generators.IGenerate; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.io.FrameReaderDelta; +import org.apache.sysds.runtime.io.FrameReaderDeltaParallel; +import org.apache.sysds.runtime.io.FrameWriterDelta; +import org.apache.sysds.test.TestUtils; +import org.apache.sysds.test.component.io.DeltaFrameTestUtils; + +/** + * Reads the SAME native Delta frame table from disk repeatedly and reports read throughput. The table is written to a + * temporary directory ONCE as (untimed) setup; every timed repetition re-opens the latest snapshot and materializes a + * fresh {@link FrameBlock}, so the numbers reflect the read path only (parquet decode + column materialization), not + * the write. + * + *

+ * This is the target for an async-profiler run: launch the perf jar under the profiler agent and this loop provides a + * long, steady-state read workload to sample. See {@code src/test/java/org/apache/sysds/performance/README.md} for how + * to run this under async-profiler. + *

+ * + *

+ * Dispatched from {@link org.apache.sysds.performance.Main} (program id 18). + *

+ */ +public class DeltaFrameRead extends APerfTest { + + // the Delta reader derives schema/names from the table metadata, so the values + // passed here are placeholders (a single detect column) and are ignored. + private static final ValueType[] DETECT_SCHEMA = new ValueType[] {ValueType.STRING}; + private static final String[] DETECT_NAMES = new String[] {"x"}; + + private final int k; + private final String mode; + private final long targetFileSize; // <=0 -> adaptive default sizing + + private String tablePath; + private Path tableDir; + private long inMemSize; + private long files; + + public DeltaFrameRead(int N, IGenerate gen, int k, String mode, long targetFileSize) { + super(N, gen); + this.k = k; + this.mode = mode; + this.targetFileSize = targetFileSize; + } + + public void run() throws Exception { + try { + setup(); + System.out.println(this); + System.out.printf("table: %s%n", tablePath); + System.out.printf("layout: files=%d, in-memory=%.1f MB, target=%s%n", files, inMemSize / 1048576.0, + targetFileSize > 0 ? (targetFileSize / 1048576) + "MB(fixed)" : "adaptive"); + + if(mode.equals("serial") || mode.equals("both")) + execute(() -> readSerial(), "Delta read serial"); + if(mode.equals("parallel") || mode.equals("both")) + execute(() -> readParallel(), "Delta read parallel(k=" + k + ")"); + } + finally { + ConfigurationManager.clearLocalConfigs(); + if(tableDir != null) + FileUtils.deleteQuietly(tableDir.toFile()); + } + } + + /** Untimed: materialize the source frame and write it to a temp Delta table once. */ + private void setup() throws Exception { + FrameBlock fb = gen.take(); + inMemSize = fb.getInMemorySize(); + + DMLConfig c = new DMLConfig(); + if(targetFileSize > 0) { + c.setTextValue(DMLConfig.DELTA_WRITER_ADAPTIVE_FILE_SIZE, "false"); + c.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(targetFileSize)); + } + ConfigurationManager.setLocalConfig(c); + + tableDir = Files.createTempDirectory("sysds_delta_frame_read_"); + tablePath = new File(tableDir.toFile(), "table").getAbsolutePath(); + new FrameWriterDelta().writeFrameToHDFS(fb, tablePath, fb.getNumRows(), fb.getNumColumns()); + files = DeltaFrameTestUtils.countParquet(tablePath); + } + + private void readSerial() { + try { + FrameBlock fb = new FrameReaderDelta().readFrameFromHDFS(tablePath, DETECT_SCHEMA, DETECT_NAMES, -1, -1); + ret.add(fb.getInMemorySize()); + } + catch(Exception e) { + throw new RuntimeException(e); + } + } + + private void readParallel() { + try { + FrameBlock fb = new FrameReaderDeltaParallel().readFrameFromHDFS(tablePath, DETECT_SCHEMA, DETECT_NAMES, -1, + -1); + ret.add(fb.getInMemorySize()); + } + catch(Exception e) { + throw new RuntimeException(e); + } + } + + @Override + protected String makeResString() { + throw new UnsupportedOperationException("Use makeResString(double[]) with the timed measurements instead."); + } + + @Override + protected String makeResString(double[] times) { + double meanMs = trimmedMean(times); + double mbPerSec = (inMemSize / 1048576.0) / (meanMs / 1000.0); + return String.format("%8.1f MB/s", mbPerSec); + } + + /** 5%-trimmed mean, matching the trimming used by the framework statistics. */ + private static double trimmedMean(double[] times) { + double[] v = times.clone(); + java.util.Arrays.sort(v); + int remove = (int) Math.floor(v.length * 0.05); + double total = 0; + int el = v.length - remove * 2; + for(int i = remove; i < v.length - remove; i++) + total += v[i]; + return total / Math.max(el, 1); + } + + @Override + public String toString() { + return super.toString() + " mode: " + mode + ", threads: " + k; + } + + /** Build a representative mixed-schema frame (string + numeric columns). */ + public static IGenerate mixedFrame(int rows, long seed) { + ValueType[] schema = new ValueType[] {ValueType.STRING, ValueType.INT64, ValueType.FP64, ValueType.BOOLEAN, + ValueType.INT32, ValueType.FP32}; + return new ConstFrame(TestUtils.generateRandomFrameBlock(rows, schema, seed)); + } +} diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaFrameReadWriteTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameReadWriteTest.java new file mode 100644 index 00000000000..7012be44426 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameReadWriteTest.java @@ -0,0 +1,737 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.io; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.Random; + +import org.apache.commons.io.FileUtils; +import org.apache.sysds.common.Types.FileFormat; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.conf.CompilerConfig; +import org.apache.sysds.conf.CompilerConfig.ConfigType; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.io.DeltaKernelUtils; +import org.apache.sysds.runtime.io.FrameReader; +import org.apache.sysds.runtime.io.FrameReaderDelta; +import org.apache.sysds.runtime.io.FrameReaderDeltaParallel; +import org.apache.sysds.runtime.io.FrameReaderFactory; +import org.apache.sysds.runtime.io.FrameWriterDelta; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.data.ColumnarBatch; +import io.delta.kernel.data.FilteredColumnarBatch; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.types.ByteType; +import io.delta.kernel.types.DataType; +import io.delta.kernel.types.DateType; +import io.delta.kernel.types.DoubleType; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.ShortType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructType; +import io.delta.kernel.utils.CloseableIterator; + +/** + * Direct (no DML) round-trip tests for the native Delta Kernel based frame reader/writer. Each test writes a FrameBlock + * to a fresh local Delta table directory and reads it back, asserting the discovered schema, column names, dimensions, + * and per-cell values match. Several tests additionally assert that the parallel reader + * ({@link FrameReaderDeltaParallel}) agrees with the serial reader cell-for-cell across a multi-file table (both its + * direct and buffered paths). + */ +public class DeltaFrameReadWriteTest { + + // nonsense schema/dims handed to the reader to confirm it discovers everything + private static final ValueType[] NO_SCHEMA = new ValueType[] {ValueType.STRING}; + private static final String[] NO_NAMES = new String[] {"x"}; + + // small target file size + enough random rows so the writer rolls multiple + // data files, exercising the per-file parallel read path rather than the + // single-file serial fallback. + private static final long SMALL_TARGET_FILE_SIZE = 512L * 1024; + private static final int ROWS_MULTI_FILE = 150_000; + + // mixed-type schema used by the multi-file round-trip tests; random data is + // generated via TestUtils rather than a bespoke per-test generator. + private static final ValueType[] MIXED_SCHEMA = {ValueType.STRING, ValueType.INT64, ValueType.FP64, + ValueType.BOOLEAN, ValueType.INT32, ValueType.FP32}; + + private static FrameBlock writeThenRead(FrameBlock in) throws Exception { + Path dir = Files.createTempDirectory("sysds_delta_frame_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + new FrameWriterDelta().writeFrameToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns()); + // pass nonsense schema/dims: the reader must discover everything from the table + return new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1); + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + private static FrameBlock alloc(ValueType[] schema, String[] names, int nrow) { + FrameBlock fb = new FrameBlock(schema, names); + fb.ensureAllocatedColumns(nrow); + return fb; + } + + @FunctionalInterface + private interface TableTest { + void accept(FrameBlock in, String tablePath) throws Exception; + } + + /** + * Write {@code in} to a fresh temp Delta table with a small target file size (so the writer rolls multiple data + * files), assert the layout really is multi-file, then run {@code body} against the table. Local config and the + * temp directory are always cleaned up. + */ + private static void withSmallTargetTable(FrameBlock in, TableTest body) throws Exception { + DMLConfig conf = new DMLConfig(); + conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(SMALL_TARGET_FILE_SIZE)); + ConfigurationManager.setLocalConfig(conf); + Path dir = Files.createTempDirectory("sysds_delta_frame_mf_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + new FrameWriterDelta().writeFrameToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns()); + assertMultiFile(tablePath); + body.accept(in, tablePath); + } + finally { + ConfigurationManager.clearLocalConfigs(); + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void roundTripMixedTypes() throws Exception { + ValueType[] schema = {ValueType.STRING, ValueType.INT64, ValueType.FP64, ValueType.BOOLEAN, ValueType.INT32, + ValueType.FP32}; + String[] names = {"name", "id", "score", "active", "count", "ratio"}; + int nrow = 5; + FrameBlock in = alloc(schema, names, nrow); + for(int r = 0; r < nrow; r++) { + in.set(r, 0, "row" + r); + in.set(r, 1, (long) (r * 1000L + 7)); + in.set(r, 2, r + 0.5); + in.set(r, 3, (r % 2 == 0)); + in.set(r, 4, r * 3); + in.set(r, 5, (float) (r / 4.0)); + } + + FrameBlock out = writeThenRead(in); + + assertEquals(nrow, out.getNumRows()); + assertEquals(schema.length, out.getNumColumns()); + // schema and names discovered from the table + for(int c = 0; c < schema.length; c++) { + assertEquals("schema col " + c, schema[c], out.getSchema()[c]); + assertEquals("name col " + c, names[c], out.getColumnNames()[c]); + } + // values (compare as strings to be type-agnostic across boxed numerics) + for(int r = 0; r < nrow; r++) + for(int c = 0; c < schema.length; c++) + assertEquals("cell (" + r + "," + c + ")", in.get(r, c).toString(), out.get(r, c).toString()); + } + + @Test + public void roundTripMultiBatch() throws Exception { + // more rows than the writer batch size (4096) to exercise chunking + ValueType[] schema = {ValueType.INT64, ValueType.STRING}; + String[] names = {"k", "v"}; + int nrow = 10000; + FrameBlock in = alloc(schema, names, nrow); + for(int r = 0; r < nrow; r++) { + in.set(r, 0, (long) r); + in.set(r, 1, "v" + r); + } + + FrameBlock out = writeThenRead(in); + assertEquals(nrow, out.getNumRows()); + assertEquals(2, out.getNumColumns()); + for(int r = 0; r < nrow; r++) { + assertEquals((long) r, ((Number) out.get(r, 0)).longValue()); + assertEquals("v" + r, out.get(r, 1).toString()); + } + } + + @Test + public void roundTripWithStringNulls() throws Exception { + // nulls are only representable in object-backed (string) columns; numeric + // frame columns store primitives and cannot carry a null. + ValueType[] schema = {ValueType.STRING, ValueType.FP64}; + String[] names = {"s", "d"}; + int nrow = 4; + FrameBlock in = alloc(schema, names, nrow); + in.set(0, 0, "a"); + in.set(0, 1, 1.0); + in.set(1, 0, null); + in.set(1, 1, 2.0); + in.set(2, 0, "c"); + in.set(2, 1, 3.0); + in.set(3, 0, null); + in.set(3, 1, 4.0); + + FrameBlock out = writeThenRead(in); + assertEquals(nrow, out.getNumRows()); + assertEquals(2, out.getNumColumns()); + assertEquals("a", out.get(0, 0).toString()); + assertEquals(1.0, ((Number) out.get(0, 1)).doubleValue(), 1e-12); + assertNull(out.get(1, 0)); + assertEquals(2.0, ((Number) out.get(1, 1)).doubleValue(), 1e-12); + assertEquals("c", out.get(2, 0).toString()); + assertEquals(3.0, ((Number) out.get(2, 1)).doubleValue(), 1e-12); + assertNull(out.get(3, 0)); + assertEquals(4.0, ((Number) out.get(3, 1)).doubleValue(), 1e-12); + } + + @Test + public void parallelReadMatchesSerialMultiFile() throws Exception { + FrameBlock in = TestUtils.generateRandomFrameBlock(ROWS_MULTI_FILE, MIXED_SCHEMA, 13); + withSmallTargetTable(in, (frame, tablePath) -> { + FrameBlock serial = new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1); + FrameBlock parallel = new FrameReaderDeltaParallel().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, + -1); + assertFramesEqual(serial, parallel); + }); + } + + @Test + public void parallelBufferedPathMatchesSerialMultiFile() throws Exception { + // the direct fast path is always taken for SystemDS-written tables (exact + // row stats, no deletion vectors); force the buffered fallback to exercise + // its per-file decode + serial concatenation and assert it matches serial. + FrameBlock in = TestUtils.generateRandomFrameBlock(ROWS_MULTI_FILE, MIXED_SCHEMA, 23); + withSmallTargetTable(in, (frame, tablePath) -> { + FrameBlock serial = new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1); + // subclass that always declines the direct path -> readBuffered() + FrameBlock buffered = new FrameReaderDeltaParallel() { + @Override + protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { + return false; + } + }.readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1); + assertFramesEqual(serial, buffered); + }); + } + + @Test + public void serialBufferedPathMatchesDirectMultiFile() throws Exception { + // the direct (pre-sized, metadata-driven) path is always taken for SystemDS- + // written tables; force the serial buffered fallback (per-batch extract + + // concatenate) to exercise it and assert it matches the direct read. + FrameBlock in = TestUtils.generateRandomFrameBlock(ROWS_MULTI_FILE, MIXED_SCHEMA, 29); + withSmallTargetTable(in, (frame, tablePath) -> { + FrameBlock direct = new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1); + // subclass that always declines the direct path -> buffered extract+concat + FrameBlock buffered = new FrameReaderDelta() { + @Override + protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { + return false; + } + }.readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1); + assertFramesEqual(direct, buffered); + }); + } + + @Test + public void adaptiveTargetFileSizeClampsAndRespectsFlag() { + // cap chosen above the 4MB floor so both clamp directions are observable + final long cap = 64L * 1024 * 1024; + DMLConfig conf = new DMLConfig(); + conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(cap)); + conf.setTextValue(DMLConfig.DELTA_WRITER_ADAPTIVE_FILE_SIZE, "true"); + ConfigurationManager.setLocalConfig(conf); + try { + assertEquals("estimatedBytes<=0 -> configured cap", cap, DeltaKernelUtils.adaptiveWriterTargetFileSize(0)); + assertEquals("negative estimate -> configured cap", cap, DeltaKernelUtils.adaptiveWriterTargetFileSize(-1)); + assertEquals("huge table -> never above the configured cap", cap, + DeltaKernelUtils.adaptiveWriterTargetFileSize(Long.MAX_VALUE / 2)); + assertEquals("tiny table -> never below the floor", DeltaKernelUtils.ADAPTIVE_WRITER_MIN_FILE_SIZE, + DeltaKernelUtils.adaptiveWriterTargetFileSize(1)); + + conf.setTextValue(DMLConfig.DELTA_WRITER_ADAPTIVE_FILE_SIZE, "false"); + assertEquals("flag OFF -> always the configured cap regardless of size", cap, + DeltaKernelUtils.adaptiveWriterTargetFileSize(1)); + } + finally { + ConfigurationManager.clearLocalConfigs(); + } + } + + @Test + public void factoryRoutesDeltaToParallelWhenEnabled() { + // the factory must pick the parallel frame reader iff parallel CP read is enabled + CompilerConfig cc = ConfigurationManager.getCompilerConfig(); + try { + cc.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, true); + ConfigurationManager.setLocalConfig(cc); + FrameReader par = FrameReaderFactory.createFrameReader(FileFormat.DELTA); + assertTrue("expected FrameReaderDeltaParallel when parallel read enabled", + par instanceof FrameReaderDeltaParallel); + + cc.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, false); + ConfigurationManager.setLocalConfig(cc); + FrameReader ser = FrameReaderFactory.createFrameReader(FileFormat.DELTA); + assertTrue("expected serial FrameReaderDelta when parallel read disabled", + ser instanceof FrameReaderDelta && !(ser instanceof FrameReaderDeltaParallel)); + } + finally { + ConfigurationManager.clearLocalConfigs(); + } + } + + @Test + public void readerBatchSizeConfigRoundTrips() throws Exception { + // a non-default reader batch size must not change the result (more, smaller + // batches exercise the per-batch extract/concatenate loop more often). + DMLConfig conf = new DMLConfig(); + conf.setTextValue(DMLConfig.DELTA_READER_BATCH_SIZE, "128"); + ConfigurationManager.setLocalConfig(conf); + Path dir = Files.createTempDirectory("sysds_delta_frame_bs_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + FrameBlock in = TestUtils.generateRandomFrameBlock(5000, MIXED_SCHEMA, 31); + new FrameWriterDelta().writeFrameToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns()); + FrameBlock out = new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1); + assertFramesEqual(in, out); + } + finally { + ConfigurationManager.clearLocalConfigs(); + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void writerTargetFileSizeConfigProducesMoreFiles() throws Exception { + // a smaller configured target file size must make the writer roll more + // data files for the same frame (the lever the parallel reader relies on); + // the multi-file layout is asserted inside withSmallTargetTable. + FrameBlock in = TestUtils.generateRandomFrameBlock(ROWS_MULTI_FILE, MIXED_SCHEMA, 41); + withSmallTargetTable(in, (frame, tablePath) -> { + // data still round-trips correctly with the custom layout + FrameBlock out = new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1); + assertFramesEqual(frame, out); + }); + } + + @Test + public void emptyFrameRoundTrip() throws Exception { + // a schema-only Delta table (no data files, 0 rows); the reader must + // rebuild empty typed columns and discover the schema/names from the table. + ValueType[] schema = {ValueType.STRING, ValueType.FP64, ValueType.INT64}; + String[] names = {"s", "d", "k"}; + DataType[] dtypes = {StringType.STRING, DoubleType.DOUBLE, LongType.LONG}; + + Path dir = Files.createTempDirectory("sysds_delta_frame_empty_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + writeEmptyTable(tablePath, names, dtypes); + FrameBlock out = new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1); + assertEquals("rows", 0, out.getNumRows()); + assertEquals("cols", schema.length, out.getNumColumns()); + for(int c = 0; c < schema.length; c++) { + assertEquals("schema col " + c, schema[c], out.getSchema()[c]); + assertEquals("name col " + c, names[c], out.getColumnNames()[c]); + } + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void readDiscoversSchemaAndDims() throws Exception { + // reader handed -1 dims and a nonsense schema must discover both from the table + ValueType[] schema = {ValueType.INT32, ValueType.FP32, ValueType.BOOLEAN, ValueType.STRING}; + String[] names = {"a", "b", "c", "d"}; + int nrow = 321; + FrameBlock in = alloc(schema, names, nrow); + Random rnd = new Random(7); + for(int r = 0; r < nrow; r++) { + in.set(r, 0, rnd.nextInt()); + in.set(r, 1, rnd.nextFloat()); + in.set(r, 2, rnd.nextBoolean()); + in.set(r, 3, "s" + r); + } + + Path dir = Files.createTempDirectory("sysds_delta_frame_disc_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + new FrameWriterDelta().writeFrameToHDFS(in, tablePath, nrow, schema.length); + FrameBlock out = new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1); + assertEquals("rows", nrow, out.getNumRows()); + assertEquals("cols", schema.length, out.getNumColumns()); + for(int c = 0; c < schema.length; c++) { + assertEquals("schema col " + c, schema[c], out.getSchema()[c]); + assertEquals("name col " + c, names[c], out.getColumnNames()[c]); + } + assertFramesEqual(in, out); + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void readNonMappableColumnRejected() throws Exception { + // a Delta column type that does not map to a frame value type (date) must + // be rejected by the reader rather than silently mis-read. + Path dir = Files.createTempDirectory("sysds_delta_frame_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + writeDateColumn(tablePath, new int[] {0, 1, 100, 18000}); + try { + new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1); + fail("expected a DMLRuntimeException for a non-mappable (date) Delta column"); + } + catch(DMLRuntimeException ex) { + assertTrue("message should mention the non-mappable column, got: " + ex.getMessage(), + ex.getMessage() != null && ex.getMessage().contains("non-mappable")); + } + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void readShortByteColumnsCoercedToInt32() throws Exception { + // the kernel can store short/byte columns; the frame reader has no narrower + // integer value type, so both must surface as INT32 with the values intact. + short[] shorts = {0, 1, -1, Short.MAX_VALUE, Short.MIN_VALUE}; + byte[] bytes = {0, 7, -7, Byte.MAX_VALUE, Byte.MIN_VALUE}; + Path dir = Files.createTempDirectory("sysds_delta_frame_sb_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + writeShortByteColumns(tablePath, shorts, bytes); + FrameBlock out = new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1); + assertEquals("rows", shorts.length, out.getNumRows()); + assertEquals("cols", 2, out.getNumColumns()); + assertEquals("short column coerced to INT32", ValueType.INT32, out.getSchema()[0]); + assertEquals("byte column coerced to INT32", ValueType.INT32, out.getSchema()[1]); + for(int r = 0; r < shorts.length; r++) { + assertEquals("short cell (" + r + ")", shorts[r], ((Number) out.get(r, 0)).intValue()); + assertEquals("byte cell (" + r + ")", bytes[r], ((Number) out.get(r, 1)).intValue()); + } + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void writerRejectsDimensionMismatch() throws Exception { + ValueType[] schema = {ValueType.STRING, ValueType.INT64}; + String[] names = {"s", "k"}; + int nrow = 3; + FrameBlock fb = alloc(schema, names, nrow); + for(int r = 0; r < nrow; r++) { + fb.set(r, 0, "r" + r); + fb.set(r, 1, (long) r); + } + Path dir = Files.createTempDirectory("sysds_delta_frame_dim_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + // declare one more row than the frame actually has -> writer must reject + new FrameWriterDelta().writeFrameToHDFS(fb, tablePath, fb.getNumRows() + 1, fb.getNumColumns()); + fail("expected an IOException for a frame/metadata dimension mismatch"); + } + catch(IOException ex) { + assertTrue("message should mention the dimension mismatch, got: " + ex.getMessage(), + ex.getMessage() != null && ex.getMessage().contains("dimensions mismatch")); + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void readFromInputStreamUnsupported() throws Exception { + // Delta is a directory-based table format; stream reads are not supported + try { + new FrameReaderDelta().readFrameFromInputStream(null, NO_SCHEMA, NO_NAMES, -1, -1); + fail("expected UnsupportedOperationException for a Delta input-stream read"); + } + catch(UnsupportedOperationException ex) { + // must throw before touching the (null) stream, for the documented reason + assertTrue("message should mention input stream, got: " + ex.getMessage(), + ex.getMessage() != null && ex.getMessage().contains("input stream")); + } + } + + @Test + public void parallelReadStringNullsMatchSerialMultiFile() throws Exception { + // string nulls across a multi-file table: the parallel direct path must + // reproduce the serial read cell-for-cell (assertFramesEqual uses + // assertEquals, so nulls are compared faithfully). + ValueType[] schema = {ValueType.STRING, ValueType.INT64}; + String[] names = {"s", "k"}; + int nrow = ROWS_MULTI_FILE; + FrameBlock in = alloc(schema, names, nrow); + for(int r = 0; r < nrow; r++) { + // interspersed string nulls (every 7th row) plus a numeric column + in.set(r, 0, (r % 7 == 0) ? null : "s" + r); + in.set(r, 1, (long) r); + } + withSmallTargetTable(in, (frame, tablePath) -> { + FrameBlock serial = new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1); + FrameBlock parallel = new FrameReaderDeltaParallel().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, + -1); + assertFramesEqual(serial, parallel); + }); + } + + private static void assertMultiFile(String tablePath) throws Exception { + long files = DeltaFrameTestUtils.countParquet(tablePath); + assertTrue("expected a multi-file Delta table to exercise the parallel path, got " + files, files > 1); + } + + private static void assertFramesEqual(FrameBlock expected, FrameBlock actual) { + assertEquals("rows", expected.getNumRows(), actual.getNumRows()); + assertEquals("cols", expected.getNumColumns(), actual.getNumColumns()); + int ncol = expected.getNumColumns(); + for(int c = 0; c < ncol; c++) { + assertEquals("schema col " + c, expected.getSchema()[c], actual.getSchema()[c]); + assertEquals("name col " + c, expected.getColumnNames()[c], actual.getColumnNames()[c]); + } + int nrow = expected.getNumRows(); + for(int r = 0; r < nrow; r++) + for(int c = 0; c < ncol; c++) + assertEquals("cell (" + r + "," + c + ")", expected.get(r, c), actual.get(r, c)); + } + + /** Commits a schema-only Delta table (no data files) to exercise the 0-row read path. */ + private static void writeEmptyTable(String tablePath, String[] names, DataType[] dtypes) throws Exception { + Engine engine = DeltaKernelUtils.createEngine(); + StructType schema = new StructType(); + for(int c = 0; c < dtypes.length; c++) + schema = schema.add(names[c], dtypes[c], true); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(tablePath), schema, empty()); + } + + private static CloseableIterator empty() { + return new CloseableIterator() { + @Override + public boolean hasNext() { + return false; + } + + @Override + public FilteredColumnarBatch next() { + throw new NoSuchElementException(); + } + + @Override + public void close() { + } + }; + } + + /** + * Writes a single date column (kernel stores dates as INT32 days) used to assert the frame reader rejects a + * non-mappable column type. + */ + private static void writeDateColumn(String tablePath, int[] days) throws Exception { + Engine engine = DeltaKernelUtils.createEngine(); + final StructType schema = new StructType().add("d", DateType.DATE, false); + ColumnarBatch batch = new ColumnarBatch() { + @Override + public StructType getSchema() { + return schema; + } + + @Override + public int getSize() { + return days.length; + } + + @Override + public ColumnVector getColumnVector(int ordinal) { + return new DateVector(days); + } + }; + FilteredColumnarBatch fcb = new FilteredColumnarBatch(batch, Optional.empty()); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(tablePath), schema, singleton(fcb)); + } + + /** + * Writes a short column and a byte column (kernel stores these as 16/8-bit integers) used to assert the frame + * reader coerces both to INT32. + */ + private static void writeShortByteColumns(String tablePath, short[] shorts, byte[] bytes) throws Exception { + Engine engine = DeltaKernelUtils.createEngine(); + final StructType schema = new StructType().add("sh", ShortType.SHORT, false).add("by", ByteType.BYTE, false); + ColumnarBatch batch = new ColumnarBatch() { + @Override + public StructType getSchema() { + return schema; + } + + @Override + public int getSize() { + return shorts.length; + } + + @Override + public ColumnVector getColumnVector(int ordinal) { + return (ordinal == 0) ? new ShortVector(shorts) : new ByteVector(bytes); + } + }; + FilteredColumnarBatch fcb = new FilteredColumnarBatch(batch, Optional.empty()); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(tablePath), schema, singleton(fcb)); + } + + private static CloseableIterator singleton(FilteredColumnarBatch fcb) { + return new CloseableIterator() { + private boolean _done = false; + + @Override + public boolean hasNext() { + return !_done; + } + + @Override + public FilteredColumnarBatch next() { + if(_done) + throw new NoSuchElementException(); + _done = true; + return fcb; + } + + @Override + public void close() { + } + }; + } + + /** Column view exposing an int[] as a Delta date column. */ + private static class DateVector implements ColumnVector { + private final int[] _days; + + DateVector(int[] days) { + _days = days; + } + + @Override + public DataType getDataType() { + return DateType.DATE; + } + + @Override + public int getSize() { + return _days.length; + } + + @Override + public boolean isNullAt(int rowId) { + return false; + } + + @Override + public int getInt(int rowId) { + return _days[rowId]; + } + + @Override + public void close() { + } + } + + /** Column view exposing a short[] as a Delta short column. */ + private static class ShortVector implements ColumnVector { + private final short[] _vals; + + ShortVector(short[] vals) { + _vals = vals; + } + + @Override + public DataType getDataType() { + return ShortType.SHORT; + } + + @Override + public int getSize() { + return _vals.length; + } + + @Override + public boolean isNullAt(int rowId) { + return false; + } + + @Override + public short getShort(int rowId) { + return _vals[rowId]; + } + + @Override + public void close() { + } + } + + /** Column view exposing a byte[] as a Delta byte column. */ + private static class ByteVector implements ColumnVector { + private final byte[] _vals; + + ByteVector(byte[] vals) { + _vals = vals; + } + + @Override + public DataType getDataType() { + return ByteType.BYTE; + } + + @Override + public int getSize() { + return _vals.length; + } + + @Override + public boolean isNullAt(int rowId) { + return false; + } + + @Override + public byte getByte(int rowId) { + return _vals[rowId]; + } + + @Override + public void close() { + } + } +} diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaFrameSparkInteropTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameSparkInteropTest.java new file mode 100644 index 00000000000..d17d9ae4005 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameSparkInteropTest.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.io; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.io.FileUtils; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.io.FrameReaderDelta; +import org.apache.sysds.runtime.io.FrameReaderDeltaParallel; +import org.apache.sysds.runtime.io.FrameWriterDelta; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Cross-engine interoperability tests for the native (Delta Kernel based) frame reader/writer against the reference + * Delta implementation (Delta's Spark connector, {@code delta-spark}, pulled in test-only). + * + *

+ * The other Delta frame tests round-trip exclusively through SystemDS' own Kernel-based read/write paths, so they + * cannot catch a table that SystemDS writes in a way other Delta engines reject (or vice versa). These tests close that + * gap by routing a mixed-type (long/double/string/boolean) frame through two independent engines: + *

    + *
  • SystemDS writes -> Spark/Delta reads (our output is spec-compliant), and
    • + *
  • Spark/Delta writes -> SystemDS reads, including a multi-file layout and a table with deletion vectors / a + * second commit that the SystemDS writer never produces itself.
    • + *
+ * + *

+ * Row order is never assumed: every table carries a unique id in column 0 and comparisons are keyed by that id, since + * neither engine guarantees row order across files. + */ +@net.jcip.annotations.NotThreadSafe +public class DeltaFrameSparkInteropTest { + + // nonsense schema/dims handed to the reader to confirm it discovers everything from the table + private static final ValueType[] NO_SCHEMA = new ValueType[] {ValueType.STRING}; + private static final String[] NO_NAMES = new String[] {"x"}; + + private static SparkSession spark; + + @BeforeClass + public static void startSpark() { + // each test class runs in its own fork (surefire reuseForks=false), so this + // is the only SparkSession in the JVM and gets the Delta extensions injected. + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + spark = SparkSession.builder().appName("sysds-delta-frame-interop").master("local[2]") + .config("spark.ui.enabled", "false").config("spark.sql.shuffle.partitions", "2") + .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog").getOrCreate(); + } + + @AfterClass + public static void stopSpark() { + if(spark != null) + spark.stop(); + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + spark = null; + } + + @Test + public void systemdsWriteSparkReadMultiFile() throws Exception { + // SystemDS writes a (forced) multi-file mixed-type frame Delta table; the + // reference Delta engine (Spark) must read every data file back with + // matching values across all four column types. + int rows = 20_000, cols = 4; + FrameBlock in = indexedFrame(rows); + + // small target file size -> multiple parquet data files (exercise that an + // external reader stitches all of our data files, not just the first). + DMLConfig conf = new DMLConfig(); + conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(16L * 1024)); + ConfigurationManager.setLocalConfig(conf); + Path dir = Files.createTempDirectory("sysds_delta_frame_s2s_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + new FrameWriterDelta().writeFrameToHDFS(in, tablePath, rows, cols); + assertTrue("writer should have produced a multi-file table", countParquet(tablePath) > 1); + + Dataset df = spark.read().format("delta").load(tablePath); + assertEquals("rows", rows, df.count()); + assertEquals("cols", cols, df.schema().fields().length); + + List read = df.collectAsList(); + assertEquals(rows, read.size()); + boolean[] seen = new boolean[rows]; + for(Row r : read) { + int id = (int) r.getLong(0); + assertTrue("id in range and unique: " + id, id >= 0 && id < rows && !seen[id]); + seen[id] = true; + assertEquals("id" + id + " c1", dval(id), r.getDouble(1), 1e-9); + assertEquals("id" + id + " c2", sval(id), r.getString(2)); + assertEquals("id" + id + " c3", Boolean.valueOf(bval(id)), Boolean.valueOf(r.getBoolean(3))); + } + } + finally { + ConfigurationManager.clearLocalConfigs(); + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void sparkWriteSystemdsReadMultiFile() throws Exception { + // the reference Delta engine writes a multi-file mixed-type table; both the + // serial and parallel SystemDS frame readers must reconstruct it cell-for-cell. + int rows = 600; + Dataset df = indexedDataFrame(rows).repartition(3); // -> multiple data files + Path dir = Files.createTempDirectory("sysds_delta_frame_p2s_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + df.write().format("delta").save(tablePath); + assertTrue("spark should have written a multi-file table", countParquet(tablePath) > 1); + + Set expected = idRange(0, rows); + assertFrameMatchesIds(new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), + expected, "serial"); + assertFrameMatchesIds( + new FrameReaderDeltaParallel().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), expected, + "parallel"); + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void sparkDeletionVectorsSystemdsRead() throws Exception { + // a Delta table with deletion vectors + a second commit (the DELETE) is a + // layout the SystemDS writer never emits; the frame readers must honor the DV + // and return only the surviving rows. With DVs present hasExactRowCounts() is + // false, so this drives the buffered (selection-mask) frame read path. + int rows = 400, deleteBelow = 50; + Path dir = Files.createTempDirectory("sysds_delta_frame_dv_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + // enable deletion vectors for tables created in this block, then delete a + // row range so Delta records a DV rather than rewriting the data files. + spark.conf().set(DV_DEFAULT, "true"); + indexedDataFrame(rows).write().format("delta").save(tablePath); + spark.sql("DELETE FROM delta.`" + tablePath + "` WHERE c0 < " + deleteBelow); + + Set expected = idRange(deleteBelow, rows); + + FrameBlock serial = new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1); + assertEquals("surviving rows (serial)", rows - deleteBelow, serial.getNumRows()); + assertFrameMatchesIds(serial, expected, "serial-dv"); + + FrameBlock parallel = new FrameReaderDeltaParallel().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, + -1); + assertEquals("surviving rows (parallel)", rows - deleteBelow, parallel.getNumRows()); + assertFrameMatchesIds(parallel, expected, "parallel-dv"); + } + finally { + // fresh fork per test class, so simply clearing the override is enough + spark.conf().unset(DV_DEFAULT); + FileUtils.deleteQuietly(dir.toFile()); + } + } + + private static final String DV_DEFAULT = "spark.databricks.delta.properties.defaults.enableDeletionVectors"; + + // deterministic, exactly-representable cell values keyed by the row id in column 0 + private static double dval(int id) { + return id * 0.5 - 1.0; + } + + private static String sval(int id) { + return "s" + id; + } + + private static boolean bval(int id) { + return id % 2 == 0; + } + + /** Frame whose column 0 is the row id and the remaining columns are exact per-id values. */ + private static FrameBlock indexedFrame(int rows) { + ValueType[] schema = {ValueType.INT64, ValueType.FP64, ValueType.STRING, ValueType.BOOLEAN}; + String[] names = {"c0", "c1", "c2", "c3"}; + FrameBlock fb = new FrameBlock(schema, names); + fb.ensureAllocatedColumns(rows); + for(int r = 0; r < rows; r++) { + fb.set(r, 0, (long) r); + fb.set(r, 1, dval(r)); + fb.set(r, 2, sval(r)); + fb.set(r, 3, bval(r)); + } + return fb; + } + + /** Spark DataFrame mirroring {@link #indexedFrame} with columns c0..c3 (long/double/string/boolean). */ + private Dataset indexedDataFrame(int rows) { + StructType schema = DataTypes + .createStructType(new StructField[] {DataTypes.createStructField("c0", DataTypes.LongType, false), + DataTypes.createStructField("c1", DataTypes.DoubleType, false), + DataTypes.createStructField("c2", DataTypes.StringType, false), + DataTypes.createStructField("c3", DataTypes.BooleanType, false)}); + + List data = new ArrayList<>(rows); + for(int r = 0; r < rows; r++) + data.add(RowFactory.create((long) r, dval(r), sval(r), bval(r))); + return spark.createDataFrame(data, schema); + } + + private static Set idRange(int fromInclusive, int toExclusive) { + Set ids = new LinkedHashSet<>(toExclusive - fromInclusive); + for(int id = fromInclusive; id < toExclusive; id++) + ids.add(id); + return ids; + } + + /** Asserts every row of {@code out} (keyed by its column-0 id) is expected and carries the exact per-id values. */ + private static void assertFrameMatchesIds(FrameBlock out, Set expectedIds, String tag) { + assertEquals(tag + " rows", expectedIds.size(), out.getNumRows()); + assertEquals(tag + " cols", 4, out.getNumColumns()); + // discovered types: long->INT64, double->FP64, string->STRING, boolean->BOOLEAN + assertEquals(tag + " c0 type", ValueType.INT64, out.getSchema()[0]); + assertEquals(tag + " c1 type", ValueType.FP64, out.getSchema()[1]); + assertEquals(tag + " c2 type", ValueType.STRING, out.getSchema()[2]); + assertEquals(tag + " c3 type", ValueType.BOOLEAN, out.getSchema()[3]); + Set seen = new HashSet<>(); + for(int r = 0; r < out.getNumRows(); r++) { + int id = ((Number) out.get(r, 0)).intValue(); + assertTrue(tag + ": unexpected/duplicate id " + id, expectedIds.contains(id) && seen.add(id)); + assertEquals(tag + " id" + id + " c1", dval(id), ((Number) out.get(r, 1)).doubleValue(), 1e-9); + assertEquals(tag + " id" + id + " c2", sval(id), out.get(r, 2).toString()); + assertEquals(tag + " id" + id + " c3", Boolean.valueOf(bval(id)), out.get(r, 3)); + } + } + + private static long countParquet(String tablePath) throws Exception { + return DeltaFrameTestUtils.countParquet(tablePath); + } +} diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaFrameTestUtils.java b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameTestUtils.java new file mode 100644 index 00000000000..fe025c40eae --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameTestUtils.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.io; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +/** Shared helpers for the native Delta frame read/write tests and benchmarks. */ +public class DeltaFrameTestUtils { + + private DeltaFrameTestUtils() { + // utility class + } + + /** Count the parquet data files under a Delta table directory. */ + public static long countParquet(String tablePath) throws Exception { + try(Stream s = Files.walk(new File(tablePath).toPath())) { + return s.filter(p -> p.toString().endsWith(".parquet")).count(); + } + } +} diff --git a/src/test/java/org/apache/sysds/test/functions/io/delta/FrameDeltaReadWriteTest.java b/src/test/java/org/apache/sysds/test/functions/io/delta/FrameDeltaReadWriteTest.java new file mode 100644 index 00000000000..2c4799d16f0 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/io/delta/FrameDeltaReadWriteTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.io.delta; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.util.HashMap; + +import org.apache.sysds.runtime.controlprogram.caching.CacheStatistics; +import org.apache.sysds.runtime.matrix.data.MatrixValue.CellIndex; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; + +/** + * End-to-end DML test of the native Delta frame read/write path. + * + *

+ * As in the matrix variant, the write and the read run as two separate SystemDS executions so the read is a genuine + * disk read rather than an in-memory cache hit. We additionally assert via {@link CacheStatistics} that the write run + * wrote (delta + text reference) and the read run read (delta + text reference) from HDFS, so a short-circuited path + * would fail the test. + *

+ */ +public class FrameDeltaReadWriteTest extends AutomatedTestBase { + + private final static String TEST_DIR = "functions/io/delta/"; + private final static String TEST_CLASS_DIR = TEST_DIR + FrameDeltaReadWriteTest.class.getSimpleName() + "/"; + private final static String WRITE_NAME = "FrameDeltaWrite"; + private final static String READ_NAME = "FrameDeltaReadCompare"; + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration(WRITE_NAME, new TestConfiguration(TEST_CLASS_DIR, WRITE_NAME, new String[] {"ref"})); + addTestConfiguration(READ_NAME, new TestConfiguration(TEST_CLASS_DIR, READ_NAME, new String[] {"R"})); + } + + @Test + public void testDenseRoundTrip() { + runFrameDeltaRoundTrip(200, 12, 1.0); + } + + @Test + public void testSparseRoundTrip() { + runFrameDeltaRoundTrip(640, 8, 0.2); + } + + @Test + public void testMultiBatchRoundTrip() { + runFrameDeltaRoundTrip(9000, 4, 1.0); + } + + private void runFrameDeltaRoundTrip(int rows, int cols, double sparsity) { + try { + String HOME = SCRIPT_DIR + TEST_DIR; + + // ---- phase 1: write the frame as a Delta table + text reference ---- + getAndLoadTestConfiguration(WRITE_NAME); + String deltaPath = output("deltaTable"); + String refPath = output("ref"); + fullDMLScriptName = HOME + WRITE_NAME + ".dml"; + programArgs = new String[] {"-stats", "-args", String.valueOf(rows), String.valueOf(cols), + String.valueOf(sparsity), deltaPath, refPath}; + runTest(true, false, null, -1); + + // the write run must materialize two objects to disk: the frame Delta + // table under test + the matrix text reference. FrameWriterDelta genuinely + // hitting HDFS is what produces the frame-side write statistic. + long hdfsWrites = CacheStatistics.getHDFSWrites(); + assertTrue("expected >= 2 HDFS writes in the write run (delta frame + reference), got " + hdfsWrites, + hdfsWrites >= 2); + // and a real Delta table (transaction log) must have been created + assertTrue("missing Delta transaction log under " + deltaPath, + new File(deltaPath, "_delta_log").isDirectory()); + + // ---- phase 2: fresh execution reads the Delta frame and compares ---- + getAndLoadTestConfiguration(READ_NAME); + fullDMLScriptName = HOME + READ_NAME + ".dml"; + programArgs = new String[] {"-stats", "-args", deltaPath, refPath, output("R")}; + runTest(true, false, null, -1); + + long hdfsReads = CacheStatistics.getHDFSHits(); + assertTrue("expected >= 2 HDFS reads in the read run (delta + reference), got " + hdfsReads, + hdfsReads >= 2); + + HashMap R = readDMLMatrixFromOutputDir("R"); + double diff = R.getOrDefault(new CellIndex(1, 1), 0.0); + double nrow = R.getOrDefault(new CellIndex(1, 2), 0.0); + double ncol = R.getOrDefault(new CellIndex(1, 3), 0.0); + + assertEquals("reconstruction error", 0.0, diff, 1e-12); + assertEquals("discovered rows", rows, (int) nrow); + assertEquals("discovered cols", cols, (int) ncol); + } + catch(Exception ex) { + throw new RuntimeException(ex); + } + } +} diff --git a/src/test/scripts/functions/io/delta/FrameDeltaReadCompare.dml b/src/test/scripts/functions/io/delta/FrameDeltaReadCompare.dml new file mode 100644 index 00000000000..cdf1f0794fc --- /dev/null +++ b/src/test/scripts/functions/io/delta/FrameDeltaReadCompare.dml @@ -0,0 +1,35 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# Reader side of the native Delta frame round-trip test. Reads the Delta table +# as a frame (schema + dimensions discovered from the transaction log) and the +# text matrix reference, both genuine HDFS reads in a fresh process, then +# reports the elementwise reconstruction error and the discovered dimensions. + +Y = read($1, data_type="frame", format="delta") +Xref = read($2, format="text") + +M = as.matrix(Y) +R = matrix(0, rows=1, cols=3) +R[1,1] = sum(abs(Xref - M)) # 0 if FrameReaderDelta reconstructed the frame exactly +R[1,2] = nrow(Y) # discovered row count +R[1,3] = ncol(Y) # discovered column count +write(R, $3) diff --git a/src/test/scripts/functions/io/delta/FrameDeltaWrite.dml b/src/test/scripts/functions/io/delta/FrameDeltaWrite.dml new file mode 100644 index 00000000000..5e152dde013 --- /dev/null +++ b/src/test/scripts/functions/io/delta/FrameDeltaWrite.dml @@ -0,0 +1,32 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# Writer side of the native Delta frame round-trip test. Generates a matrix, +# converts it to a frame, and materializes it as a Delta table (under test). +# The same matrix is also written as a plain text reference. Running the +# read/compare in a SEPARATE process prevents SystemDS from short-circuiting +# the subsequent read against the in-memory frame, so FrameReaderDelta is +# actually exercised. + +X = rand(rows=$1, cols=$2, min=-5, max=5, seed=7, sparsity=$3) +F = as.frame(X) +write(F, $4, format="delta") +write(X, $5, format="text") From 9361cbcc5af004d552ba27c95d341f7f9706d52d Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Fri, 3 Jul 2026 23:03:40 +0200 Subject: [PATCH 063/132] [SYSTEMDS-3949] Add Demo for Databricks Workspace Execution (#2520) Adds self contained guide and scripts to enable users to run SystemDS on Databricks workspaces. --- scripts/databricks/.env.example | 64 +++++ scripts/databricks/.gitignore | 3 + scripts/databricks/README.md | 186 +++++++++++++ scripts/databricks/SystemDS_Delta_E2E.scala | 236 ++++++++++++++++ .../databricks/SystemDS_MLContext_Demo.scala | 153 +++++++++++ scripts/databricks/demo.dml | 50 ++++ scripts/databricks/deploy.sh | 257 ++++++++++++++++++ 7 files changed, 949 insertions(+) create mode 100644 scripts/databricks/.env.example create mode 100644 scripts/databricks/.gitignore create mode 100644 scripts/databricks/README.md create mode 100644 scripts/databricks/SystemDS_Delta_E2E.scala create mode 100644 scripts/databricks/SystemDS_MLContext_Demo.scala create mode 100644 scripts/databricks/demo.dml create mode 100755 scripts/databricks/deploy.sh diff --git a/scripts/databricks/.env.example b/scripts/databricks/.env.example new file mode 100644 index 00000000000..c2fddae4aeb --- /dev/null +++ b/scripts/databricks/.env.example @@ -0,0 +1,64 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# Configuration for scripts/databricks/deploy.sh. +# +# Copy this file to ".env" (next to deploy.sh, at the repo root, or anywhere in +# a parent directory of the script) and fill in your own values. deploy.sh +# searches upward from its own location for the first .env it finds; you can +# also point it at a specific file with ENV_FILE=/path/to/.env. +# +# Values already exported in your shell take precedence over this file. + +# Databricks CLI profile to use (see ~/.databrickscfg). +# Authenticate once with: databricks auth login --profile +PROFILE=DEFAULT + +# Unity Catalog location for the SystemDS jar volume and demo tables. +CATALOG=main +SCHEMA=default +VOLUME=systemds + +# Compute policy that the cluster must satisfy. Leave empty for no policy. +# Example: Personal Compute policy id. +POLICY_ID= + +# Cluster shape. +# See "Choosing a node type" in README.md for NODE_TYPE options, or run: +# databricks clusters list-node-types -p "$PROFILE" -o json +SPARK_VERSION=16.4.x-scala2.12 +NODE_TYPE=i3.xlarge +NUM_WORKERS=0 +AUTOTERMINATION_MINUTES=30 +CLUSTER_NAME=systemds + +# Optional overrides (uncomment to use): +# Path to the SystemDS jar to upload (defaults to /target/SystemDS.jar). +# JAR_LOCAL=/abs/path/to/SystemDS.jar +# Workspace folder to import the notebooks into (defaults to /Users/). +# NB_DIR=/Users/me@example.com +# Notebooks to import (space-separated; language detected from extension). +# NB_FILES="SystemDS_MLContext_Demo.scala SystemDS_Delta_E2E.scala" +# Delta Kernel Maven library version installed by `deploy.sh libs` (>= 3.3.2; +# must match the delta-kernel.version in pom.xml). +# DELTA_KERNEL_VERSION=3.3.2 +# Override the auto-detected Databricks user name. +# USER_NAME=me@example.com diff --git a/scripts/databricks/.gitignore b/scripts/databricks/.gitignore new file mode 100644 index 00000000000..a8d37d17e76 --- /dev/null +++ b/scripts/databricks/.gitignore @@ -0,0 +1,3 @@ +# Local, user-specific configuration and state — never commit these. +.env +.cluster_id diff --git a/scripts/databricks/README.md b/scripts/databricks/README.md new file mode 100644 index 00000000000..04662775ff1 --- /dev/null +++ b/scripts/databricks/README.md @@ -0,0 +1,186 @@ + + +# Running SystemDS on Databricks + +Scripts and demo notebooks for deploying and running SystemDS on a Databricks +cluster. Tested against DBR 16.4 LTS (Spark 3.5.2 / Scala 2.12), where the +SystemDS jar runs unchanged. + +## Contents + +| File | Purpose | +| --- | --- | +| `deploy.sh` | Create a UC volume, upload `SystemDS.jar`, create a single-user cluster, install the Delta Kernel libraries, and import the demo notebooks. | +| `SystemDS_MLContext_Demo.scala` | Notebook: Unity Catalog round-trip using the SystemDS MLContext (Scala) API. Reads a table, runs a configurable DML script, writes the result back. | +| `SystemDS_Delta_E2E.scala` | Notebook: end-to-end Delta → linear regression on one Delta table. SystemDS reads it natively as a frame (`read(format="delta")`) → `transformencode` → `lm`; Spark ML reads the same table → `OneHotEncoder` → `LinearRegression`. Times read + encode + train for both. | +| `demo.dml` | Standalone DML smoke test: reads a matrix from storage, computes column sums and a Gram-matrix trace. | +| `.env.example` | Template for your local configuration. | + +## Prerequisites + +- The [Databricks CLI](https://docs.databricks.com/dev-tools/cli/) installed and + authenticated once interactively: + + ```bash + databricks auth login --profile + ``` + +- A built SystemDS jar at `/target/SystemDS.jar` + (`mvn -q -DskipTests package`), or set `JAR_LOCAL` to point at one. +- `python3` on your `PATH` (used to parse CLI JSON output). + +## Configuration + +All settings are read from environment variables. The easiest way is a `.env` +file: + +```bash +cp scripts/databricks/.env.example scripts/databricks/.env +# then edit scripts/databricks/.env +``` + +`deploy.sh` looks for a `.env` file by: + +1. `ENV_FILE=/abs/path/to/.env` if you set it explicitly, otherwise +2. searching upward from the script's own directory (script dir → repo root → + any parent directory) for the first `.env` it finds. + +Anything already exported in your shell overrides values from `.env`. + +| Variable | Default | Description | +| --- | --- | --- | +| `PROFILE` | `DEFAULT` | Databricks CLI profile. | +| `CATALOG` / `SCHEMA` / `VOLUME` | `main` / `default` / `systemds` | UC location for the jar volume (and notebook defaults). | +| `POLICY_ID` | _(empty)_ | Compute policy id; leave empty for none. | +| `SPARK_VERSION` | `16.4.x-scala2.12` | DBR runtime. | +| `NODE_TYPE` | `i3.xlarge` | Node type. | +| `NUM_WORKERS` | `0` | Worker count (0 = single node). | +| `AUTOTERMINATION_MINUTES` | `30` | Auto-terminate idle minutes. | +| `CLUSTER_NAME` | `systemds` | Cluster name. | +| `JAR_LOCAL` | `/target/SystemDS.jar` | Jar to upload. | +| `NB_DIR` | `/Users/` | Workspace folder to import notebooks into. | +| `NB_FILES` | _(the 2 demo notebooks)_ | Space-separated notebooks to import; language detected from extension. | +| `DELTA_KERNEL_VERSION` | `3.3.2` | Delta Kernel Maven library version installed by `deploy.sh libs` (>= 3.3.2; must match `pom.xml`). | +| `USER_NAME` | _(auto-detected)_ | Databricks user. | + +### Choosing a node type + +`NODE_TYPE` is a cloud instance type. The default `i3.xlarge` is a small, +storage-optimized AWS node (4 vCPU / 30.5 GiB RAM / 950 GB local NVMe SSD); the +fast local disk is handy because the notebook spills SystemDS scratch to +`/local_disk0`. With `NUM_WORKERS=0` this single node is both driver and +executor, so it bounds the total memory available to SystemDS. + +Some common AWS options (pick more cores/RAM for larger workloads): + +| Node type | vCPU | RAM | Local SSD | Notes | +| --- | --- | --- | --- | --- | +| `i3.xlarge` | 4 | 30.5 GiB | 950 GB | Default; storage-optimized. | +| `i3.2xlarge` | 8 | 61 GiB | 1900 GB | Same family, 2× bigger. | +| `i4i.xlarge` | 4 | 32 GiB | 937 GB | Newer gen, faster storage. | +| `m5d.xlarge` | 4 | 16 GiB | 150 GB | General purpose w/ local SSD. | +| `r5d.2xlarge` | 8 | 64 GiB | 300 GB | Memory-optimized w/ local SSD. | + +The exact set depends on your cloud (AWS / Azure / GCP) and workspace. List what +your workspace actually offers with: + +```bash +databricks clusters list-node-types -p "$PROFILE" -o json +``` + +References: +[AWS EC2 instance types](https://aws.amazon.com/ec2/instance-types/), +[Azure VM sizes](https://learn.microsoft.com/azure/virtual-machines/sizes), +[GCP machine families](https://cloud.google.com/compute/docs/machine-resource). + +## Usage + +```bash +cd scripts/databricks +./deploy.sh upload # create UC volume + copy SystemDS.jar into it +./deploy.sh cluster # create the single-user cluster + install the jar +./deploy.sh libs # install the Delta Kernel Maven libraries on the cluster +./deploy.sh import # import the demo notebooks +./deploy.sh all # all of the above +``` + +The created cluster id is written to `scripts/databricks/.cluster_id`. + +### Delta Kernel libraries + +The Delta notebook (`SystemDS_Delta_E2E`) reads Delta tables natively through the +Spark-free Delta Kernel, which is not on the DBR classpath. `./deploy.sh libs` +installs `io.delta:delta-kernel-defaults` (version `DELTA_KERNEL_VERSION`, default +`3.3.2`) as a cluster Maven library. Use **>= 3.3.2**: earlier releases trip a +classloader conflict with DBR's bundled parquet. The version must match the +`delta-kernel.version` property in the SystemDS `pom.xml`. + +## Notebook configuration (`SystemDS_MLContext_Demo`) + +The Scala notebook is driven by widgets, so nothing is hardcoded — set them in +the notebook UI or pass them as job parameters: + +| Widget | Default | Description | +| --- | --- | --- | +| `catalog` / `schema` | `main` / `default` | Where the input/output tables live. | +| `input_table` / `output_table` | `systemds_input` / `systemds_output` | Table names. | +| `dml_path` | _(blank)_ | DML script to run. Blank uses the built-in z-score demo; otherwise a path readable from the driver (UC volume, `/Workspace`, or `dbfs:`). | +| `exec_type` | `DEFAULT` | SystemDS execution mode. `DEFAULT` lets SystemDS choose the plan; `DRIVER`, `SPARK`, or `DRIVER_AND_SPARK` force a mode. | + +Custom DML contract: the script receives the input matrix as `X` and must +produce a matrix `Y` and a scalar `checksum`. + +## Notebook configuration (`SystemDS_Delta_E2E`) + +| Widget | Default | Description | +| --- | --- | --- | +| `catalog` / `schema` / `volume` | `main` / `default` / `systemds` | UC location; the Delta table is written under the volume. | +| `rows` | `1000000` | Rows in the generated Delta table. | +| `num_numeric` / `num_categorical` / `cardinality` | `100` / `20` / `30` | Feature shape. Defaults are deliberately encode-heavy (700 features) so the SystemDS-vs-Spark difference is visible. | +| `reg` | `1e-3` | L2 regularization for `lm`. | +| `recreate` | `true` | Rewrite the Delta table before running. | +| `statistics` | `true` | Print the SystemDS per-instruction breakdown. | + +The encode complexity (categoricals × cardinality), not the row count, drives the +gap: more categoricals blow up Spark's `StringIndexer` + `OneHotEncoder` (each a +shuffle stage), while SystemDS dummycodes in-memory. On a single node, raw rows +instead favor Spark, and very large tables can exhaust driver memory. + +Indicative single-node (`i3.xlarge`, 1M rows) numbers — single cold run, no warmup: + +| workload | Spark ML | SystemDS | speedup | +| --- | --- | --- | --- | +| `SystemDS_Delta_E2E`, 700 features (read + encode + train) | 116.6 s | 55.4 s | ~2.1× | + +The Spark side is the same speed on Spark 3.5.2 (DBR 16.4) and Spark 4.0.0 +(DBR 17.3 LTS), so the comparison is not an artifact of an old runtime. + +## Notes / gotchas baked into the scripts + +- UC clusters only accept JAR libraries from a **UC Volume** (not DBFS, not + `/Workspace`). +- The cluster must be **SINGLE_USER** (Assigned) mode; shared / USER_ISOLATION + blocks JAR libraries. +- SystemDS needs the Vector API module plus a full `--add-opens` set at JVM + launch (configured via `spark.{driver,executor}.extraJavaOptions`), and an + absolute scratch dir (the notebook pins `sysds.scratch`). + +`.env` and `.cluster_id` are git-ignored — they hold personal config and local +state and should never be committed. diff --git a/scripts/databricks/SystemDS_Delta_E2E.scala b/scripts/databricks/SystemDS_Delta_E2E.scala new file mode 100644 index 00000000000..cacc1c729f4 --- /dev/null +++ b/scripts/databricks/SystemDS_Delta_E2E.scala @@ -0,0 +1,236 @@ +// Databricks notebook source +//------------------------------------------------------------- +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +//------------------------------------------------------------- + +// End-to-end Delta -> linear regression, on the SAME Delta table for both engines: +// - SystemDS: read(format="delta") as a frame -> transformencode -> lm +// - Spark ML: spark.read delta -> StringIndexer + OneHotEncoder + LinearRegression +// +// SystemDS reads the Delta table natively through the Spark-free Delta Kernel +// (FrameReaderDelta), so the whole pipeline runs inside the SystemDS runtime. +// The timed region is end-to-end: read + encode + train, for both engines. +// +// Prereqs (handled by deploy.sh): +// - SystemDS.jar installed on the cluster, with the SystemDS JVM flags. +// - io.delta:delta-kernel-api + delta-kernel-defaults installed as Maven +// libraries (Delta Kernel is NOT on the DBR classpath and is NOT bundled in +// SystemDS.jar). + +// COMMAND ---------- + +// On a single-node cluster SystemDS reads the whole table into driver memory, so +// both `rows` and the encoded width (num_numeric + num_categorical*cardinality) +// are bounded by the node size. On an i3.xlarge (~30 GB) ~10M rows OOMs in +// transformencode; scale up the node or lower these on OOM. +// +// What drives the SystemDS-vs-Spark gap is encoding complexity, not row count: +// more categoricals / higher cardinality blow up Spark's StringIndexer + +// OneHotEncoder (each a shuffle stage), while SystemDS dummycodes in-memory. +// Raw rows instead favor Spark (it parallelizes across cores; single-node CP +// does not). Defaults below are deliberately encode-heavy (700 features) so the +// difference is visible; the baseline 50/4/20 config is roughly a tie at 1M rows. +dbutils.widgets.text("catalog", "main", "Unity Catalog") +dbutils.widgets.text("schema", "default", "Schema") +dbutils.widgets.text("volume", "systemds", "Volume (table is written under it)") +dbutils.widgets.text("rows", "1000000", "Number of rows") +dbutils.widgets.text("num_numeric", "100", "Numeric feature columns") +dbutils.widgets.text("num_categorical", "20", "Categorical feature columns") +dbutils.widgets.text("cardinality", "30", "Distinct values per categorical") +dbutils.widgets.text("reg", "1e-3", "L2 regularization (lambda)") +dbutils.widgets.dropdown("recreate", "true", Seq("true", "false"), "Recreate the Delta table") +dbutils.widgets.dropdown("statistics", "true", Seq("true", "false"), "Print SystemDS statistics") + +val CATALOG = dbutils.widgets.get("catalog") +val SCHEMA = dbutils.widgets.get("schema") +val VOLUME = dbutils.widgets.get("volume") +val N = dbutils.widgets.get("rows").toLong +val DNUM = dbutils.widgets.get("num_numeric").toInt +val DCAT = dbutils.widgets.get("num_categorical").toInt +val CARD = dbutils.widgets.get("cardinality").toInt +val REG = dbutils.widgets.get("reg").toDouble +val RECREATE = dbutils.widgets.get("recreate").toBoolean +val STATS = dbutils.widgets.get("statistics").toBoolean + +val numCols = (0 until DNUM).map(i => s"num_$i").toArray +val catCols = (0 until DCAT).map(j => s"cat_$j").toArray + +// The table lives on a UC volume (FUSE-mounted locally at /Volumes/...). Spark +// reads it via the same path; SystemDS reads the local FUSE mount with an +// explicit file: scheme so the Delta Kernel's Hadoop engine uses the local +// filesystem rather than the cluster default (dbfs). +val tablePath = s"/Volumes/$CATALOG/$SCHEMA/$VOLUME/delta_e2e" +val sysdsPath = "file:" + tablePath +println(s"config: rows=$N numeric=$DNUM categorical=$DCAT cardinality=$CARD reg=$REG") +println(s"table : $tablePath") + +// COMMAND ---------- + +// MAGIC %md +// MAGIC ## Setup: materialize the dataset as a Delta table (once) + +// COMMAND ---------- + +import org.apache.spark.sql.functions._ + +// Deterministic linear weights so the target is actually learnable. +val weights = (0 until DNUM).map(i => ((i % 7) - 3) * 0.5) + +def writeDeltaTable(): Unit = { + var df = spark.range(0, N).toDF("id") + for (i <- 0 until DNUM) + df = df.withColumn(s"num_$i", rand(i.toLong) * 2.0 - 1.0) + for (j <- 0 until DCAT) + df = df.withColumn(s"cat_$j", (floor(rand(100L + j) * CARD) + 1).cast("string")) + val signal = (0 until DNUM).map(i => col(s"num_$i") * lit(weights(i))).reduce(_ + _) + // Column order written to Delta is [numeric.., categorical.., y]; SystemDS + // relies on this order for the transform spec and target column. + val out = df.withColumn("y", signal + (rand(999L) * 0.2 - 0.1)).drop("id") + .select((numCols ++ catCols ++ Array("y")).map(col): _*) + // overwriteSchema so re-running with a different feature count replaces an + // existing table whose schema no longer matches. + out.write.format("delta").mode("overwrite").option("overwriteSchema", "true").save(tablePath) +} + +val exists = try { dbutils.fs.ls(tablePath); true } catch { case _: Throwable => false } +if (RECREATE || !exists) { + println(">> writing Delta table ...") + writeDeltaTable() +} +val tblRows = spark.read.format("delta").load(tablePath).count() +println(s">> Delta table ready: $tblRows rows at $tablePath") + +// COMMAND ---------- + +// MAGIC %md +// MAGIC ## SystemDS: read Delta (native Kernel) -> transformencode -> lm + +// COMMAND ---------- + +import org.apache.sysds.api.mlcontext._ +import org.apache.sysds.api.mlcontext.ScriptFactory._ + +val ml = new MLContext(sc) +// With statistics on, SystemDS prints a per-instruction breakdown (heavy hitters) +// after execute: the Delta read shows up as cache acquire-read (ACQr) time, plus +// transformencode and the lm operators (m_lm/tsmm/solve). Useful to see where the +// end-to-end time actually goes. +ml.setStatistics(STATS) +ml.setStatisticsMaxHeavyHitters(25) +ml.setConfigProperty("sysds.scratch", "/tmp/systemds_scratch") +ml.setConfigProperty("sysds.localtmpdir", "/local_disk0/tmp/systemds") +// Force single-node (CP) execution. The native Delta frame reader +// (FrameReaderDelta) is a control-program reader; under Spark execution the +// frame read is distributed and bypasses it (failing to parse the Delta parquet +// files). On a single-node cluster CP execution is the intended mode anyway. +ml.setExecutionType(MLContext.ExecutionType.DRIVER) + +// transform spec: one-hot (dummycode) the categorical columns. Delta column +// order is [numeric.., categorical.., y]; categoricals are 1-based indices +// DNUM+1 .. DNUM+DCAT. Numeric features and y pass through unchanged. +val catIdx = (DNUM + 1 to DNUM + DCAT).mkString(",") +val spec = s"""{"ids":true,"dummycode":[$catIdx]}""" + +// Whole pipeline in one script: native Delta frame read -> encode -> train. +val e2eDml = """ + F = read($path, data_type="frame", format="delta") + [X, M] = transformencode(target=F, spec=spec) + nc = ncol(X) + yv = X[, nc] + Xv = X[, 1:(nc-1)] + B = lm(X=Xv, y=yv, icpt=1, reg=reg, verbose=FALSE) + checksum = sum(B) + nfeat = ncol(Xv) + nrows = nrow(X) +""" + +def runSysds(path: String): (Double, Int, Long, Double) = { + val script = dml(e2eDml) + .in("$path", path).in("spec", spec).in("reg", REG) + .out("checksum", "nfeat", "nrows") + val t0 = System.nanoTime() + val res = ml.execute(script) + val secs = (System.nanoTime() - t0) / 1e9 + (secs, res.getDouble("nfeat").toInt, res.getDouble("nrows").toLong, res.getDouble("checksum")) +} + +val (sysdsSecs, sysdsFeat, sysdsRows, sysdsChk) = runSysds(sysdsPath) +println(f"SystemDS read+encode+train: $sysdsSecs%.2f s | rows=$sysdsRows features=$sysdsFeat checksum=$sysdsChk%.4f") + +// COMMAND ---------- + +// MAGIC %md +// MAGIC ## Spark ML: read Delta -> OneHotEncoder -> LinearRegression + +// COMMAND ---------- + +import org.apache.spark.ml.Pipeline +import org.apache.spark.ml.feature.{OneHotEncoder, StringIndexer, VectorAssembler} +import org.apache.spark.ml.regression.{LinearRegression, LinearRegressionModel} + +def sparkPipeline(): Pipeline = { + val indexers = catCols.map(c => + new StringIndexer().setInputCol(c).setOutputCol(c + "_idx").setHandleInvalid("keep")) + val ohe = new OneHotEncoder() + .setInputCols(catCols.map(_ + "_idx")).setOutputCols(catCols.map(_ + "_oh")) + .setDropLast(false) // keep all categories, matching SystemDS dummycode + val assembler = new VectorAssembler() + .setInputCols(numCols ++ catCols.map(_ + "_oh")).setOutputCol("features") + val lr = new LinearRegression() + .setLabelCol("y").setFeaturesCol("features") + .setRegParam(REG).setElasticNetParam(0.0).setFitIntercept(true) + new Pipeline().setStages(indexers ++ Array(ohe, assembler, lr)) +} + +// End-to-end: time the lazy Delta read + encode + train together (the read is +// triggered inside pipeline.fit), symmetric to the SystemDS execute above. +def runSpark(path: String): (Double, Int, Double) = { + val t0 = System.nanoTime() + val df = spark.read.format("delta").load(path) + val model = sparkPipeline().fit(df) + val secs = (System.nanoTime() - t0) / 1e9 + val lr = model.stages.last.asInstanceOf[LinearRegressionModel] + (secs, lr.numFeatures, lr.intercept) +} + +val (sparkSecs, sparkFeat, sparkIntercept) = runSpark(tablePath) +println(f"Spark ML read+encode+train: $sparkSecs%.2f s | features=$sparkFeat intercept=$sparkIntercept%.4f") + +// COMMAND ---------- + +// MAGIC %md +// MAGIC ## Result + +// COMMAND ---------- + +val speedup = sparkSecs / sysdsSecs +println(f""" +=== E2E Delta -> linear regression (read + encode + train) === +dataset : rows=$N numeric=$DNUM categorical=$DCAT cardinality=$CARD +table : $tablePath +Spark ML : $sparkSecs%6.2f s (features=$sparkFeat) +SystemDS : $sysdsSecs%6.2f s (features=$sysdsFeat, rows read=$sysdsRows) +Speedup : $speedup%6.2fx (Spark / SystemDS) +""") + +dbutils.notebook.exit( + f"rows=$N numeric=$DNUM categorical=$DCAT card=$CARD " + + f"spark_s=$sparkSecs%.2f sysds_s=$sysdsSecs%.2f speedup=$speedup%.2f " + + f"sysds_features=$sysdsFeat sysds_rows=$sysdsRows") diff --git a/scripts/databricks/SystemDS_MLContext_Demo.scala b/scripts/databricks/SystemDS_MLContext_Demo.scala new file mode 100644 index 00000000000..26a237be491 --- /dev/null +++ b/scripts/databricks/SystemDS_MLContext_Demo.scala @@ -0,0 +1,153 @@ +// Databricks notebook source +//------------------------------------------------------------- +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +//------------------------------------------------------------- + +// SystemDS on Databricks: Unity Catalog round-trip +// +// Reads a table from Unity Catalog into SystemDS, runs a DML script over it, +// and writes the result back as a UC table. Prereq: `SystemDS.jar` installed +// on the cluster (DBR 16.4 LTS, Spark 3.5.2 / Scala 2.12) with the SystemDS +// JVM flags. +// +// Everything is configured via the notebook widgets (first cell) so this works +// in any workspace: +// - catalog / schema / input_table / output_table: where data is read/written +// - dml_path: path to a DML script to run (blank = built-in z-score demo) +// - exec_type: SystemDS execution mode (DEFAULT = let SystemDS decide) +// +// The DML script contract: it receives the input matrix as `X` and must +// produce a matrix `Y` and a scalar `checksum`. + +// COMMAND ---------- + +// Widgets make the notebook portable: set these per workspace instead of +// hardcoding values. The table location defaults to main.default. +dbutils.widgets.text("catalog", "main", "Unity Catalog") +dbutils.widgets.text("schema", "default", "Schema") +dbutils.widgets.text("input_table", "systemds_input", "Input table name") +dbutils.widgets.text("output_table", "systemds_output", "Output table name") +// DML script to run. Blank uses the built-in z-score demo below. Otherwise a +// path readable from the cluster driver, e.g. a UC volume, /Workspace, or dbfs: +// /Volumes////my_script.dml +dbutils.widgets.text("dml_path", "", "DML script path (blank = built-in demo)") +// SystemDS execution mode. DEFAULT lets SystemDS choose (no forcing). +dbutils.widgets.dropdown("exec_type", "DEFAULT", + Seq("DEFAULT", "DRIVER", "SPARK", "DRIVER_AND_SPARK"), "Execution type") + +val catalog = dbutils.widgets.get("catalog") +val schema = dbutils.widgets.get("schema") +val INPUT_TABLE = s"$catalog.$schema.${dbutils.widgets.get("input_table")}" +val OUTPUT_TABLE = s"$catalog.$schema.${dbutils.widgets.get("output_table")}" +val DML_PATH = dbutils.widgets.get("dml_path") +val EXEC_TYPE = dbutils.widgets.get("exec_type") + +// COMMAND ---------- + +import org.apache.sysds.api.mlcontext._ +import org.apache.sysds.api.mlcontext.ScriptFactory._ +import org.apache.sysds.utils.Statistics +import org.apache.spark.sql.functions._ + +val ml = new MLContext(sc) +ml.setStatistics(true) +// Only override the execution mode when explicitly requested; DEFAULT leaves +// SystemDS to pick the plan (it will use Spark only when it decides to). +if (EXEC_TYPE != "DEFAULT") + ml.setExecutionType(MLContext.ExecutionType.valueOf(EXEC_TYPE)) +// SystemDS defaults to a relative scratch dir, which the Databricks default +// filesystem rejects ("Path must be absolute"). Pin both to absolute paths. +ml.setConfigProperty("sysds.scratch", "/tmp/systemds_scratch") +ml.setConfigProperty("sysds.localtmpdir", "/local_disk0/tmp/systemds") +println("Spark version: " + sc.version + " | exec_type: " + EXEC_TYPE) + +// COMMAND ---------- + +// MAGIC %md +// MAGIC ## Setup: ensure an input table exists in the catalog + +// COMMAND ---------- + +if (!spark.catalog.tableExists(INPUT_TABLE)) { + val seed = spark.range(0, 5000).select( + (rand(1) * 100).as("f1"), + (rand(2) * 10 + 5).as("f2"), + (rand(3) - 0.5).as("f3"), + (rand(4) * 1000).as("f4")) + seed.write.mode("overwrite").saveAsTable(INPUT_TABLE) +} +println(s"input table $INPUT_TABLE rows = ${spark.table(INPUT_TABLE).count()}") + +// COMMAND ---------- + +// MAGIC %md +// MAGIC ## 1. Read a table from the catalog into SystemDS + +// COMMAND ---------- + +val inDF = spark.table(INPUT_TABLE) + .select(col("f1").cast("double"), col("f2").cast("double"), + col("f3").cast("double"), col("f4").cast("double")) + +// Built-in fallback: standardize the columns (z-score). Used when no dml_path +// widget is set. A custom script must read `X` and produce `Y` and `checksum`. +val defaultScript = """ + n = nrow(X) + mu = colMeans(X) + Xc = X - mu + variance = colSums(Xc^2) / (n - 1) + sigma = sqrt(variance) + Y = Xc / sigma + checksum = sum(Y) + print("standardized " + nrow(X) + " x " + ncol(X) + " matrix") +""" + +val baseScript = if (DML_PATH.nonEmpty) { + println(s"running DML from $DML_PATH") + dmlFromFile(DML_PATH) +} else { + println("running built-in z-score demo script") + dml(defaultScript) +} +val script = baseScript.in("X", inDF).out("Y", "checksum") + +val res = ml.execute(script) +val checksum = res.getDouble("checksum") +val outDF = res.getDataFrameDoubleNoIDColumn("Y") +println(s"checksum(Y) = $checksum (≈0 for standardized data)") + +// COMMAND ---------- + +// MAGIC %md +// MAGIC ## 2. Write the SystemDS result back to the catalog + +// COMMAND ---------- + +outDF.write.mode("overwrite").saveAsTable(OUTPUT_TABLE) +val outRows = spark.table(OUTPUT_TABLE).count() +val outCols = spark.table(OUTPUT_TABLE).columns.length +println(s"wrote $OUTPUT_TABLE: $outRows rows x $outCols cols") + +// COMMAND ---------- + +val spExecuted = Statistics.getNoOfExecutedSPInst() +dbutils.notebook.exit( + s"spark=${sc.version} in=$INPUT_TABLE out=$OUTPUT_TABLE " + + s"out_rows=$outRows out_cols=$outCols checksum=$checksum sp_executed=$spExecuted") diff --git a/scripts/databricks/demo.dml b/scripts/databricks/demo.dml new file mode 100644 index 00000000000..c6195bbf3d4 --- /dev/null +++ b/scripts/databricks/demo.dml @@ -0,0 +1,50 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# SystemDS on Databricks smoke test. +# Reads a matrix from storage and computes column sums and a Gram-matrix trace, +# so the read path plus a few distributed-friendly operations are exercised. +# +# Args (all optional, with defaults): +# -nvargs in= fmt= out= +# +# `in` is any path readable from the cluster driver (a UC volume, /Workspace, +# or dbfs:) and `fmt` any SystemDS-supported matrix format (csv, binary, +# libsvm, mm, ...). Example: +# in=/Volumes////demo_input.csv fmt=csv + +in = ifdef($in, "demo_input.csv") +fmt = ifdef($fmt, "csv") +out = ifdef($out, "demo_result.txt") + +X = read(in, format=fmt) + +# Column sums and a Gram-matrix trace: both push work through Spark +# instructions for large inputs. +colSums = colSums(X) +gramTrace = sum(X * X) + +s = sum(colSums) + gramTrace + +print("rows=" + nrow(X) + " cols=" + ncol(X)) +print("result=" + s) + +write(s, out, format="text") diff --git a/scripts/databricks/deploy.sh b/scripts/databricks/deploy.sh new file mode 100755 index 00000000000..a144c6d46c1 --- /dev/null +++ b/scripts/databricks/deploy.sh @@ -0,0 +1,257 @@ +#!/usr/bin/env bash +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# Deploy + run SystemDS on a Databricks cluster. +# +# Tested against DBR 16.4 LTS (Spark 3.5.2 / Scala 2.12), where the SystemDS +# jar runs unchanged. +# +# Quick start: +# 1. Copy scripts/databricks/.env.example to .env and edit it (or place the +# .env at the root of your workspace; this script searches parent dirs). +# 2. Authenticate the Databricks CLI once, interactively: +# databricks auth login --profile +# 3. Build the SystemDS jar (mvn -q -DskipTests package) so target/SystemDS.jar +# exists, or point JAR_LOCAL at an existing jar. +# 4. Run a step: +# ./deploy.sh upload # create UC volume + copy SystemDS.jar into it +# ./deploy.sh cluster # create single-user cluster + install SystemDS.jar +# ./deploy.sh libs # install Delta Kernel Maven libraries on cluster +# ./deploy.sh import # import the demo notebook(s) +# ./deploy.sh all # upload + cluster + libs + import +# +# All configuration is read from environment variables (see .env.example). +# Anything already exported in your shell overrides the .env file. +# +# Hard-won requirements baked in below: +# - Cluster creation requires a compute policy (e.g. Personal Compute) that +# restricts node types and autotermination. +# - UC clusters only accept JAR libraries from a UC Volume (not DBFS, not +# /Workspace). +# - Must be SINGLE_USER (Assigned) mode; shared/USER_ISOLATION blocks JAR libs. +# - SystemDS needs the Vector API module + a full --add-opens set at JVM +# launch, and an absolute scratch dir (the notebook sets sysds.scratch). + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +#------------------------------------------------------------- +# Load configuration from a .env file. +# Resolution order: +# 1. $ENV_FILE if explicitly set. +# 2. The first .env found walking up from this script's directory +# (script dir -> repo root -> workspace root -> ... -> /). +#------------------------------------------------------------- +find_env_file() { + if [[ -n "${ENV_FILE:-}" ]]; then + printf '%s\n' "$ENV_FILE" + return 0 + fi + local dir="$HERE" + while [[ "$dir" != "/" ]]; do + if [[ -f "$dir/.env" ]]; then + printf '%s\n' "$dir/.env" + return 0 + fi + dir="$(dirname "$dir")" + done + return 1 +} + +if ENV_PATH="$(find_env_file)"; then + echo ">> loading config from $ENV_PATH" + set -a + # shellcheck disable=SC1090 + source "$ENV_PATH" + set +a +else + echo ">> no .env found; relying on exported environment variables / defaults" +fi + +#------------------------------------------------------------- +# Configuration (env var with sensible fallback). +#------------------------------------------------------------- +PROFILE="${PROFILE:-DEFAULT}" + +# Repo root is used to locate the default jar (target/SystemDS.jar). +REPO_ROOT="${REPO_ROOT:-$(git -C "$HERE" rev-parse --show-toplevel 2>/dev/null || echo "$HERE/../..")}" +JAR_LOCAL="${JAR_LOCAL:-$REPO_ROOT/target/SystemDS.jar}" + +db() { databricks -p "$PROFILE" "$@"; } + +# Resolve the current user lazily (only needed for the notebook import target). +resolve_user() { + if [[ -z "${USER_NAME:-}" ]]; then + USER_NAME="$(db current-user me -o json \ + | python3 -c 'import sys,json;print(json.load(sys.stdin)["userName"])')" + fi +} + +CATALOG="${CATALOG:-main}" +SCHEMA="${SCHEMA:-default}" +VOLUME="${VOLUME:-systemds}" +VOL_DIR="/Volumes/$CATALOG/$SCHEMA/$VOLUME" +JAR_REMOTE="$VOL_DIR/SystemDS.jar" + +POLICY_ID="${POLICY_ID:-}" +SPARK_VERSION="${SPARK_VERSION:-16.4.x-scala2.12}" +NODE_TYPE="${NODE_TYPE:-i3.xlarge}" +NUM_WORKERS="${NUM_WORKERS:-0}" +AUTOTERMINATION_MINUTES="${AUTOTERMINATION_MINUTES:-30}" +CLUSTER_NAME="${CLUSTER_NAME:-systemds}" +CLUSTER_ID_FILE="${CLUSTER_ID_FILE:-$HERE/.cluster_id}" + +# Delta Kernel is not on the DBR classpath and is not bundled in SystemDS.jar +# (the uber jar shades only wink + antlr). The native Delta read/write path needs +# delta-kernel-api + delta-kernel-defaults installed as cluster Maven libraries. +# Use >= 3.3.2: earlier releases (3.3.0/3.3.1) subclass parquet-mr's +# package-private InternalParquetRecordReader, which breaks across Databricks' +# library/app classloaders (IllegalAccessError, then NoSuchMethodError against +# DBR's parquet). 3.3.2 (delta PR #4494) switched to parquet's public +# ParquetReader API, so Kernel works with DBR's own parquet. 4.x targets Spark 4 +# (wrong for DBR 16.4 = Spark 3.5). Must match the delta-kernel.version in pom.xml. +DELTA_KERNEL_VERSION="${DELTA_KERNEL_VERSION:-3.3.2}" + +# SystemDS JVM flags (mirror of in pom.xml). +JVMOPTS="${JVMOPTS:---add-modules=jdk.incubator.vector \ +--add-opens=java.base/java.nio=ALL-UNNAMED \ +--add-opens=java.base/java.io=ALL-UNNAMED \ +--add-opens=java.base/java.util=ALL-UNNAMED \ +--add-opens=java.base/java.lang=ALL-UNNAMED \ +--add-opens=java.base/java.lang.ref=ALL-UNNAMED \ +--add-opens=java.base/java.util.concurrent=ALL-UNNAMED \ +--add-opens=java.base/sun.nio.ch=ALL-UNNAMED}" + +# Notebooks to import (space-separated, relative to this dir). Language is +# detected from the extension (.scala -> SCALA, .py -> PYTHON). +NB_FILES="${NB_FILES:-SystemDS_MLContext_Demo.scala SystemDS_Delta_E2E.scala}" + +#------------------------------------------------------------- +# Steps. +#------------------------------------------------------------- +upload() { + [[ -f "$JAR_LOCAL" ]] || { echo "!! jar not found: $JAR_LOCAL (build it or set JAR_LOCAL)"; exit 1; } + echo ">> ensuring UC volume $CATALOG.$SCHEMA.$VOLUME" + db volumes create "$CATALOG" "$SCHEMA" "$VOLUME" MANAGED 2>/dev/null || true + echo ">> uploading $JAR_LOCAL -> $JAR_REMOTE" + db fs cp --overwrite "$JAR_LOCAL" "dbfs:$JAR_REMOTE" +} + +cluster() { + resolve_user + echo ">> creating cluster $CLUSTER_NAME ($SPARK_VERSION, $NODE_TYPE, single-user)" + local policy_json="" + if [[ -n "$POLICY_ID" ]]; then + policy_json="\"policy_id\": \"$POLICY_ID\", \"apply_policy_default_values\": true," + fi + CID=$(db clusters create --no-wait -o json --json "{ + \"cluster_name\": \"$CLUSTER_NAME\", + $policy_json + \"spark_version\": \"$SPARK_VERSION\", + \"node_type_id\": \"$NODE_TYPE\", + \"num_workers\": $NUM_WORKERS, + \"autotermination_minutes\": $AUTOTERMINATION_MINUTES, + \"data_security_mode\": \"SINGLE_USER\", + \"single_user_name\": \"$USER_NAME\", + \"spark_conf\": { + \"spark.databricks.cluster.profile\": \"singleNode\", + \"spark.master\": \"local[*]\", + \"spark.driver.extraJavaOptions\": \"$JVMOPTS\", + \"spark.executor.extraJavaOptions\": \"$JVMOPTS\" + } + }" | python3 -c 'import sys,json;print(json.load(sys.stdin)["cluster_id"])') + echo "cluster_id=$CID" + echo "$CID" > "$CLUSTER_ID_FILE" + echo ">> installing library $JAR_REMOTE (queued; installs once RUNNING)" + db libraries install --json "{\"cluster_id\":\"$CID\",\"libraries\":[{\"jar\":\"$JAR_REMOTE\"}]}" +} + +# Resolve the target cluster id: prefer the one written by cluster(), else look +# it up by name. +resolve_cluster_id() { + if [[ -n "${CLUSTER_ID:-}" ]]; then + return 0 + fi + if [[ -f "$CLUSTER_ID_FILE" ]]; then + CLUSTER_ID="$(cat "$CLUSTER_ID_FILE")" + return 0 + fi + CLUSTER_ID="$(db clusters list -o json \ + | python3 -c 'import sys,json; +clusters=json.load(sys.stdin); +m=[c for c in clusters if c.get("cluster_name")=="'"$CLUSTER_NAME"'"]; +print(m[0]["cluster_id"] if m else "")')" + [[ -n "$CLUSTER_ID" ]] || { echo "!! could not resolve cluster id for $CLUSTER_NAME (run ./deploy.sh cluster first or set CLUSTER_ID)"; exit 1; } +} + +# Install the Delta Kernel Maven libraries on the cluster. delta-kernel-defaults +# pulls delta-kernel-api transitively; both come from Maven Central. parquet, +# hadoop and jackson are excluded so Kernel uses DBR's own copies (avoids +# duplicate-class / split-package issues across the library/app classloaders). +libs() { + resolve_cluster_id + echo ">> installing Delta Kernel $DELTA_KERNEL_VERSION Maven libs on $CLUSTER_ID" + db libraries install --json "{ + \"cluster_id\": \"$CLUSTER_ID\", + \"libraries\": [ + {\"maven\": { + \"coordinates\": \"io.delta:delta-kernel-defaults:$DELTA_KERNEL_VERSION\", + \"exclusions\": [ + \"org.apache.parquet:parquet-hadoop\", + \"org.apache.hadoop:hadoop-client-runtime\", + \"org.apache.hadoop:hadoop-client-api\", + \"com.fasterxml.jackson.core:jackson-databind\" + ] + }} + ] + }" + echo ">> queued; check status with: db libraries cluster-status $CLUSTER_ID" +} + +import() { + resolve_user + local dir="${NB_DIR:-/Users/$USER_NAME}" + local nb lang base + for nb in $NB_FILES; do + case "$nb" in + *.scala) lang="SCALA" ;; + *.py) lang="PYTHON" ;; + *.sql) lang="SQL" ;; + *.r|*.R) lang="R" ;; + *) echo "!! skipping $nb (unknown notebook language)"; continue ;; + esac + base="$(basename "${nb%.*}")" + echo ">> importing $nb -> $dir/$base ($lang)" + db workspace import --overwrite --language "$lang" --format SOURCE \ + --file "$HERE/$nb" "$dir/$base" + done +} + +case "${1:-all}" in + upload) upload ;; + cluster) cluster ;; + libs) libs ;; + import) import ;; + all) upload; cluster; libs; import ;; + *) echo "usage: $0 {upload|cluster|libs|import|all}"; exit 1 ;; +esac From e782caa96281ceaf660c924044c3131eb534b41a Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Sun, 5 Jul 2026 15:56:10 +0200 Subject: [PATCH 064/132] [OOC] IO Handler Performance Improvements and Generalization (#2532) Improve performance and generality of OOCMatrixIOHandler. Introduce experimental input/output streams that worked better in practice for OOC spilling in local experiments. Add lightweight OOCFuture to replace CompletableFuture in hot paths. --- .../spark/data/IndexedMatrixValue.java | 51 ++- .../sysds/runtime/ooc/cache/OOCFuture.java | 275 ++++++++++++++++ .../cache/io/OOCBufferedDataInputStream.java | 307 ++++++++++++++++++ .../cache/io/OOCBufferedDataOutputStream.java | 277 ++++++++++++++++ .../runtime/ooc/cache/io/OOCIOHandler.java | 3 +- .../ooc/cache/io/OOCMatrixIOHandler.java | 118 ++++--- .../runtime/ooc/cache/io/SpillableObject.java | 32 ++ .../ooc/cache/io/SpillableObjectRegistry.java | 55 ++++ .../cache/legacy/OOCLRUCacheScheduler.java | 3 +- .../ooc/cache/OOCLRUCacheSchedulerTest.java | 11 +- 10 files changed, 1060 insertions(+), 72 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObjectRegistry.java diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java index 7b20fe2f9e5..bd96bbb614f 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java @@ -20,34 +20,39 @@ package org.apache.sysds.runtime.instructions.spark.data; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import java.io.Serializable; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.runtime.matrix.data.MatrixIndexes; import org.apache.sysds.runtime.matrix.data.MatrixValue; +import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; -public class IndexedMatrixValue implements Serializable +public class IndexedMatrixValue implements SpillableObject, Serializable { private static final long serialVersionUID = 6723389820806752110L; private MatrixIndexes _indexes = null; private MatrixValue _value = null; - + public IndexedMatrixValue() { _indexes = new MatrixIndexes(); } - + public IndexedMatrixValue(Class cls) { this(); - + //create new value object for given class try { _value=cls.getDeclaredConstructor().newInstance(); - } + } catch (Exception e) { throw new RuntimeException(e); } } - + public IndexedMatrixValue(MatrixIndexes ind, MatrixValue b) { this(); _indexes.setIndexes(ind); @@ -55,14 +60,14 @@ public IndexedMatrixValue(MatrixIndexes ind, MatrixValue b) { } public IndexedMatrixValue(IndexedMatrixValue that) { - this(that._indexes, that._value); + this(that._indexes, that._value); } - + public MatrixIndexes getIndexes() { return _indexes; } - + public MatrixValue getValue() { return _value; } @@ -70,14 +75,38 @@ public MatrixValue getValue() { public void setValue(MatrixValue value) { _value = value; } - + public void set(MatrixIndexes indexes2, MatrixValue block2) { _indexes.setIndexes(indexes2); _value = block2; } - + @Override public String toString() { return "("+_indexes.getRowIndex()+", "+_indexes.getColumnIndex()+"): \n"+_value; } + + @Override + public boolean tryWrite(DataOutput dataOutput) throws IOException { + MatrixIndexes ix = _indexes; + MatrixValue value = _value; + if(ix == null || value == null) + return false; + ix.write(dataOutput); + value.write(dataOutput); + return true; + } + + @Override + public void discard() { + _value = null; + } + + @Override + public void read(DataInput dataInput) throws IOException { + _indexes = new MatrixIndexes(); + _value = new MatrixBlock(); + _indexes.readFields(dataInput); + _value.readFields(dataInput); + } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java new file mode 100644 index 00000000000..491fefaad87 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache; + +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; + +/** + * Small future implementation for OOC hot paths. It supports multiple synchronous subscribers without + * the completion-stage support of {@link java.util.concurrent.CompletableFuture}. + */ +public class OOCFuture { + private Subscriber _subscribers; + private T _value; + private Throwable _error; + private boolean _done; + + public static OOCFuture completed(T value) { + OOCFuture future = new OOCFuture<>(); + future._value = value; + future._done = true; + return future; + } + + public static OOCFuture failed(Throwable error) { + OOCFuture future = new OOCFuture<>(); + future._error = error; + future._done = true; + return future; + } + + public boolean complete(T value) { + return finish(value, null); + } + + public boolean completeExceptionally(Throwable error) { + if(error == null) + throw new NullPointerException("error"); + return finish(null, error); + } + + public void thenAccept(Consumer action) { + subscribe(null, action, null); + } + + public void whenComplete(BiConsumer action) { + subscribe(null, null, action); + } + + public OOCFuture map(Function mapper) { + return new MappedFuture<>(this, mapper); + } + + public synchronized boolean isDone() { + return _done; + } + + public synchronized T getNow(T fallback) { + if(!_done) + return fallback; + if(_error != null) + throw new CompletionException(_error); + return _value; + } + + public T get() throws InterruptedException, ExecutionException { + synchronized(this) { + while(!_done) + wait(); + if(_error != null) + throw new ExecutionException(_error); + return _value; + } + } + + public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + long remaining = unit.toNanos(timeout); + long deadline = System.nanoTime() + remaining; + synchronized(this) { + while(!_done) { + if(remaining <= 0) + throw new TimeoutException(); + TimeUnit.NANOSECONDS.timedWait(this, remaining); + remaining = deadline - System.nanoTime(); + } + if(_error != null) + throw new ExecutionException(_error); + return _value; + } + } + + private void subscribe(Function mapper, Consumer action, + BiConsumer completion) { + T value; + Throwable error; + synchronized(this) { + if(!_done) { + _subscribers = new Subscriber<>(mapper, action, completion, _subscribers); + return; + } + value = _value; + error = _error; + } + accept(mapper, action, completion, value, error); + } + + private boolean finish(T value, Throwable error) { + Subscriber subscribers; + synchronized(this) { + if(_done) + return false; + _value = value; + _error = error; + _done = true; + subscribers = _subscribers; + _subscribers = null; + notifyAll(); + } + while(subscribers != null) { + Subscriber next = subscribers.next; + subscribers.accept(value, error); + subscribers = next; + } + return true; + } + + private static void accept(Function mapper, Consumer action, + BiConsumer completion, T value, Throwable error) { + R result = null; + Throwable resultError = error; + if(resultError == null) { + try { + @SuppressWarnings("unchecked") + R mapped = mapper == null ? (R)value : mapper.apply(value); + result = mapped; + } + catch(Throwable t) { + resultError = t; + } + } + try { + if(completion != null) + completion.accept(result, resultError); + else if(resultError == null) + action.accept(result); + } + catch(Throwable ignored) { + // Subscribers are independent; one failed callback must not prevent the remaining notifications. + } + } + + private static final class Subscriber { + private final Function mapper; + private final Consumer action; + private final BiConsumer completion; + private final Subscriber next; + + @SuppressWarnings("unchecked") + private Subscriber(Function mapper, Consumer action, + BiConsumer completion, Subscriber next) { + this.mapper = mapper; + this.action = (Consumer)action; + this.completion = (BiConsumer)(BiConsumer)completion; + this.next = next; + } + + private void accept(T value, Throwable error) { + OOCFuture.accept(mapper, action, completion, value, error); + } + } + + private static final class MappedFuture extends OOCFuture { + private final OOCFuture source; + private final Function mapper; + + private MappedFuture(OOCFuture source, Function mapper) { + this.source = source; + this.mapper = mapper; + } + + @Override + public boolean complete(T value) { + throw new UnsupportedOperationException("Cannot complete a mapped OOCFuture"); + } + + @Override + public boolean completeExceptionally(Throwable error) { + throw new UnsupportedOperationException("Cannot complete a mapped OOCFuture"); + } + + @Override + public void thenAccept(Consumer action) { + source.subscribe(mapper, action, null); + } + + @Override + public void whenComplete(BiConsumer action) { + source.subscribe(mapper, null, action); + } + + @Override + public OOCFuture map(Function nextMapper) { + return new MappedFuture<>(source, value -> nextMapper.apply(mapper.apply(value))); + } + + @Override + public boolean isDone() { + return source.isDone(); + } + + @Override + public T getNow(T fallback) { + if(!source.isDone()) + return fallback; + try { + return mapper.apply(source.getNow(null)); + } + catch(CompletionException ex) { + throw ex; + } + catch(Throwable t) { + throw new CompletionException(t); + } + } + + @Override + public T get() throws InterruptedException, ExecutionException { + try { + return mapper.apply(source.get()); + } + catch(InterruptedException | ExecutionException ex) { + throw ex; + } + catch(Throwable t) { + throw new ExecutionException(t); + } + } + + @Override + public T get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + try { + return mapper.apply(source.get(timeout, unit)); + } + catch(InterruptedException | ExecutionException | TimeoutException ex) { + throw ex; + } + catch(Throwable t) { + throw new ExecutionException(t); + } + } + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java new file mode 100644 index 00000000000..ee02b28404c --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java @@ -0,0 +1,307 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache.io; + +import org.apache.sysds.runtime.data.SparseBlock; +import org.apache.sysds.runtime.data.SparseBlockCSR; +import org.apache.sysds.runtime.io.IOUtilFunctions; +import org.apache.sysds.runtime.matrix.data.MatrixBlockDataInput; + +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.DoubleBuffer; + +import jdk.incubator.vector.DoubleVector; +import jdk.incubator.vector.VectorOperators; +import jdk.incubator.vector.VectorSpecies; + +class OOCBufferedDataInputStream implements DataInput, MatrixBlockDataInput { + private static final int PAGE_SIZE = 4096; + private static final int PAGE_MASK = PAGE_SIZE - 1; + private static final int DEFAULT_BUFFER_SIZE = 64 * 1024; + private static final VectorSpecies DOUBLE_SPECIES = DoubleVector.SPECIES_PREFERRED; + private static final int DOUBLE_VECTOR_LENGTH = DOUBLE_SPECIES.length(); + + private final RandomAccessFile _in; + private final byte[] _buff; + private final byte[] _tmp; + private final DoubleBuffer[] _doubleDecodeBuffers; + private final int _bufflen; + private long _filePos; + private int _pos; + private int _count; + + OOCBufferedDataInputStream(RandomAccessFile in) throws IOException { + this(in, DEFAULT_BUFFER_SIZE); + } + + OOCBufferedDataInputStream(RandomAccessFile in, int size) throws IOException { + if(size <= 0) + throw new IllegalArgumentException("Buffer size <= 0."); + if(size % 8 != 0) + throw new IllegalArgumentException("Buffer size not a multiple of 8."); + _in = in; + _buff = new byte[size]; + _tmp = new byte[8]; + _doubleDecodeBuffers = createDoubleDecodeBuffers(_buff); + _bufflen = size; + _filePos = in.getFilePointer(); + _pos = 0; + _count = 0; + } + + @Override + public void readFully(byte[] b) throws IOException { + readFully(b, 0, b.length); + } + + @Override + public void readFully(byte [] b, int off, int len) throws IOException { + if(len < 0) + throw new IndexOutOfBoundsException(); + + while(len > 0) { + int avail = _count - _pos; + if(avail > 0) { + int n = Math.min(avail, len); + System.arraycopy(_buff, _pos, b, off, n); + _pos += n; + off += n; + len -= n; + } + else if(len >= _bufflen) { + _in.readFully(b, off, len); + _filePos += len; + return; + } + else { + refill(); + } + } + } + + @Override + public int skipBytes(int n) throws IOException { + throw new IOException("Not supported."); + } + + @Override + public boolean readBoolean() throws IOException { + return readByte() != 0; + } + + @Override + public byte readByte() throws IOException { + if(_pos >= _count) + refill(); + return _buff[_pos++]; + } + + @Override + public int readUnsignedByte() throws IOException { + return readByte() & 0xFF; + } + + @Override + public short readShort() throws IOException { + if(_count - _pos >= 2) { + short ret = (short)baToShort(_buff, _pos); + _pos += 2; + return ret; + } + readFully(_tmp, 0, 2); + return (short)baToShort(_tmp, 0); + } + + @Override + public int readUnsignedShort() throws IOException { + return readShort() & 0xFFFF; + } + + @Override + public char readChar() throws IOException { + return (char)readUnsignedShort(); + } + + @Override + public int readInt() throws IOException { + if(_count - _pos >= 4) { + int ret = baToInt(_buff, _pos); + _pos += 4; + return ret; + } + readFully(_tmp, 0, 4); + return baToInt(_tmp, 0); + } + + @Override + public long readLong() throws IOException { + if(_count - _pos >= 8) { + long ret = baToLong(_buff, _pos); + _pos += 8; + return ret; + } + readFully(_tmp, 0, 8); + return baToLong(_tmp, 0); + } + + @Override + public float readFloat() throws IOException { + return Float.intBitsToFloat(readInt()); + } + + @Override + public double readDouble() throws IOException { + return Double.longBitsToDouble(readLong()); + } + + @Override + public String readLine() throws IOException { + throw new IOException("Not supported."); + } + + @Override + public String readUTF() throws IOException { + return DataInputStream.readUTF(this); + } + + @Override + public long readDoubleArray(int len, double[] varr) throws IOException { + if(len <= 0 || len > varr.length) + throw new IndexOutOfBoundsException("len=" + len + ", varr.length=" + varr.length); + + long nnz = 0; + int ix = 0; + while(ix < len) { + int avail = _count - _pos; + if(avail <= 0) { + refill(); + continue; + } + if(avail < 8) { + readFully(_tmp, 0, 8); + double v = Double.longBitsToDouble(baToLong(_tmp, 0)); + varr[ix] = v; + nnz += (v != 0) ? 1 : 0; + ix++; + continue; + } + + int ndbl = Math.min(len - ix, avail / 8); + int end = _pos + ndbl * 8; + readDoubles(_pos, ndbl, varr, ix); + nnz += countNonZeros(varr, ix, ndbl); + ix += ndbl; + _pos = end; + } + return nnz; + } + + @Override + public long readSparseRows(int rlen, long nnz, SparseBlock rows) throws IOException { + if(rows instanceof SparseBlockCSR) { + ((SparseBlockCSR)rows).initSparse(rlen, (int)nnz, this); + return nnz; + } + + long gnnz = 0; + for(int i = 0; i < rlen; i++) { + int lnnz = readInt(); + if(lnnz > 0) { + rows.allocate(i, lnnz); + + for(int j = 0; j < lnnz; j++) { + int aix = readInt(); + double aval = readDouble(); + rows.append(i, aix, aval); + } + gnnz += lnnz; + } + } + + if(gnnz != nnz) + throw new IOException("Invalid number of read nnz: " + gnnz + " vs " + nnz); + return nnz; + } + + private void refill() throws IOException { + int len = getRefillLength(); + _count = _in.read(_buff, 0, len); + _pos = 0; + if(_count < 0) + throw new EOFException(); + _filePos += _count; + } + + private int getRefillLength() { + int pageOffset = (int)(_filePos & PAGE_MASK); + if(pageOffset == 0) + return _bufflen; + return Math.min(_bufflen, PAGE_SIZE - pageOffset); + } + + private static int baToShort(byte[] ba, final int off) { + return IOUtilFunctions.baToShort(ba, off); + } + + private static int baToInt(byte[] ba, final int off) { + return IOUtilFunctions.baToInt(ba, off); + } + + private static long baToLong(byte[] ba, final int off) { + return IOUtilFunctions.baToLong(ba, off); + } + + private void readDoubles(int srcPos, int len, double[] dest, int destPos) { + int alignment = srcPos & 7; + DoubleBuffer dbuff = _doubleDecodeBuffers[alignment]; + dbuff.position((srcPos - alignment) >>> 3); + dbuff.get(dest, destPos, len); + } + + private static DoubleBuffer[] createDoubleDecodeBuffers(byte[] buff) { + DoubleBuffer[] ret = new DoubleBuffer[8]; + for(int i = 0; i < ret.length; i++) { + ByteBuffer bbuff = ByteBuffer.wrap(buff); + bbuff.position(i); + ret[i] = bbuff.slice().order(ByteOrder.BIG_ENDIAN).asDoubleBuffer(); + } + return ret; + } + + private static long countNonZeros(double[] values, int off, int len) { + long nnz = 0; + int i = 0; + int upper = DOUBLE_SPECIES.loopBound(len); + DoubleVector vzero = DoubleVector.zero(DOUBLE_SPECIES); + for(; i < upper; i += DOUBLE_VECTOR_LENGTH) { + DoubleVector v = DoubleVector.fromArray(DOUBLE_SPECIES, values, off + i); + nnz += v.compare(VectorOperators.NE, vzero).trueCount(); + } + for(; i < len; i++) + nnz += (values[off + i] != 0) ? 1 : 0; + return nnz; + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java new file mode 100644 index 00000000000..9854a94586b --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache.io; + +import org.apache.sysds.runtime.data.SparseBlock; +import org.apache.sysds.runtime.io.IOUtilFunctions; +import org.apache.sysds.runtime.matrix.data.MatrixBlockDataOutput; + +import java.io.DataOutput; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UTFDataFormatException; + +class OOCBufferedDataOutputStream extends FilterOutputStream implements DataOutput, MatrixBlockDataOutput { + private final byte[] _buff; + private final int _bufflen; + private int _count; + private long _position; + private long _flushedPosition; + + OOCBufferedDataOutputStream(OutputStream out) { + this(out, 8192); + } + + OOCBufferedDataOutputStream(OutputStream out, int size) { + super(out); + if(size <= 0) + throw new IllegalArgumentException("Buffer size <= 0."); + if(size % 8 != 0) + throw new IllegalArgumentException("Buffer size not a multiple of 8."); + _buff = new byte[size]; + _bufflen = size; + _count = 0; + _position = 0; + _flushedPosition = 0; + } + + long getPosition() { + return _position; + } + + long getFlushedPosition() { + return _flushedPosition; + } + + @Override + public void write(int b) throws IOException { + if(_count >= _bufflen) + flushBuffer(); + _buff[_count++] = (byte)b; + _position++; + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + if(len > _bufflen) { + flushBuffer(); + out.write(b, off, len); + _position += len; + _flushedPosition += len; + } + else { + if(len > _bufflen - _count) + flushBuffer(); + System.arraycopy(b, off, _buff, _count, len); + _count += len; + _position += len; + } + } + + @Override + public void flush() throws IOException { + flushBuffer(); + out.flush(); + } + + private void flushBuffer() throws IOException { + if(_count > 0) { + out.write(_buff, 0, _count); + _flushedPosition += _count; + _count = 0; + } + } + + @Override + public void close() throws IOException { + super.close(); + } + + @Override + public void writeBoolean(boolean v) throws IOException { + if(_count >= _bufflen) + flushBuffer(); + _buff[_count++] = (byte)(v ? 1 : 0); + _position++; + } + + @Override + public void writeInt(int v) throws IOException { + if(_count + 4 > _bufflen) + flushBuffer(); + intToBa(v, _buff, _count); + _count += 4; + _position += 4; + } + + @Override + public void writeLong(long v) throws IOException { + if(_count + 8 > _bufflen) + flushBuffer(); + longToBa(v, _buff, _count); + _count += 8; + _position += 8; + } + + @Override + public void writeDouble(double v) throws IOException { + if(_count + 8 > _bufflen) + flushBuffer(); + longToBa(Double.doubleToRawLongBits(v), _buff, _count); + _count += 8; + _position += 8; + } + + @Override + public void writeFloat(float v) throws IOException { + if(_count + 4 > _bufflen) + flushBuffer(); + intToBa(Float.floatToIntBits(v), _buff, _count); + _count += 4; + _position += 4; + } + + @Override + public void writeByte(int v) throws IOException { + if(_count + 1 > _bufflen) + flushBuffer(); + _buff[_count++] = (byte)v; + _position++; + } + + @Override + public void writeShort(int v) throws IOException { + if(_count + 2 > _bufflen) + flushBuffer(); + shortToBa(v, _buff, _count); + _count += 2; + _position += 2; + } + + @Override + public void writeBytes(String s) throws IOException { + throw new IOException("Not supported."); + } + + @Override + public void writeChar(int v) throws IOException { + writeShort(v); + } + + @Override + public void writeChars(String s) throws IOException { + throw new IOException("Not supported."); + } + + @Override + public void writeUTF(String s) throws IOException { + int slen = s.length(); + int utflen = IOUtilFunctions.getUTFSize(s) - 2; + if(utflen - 2 > 65535) + throw new UTFDataFormatException("encoded string too long: " + utflen); + + writeShort(utflen); + for(int i = 0; i < slen; i++) { + if(_count + 3 > _bufflen) + flushBuffer(); + final char c = s.charAt(i); + if(c >= 0x0001 && c <= 0x007F) { + _buff[_count++] = (byte)c; + _position++; + } + else if(c >= 0x0800) { + _buff[_count++] = (byte)(0xE0 | ((c >> 12) & 0x0F)); + _buff[_count++] = (byte)(0x80 | ((c >> 6) & 0x3F)); + _buff[_count++] = (byte)(0x80 | (c & 0x3F)); + _position += 3; + } + else { + _buff[_count++] = (byte)(0xC0 | ((c >> 6) & 0x1F)); + _buff[_count++] = (byte)(0x80 | (c & 0x3F)); + _position += 2; + } + } + } + + @Override + public void writeDoubleArray(int len, double[] varr) throws IOException { + for(int i = 0; i < len; ) { + if(_count >= _bufflen) + flushBuffer(); + int lblen = Math.min(len - i, (_bufflen - _count) / 8); + if(lblen == 0) { + flushBuffer(); + continue; + } + for(int j = 0; j < lblen; j++) { + longToBa(Double.doubleToRawLongBits(varr[i + j]), _buff, _count); + _count += 8; + } + _position += 8L * lblen; + i += lblen; + if(_count >= _bufflen) + flushBuffer(); + } + } + + @Override + public void writeSparseRows(int rlen, SparseBlock rows) throws IOException { + int lrlen = Math.min(rows.numRows(), rlen); + for(int i = 0; i < lrlen; i++) { + if(!rows.isEmpty(i)) { + int apos = rows.pos(i); + int alen = rows.size(i); + int[] aix = rows.indexes(i); + double[] avals = rows.values(i); + + writeInt(alen); + + for(int j = apos; j < apos + alen; j++) { + if(_count + 12 > _bufflen) + flushBuffer(); + long tmp = Double.doubleToRawLongBits(avals[j]); + intToBa(aix[j], _buff, _count); + longToBa(tmp, _buff, _count + 4); + _count += 12; + _position += 12; + } + } + else { + writeInt(0); + } + } + + for(int i = lrlen; i < rlen; i++) + writeInt(0); + } + + private static void shortToBa(final int val, byte[] ba, final int off) { + IOUtilFunctions.shortToBa(val, ba, off); + } + + private static void intToBa(final int val, byte[] ba, final int off) { + IOUtilFunctions.intToBa(val, ba, off); + } + + private static void longToBa(final long val, byte[] ba, final int off) { + IOUtilFunctions.longToBa(val, ba, off); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java index 21085626a71..ab28df0ef0f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java @@ -25,6 +25,7 @@ import org.apache.sysds.runtime.matrix.data.MatrixIndexes; import org.apache.sysds.runtime.ooc.cache.BlockEntry; import org.apache.sysds.runtime.ooc.cache.BlockKey; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; import java.util.concurrent.CompletableFuture; import java.util.List; @@ -34,7 +35,7 @@ public interface OOCIOHandler { CompletableFuture scheduleEviction(BlockEntry block); - CompletableFuture scheduleRead(BlockEntry block); + OOCFuture scheduleRead(BlockEntry block); /** * Increase priority for a pending scheduled read if it has not started yet. diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java index 029c9e8060f..bfe1565ab3a 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java @@ -34,10 +34,9 @@ import org.apache.sysds.runtime.matrix.data.MatrixIndexes; import org.apache.sysds.runtime.ooc.cache.BlockEntry; import org.apache.sysds.runtime.ooc.cache.BlockKey; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; import org.apache.sysds.runtime.ooc.stats.OOCEventLog; import org.apache.sysds.runtime.ooc.stream.SourceOOCStream; -import org.apache.sysds.runtime.util.FastBufferedDataInputStream; -import org.apache.sysds.runtime.util.FastBufferedDataOutputStream; import org.apache.sysds.runtime.util.LocalFileUtils; import org.apache.sysds.utils.Statistics; import scala.Tuple2; @@ -46,9 +45,7 @@ import java.io.DataInput; import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; import java.io.RandomAccessFile; -import java.nio.channels.Channels; import java.nio.channels.ClosedByInterruptException; import java.util.ArrayList; import java.util.Arrays; @@ -69,13 +66,14 @@ import java.util.concurrent.atomic.AtomicReference; public class OOCMatrixIOHandler implements OOCIOHandler { - private static final int WRITER_SIZE = 4; - private static final int READER_SIZE = 10; + private static final int WRITER_SIZE = 8; + private static final int READER_SIZE = 16; private static final long OVERFLOW = 8192 * 1024; private static final long MAX_PARTITION_SIZE = 8192 * 8192; private static final long GROUP_TARGET_BYTES = 8L * 1024 * 1024; private static final long GROUP_MAX_BYTES = 16L * 1024 * 1024; private static final int GROUP_MAX_COUNT = 64; + private static final long IDLE_FLUSH_MS = 1; private final String _spillDir; private final ThreadPoolExecutor _writeExec; @@ -147,11 +145,11 @@ public void shutdown() { if (started) { try { for(int i = 0; i < WRITER_SIZE; i++) { - _q[i].close(); + if(_q[i] != null) + _q[i].close(); } } - catch(InterruptedException e) { - Thread.currentThread().interrupt(); + catch(InterruptedException ignored) { } } _writeExec.getQueue().clear(); @@ -175,18 +173,20 @@ public CompletableFuture scheduleEviction(BlockEntry block) { try { long q = _wCtr.getAndAdd(block.getSize()) / OVERFLOW; int i = (int)(q % WRITER_SIZE); - _q[i].enqueueIfOpen(new Tuple2<>(block, future)); + if(!_q[i].enqueueIfOpen(new Tuple2<>(block, future))) + future.completeExceptionally(new DMLRuntimeException("OOC writer queue is closed")); } - catch(InterruptedException e) { + catch(InterruptedException ignored) { Thread.currentThread().interrupt(); + future.completeExceptionally(new DMLRuntimeException("Interrupted while scheduling OOC eviction")); } return future; } @Override - public CompletableFuture scheduleRead(final BlockEntry block) { - final CompletableFuture future = new CompletableFuture<>(); + public OOCFuture scheduleRead(final BlockEntry block) { + final OOCFuture future = new OOCFuture<>(); int pinnedPartitionId = pinPartitionForRead(block.getKey()); try { ReadTask task = new ReadTask(block, future, _readSeq.getAndIncrement(), pinnedPartitionId); @@ -532,25 +532,23 @@ private void loadFromDisk(BlockEntry block) { String filename = partFile.filePath; - // Create an empty object to read data into. - MatrixIndexes ix = new MatrixIndexes(); - MatrixBlock mb = new MatrixBlock(); + SpillableObject obj; try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { raf.seek(sloc.offset); - DataInput dis = new FastBufferedDataInputStream(Channels.newInputStream(raf.getChannel())); + DataInput dis = new OOCBufferedDataInputStream(raf); long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; - ix.readFields(dis); // 1. Read Indexes - mb.readFields(dis); // 2. Read Block + obj = SpillableObjectRegistry.read(dis); if (DMLScript.OOC_STATISTICS) ioDuration = System.nanoTime() - ioStart; } catch (ClosedByInterruptException ignored) { + return; } catch (IOException e) { throw new RuntimeException(e); } - block.setDataUnsafe(new IndexedMatrixValue(ix, mb)); + block.setDataUnsafe(obj); if (DMLScript.OOC_STATISTICS) { Statistics.incrementOOCLoadFromDisk(); @@ -617,22 +615,27 @@ private void evictTask(CloseableQueue partFile.incrementRefCount(); // Writer pin; released when partition closes FileOutputStream fos = null; - CountableFastBufferedDataOutputStream dos = null; + OOCBufferedDataOutputStream dos = null; ConcurrentLinkedDeque>> waitingForFlush = null; try { fos = new FileOutputStream(filename); - dos = new CountableFastBufferedDataOutputStream(fos); + dos = new OOCBufferedDataOutputStream(fos); Tuple2> tpl; waitingForFlush = new ConcurrentLinkedDeque<>(); boolean closePartition = false; - while((tpl = q.take()) != null) { + while(!q.isFinished()) { + tpl = q.poll(IDLE_FLUSH_MS, TimeUnit.MILLISECONDS); + if(tpl == null) { + flushReadable(dos, waitingForFlush); + continue; + } long ioStart = DMLScript.OOC_STATISTICS || DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; - BlockEntry entry = tpl._1; - CompletableFuture future = tpl._2; - long wrote = writeOut(partitionId, entry, future, fos, dos, waitingForFlush); + BlockEntry entry = tpl._1(); + CompletableFuture future = tpl._2(); + long wrote = writeOut(partitionId, entry, future, dos, waitingForFlush); if(DMLScript.OOC_STATISTICS && wrote > 0) { Statistics.incrementOOCEvictionWrite(); @@ -654,9 +657,9 @@ private void evictTask(CloseableQueue if (!closePartition && q.close()) { while((tpl = q.take()) != null) { long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; - BlockEntry entry = tpl._1; - CompletableFuture future = tpl._2; - long wrote = writeOut(partitionId, entry, future, fos, dos, waitingForFlush); + BlockEntry entry = tpl._1(); + CompletableFuture future = tpl._2(); + long wrote = writeOut(partitionId, entry, future, dos, waitingForFlush); byteCtr += wrote; if(DMLScript.OOC_STATISTICS && wrote > 0) { @@ -685,45 +688,62 @@ private void evictTask(CloseableQueue } } - private long writeOut(int partitionId, BlockEntry entry, CompletableFuture future, FileOutputStream fos, - CountableFastBufferedDataOutputStream dos, ConcurrentLinkedDeque>> flushQueue) throws IOException { + private long writeOut(int partitionId, BlockEntry entry, CompletableFuture future, + OOCBufferedDataOutputStream dos, + ConcurrentLinkedDeque>> flushQueue) throws IOException { + String key = entry.getKey().toFileKey(); boolean alreadySpilled = _spillLocations.containsKey(key); if (!alreadySpilled) { - // 1. get the current file position. this is the offset. - // flush any buffered data to the file - //dos.flush(); - long offsetBefore = fos.getChannel().position() + dos.getCount(); + long offsetBefore = dos.getPosition(); + + if(future.isCancelled()) + return 0; // 2. write indexes and block - IndexedMatrixValue imv = (IndexedMatrixValue) entry.getDataUnsafe(); // Get data without requiring pin - if(imv == null) + SpillableObject so = (SpillableObject) entry.getDataUnsafe(); // Get data without requiring pin + if(so == null) + return 0; + if(!SpillableObjectRegistry.tryWrite(dos, so)) return 0; - imv.getIndexes().write(dos); // write Indexes - imv.getValue().write(dos); - long offsetAfter = fos.getChannel().position() + dos.getCount(); + long offsetAfter = dos.getPosition(); + if(future.isCancelled()) + return offsetAfter - offsetBefore; flushQueue.offer(new Tuple3<>(offsetBefore, offsetAfter, future)); // 3. create the spillLocation SpillLocation sloc = new SpillLocation(partitionId, offsetBefore); addSpillLocation(key, sloc); - flushQueue(fos.getChannel().position(), flushQueue); + if(future.isCancelled()) { + removeSpillLocation(key); + return offsetAfter - offsetBefore; + } + flushQueue(dos.getFlushedPosition(), flushQueue); return offsetAfter - offsetBefore; } + future.completeExceptionally(new DMLRuntimeException("Duplicate OOC spill location for: " + key)); return 0; } private void flushQueue(long offset, ConcurrentLinkedDeque>> flushQueue) { Tuple3> tmp; - while ((tmp = flushQueue.peek()) != null && tmp._2() < offset) { + while ((tmp = flushQueue.peek()) != null && tmp._2() <= offset) { flushQueue.poll(); tmp._3().complete(null); } } + private void flushReadable(OOCBufferedDataOutputStream dos, + ConcurrentLinkedDeque>> flushQueue) throws IOException { + if(dos == null || flushQueue.isEmpty()) + return; + dos.flush(); + flushQueue(dos.getFlushedPosition(), flushQueue); + } + private void addSpillLocation(String key, SpillLocation sloc) { synchronized(_spillLock) { SpillLocation existing = _spillLocations.putIfAbsent(key, sloc); @@ -806,12 +826,12 @@ private void unpinPartitionForRead(int partitionId) { private class ReadTask implements Runnable, Comparable { private final BlockEntry _block; - private final CompletableFuture _future; + private final OOCFuture _future; private final long _sequence; private final int _pinnedPartitionId; private double _priority; - private ReadTask(BlockEntry block, CompletableFuture future, long sequence, int pinnedPartitionId) { + private ReadTask(BlockEntry block, OOCFuture future, long sequence, int pinnedPartitionId) { this._block = block; this._future = future; this._sequence = sequence; @@ -880,16 +900,6 @@ int decrementRefCount() { } } - private static class CountableFastBufferedDataOutputStream extends FastBufferedDataOutputStream { - public CountableFastBufferedDataOutputStream(OutputStream out) { - super(out); - } - - public int getCount() { - return _count; - } - } - private static class SourceReadState implements SourceReadContinuation { final SourceReadRequest request; final Path[] paths; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java new file mode 100644 index 00000000000..434f70601d6 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache.io; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +public interface SpillableObject { + boolean tryWrite(DataOutput out) throws IOException; + void read(DataInput in) throws IOException; + + default void discard() { + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObjectRegistry.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObjectRegistry.java new file mode 100644 index 00000000000..6cd8fded9e7 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObjectRegistry.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache.io; + +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +public final class SpillableObjectRegistry { + private static final byte INDEXED_MATRIX_VALUE = 1; + + private SpillableObjectRegistry() { + } + + public static boolean tryWrite(DataOutput out, SpillableObject obj) throws IOException { + byte type = typeOf(obj); + out.writeByte(type); + return obj.tryWrite(out); + } + + public static SpillableObject read(DataInput in) throws IOException { + byte type = in.readByte(); + SpillableObject obj = switch(type) { + case INDEXED_MATRIX_VALUE -> new IndexedMatrixValue(); + default -> throw new IOException("Unknown spillable object type: " + type); + }; + obj.read(in); + return obj; + } + + private static byte typeOf(SpillableObject obj) throws IOException { + if(obj instanceof IndexedMatrixValue) + return INDEXED_MATRIX_VALUE; + throw new IOException("Unsupported spillable object type: " + obj.getClass().getName()); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java index c1f7058b5dc..3f5601adbae 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java @@ -28,6 +28,7 @@ import org.apache.sysds.runtime.ooc.cache.BlockEntry; import org.apache.sysds.runtime.ooc.cache.BlockKey; import org.apache.sysds.runtime.ooc.cache.BlockState; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; import org.apache.sysds.runtime.ooc.stats.OOCEventLog; @@ -860,7 +861,7 @@ else if(allReserved && reading && req.isComplete()) { for(Tuple2 tpl : toRead) { final BlockEntry entry = tpl._2; - CompletableFuture future = _ioHandler.scheduleRead(entry); + OOCFuture future = _ioHandler.scheduleRead(entry); future.whenComplete((r, t) -> { if(t != null) { BlockReadState state; diff --git a/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCLRUCacheSchedulerTest.java b/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCLRUCacheSchedulerTest.java index 002b19e57be..cd05d97b317 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCLRUCacheSchedulerTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCLRUCacheSchedulerTest.java @@ -24,6 +24,7 @@ import org.apache.sysds.runtime.ooc.cache.BlockEntry; import org.apache.sysds.runtime.ooc.cache.BlockKey; import org.apache.sysds.runtime.ooc.cache.BlockState; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; import org.apache.sysds.runtime.ooc.cache.legacy.OOCLRUCacheScheduler; import org.junit.After; @@ -302,7 +303,7 @@ private static List snapshotDeferredOrder(OOCLRUCacheScheduler schedul } private static class FakeIOHandler implements OOCIOHandler { - private final Map> _readFutures = new HashMap<>(); + private final Map> _readFutures = new HashMap<>(); private final Map _readEntries = new HashMap<>(); private final Map _readCounts = new HashMap<>(); @@ -319,8 +320,8 @@ public CompletableFuture scheduleEviction(BlockEntry block) { } @Override - public CompletableFuture scheduleRead(BlockEntry block) { - CompletableFuture future = new CompletableFuture<>(); + public OOCFuture scheduleRead(BlockEntry block) { + OOCFuture future = new OOCFuture<>(); _readFutures.put(block.getKey(), future); _readEntries.put(block.getKey(), block); _readCounts.computeIfAbsent(block.getKey(), k -> new AtomicInteger(0)).incrementAndGet(); @@ -355,7 +356,7 @@ public int getReadCount(BlockKey key) { } public void completeRead(BlockKey key) { - CompletableFuture future = _readFutures.get(key); + OOCFuture future = _readFutures.get(key); if (future == null) throw new IllegalStateException("No scheduled read for " + key); BlockEntry entry = _readEntries.get(key); @@ -364,5 +365,5 @@ public void completeRead(BlockKey key) { BlockEntryTestAccess.setDataUnsafe(entry, new Object()); future.complete(entry); } - } } +} From ddc761d945f558ae61e6afd8c19463b3df614970 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Mon, 6 Jul 2026 15:15:40 +0200 Subject: [PATCH 065/132] [CI] Add Java Format CI check for pull requests (#2527) Add a workflow that enforces dev/CodeStyle_eclipse.xml on the Java files changed by a pull request. It applies the Eclipse formatter (via formatter-maven-plugin) to only the changed src/main/java and src/test/java files and fails if that produces any diff, printing the required changes. --- .github/workflows/javaCodestyle.yml | 61 +++++- dev/format-changed.sh | 48 +++++ dev/format-exclude.txt | 35 +++ dev/format_changed.py | 324 ++++++++++++++++++++++++++++ dev/tests/test_format_changed.py | 150 +++++++++++++ 5 files changed, 615 insertions(+), 3 deletions(-) create mode 100755 dev/format-changed.sh create mode 100644 dev/format-exclude.txt create mode 100755 dev/format_changed.py create mode 100644 dev/tests/test_format_changed.py diff --git a/.github/workflows/javaCodestyle.yml b/.github/workflows/javaCodestyle.yml index 7dedc5f4865..a63e3298760 100644 --- a/.github/workflows/javaCodestyle.yml +++ b/.github/workflows/javaCodestyle.yml @@ -21,6 +21,16 @@ name: Java Codestyle +# Two Java style gates share this workflow: +# * Checkstyle -- whole-tree rule check (dev/checkstyle), on push and PR. +# * Java Format -- Eclipse formatter (dev/CodeStyle_eclipse.xml) applied to +# ONLY the lines a PR edits; fails if any edited line would +# change. The tree is not yet fully formatter-clean, so +# scoping to edited lines keeps it actionable and lets the +# codebase converge line-by-line. This gate needs the PR +# base commit, so it runs on pull_request only. +# They are separate jobs so each reports its own pass/fail status. + on: push: paths-ignore: @@ -28,7 +38,6 @@ on: - '*.md' - '*.html' - 'src/main/python/**' - - 'dev/**' branches: - main pull_request: @@ -37,16 +46,18 @@ on: - '*.md' - '*.html' - 'src/main/python/**' - - 'dev/**' branches: - main +permissions: + contents: read + concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - java_codestyle: + java_checkstyle: name: Java Checkstyle runs-on: ubuntu-latest steps: @@ -62,3 +73,47 @@ jobs: - name: Run Checkstyle run: mvn -ntp -B -Dcheckstyle.skip=false checkstyle:check + + java_format: + name: Java Format Check + # line-scoped to the PR diff -> needs the pull_request base commit + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Setup Java 17 adopt + uses: actions/setup-java@v5 + with: + distribution: adopt + java-version: '17' + cache: 'maven' + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Unit-test the format checker + run: | + python -m pip install --quiet pytest + python -m pytest dev/tests -q + + - name: Check formatting of PR-edited lines + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + # Fails only if the Eclipse formatter would change a line this PR edited. + # See dev/format_changed.py for the line-scoping logic. + if ! python3 dev/format_changed.py --check "$BASE_SHA"; then + echo "::error::Some lines edited by this PR are not formatted per dev/CodeStyle_eclipse.xml." + echo "Fix only your edited lines locally and commit the result:" + echo "" + echo " dev/format-changed.sh" + echo "" + echo "(Do NOT run a bare 'mvn formatter:format' -- it reformats the whole tree.)" + exit 1 + fi diff --git a/dev/format-changed.sh b/dev/format-changed.sh new file mode 100755 index 00000000000..a49192e3d29 --- /dev/null +++ b/dev/format-changed.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- +# +# Apply the Eclipse formatter (dev/CodeStyle_eclipse.xml) to ONLY the lines you +# edited -- the exact scope the "Java Format" CI check enforces. +# +# A bare `mvn formatter:format` would reformat EVERY .java file under the source +# roots, and even scoping to changed *files* would reformat their pre-existing +# (not-yet-clean) lines. The existing tree is not fully formatter-clean, so both +# would produce a large unrelated diff. This delegates to dev/format_changed.py, +# which formats each changed file but keeps only the changes that land on the +# lines you actually edited. +# +# "Changed" = lines that differ from the base branch (the merge target), +# including your committed-on-branch, staged, unstaged and untracked edits. Run +# `git fetch upstream main` first so the diff is accurate, or pass an explicit, +# current base ref if the default is stale/behind your branch point. The base-ref +# fallback order lives in one place: see resolve_base() in dev/format_changed.py. +# +# Usage: +# dev/format-changed.sh [base-ref] +# base-ref branch/commit to diff against (default: resolved by +# dev/format_changed.py) +# +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +exec python3 dev/format_changed.py --fix "$@" diff --git a/dev/format-exclude.txt b/dev/format-exclude.txt new file mode 100644 index 00000000000..dd3d9749eed --- /dev/null +++ b/dev/format-exclude.txt @@ -0,0 +1,35 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# Files exempt from the Java Format check (dev/format_changed.py). +# +# One pattern per line; blank lines and lines starting with '#' are ignored. +# Each pattern is matched (glob / fnmatch) against the repo-relative path AND +# against the bare file name, so both of these work: +# +# src/main/java/org/apache/sysds/conf/DMLConfig.java +# *DMLConfig.java +# +# Use this only for files whose non-standard layout is intentional and must not +# be reformatted (e.g. hand-aligned tables of constants). + +# Hand-aligned configuration constants; intentional non-standard formatting. +src/main/java/org/apache/sysds/conf/DMLConfig.java diff --git a/dev/format_changed.py b/dev/format_changed.py new file mode 100755 index 00000000000..ce8b7acccff --- /dev/null +++ b/dev/format_changed.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- +# +# Line-scoped Java formatting against dev/CodeStyle_eclipse.xml. +# +# The Eclipse formatter only works on whole files, and the existing tree is not +# fully formatter-clean, so formatting a whole edited file would flag lines the +# PR never touched. This script therefore formats each changed file but keeps +# only the formatting changes that fall on the lines the PR edited (like +# clang-format-diff): it diffs the original against the fully-formatted version +# and restricts the result to the changed line ranges. +# +# --check (default): print the changed-line formatting fixes and exit 1 if any. +# --fix : apply only the changed-line formatting fixes in place. +# +# Usage: dev/format_changed.py [--check|--fix] [base-ref] +# +import difflib +import fnmatch +import os +import re +import subprocess +import sys + +FMT_VERSION = "2.24.1" +CONFIG = "dev/CodeStyle_eclipse.xml" +EXCLUDE_FILE = "dev/format-exclude.txt" +SRC_PREFIX = r"src/(main|test)/java/" +SRC_RE = re.compile(r"^" + SRC_PREFIX + r".+\.java$") + + +# --- small process / IO helpers ------------------------------------------------ + +def git(*args, check=True): + # core.quotePath=false so non-ASCII paths are emitted verbatim (not \NNN + # escaped), otherwise SRC_RE would silently skip them and bypass the check. + proc = subprocess.run(["git", "-c", "core.quotePath=false", *args], + capture_output=True, text=True) + if check and proc.returncode != 0: + sys.exit(f"ERROR: `git {' '.join(args)}` failed:\n{proc.stderr.strip()}") + return proc.stdout + + +def read_text(path): + # newline="" keeps line endings byte-exact so a check-mode restore (or a + # fix-mode non-flagged file) round-trips without CRLF->LF rewrites. + with open(path, encoding="utf-8", newline="") as fh: + return fh.read() + + +def write_text(path, text): + with open(path, "w", encoding="utf-8", newline="") as fh: + fh.write(text) + + +def strip_src_prefix(path): + return re.sub(r"^" + SRC_PREFIX, "", path) + + +# --- exemption list ------------------------------------------------------------ + +def load_excludes(): + # glob patterns of files exempt from the style check, one per line + patterns = [] + if os.path.exists(EXCLUDE_FILE): + with open(EXCLUDE_FILE, encoding="utf-8") as fh: + for line in fh: + s = line.strip() + if s and not s.startswith("#"): + patterns.append(s) + return patterns + + +def is_excluded(path, patterns): + base = os.path.basename(path) + return any(fnmatch.fnmatch(path, p) or fnmatch.fnmatch(base, p) for p in patterns) + + +# --- git ref / file discovery -------------------------------------------------- + +def ref_exists(ref): + return subprocess.run(["git", "rev-parse", "--verify", "--quiet", ref + "^{commit}"], + capture_output=True).returncode == 0 + + +def resolve_base(explicit): + if explicit: + if not ref_exists(explicit): + sys.exit(f"Base ref not found: {explicit}") + return explicit + for ref in ("upstream/main", "origin/main", "main"): + if ref_exists(ref): + return ref + sys.exit("Could not determine a base ref; pass one explicitly.") + + +def merge_base(base): + mb = git("merge-base", base, "HEAD", check=False).strip() + return mb or base + + +def base_has(mergebase, path): + return subprocess.run(["git", "cat-file", "-e", f"{mergebase}:{path}"], + capture_output=True).returncode == 0 + + +def discover_files(mergebase): + tracked = [f for f in git("diff", "--name-only", "--diff-filter=ACMR", + mergebase, "--").splitlines() if SRC_RE.match(f)] + untracked = [f for f in git("ls-files", "--others", "--exclude-standard").splitlines() + if SRC_RE.match(f)] + seen = set(tracked) + files = tracked + [f for f in untracked if f not in seen] + + patterns = load_excludes() + skipped = [f for f in files if is_excluded(f, patterns)] + files = [f for f in files if not is_excluded(f, patterns)] + return files, skipped, set(untracked) + + +# --- changed-line ranges (pure parsing, unit-tested) --------------------------- + +def _hunk_range(header): + # parse the new-side (+) range of a `@@ -a,b +c,d @@` unified-diff header; + # returns a 1-based inclusive (start, end), or None for pure deletions / non-headers + m = re.match(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@", header) + if not m: + return None + start = int(m.group(1)) + count = int(m.group(2)) if m.group(2) is not None else 1 + return (start, start + count - 1) if count > 0 else None + + +def parse_hunks(diff_text): + # all new-side ranges in a single-file unified diff (used by tests and below) + ranges = [] + for line in diff_text.splitlines(): + r = _hunk_range(line) + if r is not None: + ranges.append(r) + return ranges + + +def changed_ranges(mergebase, files): + # one batched `git diff` for all files, split per file on the `+++ b/` header + ranges = {f: [] for f in files} + if not files: + return ranges + out = git("diff", "-U0", mergebase, "--", *files) + current = None + for line in out.splitlines(): + if line.startswith("+++ b/"): + current = line[len("+++ b/"):] + elif current is not None: + r = _hunk_range(line) + if r is not None and current in ranges: + ranges[current].append(r) + return ranges + + +def overlaps(i1, i2, ranges): + # original-side region [i1, i2) (0-based half-open) vs 1-based inclusive ranges + lo, hi = i1 + 1, i2 # convert to 1-based inclusive; insert (i1==i2) -> lo>hi + for (s, e) in ranges: + if i1 == i2: # pure insertion between original lines i1 and i1+1 + if s - 1 <= i1 <= e: + return True + elif not (hi < s or lo > e): + return True + return False + + +def line_scoped_result(original_text, formatted_text, ranges): + # reconstruct a file that keeps original content everywhere except on the + # formatting hunks that intersect the PR-edited ranges; returns the new text + # plus the kept hunks (for reporting). Pure function -- no IO. + a = original_text.splitlines(keepends=True) + b = formatted_text.splitlines(keepends=True) + sm = difflib.SequenceMatcher(None, a, b, autojunk=False) + result = [] + kept = [] + for tag, i1, i2, j1, j2 in sm.get_opcodes(): + if tag == "equal": + result.extend(a[i1:i2]) + elif overlaps(i1, i2, ranges): + result.extend(b[j1:j2]) + kept.append((i1, len(a[i1:i2]), len(b[j1:j2]), a[i1:i2], b[j1:j2])) + else: + result.extend(a[i1:i2]) + return "".join(result), kept + + +# --- formatter ----------------------------------------------------------------- + +def run_formatter(files): + includes = ",".join(strip_src_prefix(f) for f in files) + subprocess.run(["mvn", "-q", "-ntp", "-B", + f"net.revelc.code.formatter:formatter-maven-plugin:{FMT_VERSION}:format", + f"-Dconfigfile={os.getcwd()}/{CONFIG}", + "-Dmaven.compiler.source=17", "-Dmaven.compiler.target=17", + f"-Dformatter.includes={includes}"], check=True) + + +def print_report(path, kept): + print(f"\n--- a/{path}") + print(f"+++ b/{path}") + for i1, n_old, n_new, olds, news in kept: + print(f"@@ -{i1 + 1},{n_old} +{i1 + 1},{n_new} @@ (changed lines)") + for ln in olds: + print("-" + ln.rstrip("\n")) + for ln in news: + print("+" + ln.rstrip("\n")) + + +# --- CLI ----------------------------------------------------------------------- + +def parse_args(argv): + mode = "check" + positionals = [] + for a in argv: + if a == "--fix": + mode = "fix" + elif a == "--check": + mode = "check" + elif a.startswith("-"): + sys.exit(f"Unknown option: {a} (usage: format_changed.py [--check|--fix] [base-ref])") + else: + positionals.append(a) + if len(positionals) > 1: + sys.exit(f"Expected at most one base ref, got: {positionals}") + return mode, (positionals[0] if positionals else None) + + +def compute_ranges(mergebase, files, untracked, originals): + batched = changed_ranges(mergebase, files) + ranges_by_file = {} + for f in files: + r = batched[f] + if not r and (f in untracked or not base_has(mergebase, f)): + # brand-new/untracked file has no base version: treat every line as edited + r = [(1, max(1, len(originals[f].splitlines())))] + ranges_by_file[f] = r + return ranges_by_file + + +def main(): + mode, base_arg = parse_args(sys.argv[1:]) + os.chdir(git("rev-parse", "--show-toplevel").strip()) + base = resolve_base(base_arg) + mergebase = merge_base(base) + + files, skipped, untracked = discover_files(mergebase) + if skipped: + print(f"Skipping style-exempt files ({EXCLUDE_FILE}):") + for f in skipped: + print(f" {f}") + if not files: + print(f"No changed Java source files to check (base: {base}).") + return 0 + + originals = {f: read_text(f) for f in files} + ranges_by_file = compute_ranges(mergebase, files, untracked, originals) + + reports = [] + wrote_fix = False + try: + try: + run_formatter(files) + except subprocess.CalledProcessError as e: + sys.exit(f"ERROR: could not run the Eclipse formatter (is `mvn` on PATH?): {e}") + + results = {} + for f in files: + result, kept = line_scoped_result(originals[f], read_text(f), ranges_by_file[f]) + results[f] = result + if kept: + reports.append((f, kept)) + + if mode == "fix": + for f in files: + write_text(f, results[f]) + wrote_fix = True + finally: + # never leave mvn's whole-file reformat on disk: check mode is read-only, + # and fix mode must restore originals unless it fully wrote the scoped results + if mode == "check" or (mode == "fix" and not wrote_fix): + for f in files: + write_text(f, originals[f]) + + if reports and mode == "check": + for path, kept in reports: + print_report(path, kept) + print("\nERROR: the changes above are required on lines this PR edited " + "(per dev/CodeStyle_eclipse.xml).") + print("Fix locally with: dev/format-changed.sh") + return 1 + if reports and mode == "fix": + print("Applied changed-line formatting. Review and commit the result.") + else: + print("All PR-edited Java lines are correctly formatted.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dev/tests/test_format_changed.py b/dev/tests/test_format_changed.py new file mode 100644 index 00000000000..dda2c26db66 --- /dev/null +++ b/dev/tests/test_format_changed.py @@ -0,0 +1,150 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- +# +# Unit tests for the pure logic in dev/format_changed.py (the line-scoping math, +# hunk parsing, and exclude matching). Run with: python -m pytest dev/tests +# +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import format_changed as fc # noqa: E402 + + +# --- overlaps: replace/delete regions vs edited ranges ------------------------- + +def test_overlaps_replace_region_hits_and_misses(): + # replace of original lines 5..7 -> 0-based half-open [4, 7) + assert fc.overlaps(4, 7, [(6, 6)]) is True # inside + assert fc.overlaps(4, 7, [(5, 5)]) is True # first line of region + assert fc.overlaps(4, 7, [(7, 7)]) is True # last line of region + assert fc.overlaps(4, 7, [(4, 4)]) is False # one before + assert fc.overlaps(4, 7, [(8, 8)]) is False # one after + assert fc.overlaps(4, 7, [(1, 2)]) is False # entirely before + assert fc.overlaps(4, 7, [(8, 9)]) is False # entirely after + + +def test_overlaps_multiple_ranges(): + assert fc.overlaps(4, 7, [(1, 2), (7, 9)]) is True + assert fc.overlaps(4, 7, [(1, 2), (10, 11)]) is False + + +# --- overlaps: pure insertions (i1 == i2) ------------------------------------- + +def test_overlaps_pure_insertion_boundaries(): + # insertion at 0-based index 5 = between 1-based lines 5 and 6 + assert fc.overlaps(5, 5, [(5, 5)]) is True # i1 == e + assert fc.overlaps(5, 5, [(6, 6)]) is True # i1 == s - 1 + assert fc.overlaps(5, 5, [(7, 7)]) is False # below the range + assert fc.overlaps(5, 5, [(3, 4)]) is False # above the range + assert fc.overlaps(0, 0, [(1, 1)]) is True # insert before the first line + + +# --- _hunk_range / parse_hunks ------------------------------------------------ + +def test_hunk_range_multiline(): + assert fc._hunk_range("@@ -1,3 +10,4 @@") == (10, 13) + + +def test_hunk_range_missing_count_defaults_to_one(): + assert fc._hunk_range("@@ -0,0 +5 @@") == (5, 5) + + +def test_hunk_range_pure_deletion_is_none(): + assert fc._hunk_range("@@ -4,2 +3,0 @@") is None + + +def test_hunk_range_non_header_is_none(): + assert fc._hunk_range("+ some added line") is None + assert fc._hunk_range("public int mul(int a) {") is None + + +def test_parse_hunks_collects_all_ranges(): + diff = ( + "diff --git a/X.java b/X.java\n" + "--- a/X.java\n" + "+++ b/X.java\n" + "@@ -1,1 +1,1 @@\n" + "-a\n+a \n" + "@@ -10,0 +11,2 @@\n" + "+x\n+y\n" + "@@ -20,2 +22,0 @@\n" # pure deletion -> skipped + ) + assert fc.parse_hunks(diff) == [(1, 1), (11, 12)] + + +# --- exclude matching --------------------------------------------------------- + +def test_is_excluded_by_full_path(): + p = "src/main/java/org/apache/sysds/conf/DMLConfig.java" + assert fc.is_excluded(p, [p]) is True + + +def test_is_excluded_by_basename_glob(): + p = "src/main/java/org/apache/sysds/conf/DMLConfig.java" + assert fc.is_excluded(p, ["*DMLConfig.java"]) is True + + +def test_is_excluded_no_match(): + assert fc.is_excluded("src/main/java/Foo.java", ["*DMLConfig.java"]) is False + assert fc.is_excluded("src/main/java/Foo.java", []) is False + + +def test_load_excludes_skips_comments_and_blanks(tmp_path, monkeypatch): + f = tmp_path / "exclude.txt" + f.write_text("# a comment\n\n \nsrc/main/java/Foo.java\n *Bar.java \n", + encoding="utf-8") + monkeypatch.setattr(fc, "EXCLUDE_FILE", str(f)) + assert fc.load_excludes() == ["src/main/java/Foo.java", "*Bar.java"] + + +def test_load_excludes_missing_file_returns_empty(tmp_path, monkeypatch): + monkeypatch.setattr(fc, "EXCLUDE_FILE", str(tmp_path / "does-not-exist.txt")) + assert fc.load_excludes() == [] + + +# --- strip_src_prefix --------------------------------------------------------- + +def test_strip_src_prefix(): + assert fc.strip_src_prefix("src/main/java/org/A.java") == "org/A.java" + assert fc.strip_src_prefix("src/test/java/org/A.java") == "org/A.java" + + +# --- line_scoped_result: only edited-line formatting is kept ------------------ + +def test_line_scoped_result_keeps_only_edited_lines(): + # lines 1 and 3 are mis-formatted; an unchanged line 2 separates them so + # difflib yields distinct hunks. Only line 3 is "edited", so line 1 (a + # pre-existing violation the PR did not touch) must stay original. + original = "int a=1;\nint ok = 0;\nint b=2;\n" + formatted = "int a = 1;\nint ok = 0;\nint b = 2;\n" + result, kept = fc.line_scoped_result(original, formatted, [(3, 3)]) + assert result == "int a=1;\nint ok = 0;\nint b = 2;\n" + assert len(kept) == 1 + + +def test_line_scoped_result_no_edited_lines_is_noop(): + original = "int a=1;\nint b=2;\n" + formatted = "int a = 1;\nint b = 2;\n" + result, kept = fc.line_scoped_result(original, formatted, []) + assert result == original + assert kept == [] From c174b79fc9b60c003bc194c9977ecaa533c01ffc Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:40:17 +0200 Subject: [PATCH 066/132] [OOC] Add OOCCache (#2533) --- .../sysds/runtime/ooc/cache/BlockEntry.java | 65 +- .../sysds/runtime/ooc/cache/OOCCache.java | 142 ++++ .../sysds/runtime/ooc/cache/OOCCacheImpl.java | 745 ++++++++++++++++++ .../cache/collections/ConcurrentBitSet.java | 65 ++ .../collections/IndexedObjectPredicate.java | 24 + .../cache/collections/MaskedOnceArray.java | 165 ++++ .../collections/MaskedOnceArrayList.java | 225 ++++++ .../collections/SegmentedStreamTableList.java | 206 +++++ .../ooc/cache/eviction/EvictController.java | 86 ++ .../ooc/cache/eviction/IndexedObjectPair.java | 27 + .../runtime/ooc/memory/MemoryAllowance.java | 12 + .../ooc/memory/SyncMemoryAllowance.java | 207 ++++- .../component/ooc/cache/OOCCacheImplTest.java | 334 ++++++++ 13 files changed, 2254 insertions(+), 49 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCache.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheImpl.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/collections/ConcurrentBitSet.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/collections/IndexedObjectPredicate.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/collections/MaskedOnceArray.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/collections/MaskedOnceArrayList.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/collections/SegmentedStreamTableList.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/eviction/EvictController.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/eviction/IndexedObjectPair.java create mode 100644 src/test/java/org/apache/sysds/test/component/ooc/cache/OOCCacheImplTest.java diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/BlockEntry.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/BlockEntry.java index 3e040ef805e..6927ad44770 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/BlockEntry.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/BlockEntry.java @@ -19,10 +19,6 @@ package org.apache.sysds.runtime.ooc.cache; -import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; - -import java.util.List; - public final class BlockEntry { private final BlockKey _key; private final long _size; @@ -31,6 +27,19 @@ public final class BlockEntry { private Object _data; private int _retainHintCount; private int _referenceCount; // The number of references from different managing instances (e.g. CachingStream) + // Optional implementation-local cache metadata; null for cache implementations that do not need it. + private volatile Object _cacheMeta; + + public BlockEntry(BlockKey key) { + this._key = key; + this._size = -1; + this._pinCount = 0; + this._state = BlockState.COLD; + this._data = null; + this._retainHintCount = 0; + this._referenceCount = 0; + this._cacheMeta = null; + } public BlockEntry(BlockKey key, long size, Object data) { this._key = key; @@ -40,6 +49,18 @@ public BlockEntry(BlockKey key, long size, Object data) { this._data = data; this._retainHintCount = 0; this._referenceCount = 1; + this._cacheMeta = null; + } + + public BlockEntry(BlockKey key, long size, Object data, BlockState state) { + this._key = key; + this._size = size; + this._pinCount = 0; + this._state = state; + this._data = data; + this._retainHintCount = 0; + this._referenceCount = 1; + this._cacheMeta = null; } public BlockKey getKey() { @@ -56,18 +77,6 @@ public Object getData() { throw new IllegalStateException("Cannot get the data of an unpinned entry"); } - public int getGroupSize() { - if(_pinCount > 0) - return ((List)_data).size(); - throw new IllegalStateException("Cannot get the data of an unpinned entry"); - } - - public boolean isGrouped() { - if(_pinCount > 0) - return _data instanceof List; - throw new IllegalStateException("Cannot get the data of an unpinned entry"); - } - public Object getDataUnsafe() { return _data; } @@ -86,6 +95,10 @@ public boolean isPinned() { return _pinCount > 0; } + public synchronized int getPinCount() { + return _pinCount; + } + public synchronized int addReference() { return ++_referenceCount; } @@ -94,6 +107,18 @@ public synchronized int forget() { return --_referenceCount; } + public synchronized int getReferenceCount() { + return _referenceCount; + } + + public Object getCacheMeta() { + return _cacheMeta; + } + + public void setCacheMeta(Object meta) { + _cacheMeta = meta; + } + public synchronized void setState(BlockState state) { _state = state; } @@ -106,12 +131,6 @@ public synchronized void addRetainHint() { _retainHintCount++; } - public synchronized void removeRetainHint(int cnt) { - _retainHintCount -= cnt; - if(_retainHintCount < 0) - _retainHintCount = 0; - } - public synchronized void removeRetainHint() { if (_retainHintCount <= 0) return; @@ -129,8 +148,6 @@ public synchronized int getRetainHintCount() { public synchronized long clear() { if (_pinCount != 0 || _data == null) return 0; - if (_data instanceof IndexedMatrixValue) - ((IndexedMatrixValue)_data).setValue(null); // Explicitly clear _data = null; _retainHintCount = 0; return _size; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCache.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCache.java new file mode 100644 index 00000000000..7f1fdb493d0 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCache.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache; + +import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; + +import java.util.function.LongUnaryOperator; + +public interface OOCCache { + default OOCFuture pin(BlockKey key, MemoryAllowance allowance) { + return pin(key.getStreamId(), key.getSequenceNumber(), allowance); + } + + default OOCFuture pinAdmitted(BlockKey key, MemoryAllowance allowance) { + return pinAdmitted(key.getStreamId(), key.getSequenceNumber(), allowance); + } + + /** + * Adds a new pinned entry whose bytes are already owned by the given allowance. Ownership can later move only via + * pin/unpin. + */ + default BlockEntry putPinned(BlockKey key, Object data, long size, MemoryAllowance allowance) { + return putPinned(key.getStreamId(), key.getSequenceNumber(), data, size, allowance); + } + + /** + * Adds a new pinned entry whose bytes are already owned by the given allowance. Ownership can later move only via + * pin/unpin. + */ + BlockEntry putPinned(long sId, long tId, Object data, long size, MemoryAllowance allowance); + + /** + * Pins an item backed by an allowance. A successful pin transfers memory ownership from the cache to the owner of + * the allowance and guarantees data availability. While pinned, the bytes of the entry are not counted as + * cache-owned memory. + * + * @param sId + * @param tId + * @param allowance + * @return a non-null future of the pinned block entry; the future result is null if the required memory could not + * be reserved + */ + OOCFuture pin(long sId, long tId, MemoryAllowance allowance); + + default OOCFuture pinAdmitted(long sId, long tId, MemoryAllowance allowance) { + return pin(sId, tId, allowance); + } + + /** + * Pins an item backed by an allowance if it is already live in cache. A successful pin transfers memory ownership + * from the cache to the owner of the allowance and guarantees data availability. While pinned, the bytes of the + * entry are not counted as cache-owned memory. Implementations must reserve the required bytes from the allowance + * before making data available. + * + * @param sId + * @param tId + * @param allowance + * @return the pinned block entry if available. Null if the required memory could not be reserved or the block is + * not live + */ + BlockEntry pinIfLive(long sId, long tId, MemoryAllowance allowance); + + /** + * Unpins an item that is still backed by the given allowance. Unpinning tries to transfer memory ownership back to + * the cache. An ownership transfer may commit immediately only if this does not cause the cache to exceed its hard + * limit. Otherwise, the transfer is deferred and the allowance remains charged until the returned handle commits, + * is reclaimed, or is superseded by a later pin that transfers ownership to another allowance. Unpin can be viewed + * as an eventually resolving operation. + * + * @param entry + * @param allowance + * @return a handle describing the ownership transfer from allowance-owned memory back to cache-owned memory + */ + UnpinHandle unpin(BlockEntry entry, MemoryAllowance allowance); + + /** + * Referencing a pinned entry guarantees that its key remains in the cache until dereferenced. + * + * @param entry + * @return + */ + int reference(BlockEntry entry); + + /** + * Dereferencing allows an entry to be forgotten if no further reference is held. Dereferencing may not immediately + * cause entry removal if still pinned. + * + * @param entry + * @return + */ + int dereference(BlockEntry entry); + + /** + * Dereferencing allows an entry to be forgotten if no further reference is held. Dereferencing may not immediately + * cause entry removal if still pinned. + */ + int dereference(BlockKey key); + + void updateLimits(long hardLimit, long evictionLimit); + + /** + * Adds an eviction scoring policy for one logical cache stream. Larger scores are selected for eviction first. + * {@link Long#MAX_VALUE} remains reserved as "no policy score". + */ + void addEvictionPolicy(long streamId, LongUnaryOperator scoreFn); + + /** + * Returns the current cache-owned size in bytes. + */ + long getOwnedCacheSize(); + + void shutdown(); + + interface UnpinHandle { + BlockEntry entry(); + + MemoryAllowance allowance(); + + long bytes(); + + boolean isCommitted(); + + OOCFuture getCompletionFuture(); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheImpl.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheImpl.java new file mode 100644 index 00000000000..9b0008e84ab --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheImpl.java @@ -0,0 +1,745 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache; + +import org.apache.sysds.runtime.ooc.cache.collections.MaskedOnceArrayList; +import org.apache.sysds.runtime.ooc.cache.collections.SegmentedStreamTableList; +import org.apache.sysds.runtime.ooc.cache.eviction.EvictController; +import org.apache.sysds.runtime.ooc.cache.eviction.IndexedObjectPair; +import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; +import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; +import org.apache.sysds.utils.Statistics; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.PriorityQueue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.LongUnaryOperator; + +public class OOCCacheImpl implements OOCCache { + private static final int MIN_EVICTION_CANDIDATES = 1024; + private static final int MAX_EVICTION_CANDIDATES = 65536; + private static final long EVICTION_CANDIDATE_BYTE_FACTOR = 250_000; + + private final OOCIOHandler _ioHandler; + private final SegmentedStreamTableList _blocks; + private final SegmentedStreamTableList _evictControllers; + private final EvictController _defaultEvictController; + private final ConcurrentLinkedQueue _deferredUnpins; + private final Executor _collectorExecutor; + private final AtomicBoolean _evictionRunning; + + private long _hardLimit; + private long _evictionLimit; + private long _ownedBytes; + private long _evictingBytes; + private boolean _running; + + public OOCCacheImpl(OOCIOHandler ioHandler, long hardLimit, long evictionLimit) { + _ioHandler = ioHandler; + _hardLimit = hardLimit; + _evictionLimit = evictionLimit; + _ownedBytes = 0; + _evictingBytes = 0; + _running = true; + _blocks = new SegmentedStreamTableList<>(); + _evictControllers = new SegmentedStreamTableList<>(); + _defaultEvictController = new EvictController(); + _deferredUnpins = new ConcurrentLinkedQueue<>(); + _collectorExecutor = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "ooc-cache-collector"); + t.setDaemon(true); + return t; + }); + _evictionRunning = new AtomicBoolean(false); + } + + @Override + public BlockEntry putPinned(long sId, long tId, Object data, long size, MemoryAllowance allowance) { + BlockKey key = new BlockKey(sId, tId); + BlockEntry entry = new BlockEntry(key, size, data, BlockState.REMOVED); + entry.pin(); + EntryMeta meta = new EntryMeta(entry); + entry.setCacheMeta(meta); + synchronized(this) { + checkRunning(); + putEntry(entry); + } + Statistics.incrementOOCEvictionPut(); + return entry; + } + + @Override + public OOCFuture pin(long sId, long tId, MemoryAllowance allowance) { + return pinInternal(new BlockKey(sId, tId), allowance, false, false); + } + + @Override + public OOCFuture pinAdmitted(long sId, long tId, MemoryAllowance allowance) { + return pinInternal(new BlockKey(sId, tId), allowance, false, true); + } + + @Override + public BlockEntry pinIfLive(long sId, long tId, MemoryAllowance allowance) { + return pinInternal(new BlockKey(sId, tId), allowance, true, false).getNow(null); + } + + @Override + public UnpinHandle unpin(BlockEntry entry, MemoryAllowance allowance) { + if(entry.fastUnpin()) { + allowance.release(entry.getSize()); + return CacheUnpinHandle.committed(entry, allowance, entry.getSize()); + } + UnpinHandle result; + long releaseBytes; + synchronized(this) { + EntryMeta meta = getMeta(entry); + if(meta == null) + return CacheUnpinHandle.committed(entry, allowance, Math.max(0, entry.getSize())); + if(entry.getPinCount() > 1) { + entry.unpin(); + releaseBytes = entry.getSize(); + result = CacheUnpinHandle.committed(entry, allowance, releaseBytes); + } + else if(canAcceptOwnedBytes(entry.getSize())) { + releaseBytes = entry.getSize(); + result = commitLastUnpin(meta, allowance); + } + else { + CacheUnpinHandle handle = CacheUnpinHandle.deferred(entry, allowance); + meta.deferredUnpin = handle; + _deferredUnpins.offer(entry.getKey()); + return handle; + } + } + if(releaseBytes > 0) + allowance.release(releaseBytes); + return result; + } + + @Override + public synchronized int reference(BlockEntry entry) { + return entry.addReference(); + } + + @Override + public int dereference(BlockEntry entry) { + int refs; + synchronized(this) { + EntryMeta meta = getMeta(entry); + if(meta == null) + return 0; + refs = entry.forget(); + if(refs <= 0) + removeIfUnused(meta); + } + return refs; + } + + @Override + public int dereference(BlockKey key) { + BlockEntry entry = findEntry(key); + if(entry == null) + return 0; + return dereference(entry); + } + + @Override + public void updateLimits(long hardLimit, long evictionLimit) { + List completions; + synchronized(this) { + _hardLimit = hardLimit; + _evictionLimit = evictionLimit; + completions = processDeferredUnpins(); + scheduleEvictionIfNeeded(); + } + completions.forEach(this::completeDeferred); + } + + @Override + public synchronized void addEvictionPolicy(long streamId, LongUnaryOperator scoreFn) { + getOrCreateEvictController(streamId).addEvictionPolicy(scoreFn); + scheduleEvictionIfNeeded(); + } + + @Override + public synchronized long getOwnedCacheSize() { + return _ownedBytes; + } + + @Override + public synchronized void shutdown() { + _running = false; + _blocks.clear(); + _deferredUnpins.clear(); + _ownedBytes = 0; + _evictingBytes = 0; + _ioHandler.shutdown(); + } + + private OOCFuture pinInternal(BlockKey key, MemoryAllowance allowance, boolean liveOnly, + boolean waitForAdmission) { + BlockEntry deferredUnpinEntry = null; + CacheUnpinHandle deferredUnpinHandle = null; + long reserveBytes; + synchronized(this) { + checkRunning(); + BlockEntry entry = findEntry(key); + EntryMeta meta = getMeta(entry); + if(meta == null) + return OOCFuture.completed(null); + if(liveOnly && entry.getDataUnsafe() == null) + return OOCFuture.completed(null); + if(meta.deferredUnpin != null) { + if(meta.deferredUnpin.allowance == allowance) { + deferredUnpinHandle = meta.deferredUnpin; + meta.deferredUnpin = null; + deferredUnpinEntry = entry; + Statistics.incrementOOCEvictionGet(); + } + reserveBytes = entry.getSize(); + } + else if(isResidentForPin(entry)) + reserveBytes = entry.getSize(); + else if(liveOnly) + return OOCFuture.completed(null); + else + reserveBytes = entry.getSize(); + } + if(deferredUnpinEntry != null) { + deferredUnpinHandle.complete(false); + return OOCFuture.completed(deferredUnpinEntry); + } + if(!waitForAdmission) { + if(!allowance.tryReserve(reserveBytes)) + return OOCFuture.completed(null); + return pinReserved(key, allowance, reserveBytes, liveOnly); + } + + OOCFuture result = new OOCFuture<>(); + allowance.reserveAsync(reserveBytes).whenComplete((ignored, error) -> { + if(error != null) { + result.completeExceptionally(error); + return; + } + try { + pinReserved(key, allowance, reserveBytes, liveOnly).whenComplete((pinned, pinError) -> { + if(pinError != null) + result.completeExceptionally(pinError); + else + result.complete(pinned); + }); + } + catch(Throwable t) { + allowance.release(reserveBytes); + result.completeExceptionally(t); + } + }); + return result; + } + + private OOCFuture pinReserved(BlockKey key, MemoryAllowance allowance, long reservedBytes, + boolean liveOnly) { + EntryMeta meta = null; + BlockEntry deferredUnpinEntry = null; + CacheUnpinHandle deferredUnpinHandle = null; + MemoryAllowance releaseAllowance = null; + DeferredCompletion deferredCompletion = null; + BlockEntry resident = null; + long releaseBytes = 0; + boolean releaseReserved = false; + boolean returnNull = false; + synchronized(this) { + if(!_running) { + releaseReserved = true; + returnNull = true; + } + else { + BlockEntry entry = findEntry(key); + meta = getMeta(entry); + if(meta == null || (liveOnly && entry.getDataUnsafe() == null)) { + releaseReserved = true; + returnNull = true; + } + else if(meta.deferredUnpin != null) { + deferredUnpinHandle = meta.deferredUnpin; + meta.deferredUnpin = null; + deferredUnpinEntry = meta.entry; + if(deferredUnpinHandle.allowance == allowance) + releaseReserved = true; + else { + releaseAllowance = deferredUnpinHandle.allowance; + releaseBytes = meta.entry.getSize(); + } + Statistics.incrementOOCEvictionGet(); + } + else if(isResidentForPin(entry)) { + deferredCompletion = pinResident(meta); + Statistics.incrementOOCEvictionGet(); + resident = entry; + } + else if(liveOnly) { + releaseReserved = true; + returnNull = true; + } + } + } + if(releaseReserved) + allowance.release(reservedBytes); + if(releaseAllowance != null) { + releaseAllowance.release(releaseBytes); + deferredUnpinHandle.complete(false); + } + else if(deferredUnpinHandle != null) + deferredUnpinHandle.complete(false); + + completeDeferred(deferredCompletion); + if(resident != null) + return OOCFuture.completed(resident); + if(returnNull || deferredUnpinEntry != null) + return OOCFuture.completed(deferredUnpinEntry); + // Trigger read + return pinFromBackingReserved(meta, allowance, reservedBytes); + } + + private OOCFuture pinFromBackingReserved(EntryMeta meta, MemoryAllowance allowance, + long reservedBytes) { + OOCFuture readFuture; + boolean releaseReserved = false; + DeferredCompletion deferredCompletion = null; + BlockEntry resident = null; + synchronized(this) { + if(!_running || getMeta(meta.entry) != meta) { + releaseReserved = true; + readFuture = null; + } + else if(meta.entry.getDataUnsafe() != null) { + deferredCompletion = pinResident(meta); + Statistics.incrementOOCEvictionGet(); + resident = meta.entry; + readFuture = null; + } + else if(meta.readFuture == null) { + meta.entry.setState(BlockState.READING); + OOCFuture scheduled = _ioHandler.scheduleRead(meta.entry); + meta.readFuture = scheduled; + readFuture = scheduled; + scheduled.whenComplete((entry, ex) -> { + synchronized(OOCCacheImpl.this) { + if(meta.readFuture == scheduled) + meta.readFuture = null; + if(ex != null && meta.entry.getState() == BlockState.READING) + meta.entry.setState(BlockState.COLD); + } + }); + } + else + readFuture = meta.readFuture; + } + if(releaseReserved) { + allowance.release(reservedBytes); + return OOCFuture.completed(null); + } + completeDeferred(deferredCompletion); + if(resident != null) + return OOCFuture.completed(resident); + + OOCFuture result = new OOCFuture<>(); + readFuture.whenComplete((entry, ex) -> { + boolean release = false; + DeferredCompletion completion = null; + try { + if(ex != null) { + release = true; + allowance.release(reservedBytes); + result.completeExceptionally(ex); + return; + } + BlockEntry pinned; + synchronized(OOCCacheImpl.this) { + if(getMeta(meta.entry) != meta || meta.entry.getDataUnsafe() == null) { + release = true; + if(meta.entry.getState() == BlockState.READING) + meta.entry.setState(BlockState.COLD); + pinned = null; + } + else { + completion = pinResident(meta); + Statistics.incrementOOCEvictionGet(); + pinned = meta.entry; + } + } + if(release) + allowance.release(reservedBytes); + completeDeferred(completion); + result.complete(pinned); + } + catch(Throwable t) { + if(!release) + allowance.release(reservedBytes); + result.completeExceptionally(t); + } + }); + return result; + } + + private DeferredCompletion pinResident(EntryMeta meta) { + BlockEntry entry = meta.entry; + if(isCacheOwned(entry)) { + _ownedBytes -= entry.getSize(); + if(entry.getState() == BlockState.EVICTING) + _evictingBytes -= entry.getSize(); + clearLive(entry); + } + entry.setState(BlockState.REMOVED); + entry.pin(); + CacheUnpinHandle handle = meta.deferredUnpin; + if(handle == null) + return null; + long bytes = meta.entry.getSize(); + meta.deferredUnpin = null; + meta.entry.unpin(); + return new DeferredCompletion(handle, bytes, false); + } + + private UnpinHandle commitLastUnpin(EntryMeta meta, MemoryAllowance allowance) { + BlockEntry entry = meta.entry; + entry.unpin(); + if(entry.getReferenceCount() <= 0) { + removeEntry(entry.getKey()); + entry.clear(); + entry.setCacheMeta(null); + if(meta.backed) + _ioHandler.scheduleDeletion(entry); + return CacheUnpinHandle.committed(entry, allowance, entry.getSize()); + } + entry.setState(meta.backed ? BlockState.WARM : BlockState.HOT); + setLive(entry); + _ownedBytes += entry.getSize(); + scheduleEvictionIfNeeded(); + return CacheUnpinHandle.committed(entry, allowance, entry.getSize()); + } + + private List processDeferredUnpins() { + List completions = null; + while(true) { + BlockKey key = _deferredUnpins.peek(); + if(key == null) + return completions == null ? Collections.emptyList() : completions; + BlockEntry entry = findEntry(key); + EntryMeta meta = getMeta(entry); + if(meta == null || meta.deferredUnpin == null) { + _deferredUnpins.poll(); + continue; + } + if(!canAcceptOwnedBytes(meta.entry.getSize())) + return completions == null ? Collections.emptyList() : completions; + _deferredUnpins.poll(); + CacheUnpinHandle handle = meta.deferredUnpin; + meta.deferredUnpin = null; + long bytes = entry.getSize(); + entry.unpin(); + if(entry.getReferenceCount() <= 0) { + removeEntry(entry.getKey()); + entry.clear(); + entry.setCacheMeta(null); + if(meta.backed) + _ioHandler.scheduleDeletion(entry); + } + else { + entry.setState(meta.backed ? BlockState.WARM : BlockState.HOT); + setLive(entry); + _ownedBytes += entry.getSize(); + } + if(completions == null) + completions = new ArrayList<>(); + completions.add(new DeferredCompletion(handle, bytes, true)); + } + } + + private void completeDeferred(DeferredCompletion completion) { + if(completion == null) + return; + completion.handle.allowance.release(completion.bytes); + completion.handle.complete(completion.committed); + } + + private boolean canAcceptOwnedBytes(long bytes) { + return _ownedBytes + bytes <= _hardLimit; + } + + private void scheduleEvictionIfNeeded() { + if(evictionPressure() <= _evictionLimit || !_evictionRunning.compareAndSet(false, true)) + return; + _collectorExecutor.execute(this::runEviction); + } + + private void runEviction() { + try { + while(true) { + long bytes; + synchronized(this) { + bytes = evictionPressure() - _evictionLimit; + if(bytes <= 0) + return; + } + + List> candidates = collectEvictionCandidates(bytes); + if(candidates.isEmpty()) + return; + + List toWrite = new ArrayList<>(); + List completions; + boolean progress = false; + synchronized(this) { + for(IndexedObjectPair candidate : candidates) { + if(evictionPressure() <= _evictionLimit) + break; + EntryMeta meta = getMeta(candidate.obj()); + if(meta == null || candidate.obj().getPinCount() > 0 || meta.deferredUnpin != null) + continue; + BlockEntry entry = meta.entry; + if(entry.getState() == BlockState.WARM) { + entry.clear(); + entry.setState(BlockState.COLD); + clearLive(entry); + _ownedBytes -= entry.getSize(); + progress = true; + } + else if(entry.getState() == BlockState.HOT) { + entry.setState(BlockState.EVICTING); + _evictingBytes += entry.getSize(); + clearLive(entry); + toWrite.add(entry); + progress = true; + } + } + completions = processDeferredUnpins(); + } + completions.forEach(this::completeDeferred); + for(BlockEntry entry : toWrite) + _ioHandler.scheduleEviction(entry).whenComplete((ignored, ex) -> onEvicted(entry, ex)); + if(!progress) + return; + } + } + finally { + _evictionRunning.set(false); + synchronized(this) { + if(evictionPressure() > _evictionLimit) + scheduleEvictionIfNeeded(); + } + } + } + + private void onEvicted(BlockEntry entry, Throwable ex) { + List completions = null; + synchronized(this) { + EntryMeta meta = getMeta(entry); + if(meta == null) + return; + if(ex != null) { + if(entry.getState() == BlockState.EVICTING) { + entry.setState(BlockState.HOT); + _evictingBytes -= entry.getSize(); + setLive(entry); + scheduleEvictionIfNeeded(); + } + return; + } + meta.backed = true; + if(entry.getState() == BlockState.HOT) { + entry.setState(BlockState.WARM); + return; + } + if(entry.getState() != BlockState.EVICTING) + return; + entry.clear(); + entry.setState(BlockState.COLD); + _ownedBytes -= entry.getSize(); + _evictingBytes -= entry.getSize(); + removeIfUnused(meta); + completions = processDeferredUnpins(); + scheduleEvictionIfNeeded(); + } + completions.forEach(this::completeDeferred); + } + + private List> collectEvictionCandidates(long bytes) { + int k = evictionCandidateLimit(bytes); + PriorityQueue> queue = new PriorityQueue<>(); + _blocks.forEachStreamTable( + (streamId, stream) -> getEvictController(streamId).findEvictionCandidates(stream, queue, k, 0)); + + List> candidates = new ArrayList<>(queue.size()); + while(!queue.isEmpty()) + candidates.add(queue.poll()); + Collections.reverse(candidates); + return candidates; + } + + private int evictionCandidateLimit(long bytes) { + long limit = Math.max(MIN_EVICTION_CANDIDATES, + (bytes + EVICTION_CANDIDATE_BYTE_FACTOR - 1) / EVICTION_CANDIDATE_BYTE_FACTOR); + return (int) Math.min(MAX_EVICTION_CANDIDATES, limit); + } + + private EvictController getEvictController(long streamId) { + MaskedOnceArrayList controllers = _evictControllers.get(streamId); + if(controllers == null) + return _defaultEvictController; + EvictController controller = controllers.get(0); + return controller == null ? _defaultEvictController : controller; + } + + private EvictController getOrCreateEvictController(long streamId) { + MaskedOnceArrayList controllers = _evictControllers.getOrCreate(streamId); + EvictController controller = controllers.get(0); + if(controller != null) + return controller; + controller = new EvictController(); + controllers.put(0, controller); + return controller; + } + + private void removeIfUnused(EntryMeta meta) { + if(meta.entry.getReferenceCount() > 0 || meta.entry.getPinCount() > 0 || meta.deferredUnpin != null) + return; + BlockEntry entry = meta.entry; + if(isCacheOwned(entry)) + _ownedBytes -= entry.getSize(); + if(entry.getState() == BlockState.EVICTING) + _evictingBytes -= entry.getSize(); + removeEntry(entry.getKey()); + clearLive(entry); + entry.clear(); + entry.setCacheMeta(null); + if(meta.backed) + _ioHandler.scheduleDeletion(entry); + } + + private boolean isCacheOwned(BlockEntry entry) { + return entry.getState() == BlockState.HOT || entry.getState() == BlockState.WARM || + entry.getState() == BlockState.EVICTING; + } + + private boolean isResidentForPin(BlockEntry entry) { + return entry.getDataUnsafe() != null && entry.getState() != BlockState.COLD && + entry.getState() != BlockState.READING; + } + + private long evictionPressure() { + return _ownedBytes - _evictingBytes; + } + + private BlockEntry findEntry(BlockKey key) { + MaskedOnceArrayList stream = _blocks.get(key.getStreamId()); + return stream == null ? null : stream.get(blockIndex(key)); + } + + private void putEntry(BlockEntry entry) { + MaskedOnceArrayList stream = _blocks.getOrCreate(entry.getKey().getStreamId()); + int index = blockIndex(entry.getKey()); + if(stream.get(index) != null) + throw new IllegalStateException("Cache entry already exists: " + entry.getKey()); + stream.put(index, entry); + } + + private BlockEntry removeEntry(BlockKey key) { + MaskedOnceArrayList stream = _blocks.get(key.getStreamId()); + if(stream == null) + return null; + return stream.clear(blockIndex(key)) ? null : stream.get(blockIndex(key)); + } + + private void setLive(BlockEntry entry) { + MaskedOnceArrayList stream = _blocks.get(entry.getKey().getStreamId()); + if(stream != null) + stream.setLive(blockIndex(entry.getKey())); + } + + private void clearLive(BlockEntry entry) { + MaskedOnceArrayList stream = _blocks.get(entry.getKey().getStreamId()); + if(stream != null) + stream.clearLive(blockIndex(entry.getKey())); + } + + private int blockIndex(BlockKey key) { + long sequenceNumber = key.getSequenceNumber(); + if(sequenceNumber < 0 || sequenceNumber > Integer.MAX_VALUE) + throw new IndexOutOfBoundsException("Invalid block index: " + sequenceNumber); + return (int) sequenceNumber; + } + + private void checkRunning() { + if(!_running) + throw new IllegalStateException("Cache has been shut down."); + } + + private EntryMeta getMeta(BlockEntry entry) { + return entry == null ? null : (EntryMeta) entry.getCacheMeta(); + } + + private static class EntryMeta { + private final BlockEntry entry; + private boolean backed; + private OOCFuture readFuture; + private CacheUnpinHandle deferredUnpin; + + private EntryMeta(BlockEntry entry) { + this.entry = entry; + backed = entry.getState().isBackedByDisk(); + } + } + + private record DeferredCompletion(CacheUnpinHandle handle, long bytes, boolean committed) { + } + + private record CacheUnpinHandle(BlockEntry entry, MemoryAllowance allowance, long bytes, OOCFuture future) + implements UnpinHandle { + private static CacheUnpinHandle committed(BlockEntry entry, MemoryAllowance allowance, long bytes) { + return new CacheUnpinHandle(entry, allowance, bytes, OOCFuture.completed(true)); + } + + private static CacheUnpinHandle deferred(BlockEntry entry, MemoryAllowance allowance) { + return new CacheUnpinHandle(entry, allowance, entry.getSize(), new OOCFuture<>()); + } + + @Override + public boolean isCommitted() { + return future.getNow(false); + } + + @Override + public OOCFuture getCompletionFuture() { + return future; + } + + private void complete(boolean committed) { + if(future.isDone()) + return; + future.complete(committed); + } + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/collections/ConcurrentBitSet.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/collections/ConcurrentBitSet.java new file mode 100644 index 00000000000..5a1582d47d7 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/collections/ConcurrentBitSet.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache.collections; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; + +public class ConcurrentBitSet { + private static final VarHandle LONG_ARR = MethodHandles.arrayElementVarHandle(long[].class); + + private final long[] words; + + public ConcurrentBitSet(int bits) { + // (bits + 63) >>> 6 = ceil(bits / 64.0) + this.words = new long[(bits + 63) >>> 6]; + } + + public boolean get(int i) { + int w = i >>> 6; + long mask = 1L << (i & 63); + long word = (long) LONG_ARR.getAcquire(words, w); + return (word & mask) != 0; + } + + public boolean set(int i) { + int w = i >>> 6; + long mask = 1L << (i & 63); + + long prev = (long) LONG_ARR.getAndBitwiseOrRelease(words, w, mask); + return (prev & mask) == 0; // true if changed absent -> present + } + + public boolean clear(int i) { + int w = i >>> 6; + long mask = 1L << (i & 63); + + long prev = (long) LONG_ARR.getAndBitwiseAndRelease(words, w, ~mask); + return (prev & mask) != 0; // true if changed present -> absent + } + + public long getWord(int wordIndex) { + return (long) LONG_ARR.getAcquire(words, wordIndex); + } + + public int length() { + return words.length; + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/collections/IndexedObjectPredicate.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/collections/IndexedObjectPredicate.java new file mode 100644 index 00000000000..956a5d6498a --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/collections/IndexedObjectPredicate.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache.collections; + +public interface IndexedObjectPredicate { + boolean test(int idx, T value); +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/collections/MaskedOnceArray.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/collections/MaskedOnceArray.java new file mode 100644 index 00000000000..5d2d36f36ac --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/collections/MaskedOnceArray.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache.collections; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; +import java.util.concurrent.atomic.AtomicReferenceArray; +import java.util.function.Consumer; + +public class MaskedOnceArray { + private static final int RETIRED = Integer.MIN_VALUE; + private static final VarHandle NON_NULL_COUNT; + + static { + try { + NON_NULL_COUNT = MethodHandles.lookup().findVarHandle(MaskedOnceArray.class, "_nonNullCount", int.class); + } + catch(ReflectiveOperationException e) { + throw new ExceptionInInitializerError(e); + } + } + + private final AtomicReferenceArray _values; + protected final ConcurrentBitSet _liveState; + private volatile int _nonNullCount; + + public MaskedOnceArray(int length) { + _values = new AtomicReferenceArray<>(length); + _liveState = new ConcurrentBitSet(length); + _nonNullCount = 0; + } + + public boolean put(int i, T value) { + if(value == null) { + return clear(i); + } + if(!incrementNonNullCount()) + return false; + boolean changed = _values.getAndSet(i, value) == null; + if(!changed) + decrementNonNullCount(); + _liveState.set(i); + return changed; + } + + private boolean incrementNonNullCount() { + while(true) { + int count = (int) NON_NULL_COUNT.getAcquire(this); + if(count == RETIRED) + return false; + if(NON_NULL_COUNT.compareAndSet(this, count, count + 1)) + return true; + } + } + + private void decrementNonNullCount() { + while(true) { + int count = (int) NON_NULL_COUNT.getAcquire(this); + if(count <= 0) + return; + if(NON_NULL_COUNT.compareAndSet(this, count, count - 1)) + return; + } + } + + public boolean clear(int i) { + boolean changed = _values.getAndSet(i, null) != null; + if(changed) + decrementNonNullCount(); + _liveState.clear(i); + return changed; + } + + public T get(int i) { + return _values.get(i); + } + + public void forEachVisible(Consumer action) { + for(int i = 0; i < _values.length(); i++) { + T v = _values.get(i); + if(v != null) + action.accept(v); + } + } + + public boolean tryRetireIfEmpty() { + return NON_NULL_COUNT.compareAndSet(this, 0, RETIRED); + } + + public boolean isRetired() { + return (int) NON_NULL_COUNT.getAcquire(this) == RETIRED; + } + + public boolean isEmpty() { + return (int) NON_NULL_COUNT.getAcquire(this) == 0; + } + + public void setLive(int i) { + _liveState.set(i); + } + + public void clearLive(int i) { + _liveState.clear(i); + } + + public boolean forEachLive(IndexedObjectPredicate action, boolean reversed, int offset) { + if(reversed) + return forEachLiveBackward(action, offset); + else + return forEachLiveForward(action, offset); + } + + private boolean forEachLiveForward(IndexedObjectPredicate action, int offset) { + int len = _liveState.length(); + T data; + for(int word = 0; word < len; word++) { + if(_liveState.getWord(word) == 0) + continue; + int lower = word * 64; + int upper = (word + 1) * 64; + for(int i = lower; i < upper; i++) { + data = get(i); + if(data != null) + if(!action.test(offset + i, data)) + return false; + } + } + return true; + } + + private boolean forEachLiveBackward(IndexedObjectPredicate action, int offset) { + int len = _liveState.length(); + for(int word = len - 1; word >= 0; word--) { + if(_liveState.getWord(word) == 0) + continue; + int lower = word * 64; + int upper = (word + 1) * 64; + T data; + for(int i = upper - 1; i >= lower; i--) { + data = get(i); + if(data != null) + if(!action.test(offset + i, data)) + return false; + } + } + return true; + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/collections/MaskedOnceArrayList.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/collections/MaskedOnceArrayList.java new file mode 100644 index 00000000000..b4e0ed2368c --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/collections/MaskedOnceArrayList.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache.collections; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; +import java.util.function.Consumer; + +public class MaskedOnceArrayList { + private static final VarHandle PARTITIONS; + private static final VarHandle PARTITION = MethodHandles.arrayElementVarHandle(MaskedOnceArray[].class); + private static final int DEFAULT_PARTITION_SIZE = 1024; + + static { + try { + PARTITIONS = MethodHandles.lookup().findVarHandle(MaskedOnceArrayList.class, "_partitions", + MaskedOnceArray[].class); + } + catch(ReflectiveOperationException e) { + throw new ExceptionInInitializerError(e); + } + } + + private final int _partitionSize; + private final int _partitionBits; + private final int _partitionMask; + + @SuppressWarnings("rawtypes") + private volatile MaskedOnceArray[] _partitions; + + public MaskedOnceArrayList() { + this(DEFAULT_PARTITION_SIZE); + } + + public MaskedOnceArrayList(int partitionSize) { + validatePartitionSize(partitionSize); + _partitionSize = partitionSize; + _partitionBits = Integer.numberOfTrailingZeros(partitionSize); + _partitionMask = partitionSize - 1; + _partitions = new MaskedOnceArray[1]; + } + + @SuppressWarnings("rawtypes") + public boolean put(int i, T value) { + checkIndex(i); + if(value == null) + return clear(i); + int partitionIndex = partitionIndex(i); + int offset = offsetInPartition(i); + while(true) { + MaskedOnceArray[] partitions = ensurePartitionCapacity(partitionIndex); + MaskedOnceArray partition = partitionAt(partitions, partitionIndex); + boolean changed = partition.put(offset, value); + if(PARTITION.getAcquire(partitions, partitionIndex) == partition && !partition.isRetired()) + return changed; + } + } + + @SuppressWarnings("rawtypes") + public boolean clear(int i) { + checkIndex(i); + int partition = partitionIndex(i); + MaskedOnceArray[] partitions = (MaskedOnceArray[]) PARTITIONS.getAcquire(this); + if(partition < partitions.length) + return clear(partitions, partition, offsetInPartition(i)); + return false; + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + public T get(int i) { + checkIndex(i); + int partition = partitionIndex(i); + MaskedOnceArray[] partitions = (MaskedOnceArray[]) PARTITIONS.getAcquire(this); + if(partition >= partitions.length) + return null; + MaskedOnceArray p = (MaskedOnceArray) PARTITION.getAcquire(partitions, partition); + return p == null ? null : (T) p.get(offsetInPartition(i)); + } + + @SuppressWarnings("rawtypes") + public void setLive(int i) { + checkIndex(i); + int partitionIndex = partitionIndex(i); + MaskedOnceArray[] partitions = ensurePartitionCapacity(partitionIndex); + partitionAt(partitions, partitionIndex).setLive(offsetInPartition(i)); + } + + @SuppressWarnings("rawtypes") + public void clearLive(int i) { + checkIndex(i); + int partition = partitionIndex(i); + MaskedOnceArray[] partitions = (MaskedOnceArray[]) PARTITIONS.getAcquire(this); + if(partition < partitions.length) { + MaskedOnceArray p = (MaskedOnceArray) PARTITION.getAcquire(partitions, partition); + if(p != null) + p.clearLive(offsetInPartition(i)); + } + } + + @SuppressWarnings("rawtypes") + public int capacity() { + MaskedOnceArray[] partitions = (MaskedOnceArray[]) PARTITIONS.getAcquire(this); + return partitions.length * _partitionSize; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + public void forEachLive(IndexedObjectPredicate action, boolean reversed) { + MaskedOnceArray[] partitions = (MaskedOnceArray[]) PARTITIONS.getAcquire(this); + if(reversed) { + for(int i = partitions.length - 1; i >= 0; i--) { + MaskedOnceArray partition = (MaskedOnceArray) PARTITION.getAcquire(partitions, i); + if(partition != null) + partition.forEachLive(action, true, i * _partitionSize); + } + } + else { + for(int i = 0; i < partitions.length; i++) { + MaskedOnceArray partition = (MaskedOnceArray) PARTITION.getAcquire(partitions, i); + if(partition != null) + partition.forEachLive(action, false, i * _partitionSize); + } + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + public void forEachVisible(Consumer action) { + MaskedOnceArray[] partitions = (MaskedOnceArray[]) PARTITIONS.getAcquire(this); + for(int i = 0; i < partitions.length; i++) { + MaskedOnceArray partition = (MaskedOnceArray) PARTITION.getAcquire(partitions, i); + if(partition != null) + partition.forEachVisible(action); + } + } + + @SuppressWarnings("rawtypes") + private MaskedOnceArray[] ensurePartitionCapacity(int partitionIndex) { + MaskedOnceArray[] partitions = (MaskedOnceArray[]) PARTITIONS.getAcquire(this); + while(partitionIndex >= partitions.length) { + MaskedOnceArray[] bigger = growPartitions(partitions, partitionIndex + 1); + if(PARTITIONS.compareAndSet(this, partitions, bigger)) + partitions = bigger; + else + partitions = (MaskedOnceArray[]) PARTITIONS.getAcquire(this); + } + return partitions; + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private MaskedOnceArray partitionAt(MaskedOnceArray[] partitions, int partitionIndex) { + MaskedOnceArray partition; + while((partition = (MaskedOnceArray) PARTITION.getAcquire(partitions, partitionIndex)) == null || + partition.isRetired()) { + if(partition != null) { + PARTITION.compareAndSet(partitions, partitionIndex, partition, null); + continue; + } + MaskedOnceArray newPartition = new MaskedOnceArray<>(_partitionSize); + if(PARTITION.compareAndSet(partitions, partitionIndex, null, newPartition)) + return newPartition; + } + return partition; + } + + @SuppressWarnings("rawtypes") + private boolean clear(MaskedOnceArray[] partitions, int partitionIndex, int offset) { + MaskedOnceArray partition = (MaskedOnceArray) PARTITION.getAcquire(partitions, partitionIndex); + if(partition == null) + return false; + boolean changed = partition.clear(offset); + if(partition.tryRetireIfEmpty()) + PARTITION.compareAndSet(partitions, partitionIndex, partition, null); + return changed; + } + + @SuppressWarnings("rawtypes") + private MaskedOnceArray[] growPartitions(MaskedOnceArray[] partitions, int minLength) { + int newLength = partitions.length; + while(newLength < minLength) { + if(newLength > Integer.MAX_VALUE / 2) + throw new IllegalStateException("MaskedOnceArrayList capacity overflow"); + newLength <<= 1; + } + + MaskedOnceArray[] bigger = new MaskedOnceArray[newLength]; + System.arraycopy(partitions, 0, bigger, 0, partitions.length); + return bigger; + } + + private int partitionIndex(int index) { + return index >>> _partitionBits; + } + + private int offsetInPartition(int index) { + return index & _partitionMask; + } + + private static void validatePartitionSize(int partitionSize) { + if(partitionSize < 64 || (partitionSize & (partitionSize - 1)) != 0) { + throw new IllegalArgumentException( + "partitionSize must be a power of two and at least 64: " + partitionSize); + } + } + + private static void checkIndex(int i) { + if(i < 0) + throw new IndexOutOfBoundsException("Negative index: " + i); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/collections/SegmentedStreamTableList.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/collections/SegmentedStreamTableList.java new file mode 100644 index 00000000000..5dcf05bcd68 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/collections/SegmentedStreamTableList.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache.collections; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +public class SegmentedStreamTableList { + private static final VarHandle SEGMENTS; + private static final VarHandle ARRAY = MethodHandles.arrayElementVarHandle(Object[].class); + private static final int DEFAULT_SEGMENT_SIZE = 64; + + static { + try { + SEGMENTS = MethodHandles.lookup().findVarHandle(SegmentedStreamTableList.class, "_segments", + Object[].class); + } + catch(ReflectiveOperationException e) { + throw new ExceptionInInitializerError(e); + } + } + + private final int _segmentSize; + private final int _segmentBits; + private final int _segmentMask; + private final int _streamPartitionSize; + + private volatile Object[] _segments; + + public SegmentedStreamTableList() { + this(DEFAULT_SEGMENT_SIZE); + } + + public SegmentedStreamTableList(int segmentSize) { + this(segmentSize, 1024); + } + + public SegmentedStreamTableList(int segmentSize, int streamPartitionSize) { + validatePowerOfTwo(segmentSize, "segmentSize"); + _segmentSize = segmentSize; + _segmentBits = Integer.numberOfTrailingZeros(segmentSize); + _segmentMask = segmentSize - 1; + _streamPartitionSize = streamPartitionSize; + _segments = new Object[1]; + } + + public MaskedOnceArrayList get(int streamId) { + checkStreamId(streamId); + Object[] segments = (Object[]) SEGMENTS.getAcquire(this); + int segmentIndex = segmentIndex(streamId); + if(segmentIndex >= segments.length) + return null; + + Object[] segment = (Object[]) ARRAY.getAcquire(segments, segmentIndex); + if(segment == null) + return null; + + @SuppressWarnings("unchecked") + MaskedOnceArrayList streamTable = (MaskedOnceArrayList) ARRAY.getAcquire(segment, + offsetInSegment(streamId)); + return streamTable; + } + + public MaskedOnceArrayList get(long streamId) { + return get(asIntStreamId(streamId)); + } + + public MaskedOnceArrayList getOrCreate(int streamId) { + checkStreamId(streamId); + int segmentIndex = segmentIndex(streamId); + int offset = offsetInSegment(streamId); + + while(true) { + Object[] segments = ensureOuterCapacity(segmentIndex + 1); + Object[] segment = (Object[]) ARRAY.getAcquire(segments, segmentIndex); + if(segment == null) { + Object[] newSegment = new Object[_segmentSize]; + if(!ARRAY.compareAndSet(segments, segmentIndex, null, newSegment)) + continue; + segment = newSegment; + } + + @SuppressWarnings("unchecked") + MaskedOnceArrayList streamTable = (MaskedOnceArrayList) ARRAY.getAcquire(segment, offset); + if(streamTable != null) + return streamTable; + + MaskedOnceArrayList newTable = new MaskedOnceArrayList<>(_streamPartitionSize); + if(ARRAY.compareAndSet(segment, offset, null, newTable)) + return newTable; + } + } + + public MaskedOnceArrayList getOrCreate(long streamId) { + return getOrCreate(asIntStreamId(streamId)); + } + + public int capacity() { + Object[] segments = (Object[]) SEGMENTS.getAcquire(this); + return segments.length * _segmentSize; + } + + public void forEachLive(IndexedObjectPredicate action) { + forEachStreamTable(table -> table.forEachLive(action, false)); + } + + public void forEachVisible(Consumer action) { + forEachStreamTable(table -> table.forEachVisible(action)); + } + + public void forEachStreamTable(BiConsumer> action) { + Object[] segments = (Object[]) SEGMENTS.getAcquire(this); + for(int i = 0; i < segments.length; i++) { + Object[] segment = (Object[]) ARRAY.getAcquire(segments, i); + if(segment == null) + continue; + for(int j = 0; j < segment.length; j++) { + @SuppressWarnings("unchecked") + MaskedOnceArrayList table = (MaskedOnceArrayList) ARRAY.getAcquire(segment, j); + if(table != null) + action.accept((i << _segmentBits) | j, table); + } + } + } + + public void clear() { + SEGMENTS.setRelease(this, new Object[1]); + } + + private void forEachStreamTable(Consumer> action) { + Object[] segments = (Object[]) SEGMENTS.getAcquire(this); + for(int i = 0; i < segments.length; i++) { + Object[] segment = (Object[]) ARRAY.getAcquire(segments, i); + if(segment == null) + continue; + for(int j = 0; j < segment.length; j++) { + @SuppressWarnings("unchecked") + MaskedOnceArrayList table = (MaskedOnceArrayList) ARRAY.getAcquire(segment, j); + if(table != null) + action.accept(table); + } + } + } + + private Object[] ensureOuterCapacity(int minLength) { + Object[] segments = (Object[]) SEGMENTS.getAcquire(this); + while(minLength > segments.length) { + int newLength = segments.length; + while(newLength < minLength) { + if(newLength > Integer.MAX_VALUE / 2) + throw new IllegalStateException("SegmentedStreamTableList capacity overflow"); + newLength <<= 1; + } + + Object[] bigger = new Object[newLength]; + System.arraycopy(segments, 0, bigger, 0, segments.length); + if(SEGMENTS.compareAndSet(this, segments, bigger)) + return bigger; + segments = (Object[]) SEGMENTS.getAcquire(this); + } + return segments; + } + + private int segmentIndex(int streamId) { + return streamId >>> _segmentBits; + } + + private int offsetInSegment(int streamId) { + return streamId & _segmentMask; + } + + private static int asIntStreamId(long streamId) { + if(streamId < 0 || streamId > Integer.MAX_VALUE) + throw new IndexOutOfBoundsException("Invalid streamId: " + streamId); + return (int) streamId; + } + + private static void checkStreamId(int streamId) { + if(streamId < 0) + throw new IndexOutOfBoundsException("Invalid streamId: " + streamId); + } + + private static void validatePowerOfTwo(int value, String name) { + if(value <= 0 || (value & (value - 1)) != 0) + throw new IllegalArgumentException(name + " must be a power of two: " + value); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/eviction/EvictController.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/eviction/EvictController.java new file mode 100644 index 00000000000..64fefeb1160 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/eviction/EvictController.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache.eviction; + +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.BlockState; +import org.apache.sysds.runtime.ooc.cache.collections.MaskedOnceArrayList; + +import java.util.PriorityQueue; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.LongUnaryOperator; + +public class EvictController { + private final CopyOnWriteArrayList _op = new CopyOnWriteArrayList<>(); + + public void addEvictionPolicy(LongUnaryOperator op) { + if(op == null) + throw new IllegalArgumentException("Eviction policy must not be null."); + _op.add(op); + } + + public void findEvictionCandidates(MaskedOnceArrayList list, + PriorityQueue> candidates, int k, long estimatedReuseTimestamp) { + if(_op.isEmpty()) { + list.forEachLive((idx, b) -> { + if(!isEvictionCandidate(b)) + return true; + var iop = new IndexedObjectPair<>(estimatedReuseTimestamp + idx, b); + if(candidates.size() < k) { + candidates.offer(iop); + } + else if(iop.compareTo(candidates.peek()) > 0) { + candidates.poll(); + candidates.offer(iop); + } + return true; + }, true); + return; + } + list.forEachLive((idx, b) -> { + if(!isEvictionCandidate(b)) + return true; + long score = computeScore(idx); + if(score == Long.MAX_VALUE) + score = idx + estimatedReuseTimestamp; + var iop = new IndexedObjectPair<>(score, b); + if(candidates.size() < k) { + candidates.offer(iop); + } + else if(iop.compareTo(candidates.peek()) > 0) { + candidates.poll(); + candidates.offer(iop); + } + return true; + }, true); + } + + private boolean isEvictionCandidate(BlockEntry entry) { + BlockState state = entry.getState(); + return state == BlockState.HOT || state == BlockState.WARM; + } + + private long computeScore(int idx) { + long out = Long.MAX_VALUE; + for(LongUnaryOperator uop : _op) + out = Math.min(out, uop.applyAsLong(idx)); + return out; + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/eviction/IndexedObjectPair.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/eviction/IndexedObjectPair.java new file mode 100644 index 00000000000..e04c2953ddf --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/eviction/IndexedObjectPair.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache.eviction; + +public record IndexedObjectPair(long idx, T obj) implements Comparable> { + @Override + public int compareTo(IndexedObjectPair indexedObjectPair) { + return Long.compare(idx, indexedObjectPair.idx); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/memory/MemoryAllowance.java b/src/main/java/org/apache/sysds/runtime/ooc/memory/MemoryAllowance.java index 64518ded4a3..b2db5ea7533 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/memory/MemoryAllowance.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/memory/MemoryAllowance.java @@ -19,15 +19,27 @@ package org.apache.sysds.runtime.ooc.memory; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; + public interface MemoryAllowance { boolean tryReserve(long bytes); + void reserveBlocking(long bytes); + + OOCFuture reserveAsync(long bytes); + void release(long bytes); + long getUsedMemory(); + long getGrantedMemory(); + long getTargetMemory(); + void setTargetMemory(long targetMemory); + void shutdown(); + boolean isShutdown(); default void destroy() { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/memory/SyncMemoryAllowance.java b/src/main/java/org/apache/sysds/runtime/ooc/memory/SyncMemoryAllowance.java index 85d2cbfcd2b..2c4a1b7a0fa 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/memory/SyncMemoryAllowance.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/memory/SyncMemoryAllowance.java @@ -20,22 +20,51 @@ package org.apache.sysds.runtime.ooc.memory; import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; + +import java.util.ArrayDeque; +import java.util.concurrent.ExecutionException; public class SyncMemoryAllowance implements MemoryAllowance { + private static final long RELEASE_TRIM_BUFFER_BYTES = 20_000_000L; + protected final MemoryBroker _broker; + protected final long _consumptionLimit; + protected final long _minimumOperatingBytes; protected volatile long _usedBytes; protected volatile long _grantedBytes; protected volatile long _targetBytes; protected volatile boolean _shutdown; protected volatile boolean _destroyed; + private final ArrayDeque _reservationWaiters; + private boolean _drainingReservationWaiters; + private boolean _reservationDrainRequested; public SyncMemoryAllowance(MemoryBroker broker) { + this(broker, Long.MAX_VALUE); + } + + public SyncMemoryAllowance(MemoryBroker broker, long consumptionLimit) { + this(broker, consumptionLimit, 0); + } + + public SyncMemoryAllowance(MemoryBroker broker, long consumptionLimit, long minimumOperatingBytes) { + if(consumptionLimit < 0) + throw new IllegalArgumentException("Consumption limit must not be negative: " + consumptionLimit); + if(minimumOperatingBytes < 0) + throw new IllegalArgumentException( + "Minimum operating memory must not be negative: " + minimumOperatingBytes); _broker = broker; + _consumptionLimit = consumptionLimit; + _minimumOperatingBytes = Math.min(minimumOperatingBytes, consumptionLimit); _usedBytes = 0; _grantedBytes = 0; _targetBytes = 0; _shutdown = false; _destroyed = false; + _reservationWaiters = new ArrayDeque<>(); + _drainingReservationWaiters = false; + _reservationDrainRequested = false; broker.attachAllowance(this); } @@ -46,19 +75,23 @@ public boolean tryReserve(long bytes) { synchronized(this) { if(_shutdown || _destroyed) return false; - if(_usedBytes + bytes > _targetBytes) - return false; if(_usedBytes + bytes <= _grantedBytes) { _usedBytes += bytes; return true; } + if(_usedBytes + bytes > _targetBytes) + return false; minRequest = _usedBytes + bytes - _grantedBytes; maxRequest = Math.max(minRequest, Math.max(_grantedBytes, bytes) * 2); } + if(bytes > _consumptionLimit) + throw new IllegalArgumentException("Cannot reserve more memory than the consumption limit"); + long granted = _broker.requestMemory(this, minRequest, maxRequest); long refund = 0; boolean success = false; + boolean drainWaiters = false; synchronized(this) { if(_shutdown || _destroyed) refund = granted; @@ -68,36 +101,52 @@ public boolean tryReserve(long bytes) { _usedBytes += bytes; success = true; } + drainWaiters = success && !_reservationWaiters.isEmpty(); notifyAll(); } } if(refund > 0) _broker.freeMemory(this, refund); + if(drainWaiters) + requestReservationDrain(); return success; } @Override public void reserveBlocking(long bytes) { - if(_shutdown || _destroyed) - throw new IllegalStateException("Cannot reserve memory on closed allowance."); - while(true) { - if(tryReserve(bytes)) { - synchronized(this) { - notifyAll(); - } - return; - } - synchronized(this) { - if(_shutdown || _destroyed) - throw new IllegalStateException("Cannot reserve memory on closed allowance."); - try { - wait(); - } - catch(InterruptedException e) { - throw new DMLRuntimeException(e); - } + try { + reserveAsync(bytes).get(); + } + catch(InterruptedException e) { + Thread.currentThread().interrupt(); + throw new DMLRuntimeException(e); + } + catch(ExecutionException e) { + throw DMLRuntimeException.of(e.getCause()); + } + } + + @Override + public OOCFuture reserveAsync(long bytes) { + if(bytes < 0) + throw new IllegalArgumentException("Cannot reserve negative bytes: " + bytes); + if(bytes == 0) + return OOCFuture.completed(null); + if(bytes > _consumptionLimit) + return OOCFuture + .failed(new IllegalArgumentException("Cannot reserve more memory than the consumption limit")); + if(tryReserve(bytes)) + return OOCFuture.completed(null); + OOCFuture future = new OOCFuture<>(); + synchronized(this) { + if(_shutdown || _destroyed) { + future.completeExceptionally(new IllegalStateException("Cannot reserve memory on closed allowance.")); + return future; } + _reservationWaiters.addLast(new ReservationWaiter(bytes, future)); } + requestReservationDrain(); + return future; } @Override @@ -105,11 +154,23 @@ public void release(long bytes) { long freedMemory = 0; long destroyFreedMemory = 0; boolean destroy = false; + boolean drainWaiters; synchronized(this) { + if(bytes < 0) + throw new IllegalArgumentException("Cannot release negative bytes: " + bytes); + if(_usedBytes < bytes) { + throw new IllegalArgumentException("Memory allowance underflow in " + getClass().getSimpleName() + + ": release=" + bytes + ", used=" + _usedBytes + ", granted=" + _grantedBytes + ", target=" + + _targetBytes + ", shutdown=" + _shutdown + ", destroyed=" + _destroyed); + } _usedBytes -= bytes; if(_shutdown) { long oldGrantedBytes = _grantedBytes; _grantedBytes = _usedBytes; + if(_grantedBytes < 0) { + throw new IllegalArgumentException("Granted memory underflow in " + getClass().getSimpleName() + + ": granted=" + _grantedBytes + ", used=" + _usedBytes + ", released=" + bytes); + } if(_usedBytes == 0) { _destroyed = true; destroy = true; @@ -124,12 +185,20 @@ else if(_grantedBytes > _targetBytes) { _grantedBytes = Math.max(_usedBytes, _targetBytes); freedMemory = oldGrantedBytes - _grantedBytes; } + else if(_usedBytes * 3 < _grantedBytes * 2) { + long oldGrantedBytes = _grantedBytes; + _grantedBytes = Math.max(_usedBytes, Math.min(_grantedBytes, _usedBytes + RELEASE_TRIM_BUFFER_BYTES)); + freedMemory = oldGrantedBytes - _grantedBytes; + } + drainWaiters = !_reservationWaiters.isEmpty() && !_shutdown && !_destroyed; notifyAll(); } if(destroy) _broker.destroyAllowance(this, destroyFreedMemory); else if(freedMemory > 0) _broker.freeMemory(this, freedMemory); + if(drainWaiters) + requestReservationDrain(); } @Override @@ -148,11 +217,25 @@ public long getTargetMemory() { } @Override - public synchronized void setTargetMemory(long targetMemory) { - if(_shutdown || _destroyed) - return; - _targetBytes = targetMemory; - notifyAll(); + public void setTargetMemory(long targetMemory) { + long freedMemory = 0; + boolean drainWaiters = false; + synchronized(this) { + if(_shutdown || _destroyed) + return; + _targetBytes = Math.min(Math.max(targetMemory, _minimumOperatingBytes), _consumptionLimit); + if(_grantedBytes > _targetBytes) { + long oldGrantedBytes = _grantedBytes; + _grantedBytes = Math.max(_usedBytes, _targetBytes); + freedMemory = oldGrantedBytes - _grantedBytes; + } + drainWaiters = !_reservationWaiters.isEmpty(); + notifyAll(); + } + if(freedMemory > 0) + _broker.freeMemory(this, freedMemory); + if(drainWaiters) + requestReservationDrain(); } @Override @@ -160,6 +243,7 @@ public void shutdown() { long freedMemory = 0; long destroyFreedMemory = 0; boolean destroy = false; + ArrayDeque waiters; synchronized(this) { if(_shutdown || _destroyed) return; @@ -175,6 +259,8 @@ public void shutdown() { else { freedMemory = oldGrantedBytes - _grantedBytes; } + waiters = new ArrayDeque<>(_reservationWaiters); + _reservationWaiters.clear(); notifyAll(); } _broker.shutdownAllowance(this); @@ -182,10 +268,81 @@ public void shutdown() { _broker.destroyAllowance(this, destroyFreedMemory); else if(freedMemory > 0) _broker.freeMemory(this, freedMemory); + IllegalStateException ex = new IllegalStateException("Cannot reserve memory on closed allowance."); + while(!waiters.isEmpty()) + waiters.removeFirst().future.completeExceptionally(ex); } @Override public boolean isShutdown() { return _shutdown || _destroyed; } + + private void requestReservationDrain() { + synchronized(this) { + _reservationDrainRequested = true; + if(_drainingReservationWaiters) + return; + _drainingReservationWaiters = true; + } + try { + while(true) { + synchronized(this) { + _reservationDrainRequested = false; + } + drainReservationWaitersOnce(); + synchronized(this) { + if(!_reservationDrainRequested) { + _drainingReservationWaiters = false; + return; + } + } + } + } + catch(RuntimeException | Error t) { + synchronized(this) { + _drainingReservationWaiters = false; + } + throw t; + } + } + + private void drainReservationWaitersOnce() { + while(true) { + ReservationWaiter waiter; + synchronized(this) { + if(_shutdown || _destroyed) + return; + waiter = _reservationWaiters.peekFirst(); + if(waiter == null) + return; + } + boolean admitted; + try { + admitted = tryReserve(waiter.bytes); + } + catch(Throwable t) { + removeReservationWaiter(waiter); + waiter.future.completeExceptionally(t); + continue; + } + if(!admitted) + return; + if(removeReservationWaiter(waiter)) + waiter.future.complete(null); + else + release(waiter.bytes); + } + } + + private synchronized boolean removeReservationWaiter(ReservationWaiter waiter) { + if(_reservationWaiters.peekFirst() == waiter) { + _reservationWaiters.removeFirst(); + return true; + } + return _reservationWaiters.remove(waiter); + } + + private record ReservationWaiter(long bytes, OOCFuture future) { + } } diff --git a/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCCacheImplTest.java b/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCCacheImplTest.java new file mode 100644 index 00000000000..283cebe4667 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCCacheImplTest.java @@ -0,0 +1,334 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.ooc.cache; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BooleanSupplier; + +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.BlockKey; +import org.apache.sysds.runtime.ooc.cache.OOCCache; +import org.apache.sysds.runtime.ooc.cache.OOCCacheImpl; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; +import org.apache.sysds.runtime.ooc.memory.GlobalMemoryBroker; +import org.apache.sysds.runtime.ooc.memory.SyncMemoryAllowance; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class OOCCacheImplTest { + private static final long STREAM_ID = 7; + private static final long BLOCK_ID = 3; + private static final long BYTES = 1_000; + private static final long WAIT_TIMEOUT_SEC = 10; + + private RecordingIOHandler _io; + private GlobalMemoryBroker _broker; + private SyncMemoryAllowance _producer; + private SyncMemoryAllowance _reader; + private OOCCacheImpl _cache; + + @Before + public void setUp() { + _io = new RecordingIOHandler(); + _broker = new GlobalMemoryBroker(8 * BYTES); + _producer = new SyncMemoryAllowance(_broker, 4 * BYTES); + _reader = new SyncMemoryAllowance(_broker, 4 * BYTES); + _cache = new OOCCacheImpl(_io, 4 * BYTES, 4 * BYTES); + } + + @After + public void tearDown() { + if(_cache != null) + _cache.shutdown(); + if(_producer != null) + _producer.destroy(); + if(_reader != null) + _reader.destroy(); + } + + @Test + public void testPinMissingEntryReturnsNullWithoutReservation() throws Exception { + BlockEntry pinned = _cache.pin(new BlockKey(STREAM_ID, BLOCK_ID), _reader).get(WAIT_TIMEOUT_SEC, + TimeUnit.SECONDS); + + Assert.assertNull(pinned); + Assert.assertNull(_cache.pinIfLive(STREAM_ID, BLOCK_ID, _reader)); + Assert.assertEquals(0, _reader.getUsedMemory()); + Assert.assertEquals(0, _io.readCount()); + } + + @Test + public void testResidentPinTransfersOwnershipBetweenCacheAndAllowance() throws Exception { + BlockKey key = new BlockKey(STREAM_ID, BLOCK_ID); + String payload = "resident"; + + _producer.reserveBlocking(BYTES); + BlockEntry entry = _cache.putPinned(key, payload, BYTES, _producer); + Assert.assertEquals(0, _cache.getOwnedCacheSize()); + Assert.assertEquals(BYTES, _producer.getUsedMemory()); + + await(_cache.unpin(entry, _producer)); + Assert.assertEquals(BYTES, _cache.getOwnedCacheSize()); + Assert.assertEquals(0, _producer.getUsedMemory()); + + BlockEntry pinned = _cache.pin(key, _reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + Assert.assertSame(entry, pinned); + Assert.assertEquals(payload, pinned.getData()); + Assert.assertEquals(0, _cache.getOwnedCacheSize()); + Assert.assertEquals(BYTES, _reader.getUsedMemory()); + Assert.assertEquals(0, _io.readCount()); + + await(_cache.unpin(pinned, _reader)); + Assert.assertEquals(BYTES, _cache.getOwnedCacheSize()); + Assert.assertEquals(0, _reader.getUsedMemory()); + } + + @Test + public void testPinReloadsColdBackedEntry() throws Exception { + useEvictingCache(); + BlockKey key = new BlockKey(STREAM_ID, BLOCK_ID); + String payload = "payload"; + + _producer.reserveBlocking(BYTES); + BlockEntry entry = _cache.putPinned(key, payload, BYTES, _producer); + await(_cache.unpin(entry, _producer)); + waitFor(() -> _io.evictionCount() == 1 && BlockEntryTestAccess.getDataUnsafe(entry) == null); + Assert.assertEquals(0, _producer.getUsedMemory()); + + BlockEntry pinned = _cache.pin(key, _reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + + Assert.assertNotNull(pinned); + Assert.assertSame(entry, pinned); + Assert.assertEquals(payload, pinned.getData()); + Assert.assertEquals(1, _io.readCount()); + Assert.assertEquals(BYTES, _reader.getUsedMemory()); + + await(_cache.unpin(pinned, _reader)); + Assert.assertEquals(0, _reader.getUsedMemory()); + } + + @Test + public void testPinIfLiveDoesNotReadColdBackedEntry() throws Exception { + useEvictingCache(); + BlockKey key = new BlockKey(STREAM_ID, BLOCK_ID); + String payload = "cold"; + + _producer.reserveBlocking(BYTES); + BlockEntry entry = _cache.putPinned(key, payload, BYTES, _producer); + await(_cache.unpin(entry, _producer)); + waitFor(() -> _io.evictionCount() == 1 && BlockEntryTestAccess.getDataUnsafe(entry) == null); + + BlockEntry pinned = _cache.pinIfLive(STREAM_ID, BLOCK_ID, _reader); + + Assert.assertNull(pinned); + Assert.assertEquals(0, _io.readCount()); + Assert.assertEquals(0, _reader.getUsedMemory()); + } + + @Test + public void testDeferredUnpinCommitsWhenLimitsGrow() throws Exception { + useZeroHardLimitCache(); + BlockKey key = new BlockKey(STREAM_ID, BLOCK_ID); + + _producer.reserveBlocking(BYTES); + BlockEntry entry = _cache.putPinned(key, "deferred", BYTES, _producer); + OOCCache.UnpinHandle deferred = _cache.unpin(entry, _producer); + + Assert.assertFalse(deferred.isCommitted()); + Assert.assertFalse(deferred.getCompletionFuture().isDone()); + Assert.assertEquals(BYTES, _producer.getUsedMemory()); + Assert.assertEquals(0, _cache.getOwnedCacheSize()); + + _cache.updateLimits(BYTES, BYTES); + deferred.getCompletionFuture().get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + + Assert.assertTrue(deferred.isCommitted()); + Assert.assertEquals(0, _producer.getUsedMemory()); + Assert.assertEquals(BYTES, _cache.getOwnedCacheSize()); + } + + @Test + public void testDeferredUnpinCanBeAdoptedBySameAllowance() throws Exception { + useZeroHardLimitCache(); + BlockKey key = new BlockKey(STREAM_ID, BLOCK_ID); + + _producer.reserveBlocking(BYTES); + BlockEntry entry = _cache.putPinned(key, "adopt", BYTES, _producer); + OOCCache.UnpinHandle deferred = _cache.unpin(entry, _producer); + + BlockEntry repinned = _cache.pin(key, _producer).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + + Assert.assertSame(entry, repinned); + Assert.assertTrue(deferred.getCompletionFuture().isDone()); + Assert.assertFalse(deferred.isCommitted()); + Assert.assertEquals(BYTES, _producer.getUsedMemory()); + Assert.assertEquals(0, _cache.getOwnedCacheSize()); + + OOCCache.UnpinHandle cleanup = _cache.unpin(repinned, _producer); + _cache.updateLimits(BYTES, BYTES); + cleanup.getCompletionFuture().get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + Assert.assertEquals(0, _producer.getUsedMemory()); + } + + @Test + public void testDereferenceRemovesEntryAfterLastUnpin() throws Exception { + BlockKey key = new BlockKey(STREAM_ID, BLOCK_ID); + + _producer.reserveBlocking(BYTES); + BlockEntry entry = _cache.putPinned(key, "drop", BYTES, _producer); + + Assert.assertEquals(0, _cache.dereference(entry)); + await(_cache.unpin(entry, _producer)); + + Assert.assertEquals(0, _producer.getUsedMemory()); + Assert.assertNull(_cache.pin(key, _reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS)); + Assert.assertEquals(0, _reader.getUsedMemory()); + } + + @Test + public void testBackingReadFailureReleasesReservedBytes() throws Exception { + useEvictingCache(); + BlockKey key = new BlockKey(STREAM_ID, BLOCK_ID); + + _producer.reserveBlocking(BYTES); + BlockEntry entry = _cache.putPinned(key, "fail-read", BYTES, _producer); + await(_cache.unpin(entry, _producer)); + waitFor(() -> _io.evictionCount() == 1 && BlockEntryTestAccess.getDataUnsafe(entry) == null); + + _io.failReads(true); + try { + _cache.pin(key, _reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + Assert.fail("A failed backing read must fail the pin future."); + } + catch(ExecutionException expected) { + // expected + } + + Assert.assertEquals(1, _io.readCount()); + Assert.assertEquals(0, _reader.getUsedMemory()); + } + + private void useEvictingCache() { + _cache.shutdown(); + _io.reset(); + _cache = new OOCCacheImpl(_io, 4 * BYTES, 0); + } + + private void useZeroHardLimitCache() { + _cache.shutdown(); + _io.reset(); + _cache = new OOCCacheImpl(_io, 0, 0); + } + + private static void await(OOCCache.UnpinHandle handle) throws Exception { + if(!handle.isCommitted()) + handle.getCompletionFuture().get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + } + + private static void waitFor(BooleanSupplier condition) throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(WAIT_TIMEOUT_SEC); + while(!condition.getAsBoolean() && System.nanoTime() < deadline) + Thread.sleep(1); + Assert.assertTrue(condition.getAsBoolean()); + } + + private static final class RecordingIOHandler implements OOCIOHandler { + private final Map _spilled = new ConcurrentHashMap<>(); + private final AtomicInteger _evictions = new AtomicInteger(); + private final AtomicInteger _reads = new AtomicInteger(); + private volatile boolean _failReads; + + @Override + public void shutdown() { + _spilled.clear(); + } + + @Override + public CompletableFuture scheduleEviction(BlockEntry block) { + _spilled.put(block.getKey(), BlockEntryTestAccess.getDataUnsafe(block)); + _evictions.incrementAndGet(); + return CompletableFuture.completedFuture(null); + } + + @Override + public OOCFuture scheduleRead(BlockEntry block) { + _reads.incrementAndGet(); + if(_failReads) + return OOCFuture.failed(new IllegalStateException("read failed")); + Object data = _spilled.get(block.getKey()); + if(data == null) + return OOCFuture.completed(null); + BlockEntryTestAccess.setDataUnsafe(block, data); + return OOCFuture.completed(block); + } + + @Override + public void prioritizeRead(BlockKey key, double priority) { + } + + @Override + public CompletableFuture scheduleDeletion(BlockEntry block) { + _spilled.remove(block.getKey()); + return CompletableFuture.completedFuture(true); + } + + @Override + public void registerSourceLocation(BlockKey key, SourceBlockDescriptor descriptor) { + } + + @Override + public CompletableFuture scheduleSourceRead(SourceReadRequest request) { + return CompletableFuture.failedFuture(new UnsupportedOperationException()); + } + + @Override + public CompletableFuture continueSourceRead(SourceReadContinuation continuation, + long maxBytesInFlight) { + return CompletableFuture.failedFuture(new UnsupportedOperationException()); + } + + private int evictionCount() { + return _evictions.get(); + } + + private int readCount() { + return _reads.get(); + } + + private void failReads(boolean failReads) { + _failReads = failReads; + } + + private void reset() { + _spilled.clear(); + _evictions.set(0); + _reads.set(0); + _failReads = false; + } + } +} From 100dd9dbcf26459c21965b6d72f053b8c11cfb78 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:16:06 +0200 Subject: [PATCH 067/132] [OOC] Add OOCPackedCache (#2534) Adds a new OOCCache implementation that supports packing small tiles together into one logical cache entry. --- .../sysds/runtime/ooc/cache/OOCCache.java | 33 +- .../ooc/cache/io/SpillableObjectRegistry.java | 5 + .../ooc/cache/packed/OOCPackedCache.java | 761 ++++++++++++++++++ .../runtime/ooc/cache/packed/PackBuilder.java | 136 ++++ .../runtime/ooc/cache/packed/PackedBlock.java | 72 ++ .../ooc/cache/packed/PackedCacheLocation.java | 73 ++ .../ooc/cache/packed/PackedPinState.java | 234 ++++++ .../ooc/cache/packed/PackedUnpinHandle.java | 80 ++ .../component/ooc/cache/OOCCacheImplTest.java | 121 +-- .../ooc/cache/OOCCacheTestUtils.java | 130 +++ .../ooc/cache/OOCPackedCacheTest.java | 384 +++++++++ 11 files changed, 1920 insertions(+), 109 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/packed/OOCPackedCache.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackBuilder.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedBlock.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedCacheLocation.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedPinState.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedUnpinHandle.java create mode 100644 src/test/java/org/apache/sysds/test/component/ooc/cache/OOCCacheTestUtils.java create mode 100644 src/test/java/org/apache/sysds/test/component/ooc/cache/OOCPackedCacheTest.java diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCache.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCache.java index 7f1fdb493d0..aec88891595 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCache.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCache.java @@ -24,10 +24,29 @@ import java.util.function.LongUnaryOperator; public interface OOCCache { + /** + * Pins an item backed by an allowance. A successful pin transfers memory ownership from the cache to the owner of + * the allowance and guarantees data availability. While pinned, the bytes of the entry are not counted as + * cache-owned memory. + * + * @param key + * @param allowance + * @return a non-null future of the pinned block entry; the future result is null if the required memory could not + * be reserved + */ default OOCFuture pin(BlockKey key, MemoryAllowance allowance) { return pin(key.getStreamId(), key.getSequenceNumber(), allowance); } + /** + * Pins an item backed by an allowance. If the allowance cannot reserve enough memory, this method will wait until + * memory is available. A successful pin transfers memory ownership from the cache to the owner of the allowance and + * guarantees data availability. While pinned, the bytes of the entry are not counted as cache-owned memory. + * + * @param key + * @param allowance + * @return a non-null future of the pinned block entry + */ default OOCFuture pinAdmitted(BlockKey key, MemoryAllowance allowance) { return pinAdmitted(key.getStreamId(), key.getSequenceNumber(), allowance); } @@ -59,9 +78,17 @@ default BlockEntry putPinned(BlockKey key, Object data, long size, MemoryAllowan */ OOCFuture pin(long sId, long tId, MemoryAllowance allowance); - default OOCFuture pinAdmitted(long sId, long tId, MemoryAllowance allowance) { - return pin(sId, tId, allowance); - } + /** + * Pins an item backed by an allowance. If the allowance cannot reserve enough memory, this method will wait until + * memory is available. A successful pin transfers memory ownership from the cache to the owner of the allowance and + * guarantees data availability. While pinned, the bytes of the entry are not counted as cache-owned memory. + * + * @param sId + * @param tId + * @param allowance + * @return a non-null future of the pinned block entry + */ + OOCFuture pinAdmitted(long sId, long tId, MemoryAllowance allowance); /** * Pins an item backed by an allowance if it is already live in cache. A successful pin transfers memory ownership diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObjectRegistry.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObjectRegistry.java index 6cd8fded9e7..3b10a10ec54 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObjectRegistry.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObjectRegistry.java @@ -20,6 +20,7 @@ package org.apache.sysds.runtime.ooc.cache.io; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.ooc.cache.packed.PackedBlock; import java.io.DataInput; import java.io.DataOutput; @@ -27,6 +28,7 @@ public final class SpillableObjectRegistry { private static final byte INDEXED_MATRIX_VALUE = 1; + private static final byte PACKED_BLOCK = 2; private SpillableObjectRegistry() { } @@ -41,6 +43,7 @@ public static SpillableObject read(DataInput in) throws IOException { byte type = in.readByte(); SpillableObject obj = switch(type) { case INDEXED_MATRIX_VALUE -> new IndexedMatrixValue(); + case PACKED_BLOCK -> new PackedBlock(); default -> throw new IOException("Unknown spillable object type: " + type); }; obj.read(in); @@ -50,6 +53,8 @@ public static SpillableObject read(DataInput in) throws IOException { private static byte typeOf(SpillableObject obj) throws IOException { if(obj instanceof IndexedMatrixValue) return INDEXED_MATRIX_VALUE; + if(obj instanceof PackedBlock) + return PACKED_BLOCK; throw new IOException("Unsupported spillable object type: " + obj.getClass().getName()); } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/OOCPackedCache.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/OOCPackedCache.java new file mode 100644 index 00000000000..9bbdcbeae5a --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/OOCPackedCache.java @@ -0,0 +1,761 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache.packed; + +import org.apache.sysds.runtime.instructions.ooc.CachingStream; +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.BlockKey; +import org.apache.sysds.runtime.ooc.cache.BlockState; +import org.apache.sysds.runtime.ooc.cache.OOCCache; +import org.apache.sysds.runtime.ooc.cache.OOCCacheImpl; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.cache.collections.MaskedOnceArrayList; +import org.apache.sysds.runtime.ooc.cache.collections.SegmentedStreamTableList; +import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; +import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.LockSupport; +import java.util.function.LongUnaryOperator; + +public final class OOCPackedCache implements OOCCache { + private static final long PACKED_STREAM_ID = CachingStream._streamSeq.getNextID(); + private static final long DEFAULT_PACK_THRESHOLD_BYTES = 1L << 18; + private static final long DEFAULT_PACK_TARGET_BYTES = 1L << 19; // 512 KB tile packing + private static final long DEFAULT_MAX_STAGING_BYTES = 1L << 26; + private static final int DEFAULT_MAX_OPEN_BUILDERS = 64; + private static final long DEFAULT_SEAL_DELAY_MS = 5; + private static final long DEFAULT_PACK_RELEASE_DELAY_MS = 5; + + private final OOCCacheImpl _physical; + private final long _packThresholdBytes; + private final long _packTargetBytes; + private final long _maxStagingBytes; + private final int _maxOpenBuilders; + private final long _sealDelayMs; + private final long _packReleaseDelayMs; + private final SegmentedStreamTableList _locations; + private final MaskedOnceArrayList _packedStates; + private final ScheduledExecutorService _sealExecutor; + private final ExecutorService _releaseExecutor; + private final ConcurrentLinkedQueue _releaseQueue; + private final AtomicBoolean _releaseRunning; + private final AtomicBoolean _packedPolicyInstalled; + private final AtomicInteger _nextPackedId; + private final ArrayList> _logicalEvictionPolicies; + + private PackBuilder[] _builders; + private long _stagingBytes; + private int _openBuilderCount; + private boolean _running; + + public OOCPackedCache(OOCIOHandler ioHandler, long hardLimit, long evictionLimit) { + this(new OOCCacheImpl(ioHandler, hardLimit, evictionLimit), DEFAULT_PACK_THRESHOLD_BYTES, + DEFAULT_PACK_TARGET_BYTES, DEFAULT_SEAL_DELAY_MS); + } + + public OOCPackedCache(OOCCacheImpl physical) { + this(physical, DEFAULT_PACK_THRESHOLD_BYTES, DEFAULT_PACK_TARGET_BYTES, DEFAULT_SEAL_DELAY_MS); + } + + public OOCPackedCache(OOCCacheImpl physical, long packThresholdBytes, long packTargetBytes, long sealDelayMs) { + this(physical, packThresholdBytes, packTargetBytes, sealDelayMs, DEFAULT_PACK_RELEASE_DELAY_MS); + } + + public OOCPackedCache(OOCCacheImpl physical, long packThresholdBytes, long packTargetBytes, long sealDelayMs, + long packReleaseDelayMs) { + this(physical, packThresholdBytes, packTargetBytes, DEFAULT_MAX_STAGING_BYTES, DEFAULT_MAX_OPEN_BUILDERS, + sealDelayMs, packReleaseDelayMs); + } + + public OOCPackedCache(OOCCacheImpl physical, long packThresholdBytes, long packTargetBytes, long maxStagingBytes, + int maxOpenBuilders, long sealDelayMs, long packReleaseDelayMs) { + if(packThresholdBytes <= 0 || packTargetBytes < packThresholdBytes) + throw new IllegalArgumentException( + "Invalid pack sizes: threshold=" + packThresholdBytes + ", target=" + packTargetBytes); + _physical = physical; + _packThresholdBytes = packThresholdBytes; + _packTargetBytes = packTargetBytes; + _maxStagingBytes = Math.max(packTargetBytes, maxStagingBytes); + _maxOpenBuilders = Math.max(1, maxOpenBuilders); + _sealDelayMs = sealDelayMs; + _packReleaseDelayMs = packReleaseDelayMs; + _locations = new SegmentedStreamTableList<>(); + _packedStates = new MaskedOnceArrayList<>(); + _nextPackedId = new AtomicInteger(); + _releaseQueue = new ConcurrentLinkedQueue<>(); + _releaseRunning = new AtomicBoolean(false); + _packedPolicyInstalled = new AtomicBoolean(false); + _logicalEvictionPolicies = new ArrayList<>(); + _builders = new PackBuilder[16]; + _stagingBytes = 0; + _openBuilderCount = 0; + _running = true; + _sealExecutor = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "ooc-pack-sealer"); + t.setDaemon(true); + return t; + }); + _releaseExecutor = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "ooc-pack-release"); + t.setDaemon(true); + return t; + }); + } + + @Override + public BlockEntry putPinned(long sId, long tId, Object data, long size, MemoryAllowance allowance) { + if(size >= _packThresholdBytes) + return _physical.putPinned(sId, tId, data, size, allowance); + + PackBuilder builder; + int slot; + synchronized(this) { + checkRunning(); + builder = getOpenBuilder(sId, allowance, size); + slot = appendToBuilder(builder, sId, tId, data, size); + } + + BlockEntry logical = new BlockEntry(new BlockKey(sId, tId), size, data, BlockState.REMOVED); + logical.pin(); + logical.setCacheMeta(new PendingLogicalPin(builder, slot)); + return logical; + } + + public BlockEntry[] putPackPinned(long sId, long[] tIds, Object[] data, long[] sizes, int off, int len, + MemoryAllowance allowance) { + BlockEntry[] entries = new BlockEntry[len]; + synchronized(this) { + checkRunning(); + for(int i = 0; i < len; i++) { + int p = off + i; + long tId = tIds[p]; + long size = sizes[p]; + if(size >= _packThresholdBytes) { + entries[i] = _physical.putPinned(sId, tId, data[p], size, allowance); + continue; + } + PackBuilder builder = getOpenBuilder(sId, allowance, size); + int slot = appendToBuilder(builder, sId, tId, data[p], size); + BlockEntry logical = new BlockEntry(new BlockKey(sId, tId), size, data[p], BlockState.REMOVED); + logical.pin(); + logical.setCacheMeta(new PendingLogicalPin(builder, slot)); + entries[i] = logical; + } + } + return entries; + } + + public BlockEntry putSealedPackPinned(long sId, long[] tIds, Object[] data, long[] sizes, int off, int len, + MemoryAllowance allowance) { + long totalSize = 0; + Object[] packedData = new Object[len]; + long[] packedSizes = new long[len]; + for(int i = 0; i < len; i++) { + int p = off + i; + packedData[i] = data[p]; + packedSizes[i] = sizes[p]; + totalSize += sizes[p]; + } + + synchronized(this) { + checkRunning(); + BlockEntry physicalEntry = putSealedBlockPinned(new PackedBlock(packedData, packedSizes, totalSize), + allowance); + PackedPinState state = new PackedPinState(physicalEntry, sId, + Arrays.stream(tIds).mapToInt(Math::toIntExact).toArray(), off, len, len); + registerPackedState(state); + for(int i = 0; i < len; i++) + putLocation(new BlockKey(sId, tIds[off + i]), new SealedPackLocation(state, i)); + return physicalEntry; + } + } + + public PackGroup getPackGroup(long sId, long tId) { + PackedCacheLocation location = getLocation(sId, tId); + if(location instanceof PendingPackLocation pending) + location = forceSeal(pending); + return location instanceof SealedPackLocation packed ? packed.state().group : null; + } + + public int getPackGroupCount() { + return _nextPackedId.get(); + } + + public OOCFuture pinPack(PackGroup group, MemoryAllowance allowance) { + return group.state.pin(_physical, allowance, false) + .map(entry -> entry == null ? null : new PackLease(this, group, allowance)); + } + + @Override + public OOCFuture pin(long sId, long tId, MemoryAllowance allowance) { + PackedCacheLocation location = getLocation(sId, tId); + if(location == null) + return _physical.pin(sId, tId, allowance); + if(location instanceof PendingPackLocation pending) + location = forceSeal(pending); + if(!(location instanceof SealedPackLocation packed)) + return _physical.pin(sId, tId, allowance); + + return packed.state().pin(_physical, allowance, false).map(physicalEntry -> { + if(physicalEntry == null) + return null; + return createLogicalPin(new BlockKey(sId, tId), packed); + }); + } + + @Override + public OOCFuture pinAdmitted(long sId, long tId, MemoryAllowance allowance) { + PackedCacheLocation location = getLocation(sId, tId); + if(location == null) + return _physical.pinAdmitted(sId, tId, allowance); + if(location instanceof PendingPackLocation pending) + location = forceSeal(pending); + if(!(location instanceof SealedPackLocation packed)) + return _physical.pinAdmitted(sId, tId, allowance); + + return packed.state().pinAdmitted(_physical, allowance).map(physicalEntry -> { + if(physicalEntry == null) + return null; + return createLogicalPin(new BlockKey(sId, tId), packed); + }); + } + + @Override + public BlockEntry pinIfLive(long sId, long tId, MemoryAllowance allowance) { + PackedCacheLocation location = getLocation(sId, tId); + if(location == null) + return _physical.pinIfLive(sId, tId, allowance); + if(location instanceof PendingPackLocation pending) + location = forceSeal(pending); + if(!(location instanceof SealedPackLocation packed)) + return _physical.pinIfLive(sId, tId, allowance); + + if(packed.state().pinIfLive(_physical, allowance) == null) + return null; + return createLogicalPin(new BlockKey(sId, tId), packed); + } + + @Override + public UnpinHandle unpin(BlockEntry entry, MemoryAllowance allowance) { + Object meta = entry.getCacheMeta(); + if(meta instanceof PendingLogicalPin pending) + return unpinPending(entry, pending, allowance); + if(meta instanceof PackedLogicalPin packed) + return unpinPacked(entry, packed, allowance); + return _physical.unpin(entry, allowance); + } + + @Override + public int reference(BlockEntry entry) { + Object meta = entry.getCacheMeta(); + if(meta instanceof PackedLogicalPin packed) + return packed.location().retain(); + if(meta instanceof PendingLogicalPin pending) + return referencePending(pending.builder(), pending.slot()); + return _physical.reference(entry); + } + + @Override + public int dereference(BlockEntry entry) { + Object meta = entry.getCacheMeta(); + if(meta instanceof PackedLogicalPin packed) + return releaseLocation(entry.getKey(), packed.location()); + if(meta instanceof PendingLogicalPin pending) + return dereferencePending(pending.builder(), pending.slot()); + return _physical.dereference(entry); + } + + @Override + public int dereference(BlockKey key) { + PackedCacheLocation location = getLocation(key.getStreamId(), key.getSequenceNumber()); + if(location == null) + return _physical.dereference(key); + if(location instanceof PendingPackLocation pending) + return dereferencePending(pending.builder(), pending.slot()); + if(!(location instanceof SealedPackLocation packed)) + return _physical.dereference(key); + return releaseLocation(key, packed); + } + + /** + * References/dereferences on tiles in open builders are counted on the builder slot instead of forcing a seal, so + * pipelined consumers that park references (state tables, store readers) do not fragment packs into per-tile + * physical entries. Slot counts carry over into the SealedPackLocation at seal time. Only physical access (pin, + * pack group) forces a seal. + */ + private synchronized int referencePending(PackBuilder builder, int slot) { + if(!builder.sealed) + return builder.retainSlot(slot); + PackedCacheLocation location = getLocation(builder.streamIds[slot], builder.tileIds[slot]); + if(!(location instanceof SealedPackLocation packed)) + throw new IllegalStateException("Cannot retain a forgotten packed location."); + return packed.retain(); + } + + private synchronized int dereferencePending(PackBuilder builder, int slot) { + BlockKey key = new BlockKey(builder.streamIds[slot], builder.tileIds[slot]); + if(builder.sealed) { + PackedCacheLocation location = getLocation(key.getStreamId(), key.getSequenceNumber()); + return location instanceof SealedPackLocation packed ? releaseLocation(key, packed) : 0; + } + int references = builder.releaseSlot(slot); + if(references == 0) + clearLocation(key); + return references; + } + + @Override + public void updateLimits(long hardLimit, long evictionLimit) { + _physical.updateLimits(hardLimit, evictionLimit); + } + + @Override + public void addEvictionPolicy(long streamId, LongUnaryOperator scoreFn) { + _physical.addEvictionPolicy(streamId, scoreFn); + addLogicalEvictionPolicy(streamId, scoreFn); + if(_packedPolicyInstalled.compareAndSet(false, true)) + _physical.addEvictionPolicy(PACKED_STREAM_ID, this::scorePackedBlock); + } + + @Override + public long getOwnedCacheSize() { + return _physical.getOwnedCacheSize(); + } + + @Override + public synchronized void shutdown() { + if(!_running) + return; + _running = false; + for(PackBuilder builder : _builders) + if(builder != null) + sealBuilder(builder); + _physical.updateLimits(Long.MAX_VALUE, Long.MAX_VALUE); + _sealExecutor.shutdownNow(); + _releaseExecutor.shutdown(); + awaitReleaseExecutor(); + drainReleaseQueue(); + _physical.shutdown(); + } + + private void awaitReleaseExecutor() { + try { + _releaseExecutor.awaitTermination(Math.max(100, _packReleaseDelayMs * 2), TimeUnit.MILLISECONDS); + } + catch(InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + + private void drainReleaseQueue() { + PackedPinState state; + while((state = _releaseQueue.poll()) != null) { + state.clearReleaseQueued(); + state.releaseDuePins(_physical, Long.MAX_VALUE); + } + } + + public synchronized void flushPacks() { + for(PackBuilder builder : _builders) + if(builder != null) + sealBuilder(builder); + } + + private UnpinHandle unpinPending(BlockEntry entry, PendingLogicalPin pin, MemoryAllowance allowance) { + if(entry.fastUnpin()) { + allowance.release(entry.getSize()); + return PackedUnpinHandle.committed(entry, allowance, entry.getSize()); + } + synchronized(this) { + if(entry.getPinCount() > 1) { + entry.unpin(); + allowance.release(entry.getSize()); + return PackedUnpinHandle.committed(entry, allowance, entry.getSize()); + } + entry.unpin(); + entry.setCacheMeta(null); + PackedUnpinHandle handle = pin.builder().unpinProducer(entry, pin.slot(), allowance); + if(pin.builder().sealed && pin.builder().activePins == 0) + pin.builder().transferProducerOwnership(_physical); + scheduleSeal(pin.builder()); + return handle; + } + } + + private UnpinHandle unpinPacked(BlockEntry entry, PackedLogicalPin pin, MemoryAllowance allowance) { + if(entry.fastUnpin()) + return PackedUnpinHandle.committed(entry, allowance, entry.getSize()); + if(entry.getPinCount() > 1) { + entry.unpin(); + return PackedUnpinHandle.committed(entry, allowance, entry.getSize()); + } + entry.unpin(); + entry.setCacheMeta(null); + return pin.location().state().unpin(this, _packReleaseDelayMs, allowance); + } + + void enqueueRelease(PackedPinState state) { + if(!_running) + return; + if(state.markReleaseQueued()) { + _releaseQueue.offer(state); + scheduleReleaseMaintenance(); + } + } + + private void enqueueReleaseNoSchedule(PackedPinState state) { + if(state.markReleaseQueued()) + _releaseQueue.offer(state); + } + + private void scheduleReleaseMaintenance() { + if(!_releaseRunning.compareAndSet(false, true)) + return; + _releaseExecutor.execute(this::runReleaseMaintenance); + } + + private void runReleaseMaintenance() { + try { + while(_running) { + long nextDueNanos = Long.MAX_VALUE; + ArrayList delayed = null; + PackedPinState state; + long nowNanos = System.nanoTime(); + while((state = _releaseQueue.poll()) != null) { + state.clearReleaseQueued(); + long stateNextDue = state.releaseDuePins(_physical, nowNanos); + if(stateNextDue != Long.MAX_VALUE) { + if(delayed == null) + delayed = new ArrayList<>(); + delayed.add(state); + nextDueNanos = Math.min(nextDueNanos, stateNextDue); + } + } + if(delayed != null) + for(PackedPinState delayedState : delayed) + enqueueReleaseNoSchedule(delayedState); + if(nextDueNanos == Long.MAX_VALUE) + return; + long waitNanos = nextDueNanos - System.nanoTime(); + if(waitNanos > 0) + LockSupport.parkNanos(waitNanos); + } + } + finally { + _releaseRunning.set(false); + if(_running && !_releaseQueue.isEmpty()) + scheduleReleaseMaintenance(); + } + } + + private SealedPackLocation forceSeal(PendingPackLocation pending) { + synchronized(this) { + sealBuilder(pending.builder()); + PackedCacheLocation location = getLocation(pending.builder().streamIds[pending.slot()], + pending.builder().tileIds[pending.slot()]); + return (SealedPackLocation) location; + } + } + + private static BlockEntry createLogicalPin(BlockKey logicalKey, SealedPackLocation location) { + PackedBlock block = (PackedBlock) location.state().physicalEntry.getDataUnsafe(); + Object data = block.values[location.slot()]; + long size = block.sizes[location.slot()]; + BlockEntry logical = new BlockEntry(logicalKey, size, data, BlockState.REMOVED); + logical.pin(); + logical.setCacheMeta(new PackedLogicalPin(location)); + return logical; + } + + private int releaseLocation(BlockKey key, SealedPackLocation location) { + int references = location.release(); + if(references > 0) + return references; + if(!clearLocation(key)) + return 0; + if(location.state().forgetLocation()) { + _packedStates.clear((int) location.state().physicalEntry.getKey().getSequenceNumber()); + return _physical.dereference(location.state().physicalEntry); + } + return 0; + } + + private PackBuilder getOpenBuilder(long streamId, MemoryAllowance allowance, long nextSize) { + int sid = (int) streamId; + PackBuilder builder = sid < _builders.length ? _builders[sid] : null; + if(builder != null && (builder.sealed || builder.allowance != allowance)) { + sealBuilder(builder); + builder = null; + } + if(builder != null) + return builder; + while(!canOpenBuilder(nextSize)) { + PackBuilder largest = findLargestOpenBuilder(); + if(largest == null) + break; + sealBuilder(largest); + } + ensureBuilderCapacity(sid); + builder = new PackBuilder(sid, allowance, _packTargetBytes); + _builders[sid] = builder; + _openBuilderCount++; + return builder; + } + + private boolean canOpenBuilder(long nextSize) { + return _openBuilderCount < _maxOpenBuilders && _stagingBytes + nextSize <= _maxStagingBytes; + } + + private int appendToBuilder(PackBuilder builder, long streamId, long tileId, Object data, long size) { + int slot = builder.append(streamId, tileId, data, size); + _stagingBytes += size; + putLocation(new BlockKey(streamId, tileId), new PendingPackLocation(builder, slot)); + if(builder.getBytes() >= builder.packTargetBytes) + sealBuilder(builder); + else + enforceStagingBudget(); + return slot; + } + + private void enforceStagingBudget() { + while(_stagingBytes > _maxStagingBytes || _openBuilderCount > _maxOpenBuilders) { + PackBuilder builder = findLargestOpenBuilder(); + if(builder == null) + return; + sealBuilder(builder); + } + } + + private PackBuilder findLargestOpenBuilder() { + PackBuilder largest = null; + for(PackBuilder builder : _builders) + if(builder != null && !builder.sealed && (largest == null || builder.getBytes() > largest.getBytes())) + largest = builder; + return largest; + } + + private void sealBuilder(PackBuilder builder) { + if(builder.sealed || builder.count == 0) + return; + builder.sealed = true; + _stagingBytes -= builder.getBytes(); + _openBuilderCount--; + if(builder.streamSlot >= 0 && builder.streamSlot < _builders.length && _builders[builder.streamSlot] == builder) + _builders[builder.streamSlot] = null; + + PackedBlock block = builder.createBlock(); + BlockEntry physicalEntry = putSealedBlockPinned(block, builder.allowance); + int liveSlots = builder.countLiveSlots(); + PackedPinState state = new PackedPinState(physicalEntry, builder.streamIds[0], + Arrays.stream(builder.tileIds).mapToInt(Math::toIntExact).toArray(), 0, builder.count, liveSlots); + builder.state = state; + if(liveSlots > 0) + registerPackedState(state); + + // slots forgotten while pending stay in the physical pack but get no location + for(int i = 0; i < builder.count; i++) + if(builder.refCounts[i] > 0) + putLocation(new BlockKey(builder.streamIds[i], builder.tileIds[i]), + new SealedPackLocation(state, i, builder.refCounts[i])); + + if(builder.activePins == 0) + builder.transferProducerOwnership(_physical); + if(liveSlots == 0) + _physical.dereference(physicalEntry); + } + + private BlockEntry putSealedBlockPinned(PackedBlock block, MemoryAllowance allowance) { + BlockKey packedKey = new BlockKey(PACKED_STREAM_ID, _nextPackedId.getAndIncrement()); + return _physical.putPinned(packedKey, block, block.totalSize, allowance); + } + + private void registerPackedState(PackedPinState state) { + _packedStates.put((int) state.physicalEntry.getKey().getSequenceNumber(), state); + } + + private long scorePackedBlock(long packId) { + PackedPinState state = _packedStates.get((int) packId); + if(state == null) + return packId; + PackGroup group = state.group; + CopyOnWriteArrayList policies = group.streamId < _logicalEvictionPolicies + .size() ? _logicalEvictionPolicies.get((int) group.streamId) : null; + if(policies == null || policies.isEmpty()) + return packId; + long score = Long.MAX_VALUE; + for(int i = 0; i < group.size(); i++) { + long tileId = group.index(i); + for(LongUnaryOperator policy : policies) + score = Math.min(score, policy.applyAsLong(tileId)); + } + return score; + } + + private synchronized void addLogicalEvictionPolicy(long streamId, LongUnaryOperator scoreFn) { + int sid = (int) streamId; + while(sid >= _logicalEvictionPolicies.size()) + _logicalEvictionPolicies.add(null); + CopyOnWriteArrayList policies = _logicalEvictionPolicies.get(sid); + if(policies == null) { + policies = new CopyOnWriteArrayList<>(); + _logicalEvictionPolicies.set(sid, policies); + } + policies.add(scoreFn); + } + + private void scheduleSeal(PackBuilder builder) { + if(builder.sealScheduled || builder.sealed || _sealDelayMs < 0) + return; + builder.sealScheduled = true; + _sealExecutor.schedule(() -> { + synchronized(OOCPackedCache.this) { + builder.sealScheduled = false; + sealBuilder(builder); + } + }, _sealDelayMs, TimeUnit.MILLISECONDS); + } + + private PackedCacheLocation getLocation(long sId, long tId) { + MaskedOnceArrayList stream = _locations.get(sId); + return stream == null ? null : stream.get((int) tId); + } + + private void putLocation(BlockKey key, PackedCacheLocation location) { + _locations.getOrCreate(key.getStreamId()).put((int) key.getSequenceNumber(), location); + } + + private boolean clearLocation(BlockKey key) { + MaskedOnceArrayList stream = _locations.get(key.getStreamId()); + return stream != null && stream.clear((int) key.getSequenceNumber()); + } + + private void ensureBuilderCapacity(int streamId) { + if(streamId < _builders.length) + return; + int len = _builders.length; + while(streamId >= len) + len <<= 1; + PackBuilder[] bigger = new PackBuilder[len]; + System.arraycopy(_builders, 0, bigger, 0, _builders.length); + _builders = bigger; + } + + private void checkRunning() { + if(!_running) + throw new IllegalStateException("Cache has been shut down."); + } + + public static final class PackGroup { + private final PackedPinState state; + private final long streamId; + private final int firstIndex; + private final int[] indices; + private final int size; + + PackGroup(PackedPinState state, long streamId, int[] tileIds, int off, int size) { + this.state = state; + this.streamId = streamId; + this.size = size; + firstIndex = tileIds[off]; + boolean contiguous = true; + for(int i = 1; i < size; i++) { + if(tileIds[off + i] != firstIndex + i) { + contiguous = false; + break; + } + } + if(contiguous) + indices = null; + else { + indices = new int[size]; + System.arraycopy(tileIds, off, indices, 0, size); + } + } + + public int id() { + return (int) state.physicalEntry.getKey().getSequenceNumber(); + } + + public long streamId() { + return streamId; + } + + public int size() { + return size; + } + + public int index(int slot) { + if(slot < 0 || slot >= size) + throw new IndexOutOfBoundsException("Invalid pack slot: " + slot); + return indices == null ? firstIndex + slot : indices[slot]; + } + } + + public static final class PackLease implements AutoCloseable { + private final OOCPackedCache owner; + private final PackGroup group; + private final MemoryAllowance allowance; + private boolean open; + + private PackLease(OOCPackedCache owner, PackGroup group, MemoryAllowance allowance) { + this.owner = owner; + this.group = group; + this.allowance = allowance; + open = true; + } + + public PackGroup group() { + return group; + } + + public int size() { + return group.size(); + } + + public int index(int slot) { + return group.index(slot); + } + + public Object value(int slot) { + if(!open) + throw new IllegalStateException("Pack lease is closed"); + PackedBlock block = (PackedBlock) group.state.physicalEntry.getData(); + return block.values[slot]; + } + + @Override + public void close() { + if(!open) + return; + open = false; + group.state.unpin(owner, owner._packReleaseDelayMs, allowance); + } + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackBuilder.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackBuilder.java new file mode 100644 index 00000000000..9a5f998e17d --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackBuilder.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache.packed; + +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.OOCCache; +import org.apache.sysds.runtime.ooc.cache.OOCCacheImpl; +import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +final class PackBuilder { + final int streamSlot; + final MemoryAllowance allowance; + final long packTargetBytes; + final List deferredUnpins = new ArrayList<>(); + long[] streamIds = new long[16]; + long[] tileIds = new long[16]; + private Object[] values = new Object[16]; + long[] sizes = new long[16]; + int[] refCounts = new int[16]; + long bytes; + int count; + int activePins; + boolean sealed; + boolean sealScheduled; + PackedPinState state; + private boolean _producerTransferred; + + PackBuilder(int streamSlot, MemoryAllowance allowance, long packTargetBytes) { + this.streamSlot = streamSlot; + this.allowance = allowance; + this.packTargetBytes = packTargetBytes; + } + + int append(long streamId, long tileId, Object value, long size) { + ensureCapacity(count + 1); + int slot = count++; + streamIds[slot] = streamId; + tileIds[slot] = tileId; + values[slot] = value; + sizes[slot] = size; + refCounts[slot] = 1; + bytes += size; + activePins++; + return slot; + } + + int retainSlot(int slot) { + int references = refCounts[slot]; + if(references <= 0) + throw new IllegalStateException("Cannot retain a forgotten packed location."); + return refCounts[slot] = references + 1; + } + + int releaseSlot(int slot) { + int references = refCounts[slot]; + if(references <= 0) + return 0; + return refCounts[slot] = references - 1; + } + + int countLiveSlots() { + int live = 0; + for(int i = 0; i < count; i++) + if(refCounts[i] > 0) + live++; + return live; + } + + long getBytes() { + return bytes; + } + + PackedBlock createBlock() { + return new PackedBlock(Arrays.copyOf(values, count), Arrays.copyOf(sizes, count), bytes); + } + + PackedUnpinHandle unpinProducer(BlockEntry entry, int slot, MemoryAllowance owner) { + activePins--; + PackedUnpinHandle handle = PackedUnpinHandle.pendingProducerTransfer(entry, owner, sizes[slot]); + deferredUnpins.add(handle); + return handle; + } + + void transferProducerOwnership(OOCCacheImpl physical) { + if(state == null || physical == null || _producerTransferred) + return; + _producerTransferred = true; + OOCCache.UnpinHandle physicalUnpin = physical.unpin(state.physicalEntry, allowance); + if(physicalUnpin.isCommitted()) { + completeDeferredUnpins(true); + return; + } + physicalUnpin.getCompletionFuture() + .whenComplete((committed, ex) -> completeDeferredUnpins(ex == null && committed)); + } + + private void ensureCapacity(int minSize) { + if(minSize <= values.length) + return; + int len = values.length; + while(minSize > len) + len <<= 1; + streamIds = Arrays.copyOf(streamIds, len); + tileIds = Arrays.copyOf(tileIds, len); + values = Arrays.copyOf(values, len); + sizes = Arrays.copyOf(sizes, len); + refCounts = Arrays.copyOf(refCounts, len); + } + + private void completeDeferredUnpins(boolean committed) { + for(PackedUnpinHandle handle : deferredUnpins) + handle.complete(committed); + deferredUnpins.clear(); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedBlock.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedBlock.java new file mode 100644 index 00000000000..5727e2eea63 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedBlock.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache.packed; + +import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; +import org.apache.sysds.runtime.ooc.cache.io.SpillableObjectRegistry; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +public final class PackedBlock implements SpillableObject { + Object[] values; + long[] sizes; + long totalSize; + + public PackedBlock() { + values = null; + sizes = null; + totalSize = 0; + } + + PackedBlock(Object[] values, long[] sizes, long totalSize) { + this.values = values; + this.sizes = sizes; + this.totalSize = totalSize; + } + + @Override + public boolean tryWrite(DataOutput out) throws IOException { + out.writeInt(values.length); + for(int i = 0; i < values.length; i++) { + out.writeLong(sizes[i]); + Object value = values[i]; + if(!(value instanceof SpillableObject spillable)) + return false; + if(!SpillableObjectRegistry.tryWrite(out, spillable)) + return false; + } + return true; + } + + @Override + public void read(DataInput in) throws IOException { + int count = in.readInt(); + values = new Object[count]; + sizes = new long[count]; + totalSize = 0; + for(int i = 0; i < count; i++) { + sizes[i] = in.readLong(); + values[i] = SpillableObjectRegistry.read(in); + totalSize += sizes[i]; + } + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedCacheLocation.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedCacheLocation.java new file mode 100644 index 00000000000..20f429cb0bd --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedCacheLocation.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache.packed; + +interface PackedCacheLocation { +} + +record PendingPackLocation(PackBuilder builder, int slot) implements PackedCacheLocation { +} + +final class SealedPackLocation implements PackedCacheLocation { + private final PackedPinState _state; + private final int _slot; + private int _references; + + SealedPackLocation(PackedPinState state, int slot) { + this(state, slot, 1); + } + + SealedPackLocation(PackedPinState state, int slot, int references) { + if(references <= 0) + throw new IllegalArgumentException("Sealed location requires a positive reference count."); + _state = state; + _slot = slot; + _references = references; + } + + PackedPinState state() { + return _state; + } + + int slot() { + return _slot; + } + + synchronized int retain() { + if(_references <= 0) + throw new IllegalStateException("Cannot retain a forgotten packed location."); + return ++_references; + } + + synchronized int release() { + if(_references <= 0) { + // tolerated for legacy double-forget callers; assertion surfaces it in debug runs + assert false : "Packed location slot " + _slot + " dereferenced below zero."; + return 0; + } + return --_references; + } +} + +record PendingLogicalPin(PackBuilder builder, int slot) { +} + +record PackedLogicalPin(SealedPackLocation location) { +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedPinState.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedPinState.java new file mode 100644 index 00000000000..43b94c8e63e --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedPinState.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache.packed; + +import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.OOCCache; +import org.apache.sysds.runtime.ooc.cache.OOCCacheImpl; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; + +import java.util.ArrayList; +import java.util.concurrent.TimeUnit; + +final class PackedPinState { + final BlockEntry physicalEntry; + final OOCPackedCache.PackGroup group; + private MemoryAllowance[] _allowances; + private int[] _counts; + private OOCFuture[] _futures; + private long[] _releaseDueNanos; + private PackedUnpinHandle[] _releaseHandles; + private int _size; + private boolean _releaseQueued; + private int _liveLocations; + + @SuppressWarnings("unchecked") + PackedPinState(BlockEntry physicalEntry, long streamId, int[] tileIds, int off, int count, int liveLocations) { + this.physicalEntry = physicalEntry; + group = new OOCPackedCache.PackGroup(this, streamId, tileIds, off, count); + this._liveLocations = liveLocations; + _allowances = new MemoryAllowance[2]; + _counts = new int[2]; + _futures = new OOCFuture[2]; + _releaseDueNanos = new long[2]; + _releaseHandles = new PackedUnpinHandle[2]; + } + + synchronized OOCFuture pin(OOCCacheImpl physical, MemoryAllowance allowance, boolean liveOnly) { + int ix = indexOf(allowance); + if(ix >= 0) { + cancelRelease(ix); + _counts[ix]++; + return _futures[ix]; + } + OOCFuture future = liveOnly ? OOCFuture.completed( + physical.pinIfLive(physicalEntry.getKey().getStreamId(), physicalEntry.getKey().getSequenceNumber(), + allowance)) : physical.pin(physicalEntry.getKey(), allowance); + addAllowance(allowance, future); + future.whenComplete((entry, ex) -> { + if(entry == null || ex != null) + removeFailedAllowance(allowance, future); + }); + return future; + } + + synchronized OOCFuture pinAdmitted(OOCCacheImpl physical, MemoryAllowance allowance) { + int ix = indexOf(allowance); + if(ix >= 0) { + cancelRelease(ix); + _counts[ix]++; + return _futures[ix]; + } + OOCFuture future = physical.pinAdmitted(physicalEntry.getKey(), allowance); + addAllowance(allowance, future); + future.whenComplete((entry, ex) -> { + if(entry == null || ex != null) + removeFailedAllowance(allowance, future); + }); + return future; + } + + BlockEntry pinIfLive(OOCCacheImpl physical, MemoryAllowance allowance) { + try { + return pin(physical, allowance, true).getNow(null); + } + catch(RuntimeException ex) { + return null; + } + } + + synchronized OOCCache.UnpinHandle unpin(OOCPackedCache owner, long releaseDelayMs, MemoryAllowance allowance) { + int ix = indexOf(allowance); + if(ix < 0) + return PackedUnpinHandle.committed(physicalEntry, allowance, physicalEntry.getSize()); + _counts[ix]--; + if(_counts[ix] > 0) + return PackedUnpinHandle.committed(physicalEntry, allowance, physicalEntry.getSize()); + PackedUnpinHandle handle = PackedUnpinHandle.delayedPhysicalRelease(physicalEntry, allowance); + _releaseHandles[ix] = handle; + _releaseDueNanos[ix] = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(Math.max(0, releaseDelayMs)); + owner.enqueueRelease(this); + return handle; + } + + long releaseDuePins(OOCCacheImpl physical, long nowNanos) { + ArrayList due = null; + long nextDueNanos = Long.MAX_VALUE; + synchronized(this) { + for(int i = 0; i < _size;) { + PackedUnpinHandle handle = _releaseHandles[i]; + if(handle == null || _counts[i] > 0) { + i++; + continue; + } + long dueNanos = _releaseDueNanos[i]; + if(dueNanos > nowNanos) { + nextDueNanos = Math.min(nextDueNanos, dueNanos); + i++; + continue; + } + if(due == null) + due = new ArrayList<>(); + due.add(new PackedRelease(_allowances[i], handle)); + removeAt(i); + } + } + if(due != null) + for(PackedRelease release : due) + releasePhysicalPin(physical, release.allowance, release.handle); + return nextDueNanos; + } + + synchronized boolean markReleaseQueued() { + if(_releaseQueued) + return false; + _releaseQueued = true; + return true; + } + + synchronized void clearReleaseQueued() { + _releaseQueued = false; + } + + synchronized boolean forgetLocation() { + if(_liveLocations <= 0) + return false; + return --_liveLocations == 0; + } + + private int indexOf(MemoryAllowance allowance) { + for(int i = 0; i < _size; i++) + if(_allowances[i] == allowance) + return i; + return -1; + } + + private synchronized void addAllowance(MemoryAllowance allowance, OOCFuture future) { + if(_size == _allowances.length) + grow(); + _allowances[_size] = allowance; + _counts[_size] = 1; + _futures[_size] = future; + _size++; + } + + @SuppressWarnings("unchecked") + private void grow() { + int nextSize = _size * 2; + MemoryAllowance[] biggerAllowances = new MemoryAllowance[nextSize]; + int[] biggerCounts = new int[nextSize]; + OOCFuture[] biggerFutures = new OOCFuture[nextSize]; + long[] biggerReleaseDueNanos = new long[nextSize]; + PackedUnpinHandle[] biggerReleaseHandles = new PackedUnpinHandle[nextSize]; + System.arraycopy(_allowances, 0, biggerAllowances, 0, _size); + System.arraycopy(_counts, 0, biggerCounts, 0, _size); + System.arraycopy(_futures, 0, biggerFutures, 0, _size); + System.arraycopy(_releaseDueNanos, 0, biggerReleaseDueNanos, 0, _size); + System.arraycopy(_releaseHandles, 0, biggerReleaseHandles, 0, _size); + _allowances = biggerAllowances; + _counts = biggerCounts; + _futures = biggerFutures; + _releaseDueNanos = biggerReleaseDueNanos; + _releaseHandles = biggerReleaseHandles; + } + + private void cancelRelease(int ix) { + _releaseDueNanos[ix] = 0; + PackedUnpinHandle handle = _releaseHandles[ix]; + if(handle != null) { + _releaseHandles[ix] = null; + handle.complete(false); + } + } + + private void releasePhysicalPin(OOCCacheImpl physical, MemoryAllowance allowance, PackedUnpinHandle handle) { + OOCCache.UnpinHandle physicalHandle = physical.unpin(physicalEntry, allowance); + if(physicalHandle.isCommitted()) { + handle.complete(true); + return; + } + physicalHandle.getCompletionFuture() + .whenComplete((committed, ex) -> handle.complete(ex == null && Boolean.TRUE.equals(committed))); + } + + private synchronized void removeFailedAllowance(MemoryAllowance allowance, OOCFuture future) { + int ix = indexOf(allowance); + if(ix >= 0 && _futures[ix] == future) + removeAt(ix); + } + + private void removeAt(int ix) { + int last = --_size; + _allowances[ix] = _allowances[last]; + _counts[ix] = _counts[last]; + _futures[ix] = _futures[last]; + _releaseDueNanos[ix] = _releaseDueNanos[last]; + _releaseHandles[ix] = _releaseHandles[last]; + _allowances[last] = null; + _counts[last] = 0; + _futures[last] = null; + _releaseDueNanos[last] = 0; + _releaseHandles[last] = null; + } + + private record PackedRelease(MemoryAllowance allowance, PackedUnpinHandle handle) { + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedUnpinHandle.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedUnpinHandle.java new file mode 100644 index 00000000000..cb6037dc1fe --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedUnpinHandle.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.cache.packed; + +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.OOCCache; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; + +final class PackedUnpinHandle implements OOCCache.UnpinHandle { + final BlockEntry entry; + final MemoryAllowance allowance; + final long bytes; + final OOCFuture future; + + static PackedUnpinHandle committed(BlockEntry entry, MemoryAllowance allowance, long bytes) { + return new PackedUnpinHandle(entry, allowance, bytes, true); + } + + static PackedUnpinHandle pendingProducerTransfer(BlockEntry entry, MemoryAllowance allowance, long bytes) { + return new PackedUnpinHandle(entry, allowance, bytes, false); + } + + static PackedUnpinHandle delayedPhysicalRelease(BlockEntry entry, MemoryAllowance allowance) { + return new PackedUnpinHandle(entry, allowance, entry.getSize(), false); + } + + private PackedUnpinHandle(BlockEntry entry, MemoryAllowance allowance, long bytes, boolean committed) { + this.entry = entry; + this.allowance = allowance; + this.bytes = bytes; + future = committed ? OOCFuture.completed(true) : new OOCFuture<>(); + } + + @Override + public BlockEntry entry() { + return entry; + } + + @Override + public MemoryAllowance allowance() { + return allowance; + } + + @Override + public long bytes() { + return bytes; + } + + @Override + public boolean isCommitted() { + return Boolean.TRUE.equals(future.getNow(false)); + } + + @Override + public OOCFuture getCompletionFuture() { + return future; + } + + void complete(boolean committed) { + future.complete(committed); + } +} diff --git a/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCCacheImplTest.java b/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCCacheImplTest.java index 283cebe4667..5219c12b241 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCCacheImplTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCCacheImplTest.java @@ -19,22 +19,18 @@ package org.apache.sysds.test.component.ooc.cache; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; +import static org.apache.sysds.test.component.ooc.cache.OOCCacheTestUtils.await; + import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BooleanSupplier; import org.apache.sysds.runtime.ooc.cache.BlockEntry; import org.apache.sysds.runtime.ooc.cache.BlockKey; import org.apache.sysds.runtime.ooc.cache.OOCCache; import org.apache.sysds.runtime.ooc.cache.OOCCacheImpl; -import org.apache.sysds.runtime.ooc.cache.OOCFuture; -import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; import org.apache.sysds.runtime.ooc.memory.GlobalMemoryBroker; import org.apache.sysds.runtime.ooc.memory.SyncMemoryAllowance; +import org.apache.sysds.test.component.ooc.cache.OOCCacheTestUtils.RecordingOOCIOHandler; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -46,7 +42,7 @@ public class OOCCacheImplTest { private static final long BYTES = 1_000; private static final long WAIT_TIMEOUT_SEC = 10; - private RecordingIOHandler _io; + private RecordingOOCIOHandler _io; private GlobalMemoryBroker _broker; private SyncMemoryAllowance _producer; private SyncMemoryAllowance _reader; @@ -54,7 +50,7 @@ public class OOCCacheImplTest { @Before public void setUp() { - _io = new RecordingIOHandler(); + _io = new RecordingOOCIOHandler(); _broker = new GlobalMemoryBroker(8 * BYTES); _producer = new SyncMemoryAllowance(_broker, 4 * BYTES); _reader = new SyncMemoryAllowance(_broker, 4 * BYTES); @@ -92,7 +88,7 @@ public void testResidentPinTransfersOwnershipBetweenCacheAndAllowance() throws E Assert.assertEquals(0, _cache.getOwnedCacheSize()); Assert.assertEquals(BYTES, _producer.getUsedMemory()); - await(_cache.unpin(entry, _producer)); + await(_cache.unpin(entry, _producer), WAIT_TIMEOUT_SEC); Assert.assertEquals(BYTES, _cache.getOwnedCacheSize()); Assert.assertEquals(0, _producer.getUsedMemory()); @@ -103,7 +99,7 @@ public void testResidentPinTransfersOwnershipBetweenCacheAndAllowance() throws E Assert.assertEquals(BYTES, _reader.getUsedMemory()); Assert.assertEquals(0, _io.readCount()); - await(_cache.unpin(pinned, _reader)); + await(_cache.unpin(pinned, _reader), WAIT_TIMEOUT_SEC); Assert.assertEquals(BYTES, _cache.getOwnedCacheSize()); Assert.assertEquals(0, _reader.getUsedMemory()); } @@ -116,8 +112,8 @@ public void testPinReloadsColdBackedEntry() throws Exception { _producer.reserveBlocking(BYTES); BlockEntry entry = _cache.putPinned(key, payload, BYTES, _producer); - await(_cache.unpin(entry, _producer)); - waitFor(() -> _io.evictionCount() == 1 && BlockEntryTestAccess.getDataUnsafe(entry) == null); + await(_cache.unpin(entry, _producer), WAIT_TIMEOUT_SEC); + await(() -> _io.evictionCount() == 1 && BlockEntryTestAccess.getDataUnsafe(entry) == null, WAIT_TIMEOUT_SEC); Assert.assertEquals(0, _producer.getUsedMemory()); BlockEntry pinned = _cache.pin(key, _reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); @@ -128,7 +124,7 @@ public void testPinReloadsColdBackedEntry() throws Exception { Assert.assertEquals(1, _io.readCount()); Assert.assertEquals(BYTES, _reader.getUsedMemory()); - await(_cache.unpin(pinned, _reader)); + await(_cache.unpin(pinned, _reader), WAIT_TIMEOUT_SEC); Assert.assertEquals(0, _reader.getUsedMemory()); } @@ -140,8 +136,8 @@ public void testPinIfLiveDoesNotReadColdBackedEntry() throws Exception { _producer.reserveBlocking(BYTES); BlockEntry entry = _cache.putPinned(key, payload, BYTES, _producer); - await(_cache.unpin(entry, _producer)); - waitFor(() -> _io.evictionCount() == 1 && BlockEntryTestAccess.getDataUnsafe(entry) == null); + await(_cache.unpin(entry, _producer), WAIT_TIMEOUT_SEC); + await(() -> _io.evictionCount() == 1 && BlockEntryTestAccess.getDataUnsafe(entry) == null, WAIT_TIMEOUT_SEC); BlockEntry pinned = _cache.pinIfLive(STREAM_ID, BLOCK_ID, _reader); @@ -203,7 +199,7 @@ public void testDereferenceRemovesEntryAfterLastUnpin() throws Exception { BlockEntry entry = _cache.putPinned(key, "drop", BYTES, _producer); Assert.assertEquals(0, _cache.dereference(entry)); - await(_cache.unpin(entry, _producer)); + await(_cache.unpin(entry, _producer), WAIT_TIMEOUT_SEC); Assert.assertEquals(0, _producer.getUsedMemory()); Assert.assertNull(_cache.pin(key, _reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS)); @@ -217,8 +213,8 @@ public void testBackingReadFailureReleasesReservedBytes() throws Exception { _producer.reserveBlocking(BYTES); BlockEntry entry = _cache.putPinned(key, "fail-read", BYTES, _producer); - await(_cache.unpin(entry, _producer)); - waitFor(() -> _io.evictionCount() == 1 && BlockEntryTestAccess.getDataUnsafe(entry) == null); + await(_cache.unpin(entry, _producer), WAIT_TIMEOUT_SEC); + await(() -> _io.evictionCount() == 1 && BlockEntryTestAccess.getDataUnsafe(entry) == null, WAIT_TIMEOUT_SEC); _io.failReads(true); try { @@ -244,91 +240,4 @@ private void useZeroHardLimitCache() { _io.reset(); _cache = new OOCCacheImpl(_io, 0, 0); } - - private static void await(OOCCache.UnpinHandle handle) throws Exception { - if(!handle.isCommitted()) - handle.getCompletionFuture().get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); - } - - private static void waitFor(BooleanSupplier condition) throws Exception { - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(WAIT_TIMEOUT_SEC); - while(!condition.getAsBoolean() && System.nanoTime() < deadline) - Thread.sleep(1); - Assert.assertTrue(condition.getAsBoolean()); - } - - private static final class RecordingIOHandler implements OOCIOHandler { - private final Map _spilled = new ConcurrentHashMap<>(); - private final AtomicInteger _evictions = new AtomicInteger(); - private final AtomicInteger _reads = new AtomicInteger(); - private volatile boolean _failReads; - - @Override - public void shutdown() { - _spilled.clear(); - } - - @Override - public CompletableFuture scheduleEviction(BlockEntry block) { - _spilled.put(block.getKey(), BlockEntryTestAccess.getDataUnsafe(block)); - _evictions.incrementAndGet(); - return CompletableFuture.completedFuture(null); - } - - @Override - public OOCFuture scheduleRead(BlockEntry block) { - _reads.incrementAndGet(); - if(_failReads) - return OOCFuture.failed(new IllegalStateException("read failed")); - Object data = _spilled.get(block.getKey()); - if(data == null) - return OOCFuture.completed(null); - BlockEntryTestAccess.setDataUnsafe(block, data); - return OOCFuture.completed(block); - } - - @Override - public void prioritizeRead(BlockKey key, double priority) { - } - - @Override - public CompletableFuture scheduleDeletion(BlockEntry block) { - _spilled.remove(block.getKey()); - return CompletableFuture.completedFuture(true); - } - - @Override - public void registerSourceLocation(BlockKey key, SourceBlockDescriptor descriptor) { - } - - @Override - public CompletableFuture scheduleSourceRead(SourceReadRequest request) { - return CompletableFuture.failedFuture(new UnsupportedOperationException()); - } - - @Override - public CompletableFuture continueSourceRead(SourceReadContinuation continuation, - long maxBytesInFlight) { - return CompletableFuture.failedFuture(new UnsupportedOperationException()); - } - - private int evictionCount() { - return _evictions.get(); - } - - private int readCount() { - return _reads.get(); - } - - private void failReads(boolean failReads) { - _failReads = failReads; - } - - private void reset() { - _spilled.clear(); - _evictions.set(0); - _reads.set(0); - _failReads = false; - } - } } diff --git a/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCCacheTestUtils.java b/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCCacheTestUtils.java new file mode 100644 index 00000000000..8fee37646e9 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCCacheTestUtils.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.ooc.cache; + +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.BlockKey; +import org.apache.sysds.runtime.ooc.cache.OOCCache; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; +import org.apache.sysds.runtime.ooc.memory.SyncMemoryAllowance; +import org.junit.Assert; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BooleanSupplier; + +public class OOCCacheTestUtils { + + public static void await(OOCCache.UnpinHandle handle, long timeout) throws Exception { + if(!handle.isCommitted()) + handle.getCompletionFuture().get(timeout, TimeUnit.SECONDS); + } + + public static void await(BooleanSupplier condition, long timeout) throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeout); + while(!condition.getAsBoolean() && System.nanoTime() < deadline) + Thread.sleep(1); + Assert.assertTrue(condition.getAsBoolean()); + } + + public static void awaitUsedMemory(SyncMemoryAllowance allowance, long expected, long timeout) throws Exception { + await(() -> allowance.getUsedMemory() == expected, timeout); + } + + public static class RecordingOOCIOHandler implements OOCIOHandler { + private final Map _spilled = new ConcurrentHashMap<>(); + private final AtomicInteger _evictions = new AtomicInteger(); + private final AtomicInteger _reads = new AtomicInteger(); + private volatile boolean _failReads; + + @Override + public void shutdown() { + _spilled.clear(); + } + + @Override + public CompletableFuture scheduleEviction(BlockEntry block) { + _spilled.put(block.getKey(), BlockEntryTestAccess.getDataUnsafe(block)); + _evictions.incrementAndGet(); + return CompletableFuture.completedFuture(null); + } + + @Override + public OOCFuture scheduleRead(BlockEntry block) { + _reads.incrementAndGet(); + if(_failReads) + return OOCFuture.failed(new RuntimeException("Injected read failure")); + Object data = _spilled.get(block.getKey()); + if(data == null) + return OOCFuture.completed(null); + BlockEntryTestAccess.setDataUnsafe(block, data); + return OOCFuture.completed(block); + } + + @Override + public void prioritizeRead(BlockKey key, double priority) { + } + + @Override + public CompletableFuture scheduleDeletion(BlockEntry block) { + _spilled.remove(block.getKey()); + return CompletableFuture.completedFuture(true); + } + + @Override + public void registerSourceLocation(BlockKey key, OOCIOHandler.SourceBlockDescriptor descriptor) { + } + + @Override + public CompletableFuture scheduleSourceRead( + OOCIOHandler.SourceReadRequest request) { + return CompletableFuture.failedFuture(new UnsupportedOperationException()); + } + + @Override + public CompletableFuture continueSourceRead( + OOCIOHandler.SourceReadContinuation continuation, long maxBytesInFlight) { + return CompletableFuture.failedFuture(new UnsupportedOperationException()); + } + + public int evictionCount() { + return _evictions.get(); + } + + public int readCount() { + return _reads.get(); + } + + public void failReads(boolean failReads) { + _failReads = failReads; + } + + public void reset() { + _spilled.clear(); + _evictions.set(0); + _reads.set(0); + _failReads = false; + } + } +} diff --git a/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCPackedCacheTest.java b/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCPackedCacheTest.java new file mode 100644 index 00000000000..7a29f8781b4 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCPackedCacheTest.java @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.ooc.cache; + +import static org.apache.sysds.test.component.ooc.cache.OOCCacheTestUtils.await; +import static org.apache.sysds.test.component.ooc.cache.OOCCacheTestUtils.awaitUsedMemory; + +import java.util.concurrent.TimeUnit; + +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.BlockKey; +import org.apache.sysds.runtime.ooc.cache.OOCCache; +import org.apache.sysds.runtime.ooc.cache.OOCCacheImpl; +import org.apache.sysds.runtime.ooc.cache.io.OOCMatrixIOHandler; +import org.apache.sysds.runtime.ooc.cache.packed.OOCPackedCache; +import org.apache.sysds.runtime.ooc.memory.GlobalMemoryBroker; +import org.apache.sysds.runtime.ooc.memory.SyncMemoryAllowance; +import org.apache.sysds.test.component.ooc.cache.OOCCacheTestUtils.RecordingOOCIOHandler; +import org.junit.Assert; +import org.junit.Test; + +public class OOCPackedCacheTest { + private static final long STREAM_ID = 41; + private static final long BYTES = 1000; + private static final long WAIT_TIMEOUT_SEC = 10; + + @Test + public void testSmallTilesShareOnePhysicalPack() throws Exception { + GlobalMemoryBroker broker = new GlobalMemoryBroker(1L << 32); + SyncMemoryAllowance producer = new SyncMemoryAllowance(broker); + producer.setTargetMemory(1L << 30); + SyncMemoryAllowance reader = new SyncMemoryAllowance(broker); + reader.setTargetMemory(1L << 30); + OOCPackedCache cache = new OOCPackedCache(new OOCCacheImpl(new OOCMatrixIOHandler(), 1L << 30, 1L << 30), + 2 * BYTES, 10 * BYTES, -1, 0); + try { + BlockEntry[] entries = publishSmallTiles(cache, producer, STREAM_ID, 3); + unpinAndFlush(cache, producer, entries); + awaitUsedMemory(producer, 0, WAIT_TIMEOUT_SEC); + + Assert.assertEquals(1, cache.getPackGroupCount()); + OOCPackedCache.PackGroup group = cache.getPackGroup(STREAM_ID, 0); + Assert.assertNotNull(group); + Assert.assertEquals(3, group.size()); + + BlockEntry first = cache.pin(STREAM_ID, 0, reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + BlockEntry second = cache.pin(STREAM_ID, 1, reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + Assert.assertEquals(1.0, scalar(first), 0.0); + Assert.assertEquals(2.0, scalar(second), 0.0); + Assert.assertEquals("Multiple logical pins in one pack should charge the physical pack once.", 3 * BYTES, + reader.getUsedMemory()); + + await(cache.unpin(first, reader), WAIT_TIMEOUT_SEC); + Assert.assertEquals("The physical pack stays pinned while another logical pin remains.", 3 * BYTES, + reader.getUsedMemory()); + await(cache.unpin(second, reader), WAIT_TIMEOUT_SEC); + awaitUsedMemory(reader, 0, WAIT_TIMEOUT_SEC); + } + finally { + cache.shutdown(); + producer.destroy(); + reader.destroy(); + } + } + + @Test + public void testPutPackPinnedPacksSmallTilesAndBypassesLargeTile() throws Exception { + GlobalMemoryBroker broker = new GlobalMemoryBroker(1L << 32); + SyncMemoryAllowance producer = new SyncMemoryAllowance(broker); + producer.setTargetMemory(1L << 30); + SyncMemoryAllowance reader = new SyncMemoryAllowance(broker); + reader.setTargetMemory(1L << 30); + OOCPackedCache cache = new OOCPackedCache(new OOCCacheImpl(new OOCMatrixIOHandler(), 1L << 30, 1L << 30), + 2 * BYTES, 10 * BYTES, -1, 0); + try { + long largeBytes = 2 * BYTES; + long[] tileIds = new long[] {0, 1, 2}; + Object[] values = new Object[] {value(3.0), value(9.0), value(5.0)}; + long[] sizes = new long[] {BYTES, largeBytes, BYTES}; + producer.reserveBlocking(2 * BYTES + largeBytes); + BlockEntry[] entries = cache.putPackPinned(STREAM_ID, tileIds, values, sizes, 0, tileIds.length, producer); + unpinAndFlush(cache, producer, entries); + awaitUsedMemory(producer, 0, WAIT_TIMEOUT_SEC); + + OOCPackedCache.PackGroup group = cache.getPackGroup(STREAM_ID, 0); + Assert.assertNotNull(group); + Assert.assertEquals(2, group.size()); + Assert.assertEquals(0, group.index(0)); + Assert.assertEquals(2, group.index(1)); + Assert.assertNull("Large tiles in putPackPinned should bypass packing.", cache.getPackGroup(STREAM_ID, 1)); + + BlockEntry packed = cache.pin(STREAM_ID, 2, reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + BlockEntry large = cache.pin(STREAM_ID, 1, reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + Assert.assertEquals(5.0, scalar(packed), 0.0); + Assert.assertEquals(9.0, scalar(large), 0.0); + Assert.assertEquals(2 * BYTES + largeBytes, reader.getUsedMemory()); + + await(cache.unpin(packed, reader), WAIT_TIMEOUT_SEC); + await(cache.unpin(large, reader), WAIT_TIMEOUT_SEC); + awaitUsedMemory(reader, 0, WAIT_TIMEOUT_SEC); + } + finally { + cache.shutdown(); + producer.destroy(); + reader.destroy(); + } + } + + @Test + public void testPinAdmittedReusesPackedPhysicalPin() throws Exception { + GlobalMemoryBroker broker = new GlobalMemoryBroker(1L << 32); + SyncMemoryAllowance producer = new SyncMemoryAllowance(broker); + producer.setTargetMemory(1L << 30); + SyncMemoryAllowance reader = new SyncMemoryAllowance(broker); + reader.setTargetMemory(1L << 30); + OOCPackedCache cache = new OOCPackedCache(new OOCCacheImpl(new OOCMatrixIOHandler(), 1L << 30, 1L << 30), + 2 * BYTES, 10 * BYTES, -1, 1000); + try { + BlockEntry[] entries = publishSmallTiles(cache, producer, STREAM_ID, 2); + unpinAndFlush(cache, producer, entries); + awaitUsedMemory(producer, 0, WAIT_TIMEOUT_SEC); + + BlockEntry first = cache.pinAdmitted(STREAM_ID, 0, reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + Assert.assertNotNull(first); + Assert.assertEquals(1.0, scalar(first), 0.0); + Assert.assertEquals(2 * BYTES, reader.getUsedMemory()); + + OOCCache.UnpinHandle delayed = cache.unpin(first, reader); + Assert.assertFalse(delayed.isCommitted()); + Assert.assertEquals("Delayed packed release keeps the physical pack charged.", 2 * BYTES, + reader.getUsedMemory()); + + BlockEntry second = cache.pinAdmitted(STREAM_ID, 1, reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + Assert.assertNotNull(second); + Assert.assertEquals(2.0, scalar(second), 0.0); + Assert.assertFalse("Re-pinning with the same allowance should cancel the pending physical release.", + delayed.getCompletionFuture().get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS)); + Assert.assertEquals(2 * BYTES, reader.getUsedMemory()); + + await(cache.unpin(second, reader), WAIT_TIMEOUT_SEC); + awaitUsedMemory(reader, 0, WAIT_TIMEOUT_SEC); + } + finally { + cache.shutdown(); + producer.destroy(); + reader.destroy(); + } + } + + @Test + public void testReferenceAndDereferencePackedLocations() throws Exception { + GlobalMemoryBroker broker = new GlobalMemoryBroker(1L << 32); + SyncMemoryAllowance producer = new SyncMemoryAllowance(broker); + producer.setTargetMemory(1L << 30); + SyncMemoryAllowance reader = new SyncMemoryAllowance(broker); + reader.setTargetMemory(1L << 30); + OOCPackedCache cache = new OOCPackedCache(new OOCCacheImpl(new OOCMatrixIOHandler(), 1L << 30, 1L << 30), + 2 * BYTES, 10 * BYTES, -1, 0); + try { + producer.reserveBlocking(BYTES); + BlockEntry pending = cache.putPinned(STREAM_ID, 0, value(13.0), BYTES, producer); + Assert.assertEquals(2, cache.reference(pending)); + Assert.assertEquals(1, cache.dereference(pending)); + + unpinAndFlush(cache, producer, new BlockEntry[] {pending}); + awaitUsedMemory(producer, 0, WAIT_TIMEOUT_SEC); + + BlockEntry pinned = cache.pin(STREAM_ID, 0, reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + Assert.assertNotNull(pinned); + Assert.assertEquals(13.0, scalar(pinned), 0.0); + Assert.assertEquals(2, cache.reference(pinned)); + Assert.assertEquals(1, cache.dereference(pinned)); + Assert.assertEquals(0, cache.dereference(new BlockKey(STREAM_ID, 0))); + + await(cache.unpin(pinned, reader), WAIT_TIMEOUT_SEC); + awaitUsedMemory(reader, 0, WAIT_TIMEOUT_SEC); + Assert.assertNull(cache.pin(STREAM_ID, 0, reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS)); + } + finally { + cache.shutdown(); + producer.destroy(); + reader.destroy(); + } + } + + @Test + public void testLargeBlockBypassesPacking() throws Exception { + GlobalMemoryBroker broker = new GlobalMemoryBroker(1L << 32); + SyncMemoryAllowance producer = new SyncMemoryAllowance(broker); + producer.setTargetMemory(1L << 30); + SyncMemoryAllowance reader = new SyncMemoryAllowance(broker); + reader.setTargetMemory(1L << 30); + OOCPackedCache cache = new OOCPackedCache(new OOCCacheImpl(new OOCMatrixIOHandler(), 1L << 30, 1L << 30), + 2 * BYTES, 10 * BYTES, -1, 0); + long largeBytes = 2 * BYTES; + try { + producer.reserveBlocking(largeBytes); + BlockEntry entry = cache.putPinned(STREAM_ID, 0, value(5.0), largeBytes, producer); + await(cache.unpin(entry, producer), WAIT_TIMEOUT_SEC); + + Assert.assertEquals(0, cache.getPackGroupCount()); + Assert.assertNull(cache.getPackGroup(STREAM_ID, 0)); + + BlockEntry pinned = cache.pin(STREAM_ID, 0, reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + Assert.assertNotNull(pinned); + Assert.assertEquals(5.0, scalar(pinned), 0.0); + Assert.assertEquals(largeBytes, reader.getUsedMemory()); + await(cache.unpin(pinned, reader), WAIT_TIMEOUT_SEC); + awaitUsedMemory(reader, 0, WAIT_TIMEOUT_SEC); + } + finally { + cache.shutdown(); + producer.destroy(); + reader.destroy(); + } + } + + @Test + public void testPinPackExposesWholePack() throws Exception { + GlobalMemoryBroker broker = new GlobalMemoryBroker(1L << 32); + SyncMemoryAllowance producer = new SyncMemoryAllowance(broker); + producer.setTargetMemory(1L << 30); + SyncMemoryAllowance reader = new SyncMemoryAllowance(broker); + reader.setTargetMemory(1L << 30); + OOCPackedCache cache = new OOCPackedCache(new OOCCacheImpl(new OOCMatrixIOHandler(), 1L << 30, 1L << 30), + 2 * BYTES, 10 * BYTES, -1, 0); + try { + long[] tileIds = new long[] {2, 5}; + Object[] values = new Object[] {value(7.0), value(11.0)}; + long[] sizes = new long[] {BYTES, BYTES}; + producer.reserveBlocking(2 * BYTES); + BlockEntry physical = cache.putSealedPackPinned(STREAM_ID, tileIds, values, sizes, 0, tileIds.length, + producer); + await(cache.unpin(physical, producer), WAIT_TIMEOUT_SEC); + awaitUsedMemory(producer, 0, WAIT_TIMEOUT_SEC); + + OOCPackedCache.PackGroup group = cache.getPackGroup(STREAM_ID, 5); + Assert.assertNotNull(group); + Assert.assertEquals(2, group.size()); + Assert.assertEquals(2, group.index(0)); + Assert.assertEquals(5, group.index(1)); + + OOCPackedCache.PackLease lease = cache.pinPack(group, reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + Assert.assertNotNull(lease); + try(lease) { + Assert.assertEquals(7.0, scalar((IndexedMatrixValue) lease.value(0)), 0.0); + Assert.assertEquals(11.0, scalar((IndexedMatrixValue) lease.value(1)), 0.0); + Assert.assertEquals(2 * BYTES, reader.getUsedMemory()); + } + awaitUsedMemory(reader, 0, WAIT_TIMEOUT_SEC); + } + finally { + cache.shutdown(); + producer.destroy(); + reader.destroy(); + } + } + + @Test + public void testLogicalEvictionPolicyScoresPackedBlocks() throws Exception { + RecordingOOCIOHandler io = new RecordingOOCIOHandler(); + GlobalMemoryBroker broker = new GlobalMemoryBroker(1L << 32); + SyncMemoryAllowance producer = new SyncMemoryAllowance(broker); + producer.setTargetMemory(1L << 30); + SyncMemoryAllowance reader = new SyncMemoryAllowance(broker); + reader.setTargetMemory(1L << 30); + OOCPackedCache cache = new OOCPackedCache(new OOCCacheImpl(io, 4 * BYTES, 2 * BYTES), 2 * BYTES, 2 * BYTES, -1, + 0); + try { + cache.addEvictionPolicy(STREAM_ID, tileId -> tileId < 2 ? 100 : 0); + BlockEntry[] entries = publishSmallTiles(cache, producer, STREAM_ID, 4); + unpinAndFlush(cache, producer, entries); + awaitUsedMemory(producer, 0, WAIT_TIMEOUT_SEC); + await(() -> io.evictionCount() == 1 && cache.getOwnedCacheSize() == 2 * BYTES, WAIT_TIMEOUT_SEC); + + int readsBefore = io.readCount(); + BlockEntry retained = cache.pin(STREAM_ID, 2, reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + Assert.assertNotNull(retained); + Assert.assertEquals(3.0, scalar(retained), 0.0); + Assert.assertEquals("The lower-scored packed group should remain resident.", readsBefore, io.readCount()); + await(cache.unpin(retained, reader), WAIT_TIMEOUT_SEC); + awaitUsedMemory(reader, 0, WAIT_TIMEOUT_SEC); + + readsBefore = io.readCount(); + BlockEntry evicted = cache.pin(STREAM_ID, 0, reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + Assert.assertNotNull(evicted); + Assert.assertEquals(1.0, scalar(evicted), 0.0); + Assert.assertTrue("The higher-scored packed group should be evicted first.", io.readCount() > readsBefore); + await(cache.unpin(evicted, reader), WAIT_TIMEOUT_SEC); + awaitUsedMemory(reader, 0, WAIT_TIMEOUT_SEC); + } + finally { + cache.shutdown(); + producer.destroy(); + reader.destroy(); + } + } + + @Test + public void testEvictedPackReplaysThroughLogicalPin() throws Exception { + RecordingOOCIOHandler io = new RecordingOOCIOHandler(); + GlobalMemoryBroker broker = new GlobalMemoryBroker(1L << 32); + SyncMemoryAllowance producer = new SyncMemoryAllowance(broker); + producer.setTargetMemory(1L << 30); + SyncMemoryAllowance reader = new SyncMemoryAllowance(broker); + reader.setTargetMemory(1L << 30); + OOCPackedCache cache = new OOCPackedCache(new OOCCacheImpl(io, 4 * BYTES, 0), 2 * BYTES, 10 * BYTES, -1, 0); + try { + BlockEntry[] entries = publishSmallTiles(cache, producer, STREAM_ID, 4); + unpinAndFlush(cache, producer, entries); + awaitUsedMemory(producer, 0, WAIT_TIMEOUT_SEC); + await(() -> io.evictionCount() > 0 && cache.getOwnedCacheSize() == 0, WAIT_TIMEOUT_SEC); + + int readsBefore = io.readCount(); + BlockEntry pinned = cache.pin(STREAM_ID, 3, reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); + + Assert.assertNotNull(pinned); + Assert.assertEquals(4.0, scalar(pinned), 0.0); + Assert.assertTrue("Pinning an evicted logical tile should read the physical pack.", + io.readCount() > readsBefore); + Assert.assertEquals(4 * BYTES, reader.getUsedMemory()); + + await(cache.unpin(pinned, reader), WAIT_TIMEOUT_SEC); + awaitUsedMemory(reader, 0, WAIT_TIMEOUT_SEC); + } + finally { + cache.shutdown(); + producer.destroy(); + reader.destroy(); + } + } + + private static BlockEntry[] publishSmallTiles(OOCPackedCache cache, SyncMemoryAllowance producer, long streamId, + int count) { + BlockEntry[] entries = new BlockEntry[count]; + for(int i = 0; i < count; i++) { + producer.reserveBlocking(BYTES); + entries[i] = cache.putPinned(streamId, i, value(i + 1.0), BYTES, producer); + } + return entries; + } + + private static void unpinAndFlush(OOCPackedCache cache, SyncMemoryAllowance producer, BlockEntry[] entries) + throws Exception { + OOCCache.UnpinHandle[] handles = new OOCCache.UnpinHandle[entries.length]; + for(int i = 0; i < entries.length; i++) + handles[i] = cache.unpin(entries[i], producer); + cache.flushPacks(); + for(OOCCache.UnpinHandle handle : handles) + await(handle, WAIT_TIMEOUT_SEC); + } + + private static IndexedMatrixValue value(double scalar) { + return new IndexedMatrixValue(new MatrixIndexes(1, 1), new MatrixBlock(1, 1, scalar)); + } + + private static double scalar(BlockEntry entry) { + return scalar((IndexedMatrixValue) entry.getData()); + } + + private static double scalar(IndexedMatrixValue value) { + return value.getValue().get(0, 0); + } +} From 3e184e9401963fa8ab0823c2f45220c2d29aa287 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:27:01 +0200 Subject: [PATCH 068/132] [OOC] Adapt OOCStream to Forward QueueCallbacks (#2537) --- .../runtime/instructions/ooc/OOCStream.java | 4 + .../instructions/ooc/PlaybackStream.java | 37 ++++-- .../ooc/SubscribableTaskQueue.java | 115 +++++++++++------- .../runtime/ooc/stream/FilteredOOCStream.java | 25 +++- .../runtime/ooc/stream/MergedOOCStream.java | 19 ++- .../ooc/stream/SplittingOOCStream.java | 10 ++ .../runtime/ooc/stream/SubOOCStream.java | 15 ++- 7 files changed, 166 insertions(+), 59 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStream.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStream.java index ce53e5f0949..b4ffbbbaedb 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStream.java @@ -30,8 +30,12 @@ static QueueCallback eos(DMLRuntimeException e) { void enqueue(T t); + void enqueue(QueueCallback callback); + T dequeue(); + QueueCallback dequeueCB(); + void closeInput(); void propagateFailure(DMLRuntimeException re); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/PlaybackStream.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/PlaybackStream.java index 6a67c6602b6..3de438bf17b 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/PlaybackStream.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/PlaybackStream.java @@ -52,6 +52,11 @@ public void enqueue(IndexedMatrixValue t) { throw new DMLRuntimeException("Cannot enqueue to a playback stream"); } + @Override + public void enqueue(QueueCallback callback) { + throw new DMLRuntimeException("Cannot enqueue to a playback stream"); + } + @Override public void closeInput() { throw new DMLRuntimeException("Cannot close a playback stream"); @@ -59,11 +64,11 @@ public void closeInput() { @Override public synchronized IndexedMatrixValue dequeue() { - if (_subscriberSet.get()) + if(_subscriberSet.get()) throw new IllegalStateException("Cannot dequeue from a playback stream if a subscriber has been set"); try { - if (_lastDequeue != null) + if(_lastDequeue != null) _lastDequeue.close(); _lastDequeue = _streamCache.get(_streamIdx.getAndIncrement()).get(); return _lastDequeue.get(); @@ -72,6 +77,22 @@ public synchronized IndexedMatrixValue dequeue() { } } + @Override + public synchronized QueueCallback dequeueCB() { + if(_subscriberSet.get()) + throw new IllegalStateException("Cannot dequeue from a playback stream if a subscriber has been set"); + + try { + if(_lastDequeue != null) + _lastDequeue.close(); + _lastDequeue = _streamCache.get(_streamIdx.getAndIncrement()).get(); + return _lastDequeue; + } + catch(InterruptedException | ExecutionException e) { + throw new DMLRuntimeException(e); + } + } + @Override public OOCStream getReadStream() { return _streamCache.getReadStream(); @@ -114,9 +135,9 @@ public void messageDownstream(OOCStreamMessage msg) { if(msg.isCancelled()) return; CopyOnWriteArrayList> relays = _downstreamRelays; - if (relays != null) { - for (Consumer relay : relays) { - if (msg.isCancelled()) + if(relays != null) { + for(Consumer relay : relays) { + if(msg.isCancelled()) break; relay.accept(msg); } @@ -125,7 +146,7 @@ public void messageDownstream(OOCStreamMessage msg) { @Override public void setSubscriber(Consumer> subscriber) { - if (!_subscriberSet.compareAndSet(false, true)) + if(!_subscriberSet.compareAndSet(false, true)) throw new IllegalArgumentException("Subscriber cannot be set multiple times"); _streamCache.setSubscriber(subscriber, false); @@ -163,10 +184,10 @@ public void addUpstreamMessageRelay(Consumer relay) { @Override public void addDownstreamMessageRelay(Consumer relay) { - if (relay == null) + if(relay == null) throw new IllegalArgumentException("Cannot set downstream relay to null"); CopyOnWriteArrayList> relays = _downstreamRelays; - if (relays == null) { + if(relays == null) { synchronized(this) { if (_downstreamRelays == null) _downstreamRelays = new CopyOnWriteArrayList<>(); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/SubscribableTaskQueue.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/SubscribableTaskQueue.java index a2030f5a4a4..67746b64257 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/SubscribableTaskQueue.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/SubscribableTaskQueue.java @@ -35,11 +35,12 @@ import java.util.function.BiFunction; import java.util.function.Consumer; -public class SubscribableTaskQueue extends LocalTaskQueue implements OOCStream { +public class SubscribableTaskQueue extends LocalTaskQueue> implements OOCStream { private final AtomicInteger _availableCtr = new AtomicInteger(1); private final AtomicBoolean _closed = new AtomicBoolean(false); private final AtomicInteger _blockCount = new AtomicInteger(0); + private QueueCallback _lastDequeued = null; private CacheableData _cdata; private volatile Consumer> _subscriber = null; private volatile CopyOnWriteArrayList> _upstreamMsgRelays = null; @@ -48,7 +49,7 @@ public class SubscribableTaskQueue extends LocalTaskQueue implements OOCSt private String _watchdogId; public SubscribableTaskQueue() { - if (OOCWatchdog.WATCH) { + if(OOCWatchdog.WATCH) { _watchdogId = "STQ-" + hashCode(); // Capture a short context to help identify origin OOCWatchdog.registerOpen(_watchdogId, "SubscribableTaskQueue@" + hashCode(), getCtxMsg(), this); @@ -71,12 +72,16 @@ private String getCtxMsg() { @Override public void enqueue(T t) { - if (t == NO_MORE_TASKS) - throw new DMLRuntimeException("Cannot enqueue NO_MORE_TASKS item"); + enqueue(new SimpleQueueCallback<>(t, _failure)); + } + @Override + public void enqueue(QueueCallback cb) { + if(cb == NO_MORE_TASKS) + throw new DMLRuntimeException("Cannot enqueue NO_MORE_TASKS item"); int cnt = _availableCtr.incrementAndGet(); - if (cnt <= 1) { // Then the queue was already closed and we disallow further enqueues + if(cnt <= 1) { // Then the queue was already closed and we disallow further enqueues _availableCtr.decrementAndGet(); // Undo increment throw new DMLRuntimeException("Cannot enqueue into closed SubscribableTaskQueue"); } @@ -86,17 +91,17 @@ public void enqueue(T t) { Consumer> s = _subscriber; final Consumer> fS = s; - if (fS != null) { - fS.accept(new SimpleQueueCallback<>(t, _failure)); + if(fS != null) { + fS.accept(cb); onDeliveryFinished(); return; } - synchronized (this) { + synchronized(this) { // Re-check that subscriber is really null to avoid race conditions - if (_subscriber == null) { + if(_subscriber == null) { try { - super.enqueueTask(t); + super.enqueueTask(cb); } catch(InterruptedException e) { throw new DMLRuntimeException(e); @@ -108,16 +113,16 @@ public void enqueue(T t) { } // Last case if due to race a subscriber has been set - s.accept(new SimpleQueueCallback<>(t, _failure)); + s.accept(cb); onDeliveryFinished(); } protected boolean tryDeliverCallback(QueueCallback cb, int blockCount) { Consumer> s = _subscriber; - if (s == null) + if(s == null) return false; int cnt = _availableCtr.incrementAndGet(); - if (cnt <= 1) { // Then the queue was already closed and we disallow further enqueues + if(cnt <= 1) { // Then the queue was already closed and we disallow further enqueues _availableCtr.decrementAndGet(); // Undo increment throw new DMLRuntimeException("Cannot enqueue into closed SubscribableTaskQueue"); } @@ -128,19 +133,47 @@ protected boolean tryDeliverCallback(QueueCallback cb, int blockCount) { } @Override - public synchronized void enqueueTask(T t) { + public synchronized void enqueueTask(OOCStream.QueueCallback t) { enqueue(t); } @Override public T dequeue() { try { - if (OOCWatchdog.WATCH) + if(OOCWatchdog.WATCH) + OOCWatchdog.addEvent(_watchdogId, "dequeue -- " + getCtxMsg()); + if(_lastDequeued != null) { + _lastDequeued.close(); + _lastDequeued = null; + } + OOCStream.QueueCallback deq = super.dequeueTask(); + if(deq != NO_MORE_TASKS) { + onDeliveryFinished(); + _lastDequeued = deq; + return deq.get(); + } + return null; + } + catch(InterruptedException e) { + throw new DMLRuntimeException(e); + } + } + + @Override + public OOCStream.QueueCallback dequeueCB() { + try { + if(OOCWatchdog.WATCH) OOCWatchdog.addEvent(_watchdogId, "dequeue -- " + getCtxMsg()); - T deq = super.dequeueTask(); - if (deq != NO_MORE_TASKS) + if(_lastDequeued != null) { + _lastDequeued.close(); + _lastDequeued = null; + } + OOCStream.QueueCallback deq = super.dequeueTask(); + if(deq != NO_MORE_TASKS) { onDeliveryFinished(); - return deq; + _lastDequeued = deq; + } + return deq == NO_MORE_TASKS ? null : deq; } catch(InterruptedException e) { throw new DMLRuntimeException(e); @@ -148,29 +181,30 @@ public T dequeue() { } @Override - public synchronized T dequeueTask() { - return dequeue(); + public synchronized OOCStream.QueueCallback dequeueTask() { + return dequeueCB(); } @Override public synchronized void closeInput() { - if (_closed.compareAndSet(false, true)) { + if(_closed.compareAndSet(false, true)) { super.closeInput(); onDeliveryFinished(); _upstreamMsgRelays = null; _downstreamMsgRelays = null; - } else { + } + else { throw new IllegalStateException("Multiple close input calls"); } } private void validateBlockCountOnClose() { DataCharacteristics dc = getDataCharacteristics(); - if (dc != null && dc.dimsKnown() && dc.getBlocksize() > 0) { + if(dc != null && dc.dimsKnown() && dc.getBlocksize() > 0) { long expected = OOCUtils.getNumBlocks(dc); - if (expected >= 0 && _blockCount.get() != expected) { - throw new DMLRuntimeException("OOCStream block count mismatch: expected " - + expected + " but saw " + _blockCount.get() + " (" + dc.getRows() + "x" + dc.getCols() + ")"); + if(expected >= 0 && _blockCount.get() != expected) { + throw new DMLRuntimeException("OOCStream block count mismatch: expected " + expected + " but saw " + + _blockCount.get() + " (" + dc.getRows() + "x" + dc.getCols() + ")"); } } } @@ -180,7 +214,7 @@ public void setSubscriber(Consumer> subscriber) { if(subscriber == null) throw new IllegalArgumentException("Cannot set subscriber to null"); - LinkedList data; + LinkedList> data; boolean needsEos; synchronized(this) { @@ -198,8 +232,8 @@ public void setSubscriber(Consumer> subscriber) { _availableCtr.incrementAndGet(); // route terminal emission via onDeliveryFinished } - for (T t : data) { - subscriber.accept(new SimpleQueueCallback<>(t, _failure)); + for(QueueCallback t : data) { + subscriber.accept(t); onDeliveryFinished(); } @@ -207,17 +241,16 @@ public void setSubscriber(Consumer> subscriber) { onDeliveryFinished(); } - @SuppressWarnings("unchecked") private void onDeliveryFinished() { int ctr = _availableCtr.decrementAndGet(); - if (ctr == 0) { + if(ctr == 0) { validateBlockCountOnClose(); Consumer> s = _subscriber; - if (s != null) - s.accept(new SimpleQueueCallback<>((T) LocalTaskQueue.NO_MORE_TASKS, _failure)); + if(s != null) + s.accept(OOCStream.eos(_failure)); - if (OOCWatchdog.WATCH) + if(OOCWatchdog.WATCH) OOCWatchdog.registerClose(_watchdogId); } } @@ -248,17 +281,17 @@ public void messageUpstream(OOCStreamMessage msg) { if(msg.isCancelled()) return; msg.addIXTransform(_ixTransform); - if (msg.isCancelled()) + if(msg.isCancelled()) return; - if (msg instanceof OOCGetStreamTypeMessage) { - if (_cdata != null) + if(msg instanceof OOCGetStreamTypeMessage) { + if(_cdata != null) ((OOCGetStreamTypeMessage) msg).setInMemoryType(); return; } CopyOnWriteArrayList> relays = _upstreamMsgRelays; if(relays != null) { - for (Consumer relay : relays) { - if (msg.isCancelled()) + for(Consumer relay : relays) { + if(msg.isCancelled()) break; relay.accept(msg); } @@ -272,8 +305,8 @@ public void messageDownstream(OOCStreamMessage msg) { msg.addIXTransform(_ixTransform); CopyOnWriteArrayList> relays = _downstreamMsgRelays; if(relays != null) { - for (Consumer relay : relays) { - if (msg.isCancelled()) + for(Consumer relay : relays) { + if(msg.isCancelled()) break; relay.accept(msg); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/FilteredOOCStream.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/FilteredOOCStream.java index 4ad28ae6162..f7f57744390 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/FilteredOOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/FilteredOOCStream.java @@ -35,6 +35,7 @@ public class FilteredOOCStream implements OOCStream { private final OOCStream _sourceStream; private final Function _predicate; private CacheableData _data; + private QueueCallback _last; public FilteredOOCStream(OOCStream sourceStream, Function predicate) { _sourceStream = sourceStream; @@ -46,12 +47,26 @@ public void enqueue(T t) { _sourceStream.enqueue(t); } + @Override + public void enqueue(QueueCallback callback) { + _sourceStream.enqueue(callback); + } + @Override public synchronized T dequeue() { - T next; - while((next = _sourceStream.dequeue()) != null) { - if(_predicate.apply(next)) - return next; + QueueCallback cb = dequeueCB(); + return cb == null ? null : cb.get(); + } + + @Override + public synchronized QueueCallback dequeueCB() { + if(_last != null) + _last.close(); + while((_last = _sourceStream.dequeueCB()) != null) { + if(_predicate.apply(_last.get())) + return _last; + _last.close(); + _last = null; } return null; } @@ -101,6 +116,8 @@ public void setSubscriber(Consumer> subscriber) { if(_predicate.apply(cb.get())) subscriber.accept(cb); + else + cb.close(); }); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/MergedOOCStream.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/MergedOOCStream.java index 8ea13842177..51e38c8cce9 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/MergedOOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/MergedOOCStream.java @@ -37,7 +37,7 @@ public class MergedOOCStream implements OOCStream { private final List> _sources; - private final SubscribableTaskQueue> _taskQueue; + private final SubscribableTaskQueue _taskQueue; private final AtomicInteger _openSources; private final AtomicBoolean _failed; private final CachingStream _sharedCache; @@ -132,16 +132,29 @@ public void enqueue(T t) { throw new UnsupportedOperationException(); } + @Override + public void enqueue(QueueCallback callback) { + throw new UnsupportedOperationException(); + } + @Override public synchronized T dequeue() { if(_last != null) _last.close(); - _last = _taskQueue.dequeue(); + _last = _taskQueue.dequeueCB(); if(_last == null) return null; return _last.get(); } + @Override + public synchronized QueueCallback dequeueCB() { + if(_last != null) + _last.close(); + _last = _taskQueue.dequeueCB(); + return _last; + } + @Override public void closeInput() { throw new UnsupportedOperationException(); @@ -183,7 +196,7 @@ public void setSubscriber(Consumer> subscriber) { } return; } - subscriber.accept(cb.get()); + subscriber.accept(cb); }); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/SplittingOOCStream.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/SplittingOOCStream.java index fb19611c253..b7fd9a1d1a4 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/SplittingOOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/SplittingOOCStream.java @@ -110,11 +110,21 @@ public void enqueue(T t) { throw new UnsupportedOperationException(); } + @Override + public void enqueue(QueueCallback callback) { + throw new UnsupportedOperationException(); + } + @Override public T dequeue() { throw new UnsupportedOperationException(); } + @Override + public QueueCallback dequeueCB() { + throw new UnsupportedOperationException(); + } + @Override public void closeInput() { throw new UnsupportedOperationException(); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/SubOOCStream.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/SubOOCStream.java index e5908c18b04..9a231654c26 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/SubOOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/SubOOCStream.java @@ -33,7 +33,7 @@ public class SubOOCStream implements OOCStream { private OOCStream _sourceStream; - private SubscribableTaskQueue> _taskQueue; + private SubscribableTaskQueue _taskQueue; private QueueCallback _last; public SubOOCStream(OOCStream sourceStream) { @@ -42,6 +42,7 @@ public SubOOCStream(OOCStream sourceStream) { _taskQueue.setUpstreamMessageRelay(_sourceStream::messageUpstream); } + @Override public void enqueue(QueueCallback callback) { _taskQueue.enqueue(callback); } @@ -55,12 +56,20 @@ public void enqueue(T t) { public synchronized T dequeue() { if(_last != null) _last.close(); - _last = _taskQueue.dequeue(); + _last = _taskQueue.dequeueCB(); if(_last != null) return _last.get(); return null; } + @Override + public synchronized QueueCallback dequeueCB() { + if(_last != null) + _last.close(); + _last = _taskQueue.dequeueCB(); + return _last; + } + @Override public void closeInput() { _taskQueue.closeInput(); @@ -98,7 +107,7 @@ public void setSubscriber(Consumer> subscriber) { } } else - subscriber.accept(cb.get()); + subscriber.accept(cb); }); } From 2de79f5225d5dd6f0f30b1f8c05033d216892716 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 12 Jul 2026 22:05:15 +0200 Subject: [PATCH 069/132] Bump actions/setup-python from 5 to 6 (#2536) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/javaCodestyle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/javaCodestyle.yml b/.github/workflows/javaCodestyle.yml index a63e3298760..39593e64563 100644 --- a/.github/workflows/javaCodestyle.yml +++ b/.github/workflows/javaCodestyle.yml @@ -93,7 +93,7 @@ jobs: cache: 'maven' - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.11' From f0043b6096373ad4e3be89f3cc46b75b67fa404d Mon Sep 17 00:00:00 2001 From: Jakob-al28 <149481651+Jakob-al28@users.noreply.github.com> Date: Sun, 12 Jul 2026 22:38:03 +0200 Subject: [PATCH 070/132] [SYSTEMDS-3949] Column API parquet decode for Delta frame reads (#2535) Upgrade the underlying Parquet readers for the Delta file format to use the Column API parquet decode. --- .../sysds/runtime/io/DeltaKernelUtils.java | 267 +++++++++++++++++ .../sysds/runtime/io/FrameReaderDelta.java | 83 +++--- .../runtime/io/FrameReaderDeltaParallel.java | 24 +- .../component/io/DeltaFrameReadWriteTest.java | 218 ++++++++++++++ .../io/DeltaFrameShapeCoverageTest.java | 205 +++++++++++++ .../io/DeltaFrameSparkContractTest.java | 271 ++++++++++++++++++ 6 files changed, 1014 insertions(+), 54 deletions(-) create mode 100644 src/test/java/org/apache/sysds/test/component/io/DeltaFrameShapeCoverageTest.java create mode 100644 src/test/java/org/apache/sysds/test/component/io/DeltaFrameSparkContractTest.java diff --git a/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java index bbca857a1cd..c3b9351d3d3 100644 --- a/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java +++ b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java @@ -22,7 +22,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.function.Function; @@ -30,6 +32,19 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.ColumnReader; +import org.apache.parquet.column.impl.ColumnReadStoreImpl; +import org.apache.parquet.column.page.PageReadStore; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.metadata.FileMetaData; +import org.apache.parquet.hadoop.util.HadoopInputFile; +import org.apache.parquet.io.api.Converter; +import org.apache.parquet.io.api.GroupConverter; +import org.apache.parquet.io.api.PrimitiveConverter; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.apache.parquet.schema.Type.Repetition; import org.apache.sysds.conf.ConfigurationManager; import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.runtime.DMLRuntimeException; @@ -105,6 +120,14 @@ public class DeltaKernelUtils { public static final int T_BOOLEAN = 6; public static final int T_STRING = 7; + /** + * Parquet physical type each {@code T_*} column is stored as, indexed by type code (delta int/short/byte columns + * are all stored as annotated parquet INT32). + */ + private static final PrimitiveTypeName[] T_PHYSICAL = {PrimitiveTypeName.DOUBLE, PrimitiveTypeName.FLOAT, + PrimitiveTypeName.INT64, PrimitiveTypeName.INT32, PrimitiveTypeName.INT32, PrimitiveTypeName.INT32, + PrimitiveTypeName.BOOLEAN, PrimitiveTypeName.BINARY}; + //derived configuration cached to avoid copying the (large) base conf on every //engine creation (createEngine is called once per data file in parallel reads); //rebuilt whenever the base conf or the relevant SystemDS settings change. @@ -162,6 +185,250 @@ public static int countSelected(int size, boolean[] selected) { return n; } + // ------------------------------------------ + // direct parquet decode of Delta data files + // ------------------------------------------ + + /** Physical-schema metadata key carrying the parquet field id (column mapping mode {@code id}). */ + private static final String PARQUET_FIELD_ID_KEY = "parquet.field.id"; + + /** + * Whether data files can be decoded directly into pre-allocated output columns: the physical read schema must be a + * positional 1:1 image of the logical schema, i.e. no partition columns (not stored in the data files, spliced back + * in by the kernel) and no kernel metadata columns such as {@code row_index} (only requested for deletion-vector + * reads). Deletion vectors themselves are excluded separately via the exact-row-count check. + * + * @param logicalSchema the table's logical schema + * @param physicalSchema the physical read schema from the scan state + * @return true if data files can be decoded directly + */ + public static boolean supportsDirectDecode(StructType logicalSchema, StructType physicalSchema) { + if(physicalSchema.length() != logicalSchema.length()) + return false; + for(int c = 0; c < physicalSchema.length(); c++) + if(physicalSchema.at(c).isMetadataColumn()) + return false; + return true; + } + + /** @param scanFileRow a scan-file row @return the fully-qualified path of its data file */ + public static String dataFilePath(Row scanFileRow) { + return InternalScanFileUtils.getAddFileStatus(scanFileRow).getPath(); + } + + /** + * Thrown when a data file's parquet layout cannot be decoded directly into the typed output columns, e.g. a + * physical type narrower than the Delta column type (left behind by type widening). Raised before anything is + * written to the output arrays, so callers can re-read the file through the kernel engine (which performs those + * conversions) instead. + */ + public static final class UnsupportedDirectDecodeException extends DMLRuntimeException { + private static final long serialVersionUID = 1L; + + public UnsupportedDirectDecodeException(String msg) { + super(msg); + } + } + + /** + * Decode one Delta data file into pre-allocated typed column arrays at the given absolute row offset, through + * parquet-mr's column API ({@link ColumnReadStoreImpl}/{@link ColumnReader}) with no kernel engine or intermediate + * batch vectors in the path. Columns are resolved by parquet field id first (column mapping mode {@code id}) and + * physical name second; columns absent from the file (schema evolution) keep the array defaults (0 for numerics, + * null for strings), matching the kernel-path null semantics. + * + * @param filePath fully-qualified path of the parquet data file + * @param physicalSchema physical read schema (positionally 1:1 with the output columns) + * @param readCodes per-column type codes (see the {@code T_*} constants) + * @param dest pre-allocated per-column backing arrays + * @param destOff absolute row offset of this file's first row + * @param limit exclusive upper row bound of this file's slice + * @param tablePath table path for error messages + * @return the number of rows decoded + * @throws IOException on read failure + * @throws UnsupportedDirectDecodeException if a column's parquet layout does not match the Delta column type + * (thrown before any output is written, so the caller can fall back to the + * kernel for this file) + */ + public static int decodeDataFileInto(String filePath, StructType physicalSchema, int[] readCodes, Object[] dest, + int destOff, int limit, String tablePath) throws IOException { + final Configuration conf = ConfigurationManager.getCachedJobConf(); + final int ncol = physicalSchema.length(); + int off = destOff; + try(ParquetFileReader reader = ParquetFileReader.open(HadoopInputFile.fromPath(new Path(filePath), conf))) { + FileMetaData meta = reader.getFooter().getFileMetaData(); + MessageType parquetSchema = meta.getSchema(); + String createdBy = meta.getCreatedBy(); + String[] colNames = resolveParquetColumns(physicalSchema, parquetSchema); + // validate every column before decoding anything: a file whose physical types + // do not match the Delta schema (type widening) must be left to the kernel + final ColumnDescriptor[] descs = new ColumnDescriptor[ncol]; + for(int c = 0; c < ncol; c++) + if(colNames[c] != null) // absent columns keep the array defaults (nulls) + descs[c] = validateDecodable(parquetSchema, colNames[c], readCodes[c], filePath); + GroupConverter root = dummyConverter(parquetSchema.getFieldCount()); + PageReadStore pages; + while((pages = reader.readNextRowGroup()) != null) { + int nrow = (int) pages.getRowCount(); + checkSliceLimit(off, nrow, limit, tablePath); + ColumnReadStoreImpl store = new ColumnReadStoreImpl(pages, root, parquetSchema, createdBy); + for(int c = 0; c < ncol; c++) { + if(descs[c] == null) + continue; + decodeColumnInto(store.getColumnReader(descs[c]), descs[c].getMaxDefinitionLevel(), nrow, + readCodes[c], dest[c], off); + } + off += nrow; + } + } + return off - destOff; + } + + /** + * Guard before writing {@code n} rows at {@code off}: writing past {@code limit} would overflow into the next + * file's slice (or off the array) in the pre-allocated output. Shared by the direct decode and the per-file kernel + * fallback so the check and its message live in one place. + */ + static void checkSliceLimit(int off, int n, int limit, String tablePath) { + if(off + n > limit) + throw new DMLRuntimeException( + "Delta file produced more rows than its numRecords statistic; refusing direct read of " + tablePath); + } + + /** + * Resolve the descriptor of one parquet column and verify it is decodable as the given read code: a non-repeated + * primitive whose physical type is the one the Delta column type is stored as. A mismatch (e.g. an INT32 data file + * under a schema widened to {@code bigint}, or a nested/repeated field where a primitive is expected) is readable + * through the kernel engine but not by the typed direct decode. + */ + private static ColumnDescriptor validateDecodable(MessageType parquetSchema, String colName, int readCode, + String filePath) { + org.apache.parquet.schema.Type t = parquetSchema.getType(colName); + if(!t.isPrimitive() || t.isRepetition(Repetition.REPEATED)) + throw new UnsupportedDirectDecodeException( + "Parquet column '" + colName + "' in " + filePath + " is not a non-repeated primitive"); + PrimitiveTypeName actual = t.asPrimitiveType().getPrimitiveTypeName(); + PrimitiveTypeName expected = T_PHYSICAL[readCode]; + if(actual != expected) + throw new UnsupportedDirectDecodeException("Parquet column '" + colName + "' in " + filePath + " stores " + + actual + " but the Delta schema requires " + expected + " (e.g. a type-widened table)"); + return parquetSchema.getColumnDescription(new String[] {colName}); + } + + /** + * Resolve each physical-schema column to the parquet column name of the given file: by parquet field id when the + * schema carries one, by name otherwise, or null when the file does not contain the column at all. + */ + private static String[] resolveParquetColumns(StructType schema, MessageType parquetSchema) { + Map idToName = new HashMap<>(); + Map names = new HashMap<>(); + for(int i = 0; i < parquetSchema.getFieldCount(); i++) { + org.apache.parquet.schema.Type t = parquetSchema.getType(i); + names.put(t.getName(), t.getName()); + if(t.getId() != null) + idToName.put(t.getId().intValue(), t.getName()); + } + String[] resolved = new String[schema.length()]; + for(int c = 0; c < schema.length(); c++) { + Object fid = schema.at(c).getMetadata().get(PARQUET_FIELD_ID_KEY); + String byId = (fid instanceof Number) ? idToName.get(((Number) fid).intValue()) : null; + resolved[c] = (byId != null) ? byId : names.get(schema.at(c).getName()); + } + return resolved; + } + + /** No-op converter tree; the column API requires one, but values are pulled via the typed getters. */ + private static GroupConverter dummyConverter(int nFields) { + final PrimitiveConverter[] leaves = new PrimitiveConverter[nFields]; + for(int i = 0; i < nFields; i++) + leaves[i] = new PrimitiveConverter() { + }; + return new GroupConverter() { + @Override + public Converter getConverter(int fieldIndex) { + return leaves[fieldIndex]; + } + + @Override + public void start() { + } + + @Override + public void end() { + } + }; + } + + /** + * Decode one parquet column of the current row group into a pre-allocated typed array at the given offset. Null + * cells (definition level below max) keep the array default (0 for numerics, null for strings). + */ + private static void decodeColumnInto(ColumnReader creader, int maxDef, int nrow, int readCode, Object dest, + int off) { + final int end = off + nrow; + switch(readCode) { + case T_DOUBLE: { + double[] a = (double[]) dest; + for(int r = off; r < end; r++) { + if(creader.getCurrentDefinitionLevel() == maxDef) + a[r] = creader.getDouble(); + creader.consume(); + } + break; + } + case T_FLOAT: { + float[] a = (float[]) dest; + for(int r = off; r < end; r++) { + if(creader.getCurrentDefinitionLevel() == maxDef) + a[r] = creader.getFloat(); + creader.consume(); + } + break; + } + case T_LONG: { + long[] a = (long[]) dest; + for(int r = off; r < end; r++) { + if(creader.getCurrentDefinitionLevel() == maxDef) + a[r] = creader.getLong(); + creader.consume(); + } + break; + } + case T_INT: + case T_SHORT: + case T_BYTE: { + // delta short/byte columns are stored as annotated parquet INT32 + int[] a = (int[]) dest; + for(int r = off; r < end; r++) { + if(creader.getCurrentDefinitionLevel() == maxDef) + a[r] = creader.getInteger(); + creader.consume(); + } + break; + } + case T_BOOLEAN: { + boolean[] a = (boolean[]) dest; + for(int r = off; r < end; r++) { + if(creader.getCurrentDefinitionLevel() == maxDef) + a[r] = creader.getBoolean(); + creader.consume(); + } + break; + } + case T_STRING: { + String[] a = (String[]) dest; + for(int r = off; r < end; r++) { + if(creader.getCurrentDefinitionLevel() == maxDef) + a[r] = creader.getBinary().toStringUsingUTF8(); + creader.consume(); + } + break; + } + default: + throw new DMLRuntimeException("Unsupported read code for direct decode: " + readCode); + } + } + /** Floor on the adaptive writer target file size. Below this the per-file metadata/open * overhead (and tiny-file proliferation) outweighs the extra read parallelism. */ public static final long ADAPTIVE_WRITER_MIN_FILE_SIZE = 4L * 1024 * 1024; diff --git a/src/main/java/org/apache/sysds/runtime/io/FrameReaderDelta.java b/src/main/java/org/apache/sysds/runtime/io/FrameReaderDelta.java index 9e8823f7ecf..0127250247d 100644 --- a/src/main/java/org/apache/sysds/runtime/io/FrameReaderDelta.java +++ b/src/main/java/org/apache/sysds/runtime/io/FrameReaderDelta.java @@ -36,9 +36,10 @@ /** * Single-threaded native Delta Lake reader for frames, built on the Spark-free Delta Kernel library. It opens the - * latest snapshot of a Delta table, reads its parquet data files through the kernel's default engine (honoring deletion - * vectors), and materializes the columns into a {@link FrameBlock} whose schema and column names are derived from the - * Delta table schema. + * latest snapshot of a Delta table through the kernel (log replay, schema, data-file listing) and decodes the parquet + * data files directly into pre-allocated columns via parquet-mr's column API; tables with deletion vectors, partition + * columns or missing row statistics are read through the kernel's default engine instead (which applies deletion + * vectors and splices partition values). Schema and column names are derived from the Delta table schema. * *

* Data is extracted column-at-a-time into primitive arrays (no per-cell boxing or {@code FrameBlock.set} dispatch) and @@ -92,7 +93,7 @@ protected FrameBlock readWithHandle(String fname, Engine engine, DeltaKernelUtil if(total == 0) return new FrameBlock(plan.vt, plan.cnames, 0); if(total <= Integer.MAX_VALUE) - return readDirect(fname, engine, handle, plan, (int) total); + return readDirect(fname, handle, plan, (int) total); } // fallback: row counts unknown or deletion vectors present -> decode into @@ -135,24 +136,27 @@ protected static ReadPlan planColumns(DeltaKernelUtils.ScanHandle handle) { } /** - * Whether the metadata-driven direct read fast path can be used for this table (exact per-file row counts and no - * deletion vectors, so the output can be pre-sized and each file decoded straight into its row offset). Visible for - * testing: the buffered fallback is otherwise only reachable for tables lacking row statistics or carrying deletion - * vectors, which the SystemDS Delta writer never produces. + * Whether the metadata-driven direct read fast path can be used for this table: exact per-file row counts (no + * deletion vectors), so the output can be pre-sized, and a physical read schema that maps 1:1 onto the output + * columns (no partition columns or kernel metadata columns to splice back in), so each data file can be decoded + * straight into its row offset without the kernel engine. The buffered kernel-path fallback covers everything else + * (deletion vectors, missing statistics, partitioned tables). * * @param handle the opened scan handle * @return true if the direct path is applicable */ protected boolean useDirectPath(DeltaKernelUtils.ScanHandle handle) { - return handle.hasExactRowCounts(); + return handle.hasExactRowCounts() && + DeltaKernelUtils.supportsDirectDecode(handle.schema, handle.physicalReadSchema); } /** - * Fast path: decode each data file straight into pre-sized typed column arrays at a metadata-derived row offset. - * One allocation per column, single pass, no intermediate per-batch buffers or serial concatenation. + * Fast path: decode each data file straight into pre-sized typed column arrays at a metadata-derived row offset, + * through parquet-mr's column API with no kernel engine or intermediate batch vectors in the path. One allocation + * per column, single pass, no per-batch buffers or serial concatenation. */ - private FrameBlock readDirect(String fname, Engine engine, DeltaKernelUtils.ScanHandle handle, ReadPlan plan, - int nrow) throws IOException { + private FrameBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, ReadPlan plan, int nrow) + throws IOException { final int ncol = plan.ncol; final int[] readCodes = plan.readCodes; final Object[] dest = new Object[ncol]; @@ -161,27 +165,8 @@ private FrameBlock readDirect(String fname, Engine engine, DeltaKernelUtils.Scan int base = 0; for(int i = 0; i < handle.scanFiles.size(); i++) { - // exclusive upper row bound for this file's slice; a file decoding more - // rows than its numRecords statistic would otherwise overflow into the - // next file's region or off the array final int limit = base + (int) handle.numRecords[i]; - final int[] cur = new int[] {base}; - DeltaKernelUtils.readScanFile(engine, handle.scanState, handle.physicalReadSchema, handle.scanFiles.get(i), - (cols, size, selected) -> { - int n = DeltaKernelUtils.countSelected(size, selected); - if(cur[0] + n > limit) - throw new DMLRuntimeException("Delta file produced more rows than its " - + "numRecords statistic; refusing direct read of " + fname); - for(int c = 0; c < ncol; c++) - extractColumnInto(cols[c], size, selected, readCodes[c], dest[c], cur[0]); - cur[0] += n; - }); - // also fail loud on underflow: a file decoding fewer rows than its - // numRecords statistic would leave the tail of the slice at the array - // default (0/null) while nrow still reports the (inflated) statistic. - if(cur[0] != limit) - throw new DMLRuntimeException("Delta file produced " + (cur[0] - base) + " rows, expected " - + (limit - base) + " from its numRecords statistic; refusing direct read of " + fname); + decodeFileSlice(handle, handle.scanFiles.get(i), readCodes, dest, base, limit, fname); base = limit; } @@ -259,6 +244,38 @@ static Array concatColumn(ValueType vt, int nrow, ArrayList batchCo return ArrayFactory.create(vt, full); } + /** + * Decode one data file into its pre-sized slice {@code [base, limit)} of the destination arrays: through the direct + * parquet column decode by default, or through the kernel engine (which performs the physical-type conversions the + * typed decode declines, e.g. for files left behind by type widening) as a per-file fallback. Fails loud when the + * file produces more or fewer rows than its {@code numRecords} statistic promised, so a lying statistic can neither + * overflow into the next file's slice nor leave a silent gap of array defaults. Thread-safe for distinct files, so + * the parallel reader shares it. + */ + static void decodeFileSlice(DeltaKernelUtils.ScanHandle handle, Row scanFileRow, int[] readCodes, Object[] dest, + int base, int limit, String fname) throws IOException { + int n; + try { + n = DeltaKernelUtils.decodeDataFileInto(DeltaKernelUtils.dataFilePath(scanFileRow), + handle.physicalReadSchema, readCodes, dest, base, limit, fname); + } + catch(DeltaKernelUtils.UnsupportedDirectDecodeException e) { + final int[] off = {base}; + DeltaKernelUtils.readScanFile(DeltaKernelUtils.createEngine(), handle.scanState, handle.physicalReadSchema, + scanFileRow, (cols, size, selected) -> { + int m = DeltaKernelUtils.countSelected(size, selected); + DeltaKernelUtils.checkSliceLimit(off[0], m, limit, fname); + for(int c = 0; c < readCodes.length; c++) + extractColumnInto(cols[c], size, selected, readCodes[c], dest[c], off[0]); + off[0] += m; + }); + n = off[0] - base; + } + if(base + n != limit) + throw new DMLRuntimeException("Delta file produced " + n + " rows, expected " + (limit - base) + + " from its numRecords statistic; refusing direct read of " + fname); + } + static int readCode(DataType dt, String name) { // reuse the shared Delta type -> code mapping; frames additionally reject the // types the matrix reader also cannot map (typeCode returns -1) diff --git a/src/main/java/org/apache/sysds/runtime/io/FrameReaderDeltaParallel.java b/src/main/java/org/apache/sysds/runtime/io/FrameReaderDeltaParallel.java index 106264afe6c..0e0625824e1 100644 --- a/src/main/java/org/apache/sysds/runtime/io/FrameReaderDeltaParallel.java +++ b/src/main/java/org/apache/sysds/runtime/io/FrameReaderDeltaParallel.java @@ -89,7 +89,8 @@ public FrameBlock readFrameFromHDFS(String fname, ValueType[] schema, String[] n /** * Fast path: each thread decodes one data file straight into the final typed column arrays at a metadata-derived - * row offset. Single allocation per column, fully parallel. + * row offset, through parquet-mr's column API with no kernel engine in the path (and hence no per-file engine + * creation). Single allocation per column, fully parallel. */ private FrameBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, ReadPlan plan, int nrow) throws IOException { @@ -112,28 +113,9 @@ private FrameBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, for(int i = 0; i < nfiles; i++) { final Row scanFileRow = handle.scanFiles.get(i); final int base = rowOffset[i]; - // exclusive upper row bound for this file's slice; a file decoding more - // rows than its numRecords statistic would otherwise overflow into the - // next file's region (concurrent overlapping writes) or off the array final int limit = base + (int) handle.numRecords[i]; tasks.add(() -> { - int[] cur = new int[] {base}; - Engine eng = DeltaKernelUtils.createEngine(); - DeltaKernelUtils.readScanFile(eng, handle.scanState, handle.physicalReadSchema, scanFileRow, - (cols, size, selected) -> { - int n = DeltaKernelUtils.countSelected(size, selected); - if(cur[0] + n > limit) - throw new DMLRuntimeException("Delta file produced more rows than its " - + "numRecords statistic; refusing parallel direct read of " + fname); - for(int c = 0; c < ncol; c++) - extractColumnInto(cols[c], size, selected, readCodes[c], dest[c], cur[0]); - cur[0] += n; - }); - // fail loud on underflow too: fewer decoded rows than the statistic - // would leave this slice's tail at the array default (0/null). - if(cur[0] != limit) - throw new DMLRuntimeException("Delta file produced " + (cur[0] - base) + " rows, expected " - + (limit - base) + " from its numRecords statistic; refusing parallel direct read of " + fname); + decodeFileSlice(handle, scanFileRow, readCodes, dest, base, limit, fname); return null; }); } diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaFrameReadWriteTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameReadWriteTest.java index 7012be44426..752ef190ca8 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaFrameReadWriteTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameReadWriteTest.java @@ -28,11 +28,18 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.NoSuchElementException; import java.util.Optional; import java.util.Random; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.util.HadoopInputFile; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; import org.apache.sysds.common.Types.FileFormat; import org.apache.sysds.common.Types.ValueType; import org.apache.sysds.conf.CompilerConfig; @@ -50,19 +57,28 @@ import org.apache.sysds.test.TestUtils; import org.junit.Test; +import io.delta.kernel.DataWriteContext; +import io.delta.kernel.Operation; +import io.delta.kernel.Table; +import io.delta.kernel.Transaction; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.FilteredColumnarBatch; +import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; +import io.delta.kernel.internal.util.Utils; import io.delta.kernel.types.ByteType; import io.delta.kernel.types.DataType; import io.delta.kernel.types.DateType; import io.delta.kernel.types.DoubleType; +import io.delta.kernel.types.IntegerType; import io.delta.kernel.types.LongType; import io.delta.kernel.types.ShortType; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructType; +import io.delta.kernel.utils.CloseableIterable; import io.delta.kernel.utils.CloseableIterator; +import io.delta.kernel.utils.DataFileStatus; /** * Direct (no DML) round-trip tests for the native Delta Kernel based frame reader/writer. Each test writes a FrameBlock @@ -451,6 +467,87 @@ public void readShortByteColumnsCoercedToInt32() throws Exception { } } + @Test + public void readTypeWidenedIntFilesAsLongColumn() throws Exception { + // what Delta type widening leaves behind: the schema declares bigint while an + // older data file still physically stores INT32. The direct decode must detect + // the narrower physical type and fall back to the kernel engine for that file + // only (the INT64 file stays on the direct path), in both readers. + long[] oldInts = {1, -2, 0, Integer.MAX_VALUE, Integer.MIN_VALUE}; + long[] newLongs = {10L, -20L, 5_000_000_000L, Long.MAX_VALUE, Long.MIN_VALUE}; + Path dir = Files.createTempDirectory("sysds_delta_frame_tw_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + writeWidenedLongTable(tablePath, oldInts, newLongs); + // pin the fixture: stats present (so the pre-sized direct path is what runs, + // not the buffered fallback) and exactly one file physically narrower than + // the schema (so the per-file kernel fallback really triggers) + DeltaKernelUtils.ScanHandle handle = DeltaKernelUtils.openScan(DeltaKernelUtils.createEngine(), + DeltaKernelUtils.qualify(tablePath)); + assertTrue("fixture must carry exact row counts", handle.hasExactRowCounts()); + assertEquals("fixture must span two data files", 2, handle.scanFiles.size()); + assertEquals("fixture must contain exactly one INT32-physical data file", 1, countInt32Files(tablePath)); + FrameReader[] readers = {new FrameReaderDelta(), new FrameReaderDeltaParallel()}; + for(FrameReader reader : readers) { + FrameBlock out = reader.readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1); + assertEquals("rows", oldInts.length + newLongs.length, out.getNumRows()); + assertEquals("cols", 1, out.getNumColumns()); + assertEquals("widened column surfaces as INT64", ValueType.INT64, out.getSchema()[0]); + for(int r = 0; r < oldInts.length; r++) + assertEquals("int-file cell " + r, oldInts[r], ((Number) out.get(r, 0)).longValue()); + for(int r = 0; r < newLongs.length; r++) + assertEquals("long-file cell " + r, newLongs[r], + ((Number) out.get(oldInts.length + r, 0)).longValue()); + } + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void readSchemaEvolvedFilesMissingAndReorderedColumns() throws Exception { + // schema evolution leftovers: one data file written before column 'w' existed + // (its cells must keep the column default, 0 for numerics), and one whose + // parquet column order is reversed relative to the table schema (values must + // land by name, not by position), identically on all three read paths. + StructType tableSchema = new StructType().add("v", LongType.LONG, true).add("w", LongType.LONG, true); + StructType vOnly = new StructType().add("v", LongType.LONG, true); + StructType reversed = new StructType().add("w", LongType.LONG, true).add("v", LongType.LONG, true); + long[] v1 = {1L, 2L, 3L}; + long[] v2 = {10L, 20L}; + long[] w2 = {100L, 200L}; + + Path dir = Files.createTempDirectory("sysds_delta_frame_se_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + commitFiles(tablePath, tableSchema, batchOf(vOnly, new LongBackedVector(LongType.LONG, v1)), + batchOf(reversed, new LongBackedVector(LongType.LONG, w2), new LongBackedVector(LongType.LONG, v2))); + FrameReader[] readers = {new FrameReaderDelta(), new FrameReaderDeltaParallel(), new FrameReaderDelta() { + @Override + protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { + return false; + } + }}; + for(FrameReader reader : readers) { + FrameBlock out = reader.readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1); + assertEquals("rows", v1.length + v2.length, out.getNumRows()); + assertEquals("cols", 2, out.getNumColumns()); + for(int r = 0; r < v1.length; r++) { + assertEquals("old-file v " + r, v1[r], ((Number) out.get(r, 0)).longValue()); + assertEquals("old-file w (missing -> default) " + r, 0L, ((Number) out.get(r, 1)).longValue()); + } + for(int r = 0; r < v2.length; r++) { + assertEquals("reordered-file v " + r, v2[r], ((Number) out.get(v1.length + r, 0)).longValue()); + assertEquals("reordered-file w " + r, w2[r], ((Number) out.get(v1.length + r, 1)).longValue()); + } + } + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + @Test public void writerRejectsDimensionMismatch() throws Exception { ValueType[] schema = {ValueType.STRING, ValueType.INT64}; @@ -613,6 +710,87 @@ public ColumnVector getColumnVector(int ordinal) { DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(tablePath), schema, singleton(fcb)); } + /** + * Creates what Delta type widening leaves behind: a table whose schema declares {@code bigint} while its first data + * file physically stores INT32 (written before the widen) and its second INT64. + */ + private static void writeWidenedLongTable(String tablePath, long[] oldInts, long[] newLongs) throws Exception { + StructType tableSchema = new StructType().add("v", LongType.LONG, true); + StructType intSchema = new StructType().add("v", IntegerType.INTEGER, true); + commitFiles(tablePath, tableSchema, batchOf(intSchema, new LongBackedVector(IntegerType.INTEGER, oldInts)), + batchOf(tableSchema, new LongBackedVector(LongType.LONG, newLongs))); + } + + /** + * Creates a table with the given schema and commits one physically-written data file per batch. The files are + * written through the kernel's parquet handler directly (bypassing the logical-data transform, which would reject + * batch schemas deviating from the table schema) and committed with their statistics, so reads take the pre-sized + * direct path. + */ + private static void commitFiles(String tablePath, StructType tableSchema, ColumnarBatch... batches) + throws Exception { + Engine engine = DeltaKernelUtils.createEngine(); + Table table = Table.forPath(engine, DeltaKernelUtils.qualify(tablePath)); + Transaction txn = table.createTransactionBuilder(engine, "SystemDS-test", Operation.CREATE_TABLE) + .withSchema(engine, tableSchema).build(engine); + Row txnState = txn.getTransactionState(engine); + DataWriteContext ctx = Transaction.getWriteContext(engine, txnState, Collections.emptyMap()); + + List files = new ArrayList<>(); + for(ColumnarBatch batch : batches) + drain(engine.getParquetHandler().writeParquetFiles(ctx.getTargetDirectory(), + singleton(new FilteredColumnarBatch(batch, Optional.empty())), ctx.getStatisticsColumns()), files); + + CloseableIterator actions = Transaction.generateAppendActions(engine, txnState, + Utils.toCloseableIterator(files.iterator()), ctx); + txn.commit(engine, CloseableIterable.inMemoryIterable(actions)); + } + + /** Count the parquet data files whose single column is physically stored as INT32. */ + private static int countInt32Files(String tablePath) throws Exception { + Configuration conf = new Configuration(); + int n = 0; + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { + for(Path p : (Iterable) s.filter(f -> f.toString().endsWith(".parquet"))::iterator) { + try(ParquetFileReader r = ParquetFileReader + .open(HadoopInputFile.fromPath(new org.apache.hadoop.fs.Path(p.toString()), conf))) { + PrimitiveTypeName t = r.getFooter().getFileMetaData().getSchema().getType(0).asPrimitiveType() + .getPrimitiveTypeName(); + if(t == PrimitiveTypeName.INT32) + n++; + } + } + } + return n; + } + + private static void drain(CloseableIterator it, List into) throws IOException { + try(CloseableIterator i = it) { + while(i.hasNext()) + into.add(i.next()); + } + } + + /** Batch view over per-column vectors (positionally matching the given schema). */ + private static ColumnarBatch batchOf(StructType schema, ColumnVector... cols) { + return new ColumnarBatch() { + @Override + public StructType getSchema() { + return schema; + } + + @Override + public int getSize() { + return cols[0].getSize(); + } + + @Override + public ColumnVector getColumnVector(int ordinal) { + return cols[ordinal]; + } + }; + } + private static CloseableIterator singleton(FilteredColumnarBatch fcb) { return new CloseableIterator() { private boolean _done = false; @@ -702,6 +880,46 @@ public void close() { } } + /** Column view exposing a long[] as a Delta integer or long column ({@code getInt} narrows). */ + private static class LongBackedVector implements ColumnVector { + private final DataType _dt; + private final long[] _vals; + + LongBackedVector(DataType dt, long[] vals) { + _dt = dt; + _vals = vals; + } + + @Override + public DataType getDataType() { + return _dt; + } + + @Override + public int getSize() { + return _vals.length; + } + + @Override + public boolean isNullAt(int rowId) { + return false; + } + + @Override + public int getInt(int rowId) { + return (int) _vals[rowId]; + } + + @Override + public long getLong(int rowId) { + return _vals[rowId]; + } + + @Override + public void close() { + } + } + /** Column view exposing a byte[] as a Delta byte column. */ private static class ByteVector implements ColumnVector { private final byte[] _vals; diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaFrameShapeCoverageTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameShapeCoverageTest.java new file mode 100644 index 00000000000..b902ad22b42 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameShapeCoverageTest.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.io; + +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.commons.io.FileUtils; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.io.DeltaKernelUtils; +import org.apache.sysds.runtime.io.FrameReaderDelta; +import org.apache.sysds.runtime.io.FrameReaderDeltaParallel; +import org.apache.sysds.runtime.io.FrameWriterDelta; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; + +/** + * Shape/size coverage for the native Delta frame readers: single-cell tables, the writer batch boundary (4096 rows), 1 + * to 1000 columns, a multi-file layout, all-null columns, and adversarial cell values (NaN/infinities, signed zeros, + * integer extremes, empty/unicode/very long strings). + * + * Every shape is verified on all three read paths (serial direct decode as the default, parallel direct decode, and the + * forced kernel-engine buffered fallback) against the in-memory input, cell for cell. Column types cycle through all + * six writable frame value types so each typed decode loop is exercised at every shape; string nulls, empty strings and + * multi-byte unicode are covered by the all-null column and extreme value tests. + */ +public class DeltaFrameShapeCoverageTest { + + // nonsense schema/dims handed to the readers to confirm discovery from the table + private static final ValueType[] NO_SCHEMA = new ValueType[] {ValueType.STRING}; + private static final String[] NO_NAMES = new String[] {"x"}; + + // small target file size so large frames roll multiple data files and the + // per-file parallel path really splits (mirrors DeltaFrameReadWriteTest) + private static final long SMALL_TARGET_FILE_SIZE = 512L * 1024; + + // all value types the Delta frame writer can emit, cycled across columns so + // every typed decode loop (string/long/double/boolean/int/float) is hit at + // every covered shape + private static final ValueType[] TYPE_CYCLE = {ValueType.STRING, ValueType.INT64, ValueType.FP64, ValueType.BOOLEAN, + ValueType.INT32, ValueType.FP32}; + + @Test + public void tinyShapesRoundTrip() throws Exception { + assertRoundTrip(1, 1, 101, false); + assertRoundTrip(7, 6, 102, false); + } + + @Test + public void writerBatchBoundaryRoundTrip() throws Exception { + assertRoundTrip(4096, 2, 201, false); + } + + @Test + public void columnScalingRoundTrip() throws Exception { + // the 1-column and 1000-column endpoints at small row counts + assertRoundTrip(1000, 1, 301, false); + assertRoundTrip(100, 1000, 302, false); + } + + @Test + public void multiFileRoundTrip() throws Exception { + // a table large enough to roll multiple data files so the per-file slicing + // of the direct and parallel paths is covered + assertRoundTrip(100_000, 6, 401, true); + } + + @Test + public void allNullStringColumnRoundTrip() throws Exception { + // a string column that is entirely null (definition level 0 for every row, + // no data bytes in any page) next to a fully-live numeric column + int nrow = 1000; + FrameBlock in = TestUtils.generateRandomFrameBlock(nrow, new ValueType[] {ValueType.STRING, ValueType.FP64}, + 61); + for(int r = 0; r < nrow; r++) + in.set(r, 0, null); + roundTripAllReaders(in, false); + } + + @Test + public void extremeValuesRoundTrip() throws Exception { + // adversarial cell values for every type: NaN, infinities, signed zeros, + // subnormals, MIN/MAX of each numeric width, and empty / whitespace / + // multi-byte unicode / control-character / very long strings + ValueType[] schema = {ValueType.FP64, ValueType.FP32, ValueType.INT64, ValueType.INT32, ValueType.BOOLEAN, + ValueType.STRING}; + String[] names = {"d", "f", "l", "i", "b", "s"}; + double[] d = {Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, Double.MAX_VALUE, + -Double.MAX_VALUE, Double.MIN_VALUE, 0.0, -0.0}; + float[] f = {Float.NaN, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY, Float.MAX_VALUE, -Float.MAX_VALUE, + Float.MIN_VALUE, 0.0f, -0.0f}; + long[] l = {Long.MAX_VALUE, Long.MIN_VALUE, 0L, -1L, 1L, 1L << 40, -(1L << 40), 42L}; + int[] i = {Integer.MAX_VALUE, Integer.MIN_VALUE, 0, -1, 1, 1 << 20, -(1 << 20), 42}; + String longStr = new String(new char[10_000]).replace('\0', 'x'); + String[] s = {null, "", " ", "ü€𐍈", longStr, "line\nbreak", "tab\tsep", "quote\"'"}; + + int nrow = d.length; + FrameBlock in = new FrameBlock(schema, names); + in.ensureAllocatedColumns(nrow); + for(int r = 0; r < nrow; r++) { + in.set(r, 0, d[r]); + in.set(r, 1, f[r]); + in.set(r, 2, l[r]); + in.set(r, 3, i[r]); + in.set(r, 4, r % 2 == 0); + in.set(r, 5, s[r]); + } + roundTripAllReaders(in, false); + } + + // ------------------------------------------ + // helpers + // ------------------------------------------ + + private static ValueType[] cycleSchema(int ncol) { + ValueType[] schema = new ValueType[ncol]; + for(int c = 0; c < ncol; c++) + schema[c] = TYPE_CYCLE[c % TYPE_CYCLE.length]; + return schema; + } + + @FunctionalInterface + private interface TableBody { + void accept(String tablePath) throws Exception; + } + + /** + * Write {@code in} to a fresh temp Delta table and run {@code body} against it. With {@code multiFile} a small + * target file size is configured and the resulting layout is asserted to really span multiple data files. Local + * config and the temp directory are always cleaned up. + */ + private static void withTable(FrameBlock in, boolean multiFile, TableBody body) throws Exception { + if(multiFile) { + DMLConfig conf = new DMLConfig(); + conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(SMALL_TARGET_FILE_SIZE)); + ConfigurationManager.setLocalConfig(conf); + } + Path dir = Files.createTempDirectory("sysds_delta_shape_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + new FrameWriterDelta().writeFrameToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns()); + if(multiFile) + assertTrue("expected a multi-file Delta table for this shape", + DeltaFrameTestUtils.countParquet(tablePath) > 1); + body.accept(tablePath); + } + finally { + if(multiFile) + ConfigurationManager.clearLocalConfigs(); + FileUtils.deleteQuietly(dir.toFile()); + } + } + + private static void assertRoundTrip(int nrow, int ncol, long seed, boolean multiFile) throws Exception { + roundTripAllReaders(TestUtils.generateRandomFrameBlock(nrow, cycleSchema(ncol), seed), multiFile); + } + + /** + * Write the frame and assert that the serial direct read (the default path), the parallel direct read, and the + * forced kernel-engine buffered fallback each reproduce the input cell for cell. + */ + private static void roundTripAllReaders(FrameBlock in, boolean multiFile) throws Exception { + withTable(in, multiFile, tablePath -> { + TestUtils.compareFrames(in, + new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), true); + TestUtils.compareFrames(in, + new FrameReaderDeltaParallel().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), true); + TestUtils.compareFrames(in, newBufferedReader().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), + true); + }); + } + + /** Serial reader that always declines the direct path, forcing the kernel-engine buffered read. */ + private static FrameReaderDelta newBufferedReader() { + return new FrameReaderDelta() { + @Override + protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { + return false; + } + }; + } +} diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaFrameSparkContractTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameSparkContractTest.java new file mode 100644 index 00000000000..5687461c851 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameSparkContractTest.java @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.io; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.io.FileUtils; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.io.FrameReaderDelta; +import org.apache.sysds.runtime.io.FrameReaderDeltaParallel; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Regression tests pinning the Delta table layouts that the direct column-API decode path (and its kernel-engine + * fallback for deletion vectors and partitioned tables) must honor beyond plain flat reads. Each case is a table layout + * the SystemDS writer never produces itself, so it must be created by the reference engine (Spark/Delta). + * + * dvFeatureEnabledNoDeleteRead covers a table whose protocol carries the {@code deletionVectors} reader feature but has + * no deleted rows - a distinct case from {@link DeltaFrameSparkInteropTest#sparkDeletionVectorsSystemdsRead}, which + * covers row filtering once rows are actually deleted. Here the kernel still appends the {@code _metadata.row_index} + * metadata column to every read once the feature is enabled, which does not exist in the data files and must not be + * requested from them. + * + * schemaEvolutionAddedColumnRead covers a column added after the first commit, so older data files lack it and the + * handler must surface it as nulls rather than fail. partitionedTableRead covers partition values, which are not stored + * in the data files and must be spliced back in. idColumnMappingRead covers a table using + * {@code delta.columnMapping.mode = id}, where columns must be resolvable by parquet field id rather than logical name. + */ +@net.jcip.annotations.NotThreadSafe +public class DeltaFrameSparkContractTest { + + // nonsense schema/dims handed to the reader to confirm it discovers everything from the table + private static final ValueType[] NO_SCHEMA = new ValueType[] {ValueType.STRING}; + private static final String[] NO_NAMES = new String[] {"x"}; + + private static final String DV_DEFAULT = "spark.databricks.delta.properties.defaults.enableDeletionVectors"; + + private static SparkSession spark; + + @BeforeClass + public static void startSpark() { + // each test class runs in its own fork (surefire reuseForks=false), so this + // is the only SparkSession in the JVM and gets the Delta extensions injected. + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + spark = SparkSession.builder().appName("sysds-delta-frame-contract").master("local[2]") + .config("spark.ui.enabled", "false").config("spark.sql.shuffle.partitions", "2") + .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog").getOrCreate(); + } + + @AfterClass + public static void stopSpark() { + if(spark != null) + spark.stop(); + SparkSession.clearActiveSession(); + SparkSession.clearDefaultSession(); + spark = null; + } + + @Test + public void dvFeatureEnabledNoDeleteRead() throws Exception { + // enabling deletion vectors adds the feature to the table protocol, which + // makes the kernel append the _metadata.row_index metadata column to the physical + // read schema of every read, no row has to be deleted. The parquet handler must + // populate that column (it is not stored in the data files) instead of failing. + int rows = 500; + Path dir = Files.createTempDirectory("sysds_delta_frame_dvfeat_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + spark.conf().set(DV_DEFAULT, "true"); + indexedDataFrame(rows).write().format("delta").save(tablePath); + + assertFrameMatchesIds(new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), + rows, "serial-dvfeat"); + assertFrameMatchesIds( + new FrameReaderDeltaParallel().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), rows, + "parallel-dvfeat"); + } + finally { + spark.conf().unset(DV_DEFAULT); + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void schemaEvolutionAddedColumnRead() throws Exception { + // column c4 is added by the second commit, so the data files of the first commit + // do not contain it; the parquet handler must return nulls for it there (the + // kernel hands every file the same table-level physical read schema). + int oldRows = 200, allRows = 300; + Path dir = Files.createTempDirectory("sysds_delta_frame_evo_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + indexedDataFrame(oldRows).write().format("delta").save(tablePath); + evolvedDataFrame(oldRows, allRows).write().format("delta").mode("append").option("mergeSchema", "true") + .save(tablePath); + + for(FrameBlock out : new FrameBlock[] { + new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), + new FrameReaderDeltaParallel().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1)}) { + assertEquals("rows", allRows, out.getNumRows()); + assertEquals("cols", 5, out.getNumColumns()); + assertEquals("c4 type", ValueType.STRING, out.getSchema()[4]); + Set seen = new HashSet<>(); + for(int r = 0; r < out.getNumRows(); r++) { + int id = ((Number) out.get(r, 0)).intValue(); + assertTrue("unexpected/duplicate id " + id, id >= 0 && id < allRows && seen.add(id)); + assertEquals("id" + id + " c1", dval(id), ((Number) out.get(r, 1)).doubleValue(), 1e-9); + assertEquals("id" + id + " c2", sval(id), out.get(r, 2).toString()); + Object c4 = out.get(r, 4); + if(id < oldRows) + assertNull("id" + id + " c4 must be null (file predates the column)", c4); + else + assertEquals("id" + id + " c4", vval(id), c4.toString()); + } + assertEquals(allRows, seen.size()); + } + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void partitionedTableRead() throws Exception { + // partition values are not stored in the data files; the kernel splices them back + // into every batch (ColumnarBatch.withNewColumn), so this pins the batch-reshaping + // side of the contract that plain unpartitioned round-trips never touch. + int rows = 300; + Path dir = Files.createTempDirectory("sysds_delta_frame_part_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + indexedDataFrame(rows).write().format("delta").partitionBy("c3").save(tablePath); + + assertFrameMatchesIds(new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), + rows, "serial-part"); + assertFrameMatchesIds( + new FrameReaderDeltaParallel().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), rows, + "parallel-part"); + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void idColumnMappingRead() throws Exception { + // with delta.columnMapping.mode=id the parquet columns carry field ids and + // physical names; the handler must resolve columns through the + // mapped physical schema, preferring field ids per the ParquetHandler contract. + int rows = 400; + Path dir = Files.createTempDirectory("sysds_delta_frame_idmap_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + spark.sql("CREATE TABLE delta.`" + tablePath + "` (c0 BIGINT, c1 DOUBLE, c2 STRING, c3 BOOLEAN) " + + "USING delta TBLPROPERTIES ('delta.columnMapping.mode'='id')"); + indexedDataFrame(rows).write().format("delta").mode("append").save(tablePath); + + assertFrameMatchesIds(new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), + rows, "serial-idmap"); + assertFrameMatchesIds( + new FrameReaderDeltaParallel().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), rows, + "parallel-idmap"); + } + finally { + spark.sql("DROP TABLE IF EXISTS delta.`" + tablePath + "`"); + FileUtils.deleteQuietly(dir.toFile()); + } + } + + // deterministic, exactly-representable cell values keyed by the row id in column 0 + private static double dval(int id) { + return id * 0.5 - 1.0; + } + + private static String sval(int id) { + return "s" + id; + } + + private static boolean bval(int id) { + return id % 2 == 0; + } + + private static String vval(int id) { + return "v" + id; + } + + /** Spark DataFrame with columns c0..c3 (long/double/string/boolean) keyed by the row id in c0. */ + private Dataset indexedDataFrame(int rows) { + StructType schema = DataTypes + .createStructType(new StructField[] {DataTypes.createStructField("c0", DataTypes.LongType, false), + DataTypes.createStructField("c1", DataTypes.DoubleType, false), + DataTypes.createStructField("c2", DataTypes.StringType, false), + DataTypes.createStructField("c3", DataTypes.BooleanType, false)}); + List data = new ArrayList<>(rows); + for(int r = 0; r < rows; r++) + data.add(RowFactory.create((long) r, dval(r), sval(r), bval(r))); + return spark.createDataFrame(data, schema); + } + + /** Like {@link #indexedDataFrame} for ids [from,to) but with an additional string column c4. */ + private Dataset evolvedDataFrame(int from, int to) { + StructType schema = DataTypes + .createStructType(new StructField[] {DataTypes.createStructField("c0", DataTypes.LongType, false), + DataTypes.createStructField("c1", DataTypes.DoubleType, false), + DataTypes.createStructField("c2", DataTypes.StringType, false), + DataTypes.createStructField("c3", DataTypes.BooleanType, false), + DataTypes.createStructField("c4", DataTypes.StringType, true)}); + List data = new ArrayList<>(to - from); + for(int r = from; r < to; r++) + data.add(RowFactory.create((long) r, dval(r), sval(r), bval(r), vval(r))); + return spark.createDataFrame(data, schema); + } + + /** Asserts {@code out} holds exactly ids [0,rows) with the exact per-id values in c1..c3. */ + private static void assertFrameMatchesIds(FrameBlock out, int rows, String tag) { + assertEquals(tag + " rows", rows, out.getNumRows()); + assertEquals(tag + " cols", 4, out.getNumColumns()); + assertEquals(tag + " c0 type", ValueType.INT64, out.getSchema()[0]); + assertEquals(tag + " c1 type", ValueType.FP64, out.getSchema()[1]); + assertEquals(tag + " c2 type", ValueType.STRING, out.getSchema()[2]); + assertEquals(tag + " c3 type", ValueType.BOOLEAN, out.getSchema()[3]); + boolean[] seen = new boolean[rows]; + for(int r = 0; r < rows; r++) { + int id = ((Number) out.get(r, 0)).intValue(); + assertTrue(tag + ": unexpected/duplicate id " + id, id >= 0 && id < rows && !seen[id]); + seen[id] = true; + assertEquals(tag + " id" + id + " c1", dval(id), ((Number) out.get(r, 1)).doubleValue(), 1e-9); + assertEquals(tag + " id" + id + " c2", sval(id), out.get(r, 2).toString()); + assertEquals(tag + " id" + id + " c3", Boolean.valueOf(bval(id)), out.get(r, 3)); + } + } +} From 84436aba97ae9aea0354685eec7b62cdc7daf46e Mon Sep 17 00:00:00 2001 From: Jannik Lindemann Date: Mon, 13 Jul 2026 12:46:10 +0200 Subject: [PATCH 071/132] [MINOR] Prepare Release Metadata for SystemDS 3.4.0 --- NOTICE | 2 +- src/assembly/bin/NOTICE | 2 +- src/assembly/extra/NOTICE | 2 +- src/main/python/docs/source/conf.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE b/NOTICE index 1b034eefd04..86506b75718 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache SystemDS -Copyright [2015-2024] The Apache Software Foundation +Copyright [2015-2026] The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/src/assembly/bin/NOTICE b/src/assembly/bin/NOTICE index 1b034eefd04..86506b75718 100644 --- a/src/assembly/bin/NOTICE +++ b/src/assembly/bin/NOTICE @@ -1,5 +1,5 @@ Apache SystemDS -Copyright [2015-2024] The Apache Software Foundation +Copyright [2015-2026] The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/src/assembly/extra/NOTICE b/src/assembly/extra/NOTICE index 1b034eefd04..86506b75718 100644 --- a/src/assembly/extra/NOTICE +++ b/src/assembly/extra/NOTICE @@ -1,5 +1,5 @@ Apache SystemDS -Copyright [2015-2024] The Apache Software Foundation +Copyright [2015-2026] The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/src/main/python/docs/source/conf.py b/src/main/python/docs/source/conf.py index c6e69b3de23..6823deb9cc2 100644 --- a/src/main/python/docs/source/conf.py +++ b/src/main/python/docs/source/conf.py @@ -34,11 +34,11 @@ # -- Project information ----------------------------------------------------- project = 'SystemDS' -copyright = '2024, Apache SystemDS' +copyright = '2026, Apache SystemDS' author = 'Apache SystemDS' # The full version, including alpha/beta/rc tags -release = '3.3.0-dev' +release = '3.4.0-dev' # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. From 1ee76a87498b1e426cfb82474f54b7cda867edcf Mon Sep 17 00:00:00 2001 From: Jannik Lindemann Date: Mon, 13 Jul 2026 13:19:57 +0200 Subject: [PATCH 072/132] [MINOR] Add Java 17 Module Access Options --- bin/systemds | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/bin/systemds b/bin/systemds index f0cb0b729b0..b9dcfa6fbcb 100755 --- a/bin/systemds +++ b/bin/systemds @@ -75,6 +75,21 @@ else SYSTEMDS_STANDALONE_OPTS="-Xmx4g -Xms4g -Xmn400m " fi +if [ -n "$SYSTEMDS_JAVA_OPTS" ]; then + print_out "Overriding SYSTEMDS_JAVA_OPTS with env var: $SYSTEMDS_JAVA_OPTS" +else + # Java 17 module opens required by Spark/SystemDS reflective access paths. + SYSTEMDS_JAVA_OPTS="\ + --add-modules=jdk.incubator.vector \ + --add-opens=java.base/java.nio=ALL-UNNAMED \ + --add-opens=java.base/java.io=ALL-UNNAMED \ + --add-opens=java.base/java.util=ALL-UNNAMED \ + --add-opens=java.base/java.lang=ALL-UNNAMED \ + --add-opens=java.base/java.lang.ref=ALL-UNNAMED \ + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED \ + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED " +fi + if [ -n "$SYSTEMDS_REMOTE_DEBUGGING" ]; then print_out "Overriding SYSTEMDS_REMOTE_DEBUGGING with env var: $SYSTEMDS_REMOTE_DEBUGGING" else @@ -107,8 +122,8 @@ else --master yarn \ --deploy-mode client \ --driver-memory 100g \ - --conf spark.driver.extraJavaOptions=\"-Xms100g -Xmn10g -Dlog4j.configuration=file:$LOG4JPROP\" \ - --conf spark.executor.extraJavaOptions=\"-Dlog4j.configuration=file:$LOG4JPROP\" \ + --conf spark.driver.extraJavaOptions=\"-Xms100g -Xmn10g -Dlog4j.configuration=file:$LOG4JPROP $SYSTEMDS_JAVA_OPTS\" \ + --conf spark.executor.extraJavaOptions=\"-Dlog4j.configuration=file:$LOG4JPROP $SYSTEMDS_JAVA_OPTS\" \ --conf spark.executor.heartbeatInterval=100s \ --files $LOG4JPROP \ --conf spark.network.timeout=512s \ @@ -413,7 +428,7 @@ if [ $WORKER == 1 ]; then print_out "# starting Federated worker on port $PORT" CMD=" \ java $SYSTEMDS_STANDALONE_OPTS \ - --add-modules=jdk.incubator.vector \ + $SYSTEMDS_JAVA_OPTS \ $LOG4JPROPFULL \ -jar $SYSTEMDS_JAR_FILE \ -w $PORT \ @@ -423,7 +438,7 @@ elif [ "$FEDMONITORING" == 1 ]; then print_out "# starting Federated backend monitoring on port $PORT" CMD=" \ java $SYSTEMDS_STANDALONE_OPTS \ - --add-modules=jdk.incubator.vector \ + $SYSTEMDS_JAVA_OPTS \ $LOG4JPROPFULL \ -jar $SYSTEMDS_JAR_FILE \ -fedMonitoring $PORT \ @@ -434,8 +449,8 @@ elif [ $SYSDS_DISTRIBUTED == 0 ]; then CMD=" \ java $SYSTEMDS_STANDALONE_OPTS \ + $SYSTEMDS_JAVA_OPTS \ $LOG4JPROPFULL \ - --add-modules=jdk.incubator.vector \ -jar $SYSTEMDS_JAR_FILE \ -f $SCRIPT_FILE \ -exec $SYSDS_EXEC_MODE \ @@ -445,7 +460,6 @@ else print_out "# Running script $SCRIPT_FILE distributed with opts: $*" CMD=" \ spark-submit $SYSTEMDS_DISTRIBUTED_OPTS \ - --add-modules=jdk.incubator.vector \ $SYSTEMDS_JAR_FILE \ -f $SCRIPT_FILE \ -exec $SYSDS_EXEC_MODE \ From 193c261b569acde43dfa172edc847a20c6d222b4 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann Date: Mon, 13 Jul 2026 14:20:36 +0000 Subject: [PATCH 073/132] [maven-release-plugin] prepare release 3.4.0-rc1 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 068bed2e8ea..1945bece0ee 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ 24 org.apache.systemds - 3.4.0-SNAPSHOT + 3.4.0 systemds jar Apache SystemDS @@ -118,7 +118,7 @@ scm:git:https://github.com/apache/systemds.git - HEAD + 3.4.0-rc1 From b49ebf5cffa3a5d490381608f0baa74df38bdebf Mon Sep 17 00:00:00 2001 From: Jannik Lindemann Date: Mon, 13 Jul 2026 14:20:50 +0000 Subject: [PATCH 074/132] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1945bece0ee..be50b05a92c 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ 24 org.apache.systemds - 3.4.0 + 3.5.0-SNAPSHOT systemds jar Apache SystemDS @@ -118,7 +118,7 @@ scm:git:https://github.com/apache/systemds.git - 3.4.0-rc1 + HEAD From a4642e770722fcaf75e113a5b3ae7fc3598025e1 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann Date: Mon, 13 Jul 2026 18:17:25 +0200 Subject: [PATCH 075/132] [MINOR][FIX] Add SciPy as Dependency --- src/main/python/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/python/setup.py b/src/main/python/setup.py index 2fab35f58f9..9d80f817972 100755 --- a/src/main/python/setup.py +++ b/src/main/python/setup.py @@ -39,7 +39,8 @@ 'numpy >= 1.8.2', 'py4j >= 0.10.9', 'requests >= 2.24.0', - 'pandas >= 1.2.2' + 'pandas >= 1.2.2', + 'scipy >= 1.5.0' ] LONG_DESCRIPTION= '''This package provides a Pythonic interface for working with Apache SystemDS. From 1c23a166a9b702677681752ad6794cf660fd15b6 Mon Sep 17 00:00:00 2001 From: Elias Strauss Date: Mon, 13 Jul 2026 19:06:43 +0200 Subject: [PATCH 076/132] [Minor] Update python api README.md --- src/main/python/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/python/README.md b/src/main/python/README.md index 3ab4b2a7528..9fa958f4a13 100644 --- a/src/main/python/README.md +++ b/src/main/python/README.md @@ -49,6 +49,12 @@ The following steps have to be done for both the cases ### Building python package +- Make sure you have `setuptools` installed + +```bash +pip install setuptools +``` + - Run `create_python_dist.py` ```bash From 6808f5c93ff4f2caf09783115eb8fa9daaf7e130 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann Date: Mon, 13 Jul 2026 19:45:38 +0000 Subject: [PATCH 077/132] [maven-release-plugin] prepare release 3.4.0-rc2 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index be50b05a92c..5c96ea026e9 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ 24 org.apache.systemds - 3.5.0-SNAPSHOT + 3.4.0 systemds jar Apache SystemDS @@ -118,7 +118,7 @@ scm:git:https://github.com/apache/systemds.git - HEAD + 3.4.0-rc2 From 5f33cd1f7dda4e10efae3c039578d2a72cdbd02f Mon Sep 17 00:00:00 2001 From: Jannik Lindemann Date: Mon, 13 Jul 2026 19:46:01 +0000 Subject: [PATCH 078/132] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 5c96ea026e9..be50b05a92c 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ 24 org.apache.systemds - 3.4.0 + 3.5.0-SNAPSHOT systemds jar Apache SystemDS @@ -118,7 +118,7 @@ scm:git:https://github.com/apache/systemds.git - 3.4.0-rc2 + HEAD From a060404dca4480ba1979cc4189832a991b2d9a20 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Thu, 16 Jul 2026 10:30:53 +0200 Subject: [PATCH 079/132] [SYSTEMDS-3891] Add Materialized OOC Structures --- .../ooc/memory/InMemoryQueueCallback.java | 22 ++ .../runtime/ooc/memory/ManagedPayload.java | 68 ++++ .../store/IndexedMaterializedStoreReader.java | 117 ++++++ .../ooc/store/MaterializedCallback.java | 84 ++++ .../runtime/ooc/store/MaterializedStore.java | 233 ++++++++++++ .../ooc/store/OOCStreamMaterializer.java | 151 ++++++++ .../store/OrderedMaterializedStoreReader.java | 253 ++++++++++++ .../sysds/runtime/ooc/store/StoreLease.java | 78 ++++ .../sysds/runtime/ooc/util/OOCUtils.java | 36 ++ .../component/ooc/MaterializedStoreTest.java | 359 ++++++++++++++++++ 10 files changed, 1401 insertions(+) create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/memory/ManagedPayload.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/store/IndexedMaterializedStoreReader.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedCallback.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedStore.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/store/OOCStreamMaterializer.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/store/OrderedMaterializedStoreReader.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/store/StoreLease.java create mode 100644 src/test/java/org/apache/sysds/test/component/ooc/MaterializedStoreTest.java diff --git a/src/main/java/org/apache/sysds/runtime/ooc/memory/InMemoryQueueCallback.java b/src/main/java/org/apache/sysds/runtime/ooc/memory/InMemoryQueueCallback.java index 7496279d3e3..7fafc042e41 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/memory/InMemoryQueueCallback.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/memory/InMemoryQueueCallback.java @@ -191,4 +191,26 @@ private void closeFinal() { public IndexedMatrixValue takeManagedResultForHandover() { return _handle.takeManagedResultForHandover(); } + + public synchronized ManagedPayload extractManagedPayload() { + if(_closed) + throw new IllegalStateException("Cannot extract a managed payload from a closed callback."); + CallbackHandle handle = _handle; + synchronized(handle) { + if(handle._failure != null) + throw handle._failure; + if(!handle.isExclusiveToRoot()) + throw new IllegalStateException("Cannot extract a managed payload while callback aliases exist."); + if(handle._cacheIdx >= 0) + throw new IllegalStateException("Cannot extract a managed payload from a cached-slot callback."); + IndexedMatrixValue result = handle._result; + if(result == null) + throw new IllegalStateException("Cannot extract a managed payload from an empty callback."); + long bytes = handle._reservedBytes; + MemoryAllowance owner = handle._allow; + handle._result = null; + handle._reservedBytes = 0; + return new ManagedPayload<>(result, bytes, owner); + } + } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/memory/ManagedPayload.java b/src/main/java/org/apache/sysds/runtime/ooc/memory/ManagedPayload.java new file mode 100644 index 00000000000..494ced159a0 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/memory/ManagedPayload.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.memory; + +import java.util.concurrent.atomic.AtomicBoolean; + +public final class ManagedPayload { + private final T _value; + private final long _bytes; + private final MemoryAllowance _owner; + private final AtomicBoolean _transferred; + + public ManagedPayload(T value, long bytes, MemoryAllowance owner) { + if(value == null) + throw new IllegalArgumentException("Managed payload requires a value."); + if(bytes < 0) + throw new IllegalArgumentException("Managed payload bytes must not be negative: " + bytes); + if(bytes > 0 && owner == null) + throw new IllegalArgumentException("Managed payload with charged bytes requires an owning allowance."); + _value = value; + _bytes = bytes; + _owner = owner; + _transferred = new AtomicBoolean(false); + } + + public T value() { + return _value; + } + + public long bytes() { + return _bytes; + } + + public MemoryAllowance owner() { + return _owner; + } + + public void transfer() { + if(!_transferred.compareAndSet(false, true)) + throw new IllegalStateException("Managed payload was already settled."); + } + + public void release() { + if(_transferred.compareAndSet(false, true) && _bytes > 0) + _owner.release(_bytes); + } + + public boolean isTransferred() { + return _transferred.get(); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/IndexedMaterializedStoreReader.java b/src/main/java/org/apache/sysds/runtime/ooc/store/IndexedMaterializedStoreReader.java new file mode 100644 index 00000000000..3106705a34a --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/IndexedMaterializedStoreReader.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.store; + +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.OOCCache; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; +import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; +import org.apache.sysds.runtime.ooc.util.OOCUtils; + +import java.util.function.IntConsumer; +import java.util.function.IntSupplier; + +public final class IndexedMaterializedStoreReader implements MaterializedStore.StoreReader { + private final OOCCache _cache; + private final long _streamId; + private final IntSupplier _completedSize; + private final MaterializedStore.Liveness _liveness; + private final Runnable _afterClose; + private final IntConsumer _afterRelease; + private volatile boolean _closed; + + IndexedMaterializedStoreReader(OOCCache cache, long streamId, IntSupplier completedSize, + MaterializedStore.Liveness liveness, Runnable afterClose, IntConsumer afterRelease) { + _cache = cache; + _streamId = streamId; + _completedSize = completedSize; + _liveness = liveness; + _afterClose = afterClose; + _afterRelease = afterRelease; + } + + @Override + public MaterializedStore.Liveness liveness() { + return _liveness; + } + + @Override + public boolean isClosed() { + return _closed; + } + + @Override + public void close() { + if(_closed) + return; + _closed = true; + _afterClose.run(); + } + + public OOCFuture> request(int index, MemoryAllowance requestAllowance) { + checkReady(index); + reserve(index); + OOCFuture pinned = OOCUtils.pinAdmitted(_cache, _streamId, index, requestAllowance, () -> _closed); + OOCFuture> result = new OOCFuture<>(); + pinned.whenComplete((entry, error) -> { + if(error != null) { + _liveness.unreserve(index); + result.completeExceptionally(error); + } + else if(entry == null) { + _liveness.unreserve(index); + result.complete(null); + } + else + result.complete(new StoreLease<>(entry, () -> release(index, entry, requestAllowance))); + }); + return result; + } + + public StoreLease requestIfLive(int index, MemoryAllowance requestAllowance) { + checkReady(index); + reserve(index); + BlockEntry entry = _cache.pinIfLive(_streamId, index, requestAllowance); + if(entry == null) { + _liveness.unreserve(index); + return null; + } + return new StoreLease<>(entry, () -> release(index, entry, requestAllowance)); + } + + private void release(int index, BlockEntry entry, MemoryAllowance requestAllowance) { + _cache.unpin(entry, requestAllowance); + _liveness.consumed(index); + _afterRelease.accept(index); + } + + private void reserve(int index) { + if(!_liveness.reserve(index)) + throw new IllegalStateException("Index is no longer live for this reader: " + index); + } + + private void checkReady(int index) { + if(_closed) + throw new IllegalStateException("Reader is closed"); + if(index < 0 || index >= _completedSize.getAsInt()) + throw new IndexOutOfBoundsException("Invalid requested index: " + index); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedCallback.java b/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedCallback.java new file mode 100644 index 00000000000..829e60b32ea --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedCallback.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.store; + +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.ooc.cache.BlockEntry; + +import java.util.concurrent.atomic.AtomicReference; + +public final class MaterializedCallback implements OOCStream.QueueCallback { + private final StoreLease _lease; + private final AtomicReference _failure; + private boolean _closed; + + public MaterializedCallback(StoreLease lease) { + this(lease, new AtomicReference<>()); + } + + private MaterializedCallback(StoreLease lease, AtomicReference failure) { + _lease = lease; + _failure = failure; + } + + public BlockEntry pinnedEntry() { + return _lease != null ? _lease.entry() : null; + } + + @Override + public IndexedMatrixValue get() { + DMLRuntimeException failure = _failure.get(); + if(failure != null) + throw failure; + return _lease.value(); + } + + @Override + public synchronized OOCStream.QueueCallback keepOpen() { + if(_closed) + throw new IllegalStateException("Cannot keep open a closed callback"); + return new MaterializedCallback(_lease.retain(), _failure); + } + + @Override + public synchronized void close() { + if(_closed) + return; + _closed = true; + _lease.close(); + } + + @Override + public void fail(DMLRuntimeException failure) { + _failure.set(failure); + } + + @Override + public boolean isEos() { + return false; + } + + @Override + public boolean isFailure() { + return _failure.get() != null; + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedStore.java b/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedStore.java new file mode 100644 index 00000000000..d1ed01ffa3e --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedStore.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.store; + +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.BlockKey; +import org.apache.sysds.runtime.ooc.cache.OOCCache; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; +import org.apache.sysds.runtime.ooc.memory.ManagedPayload; +import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; +import org.apache.sysds.runtime.ooc.util.OOCUtils; + +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +public final class MaterializedStore { + private final OOCCache _cache; + private final long _streamId; + private final ArrayList _registeredReaders; + private final BitSet _forgotten; + private final AtomicInteger _published; + private final AtomicInteger _publishedCount; + + private volatile List _readers; + private volatile int _completedSize; + private volatile boolean _complete; + private volatile boolean _readersSealed; + private volatile boolean _closed; + + public MaterializedStore(OOCCache cache, long streamId) { + _cache = cache; + _streamId = streamId; + _registeredReaders = new ArrayList<>(); + _forgotten = new BitSet(); + _published = new AtomicInteger(); + _publishedCount = new AtomicInteger(); + _readers = Collections.emptyList(); + } + + StoreLease publishPinnedLive(int index, T value, long bytes, MemoryAllowance allowance) { + BlockEntry entry; + try { + if(_complete || _closed) + throw new IllegalStateException("Store no longer accepts published items"); + if(index < 0 || index == Integer.MAX_VALUE) + throw new IndexOutOfBoundsException("Invalid index: " + index); + entry = _cache.putPinned(_streamId, index, value, bytes, allowance); + } + catch(RuntimeException ex) { + if(bytes > 0) + allowance.release(bytes); + throw ex; + } + _publishedCount.incrementAndGet(); + updatePublished(index + 1); + return new StoreLease<>(entry, () -> { + _cache.unpin(entry, allowance); + tryForget(index); + }); + } + + StoreLease publishPinnedLive(int index, ManagedPayload payload) { + payload.transfer(); + return publishPinnedLive(index, payload.value(), payload.bytes(), payload.owner()); + } + + public synchronized void complete() { + if(_complete) + return; + _completedSize = _published.get(); + if(_publishedCount.get() != _completedSize) + throw new IllegalStateException("Incomplete publication: " + _publishedCount.get() + + " published items for logical range [0, " + _completedSize + ")"); + _complete = true; + } + + public synchronized OrderedMaterializedStoreReader openReader(AccessPattern pattern, MemoryAllowance allowance, + int maxPrefetch) { + return openReader(pattern, allowance, maxPrefetch, true); + } + + public synchronized OrderedMaterializedStoreReader openReader(AccessPattern pattern, MemoryAllowance allowance, + int maxPrefetch, boolean softOrdering) { + if(!_complete || _closed) + throw new IllegalStateException("Readers require a completed store"); + if(_readersSealed) + throw new IllegalStateException("Store no longer accepts new readers"); + OrderedMaterializedStoreReader reader = new OrderedMaterializedStoreReader<>(_cache, _streamId, pattern, + allowance, Math.max(1, maxPrefetch), softOrdering, this::forgetAfterReaderClose, this::tryForget); + _registeredReaders.add(reader); + return reader; + } + + public synchronized IndexedMaterializedStoreReader openIndexedReader(Liveness liveness) { + if(!_complete || _closed) + throw new IllegalStateException("Readers require a completed store"); + if(_readersSealed) + throw new IllegalStateException("Store no longer accepts new readers"); + IndexedMaterializedStoreReader reader = new IndexedMaterializedStoreReader<>(_cache, _streamId, + () -> _completedSize, liveness, this::forgetAfterReaderClose, this::tryForget); + _registeredReaders.add(reader); + return reader; + } + + public OOCFuture> requestPublished(int index, MemoryAllowance allowance) { + if(_closed) + throw new IllegalStateException("Store is closed"); + if(index < 0 || index >= _published.get()) + throw new IndexOutOfBoundsException("Invalid requested index: " + index); + OOCFuture pinned = OOCUtils.pinAdmitted(_cache, _streamId, index, allowance, () -> _closed); + OOCFuture> result = new OOCFuture<>(); + pinned.whenComplete((entry, error) -> { + if(error != null) + result.completeExceptionally(error); + else if(entry == null) + result.complete(null); + else + result.complete(new StoreLease<>(entry, () -> _cache.unpin(entry, allowance))); + }); + return result; + } + + public synchronized void sealReaders() { + if(_closed) + throw new IllegalStateException("Cannot seal readers for a closed store"); + if(_readersSealed) + return; + _readers = new ArrayList<>(_registeredReaders); + _readersSealed = true; + int publishedSize = _complete ? _completedSize : _published.get(); + for(int i = 0; i < publishedSize; i++) + tryForget(i); + } + + public int size() { + return _complete ? _completedSize : _published.get(); + } + + public void close() { + List localReaders; + synchronized(this) { + if(_closed) + return; + _closed = true; + localReaders = _readersSealed ? _readers : new ArrayList<>(_registeredReaders); + } + for(StoreReader localReader : localReaders) + localReader.close(); + for(int i = 0; i < size(); i++) + if(markForgotten(i)) + _cache.dereference(new BlockKey(_streamId, i)); + } + + private void tryForget(int index) { + if(!_readersSealed) + return; + List localReaders = _readers; + for(StoreReader reader : localReaders) + if(!reader.isClosed() && reader.liveness().needs(index)) + return; + if(markForgotten(index)) + _cache.dereference(new BlockKey(_streamId, index)); + } + + private synchronized boolean markForgotten(int index) { + if(_forgotten.get(index)) + return false; + _forgotten.set(index); + return true; + } + + private void forgetAfterReaderClose() { + if(_closed || !_readersSealed) + return; + for(int i = 0; i < _completedSize; i++) + tryForget(i); + } + + private void updatePublished(int size) { + int current = _published.get(); + while(current < size && !_published.compareAndSet(current, size)) + current = _published.get(); + } + + public interface Liveness { + boolean needs(int index); + + void consumed(int index); + + default boolean reserve(int index) { + return needs(index); + } + + default void unreserve(int index) { + } + } + + public interface AccessPattern extends Liveness { + boolean hasNext(); + + int next(); + } + + public interface StoreReader extends AutoCloseable { + Liveness liveness(); + + boolean isClosed(); + + @Override + void close(); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/OOCStreamMaterializer.java b/src/main/java/org/apache/sysds/runtime/ooc/store/OOCStreamMaterializer.java new file mode 100644 index 00000000000..2cede71f804 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/OOCStreamMaterializer.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.store; + +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; +import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; + +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import java.util.function.ToIntFunction; + +public final class OOCStreamMaterializer implements Consumer> { + private final MaterializedStore _store; + private final ToIntFunction _linearize; + private final MemoryAllowance _allowance; + private final List>> _liveConsumers; + private final OOCFuture _completion; + private final AtomicBoolean _done; + + public OOCStreamMaterializer(MaterializedStore store, ToIntFunction linearize, + MemoryAllowance allowance) { + this(store, linearize, allowance, List.of()); + } + + public OOCStreamMaterializer(MaterializedStore store, ToIntFunction linearize, + MemoryAllowance allowance, List>> liveConsumers) { + _store = store; + _linearize = linearize; + _allowance = allowance; + _liveConsumers = List.copyOf(liveConsumers); + _completion = new OOCFuture<>(); + _done = new AtomicBoolean(false); + } + + public void attach(OOCStream source) { + source.setSubscriber(this); + } + + public OOCFuture completion() { + return _completion; + } + + @Override + public void accept(OOCStream.QueueCallback callback) { + if(_done.get()) { + callback.close(); + return; + } + try(callback) { + if(callback.isFailure()) { + try { + callback.get(); + } + catch(DMLRuntimeException ex) { + fail(ex); + } + return; + } + if(callback.isEos()) { + finish(); + return; + } + publish(callback); + } + catch(RuntimeException ex) { + fail(DMLRuntimeException.of(ex)); + } + } + + private void publish(OOCStream.QueueCallback callback) { + IndexedMatrixValue value = callback.get(); + int index = _linearize.applyAsInt(value.getIndexes()); + StoreLease lease; + if(callback instanceof InMemoryQueueCallback managed && managed.getManagedBytes() > 0) { + lease = _store.publishPinnedLive(index, managed.extractManagedPayload()); + } + else { + // To handle not-yet managed callbacks + long bytes = serializedSize(value); + _allowance.reserveBlocking(bytes); + lease = _store.publishPinnedLive(index, value, bytes, _allowance); + } + try(lease) { + for(Consumer> liveConsumer : _liveConsumers) { + try(OOCStream.QueueCallback alias = new MaterializedCallback(lease.retain())) { + liveConsumer.accept(alias); + } + } + } + } + + private void finish() { + if(!_done.compareAndSet(false, true)) + return; + try { + _store.complete(); + } + catch(RuntimeException ex) { + deliverEos(DMLRuntimeException.of(ex)); + _completion.completeExceptionally(ex); + return; + } + deliverEos(null); + _completion.complete(null); + } + + private void fail(DMLRuntimeException failure) { + if(!_done.compareAndSet(false, true)) + return; + deliverEos(failure); + _completion.completeExceptionally(failure); + } + + private void deliverEos(DMLRuntimeException failure) { + for(Consumer> liveConsumer : _liveConsumers) { + try { + liveConsumer.accept(OOCStream.eos(failure)); + } + catch(RuntimeException ignored) { + } + } + } + + private static long serializedSize(IndexedMatrixValue value) { + return ((MatrixBlock) value.getValue()).getExactSerializedSize(); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/OrderedMaterializedStoreReader.java b/src/main/java/org/apache/sysds/runtime/ooc/store/OrderedMaterializedStoreReader.java new file mode 100644 index 00000000000..2dd048750c1 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/OrderedMaterializedStoreReader.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.store; + +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.OOCCache; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; +import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; +import org.apache.sysds.runtime.ooc.util.OOCUtils; + +import java.util.ArrayDeque; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.IntConsumer; + +public final class OrderedMaterializedStoreReader implements MaterializedStore.StoreReader { + private static final Request CLOSED = new Request(-1, OOCFuture.completed(null)); + + private final OOCCache _cache; + private final long _streamId; + private final MaterializedStore.AccessPattern _pattern; + private final MemoryAllowance _allowance; + private final int _maxPrefetch; + private final boolean _softOrdering; + private final Runnable _afterClose; + private final IntConsumer _afterRelease; + private final BlockingQueue _requests; + private final AtomicInteger _inFlightRequests; + private volatile boolean _closed; + + OrderedMaterializedStoreReader(OOCCache cache, long streamId, MaterializedStore.AccessPattern pattern, + MemoryAllowance allowance, int maxPrefetch, boolean softOrdering, Runnable afterClose, + IntConsumer afterRelease) { + _cache = cache; + _streamId = streamId; + _pattern = pattern; + _allowance = allowance; + _maxPrefetch = maxPrefetch; + _softOrdering = softOrdering; + _afterClose = afterClose; + _afterRelease = afterRelease; + _requests = new LinkedBlockingQueue<>(); + _inFlightRequests = new AtomicInteger(); + } + + @Override + public MaterializedStore.Liveness liveness() { + return _pattern; + } + + @Override + public boolean isClosed() { + return _closed; + } + + @Override + public void close() { + ArrayDeque pending = new ArrayDeque<>(); + synchronized(this) { + if(_closed) + return; + _closed = true; + _requests.drainTo(pending); + } + for(Request request : pending) { + releaseWhenReady(request); + if(_softOrdering) + _inFlightRequests.decrementAndGet(); + } + if(_softOrdering) + _requests.offer(CLOSED); + _afterClose.run(); + } + + public boolean hasNext() { + if(_softOrdering) { + checkReady(); + fillSoft(); + return _inFlightRequests.get() > 0; + } + synchronized(this) { + checkReady(); + fillStrict(); + return !_requests.isEmpty(); + } + } + + public StoreLease next() throws InterruptedException { + if(_softOrdering) { + checkReady(); + return nextSoft(); + } + Request request; + synchronized(this) { + checkReady(); + fillStrict(); + if(_requests.isEmpty()) + throw new IllegalStateException("No remaining item"); + request = _requests.remove(); + } + BlockEntry entry; + try { + entry = awaitEntry(request); + } + catch(InterruptedException | RuntimeException ex) { + releaseWhenReady(request); + throw ex; + } + synchronized(this) { + if(_closed) { + _cache.unpin(entry, _allowance); + throw new IllegalStateException("Reader is closed"); + } + try { + fillStrict(); + } + catch(RuntimeException ex) { + _cache.unpin(entry, _allowance); + throw ex; + } + } + return new StoreLease<>(entry, () -> release(request._index, entry)); + } + + public void release(int index, BlockEntry entry) { + _cache.unpin(entry, _allowance); + _pattern.consumed(index); + _afterRelease.accept(index); + } + + private void checkReady() { + if(_closed) + throw new IllegalStateException("Reader is closed"); + } + + private StoreLease nextSoft() throws InterruptedException { + fillSoft(); + if(_inFlightRequests.get() <= 0) + throw new IllegalStateException("No remaining item"); + Request request = _requests.take(); + if(request == CLOSED) + throw new IllegalStateException("Reader is closed"); + _inFlightRequests.decrementAndGet(); + BlockEntry entry; + try { + entry = request._future.get(); + } + catch(ExecutionException e) { + throw DMLRuntimeException.of(e.getCause()); + } + if(entry == null) + throw new IllegalStateException("Reader is closed"); + fillSoft(); + return new StoreLease<>(entry, () -> release(request._index, entry)); + } + + private void fillStrict() { + while(_requests.size() < _maxPrefetch && _pattern.hasNext()) { + int index = _pattern.next(); + OOCFuture future = _cache.pin(_streamId, index, _allowance); + _requests.offer(new Request(index, future)); + } + } + + private void fillSoft() { + while(_inFlightRequests.get() < _maxPrefetch && _pattern.hasNext()) { + int index = _pattern.next(); + OOCFuture future = _cache.pin(_streamId, index, _allowance); + _inFlightRequests.incrementAndGet(); + registerSoftRequest(new Request(index, future)); + } + } + + private void registerSoftRequest(Request request) { + request._future.whenComplete((entry, error) -> { + if(error != null || entry != null) { + completeSoft(request); + return; + } + request._future = OOCUtils.pinAdmitted(_cache, _streamId, request._index, _allowance, () -> _closed); + request._future.whenComplete((admittedEntry, admittedError) -> completeSoft(request)); + }); + } + + private void completeSoft(Request request) { + if(_closed) { + releaseWhenReady(request); + _inFlightRequests.decrementAndGet(); + return; + } + _requests.offer(request); + if(_closed && _requests.remove(request)) { + releaseWhenReady(request); + _inFlightRequests.decrementAndGet(); + } + } + + private BlockEntry awaitEntry(Request request) throws InterruptedException { + try { + BlockEntry entry = request._future.get(); + if(entry == null) { + OOCFuture retried = OOCUtils.pinAdmitted(_cache, _streamId, request._index, _allowance, + () -> _closed); + request._future = retried; + entry = retried.get(); + if(entry == null) + throw new IllegalStateException("Reader is closed"); + } + return entry; + } + catch(ExecutionException e) { + throw DMLRuntimeException.of(e.getCause()); + } + } + + private void releaseWhenReady(Request request) { + request._future.whenComplete((entry, error) -> { + if(entry != null) + _cache.unpin(entry, _allowance); + }); + } + + private static final class Request { + private final int _index; + private OOCFuture _future; + + private Request(int index, OOCFuture future) { + _index = index; + _future = future; + } + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/StoreLease.java b/src/main/java/org/apache/sysds/runtime/ooc/store/StoreLease.java new file mode 100644 index 00000000000..814c95cb56a --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/StoreLease.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.store; + +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; + +import java.util.concurrent.atomic.AtomicInteger; + +public final class StoreLease implements AutoCloseable { + private final Runnable _releaser; + private final T _value; + private final BlockEntry _entry; + private final AtomicInteger _shared; + private boolean _open; + + StoreLease(BlockEntry entry, Runnable releaser) { + this(null, entry, releaser, new AtomicInteger(1)); + } + + StoreLease(T value, Runnable releaser) { + this(value, null, releaser, new AtomicInteger(1)); + } + + private StoreLease(T value, BlockEntry entry, Runnable releaser, AtomicInteger shared) { + _releaser = releaser; + _value = value; + _entry = entry; + _shared = shared; + _open = true; + } + + @SuppressWarnings("unchecked") + public synchronized T value() { + if(!_open) + throw new IllegalStateException("Lease is closed"); + return _entry == null ? _value : (T) _entry.getData(); + } + + synchronized BlockEntry entry() { + if(!_open) + throw new IllegalStateException("Lease is closed"); + return _entry; + } + + public synchronized StoreLease retain() { + if(!_open) + throw new IllegalStateException("Lease is closed"); + _shared.incrementAndGet(); + return new StoreLease<>(_value, _entry, _releaser, _shared); + } + + @Override + public synchronized void close() { + if(!_open) + return; + _open = false; + if(_shared.decrementAndGet() == 0) + _releaser.run(); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java index f33d17ea132..74e6ed30a8f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java @@ -21,14 +21,50 @@ import org.apache.sysds.runtime.matrix.data.MatrixIndexes; import org.apache.sysds.runtime.meta.DataCharacteristics; +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.OOCCache; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; import org.apache.sysds.runtime.util.IndexRange; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.function.BooleanSupplier; public class OOCUtils { + public static OOCFuture pinAdmitted(OOCCache cache, long streamId, long sequenceNumber, + MemoryAllowance allowance, BooleanSupplier cancelled) { + if(cancelled.getAsBoolean()) + return OOCFuture.completed(null); + if(allowance.isShutdown()) + return OOCFuture + .failed(new IllegalStateException("Allowance was shut down while a pin admission was pending.")); + + OOCFuture admitted; + try { + admitted = cache.pinAdmitted(streamId, sequenceNumber, allowance); + } + catch(RuntimeException ex) { + return OOCFuture.failed(ex); + } + OOCFuture result = new OOCFuture<>(); + admitted.whenComplete((entry, error) -> { + if(error != null) { + result.completeExceptionally(error); + return; + } + if(entry != null && cancelled.getAsBoolean()) { + cache.unpin(entry, allowance); + result.complete(null); + return; + } + result.complete(entry); + }); + return result; + } + public static IndexRange getRangeOfTile(MatrixIndexes tileIdx, long blen) { long rs = 1 + tileIdx.getRowIndex() * blen; long re = (tileIdx.getRowIndex() + 1) * blen; diff --git a/src/test/java/org/apache/sysds/test/component/ooc/MaterializedStoreTest.java b/src/test/java/org/apache/sysds/test/component/ooc/MaterializedStoreTest.java new file mode 100644 index 00000000000..d8ea0e3986f --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/ooc/MaterializedStoreTest.java @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.ooc; + +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.instructions.ooc.CachingStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.ooc.cache.OOCCacheImpl; +import org.apache.sysds.runtime.ooc.memory.GlobalMemoryBroker; +import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; +import org.apache.sysds.runtime.ooc.memory.SyncMemoryAllowance; +import org.apache.sysds.runtime.ooc.store.IndexedMaterializedStoreReader; +import org.apache.sysds.runtime.ooc.store.MaterializedStore; +import org.apache.sysds.runtime.ooc.store.OOCStreamMaterializer; +import org.apache.sysds.runtime.ooc.store.OrderedMaterializedStoreReader; +import org.apache.sysds.runtime.ooc.store.StoreLease; +import org.apache.sysds.test.component.ooc.cache.OOCCacheTestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class MaterializedStoreTest { + private static final long MEMORY_LIMIT = 100_000_000; + private static final long WAIT_SECONDS = 10; + private static final long TILE_BYTES = new MatrixBlock(4, 4, 1.0).getExactSerializedSize(); + + private GlobalMemoryBroker _broker; + private SyncMemoryAllowance _producer; + private SyncMemoryAllowance _materializerAllowance; + private SyncMemoryAllowance _readerAllowance; + private OOCCacheImpl _cache; + private MaterializedStore _store; + + @Before + public void setUp() { + _broker = new GlobalMemoryBroker(1_000_000_000); + _producer = new SyncMemoryAllowance(_broker); + _materializerAllowance = new SyncMemoryAllowance(_broker); + _readerAllowance = new SyncMemoryAllowance(_broker); + _producer.setTargetMemory(MEMORY_LIMIT); + _materializerAllowance.setTargetMemory(MEMORY_LIMIT); + _readerAllowance.setTargetMemory(MEMORY_LIMIT); + _cache = new OOCCacheImpl(new OOCCacheTestUtils.RecordingOOCIOHandler(), MEMORY_LIMIT, MEMORY_LIMIT); + _store = new MaterializedStore<>(_cache, CachingStream._streamSeq.getNextID()); + } + + @After + public void tearDown() { + _store.close(); + _cache.shutdown(); + _producer.destroy(); + _materializerAllowance.destroy(); + _readerAllowance.destroy(); + } + + @Test + public void testMaterializationReadersAndForgetting() throws Exception { + OOCStreamMaterializer materializer = new OOCStreamMaterializer(_store, + indexes -> (int) indexes.getRowIndex() - 1, _materializerAllowance); + _producer.reserveBlocking(TILE_BYTES); + materializer.accept(new InMemoryQueueCallback(tile(0, 1.0), null, _producer, TILE_BYTES)); + materializer.accept(new OOCStream.SimpleQueueCallback<>(tile(1, 2.0), null)); + _producer.reserveBlocking(TILE_BYTES); + materializer.accept(new InMemoryQueueCallback(tile(2, 3.0), null, _producer, TILE_BYTES)); + materializer.accept(OOCStream.eos(null)); + materializer.completion().get(WAIT_SECONDS, TimeUnit.SECONDS); + + Assert.assertEquals(0, _producer.getUsedMemory()); + Assert.assertEquals(0, _materializerAllowance.getUsedMemory()); + Assert.assertEquals(3, _store.size()); + + MaterializedStore.AccessPattern pattern = sequentialPattern(3); + boolean[] indexedConsumed = new boolean[3]; + MaterializedStore.Liveness liveness = new MaterializedStore.Liveness() { + @Override + public boolean needs(int index) { + return !indexedConsumed[index]; + } + + @Override + public void consumed(int index) { + indexedConsumed[index] = true; + } + }; + + OrderedMaterializedStoreReader ordered = _store.openReader(pattern, _readerAllowance, 2); + IndexedMaterializedStoreReader indexed = _store.openIndexedReader(liveness); + _store.sealReaders(); + + int index = 0; + while(ordered.hasNext()) { + try(StoreLease lease = ordered.next()) { + Assert.assertEquals(index + 1L, lease.value().getIndexes().getRowIndex()); + index++; + } + } + ordered.close(); + Assert.assertEquals(3, index); + Assert.assertTrue(_cache.getOwnedCacheSize() > 0); + + for(index = 2; index > 0; index--) { + try(StoreLease lease = indexed.request(index, _readerAllowance).get(WAIT_SECONDS, + TimeUnit.SECONDS)) { + Assert.assertEquals(index + 1L, lease.value().getIndexes().getRowIndex()); + } + } + Assert.assertTrue(_cache.getOwnedCacheSize() > 0); + indexed.close(); + OOCCacheTestUtils.await(() -> _cache.getOwnedCacheSize() == 0, WAIT_SECONDS); + Assert.assertEquals(0, _readerAllowance.getUsedMemory()); + } + + @Test + public void testOrderedReaderRetries() throws Exception { + OOCStreamMaterializer materializer = new OOCStreamMaterializer(_store, + indexes -> (int) indexes.getRowIndex() - 1, _materializerAllowance); + for(int i = 0; i < 2; i++) { + _producer.reserveBlocking(TILE_BYTES); + materializer.accept(new InMemoryQueueCallback(tile(i, i + 1.0), null, _producer, TILE_BYTES)); + } + materializer.accept(OOCStream.eos(null)); + materializer.completion().get(WAIT_SECONDS, TimeUnit.SECONDS); + + _readerAllowance.destroy(); + _readerAllowance = new SyncMemoryAllowance(_broker, TILE_BYTES); + _readerAllowance.setTargetMemory(TILE_BYTES); + OrderedMaterializedStoreReader reader = _store.openReader(sequentialPattern(2), + _readerAllowance, 2, false); + _store.sealReaders(); + + Assert.assertTrue(reader.hasNext()); + try(StoreLease first = reader.next()) { + Assert.assertEquals(1L, first.value().getIndexes().getRowIndex()); + } + try(StoreLease second = reader.next()) { + Assert.assertEquals(2L, second.value().getIndexes().getRowIndex()); + } + reader.close(); + Assert.assertEquals(0, _readerAllowance.getUsedMemory()); + } + + @Test + public void testSoftOrderingReturnsReadyRequestFirst() throws Exception { + MatrixBlock largeBlock = new MatrixBlock(16, 16, 1.0); + long largeBytes = largeBlock.getExactSerializedSize(); + OOCStreamMaterializer materializer = new OOCStreamMaterializer(_store, + indexes -> (int) indexes.getRowIndex() - 1, _materializerAllowance); + _producer.reserveBlocking(largeBytes); + materializer.accept(new InMemoryQueueCallback(new IndexedMatrixValue(new MatrixIndexes(1, 1), largeBlock), null, + _producer, largeBytes)); + _producer.reserveBlocking(TILE_BYTES); + materializer.accept(new InMemoryQueueCallback(tile(1, 2.0), null, _producer, TILE_BYTES)); + materializer.accept(OOCStream.eos(null)); + materializer.completion().get(WAIT_SECONDS, TimeUnit.SECONDS); + + _readerAllowance.destroy(); + _readerAllowance = new SyncMemoryAllowance(_broker, largeBytes); + _readerAllowance.setTargetMemory(largeBytes); + long heldBytes = largeBytes - TILE_BYTES; + _readerAllowance.reserveBlocking(heldBytes); + OrderedMaterializedStoreReader reader = _store.openReader(sequentialPattern(2), + _readerAllowance, 2); + _store.sealReaders(); + + Assert.assertTrue(reader.hasNext()); + try(StoreLease first = reader.next()) { + Assert.assertEquals(2L, first.value().getIndexes().getRowIndex()); + } + _readerAllowance.release(heldBytes); + try(StoreLease second = reader.next()) { + Assert.assertEquals(1L, second.value().getIndexes().getRowIndex()); + } + Assert.assertFalse(reader.hasNext()); + reader.close(); + Assert.assertEquals(0, _readerAllowance.getUsedMemory()); + } + + @Test + public void testDirectRequests() throws Exception { + OOCStreamMaterializer materializer = new OOCStreamMaterializer(_store, + indexes -> (int) indexes.getRowIndex() - 1, _materializerAllowance); + _producer.reserveBlocking(TILE_BYTES); + materializer.accept(new InMemoryQueueCallback(tile(0, 1.0), null, _producer, TILE_BYTES)); + materializer.accept(OOCStream.eos(null)); + materializer.completion().get(WAIT_SECONDS, TimeUnit.SECONDS); + + boolean[] needed = {true}; + IndexedMaterializedStoreReader reader = _store + .openIndexedReader(new MaterializedStore.Liveness() { + @Override + public boolean needs(int index) { + return needed[index]; + } + + @Override + public void consumed(int index) { + needed[index] = false; + } + }); + _store.sealReaders(); + + try(StoreLease published = _store.requestPublished(0, _readerAllowance).get(WAIT_SECONDS, + TimeUnit.SECONDS)) { + Assert.assertEquals(1L, published.value().getIndexes().getRowIndex()); + } + StoreLease live = reader.requestIfLive(0, _readerAllowance); + Assert.assertNotNull(live); + StoreLease retained = live.retain(); + live.close(); + Assert.assertEquals(TILE_BYTES, _readerAllowance.getUsedMemory()); + Assert.assertEquals(1L, retained.value().getIndexes().getRowIndex()); + retained.close(); + Assert.assertEquals(0, _readerAllowance.getUsedMemory()); + reader.close(); + Assert.assertTrue(reader.isClosed()); + } + + @Test + public void testCompletionMissingPublications() throws Exception { + AtomicInteger failures = new AtomicInteger(); + OOCStreamMaterializer materializer = new OOCStreamMaterializer(_store, + indexes -> (int) indexes.getRowIndex() - 1, _materializerAllowance, List.of(callback -> { + if(callback.isFailure()) + failures.incrementAndGet(); + })); + for(int index : new int[] {0, 2}) { + _producer.reserveBlocking(TILE_BYTES); + materializer.accept(new InMemoryQueueCallback(tile(index, 1.0), null, _producer, TILE_BYTES)); + } + materializer.accept(OOCStream.eos(null)); + + try { + materializer.completion().get(WAIT_SECONDS, TimeUnit.SECONDS); + Assert.fail("Completion must reject a missing publication index"); + } + catch(ExecutionException ex) { + Assert.assertTrue(ex.getCause() instanceof IllegalStateException); + } + Assert.assertEquals(1, failures.get()); + Assert.assertEquals(0, _producer.getUsedMemory()); + } + + @Test + public void testLiveCallbackKeepsPublicationPinned() throws Exception { + AtomicReference> retained = new AtomicReference<>(); + AtomicInteger eos = new AtomicInteger(); + OOCStreamMaterializer materializer = new OOCStreamMaterializer(_store, + indexes -> (int) indexes.getRowIndex() - 1, _materializerAllowance, List.of(callback -> { + if(callback.isEos()) + eos.incrementAndGet(); + else + retained.set(callback.keepOpen()); + })); + + _producer.reserveBlocking(TILE_BYTES); + materializer.accept(new InMemoryQueueCallback(tile(0, 1.0), null, _producer, TILE_BYTES)); + Assert.assertEquals(TILE_BYTES, _producer.getUsedMemory()); + Assert.assertNotNull(retained.get()); + retained.get().close(); + retained.get().close(); + Assert.assertEquals(0, _producer.getUsedMemory()); + + materializer.accept(OOCStream.eos(null)); + materializer.completion().get(WAIT_SECONDS, TimeUnit.SECONDS); + Assert.assertEquals(1, eos.get()); + } + + @Test + public void testFailurePropagation() throws Exception { + DMLRuntimeException sourceFailure = new DMLRuntimeException("injected failure"); + AtomicInteger failures = new AtomicInteger(); + OOCStreamMaterializer materializer = new OOCStreamMaterializer(_store, + indexes -> (int) indexes.getRowIndex() - 1, _materializerAllowance, List.of(callback -> { + if(callback.isFailure()) + failures.incrementAndGet(); + })); + materializer.accept(OOCStream.eos(sourceFailure)); + try { + materializer.completion().get(WAIT_SECONDS, TimeUnit.SECONDS); + Assert.fail("The source failure must reach materializer completion"); + } + catch(ExecutionException ex) { + Assert.assertSame(sourceFailure, ex.getCause()); + } + Assert.assertEquals(1, failures.get()); + + _store.close(); + _store = new MaterializedStore<>(_cache, CachingStream._streamSeq.getNextID()); + _store.close(); + materializer = new OOCStreamMaterializer(_store, indexes -> (int) indexes.getRowIndex() - 1, + _materializerAllowance); + _producer.reserveBlocking(TILE_BYTES); + materializer.accept(new InMemoryQueueCallback(tile(0, 1.0), null, _producer, TILE_BYTES)); + try { + materializer.completion().get(WAIT_SECONDS, TimeUnit.SECONDS); + Assert.fail("Publishing into a closed store must fail"); + } + catch(ExecutionException ex) { + Assert.assertTrue(ex.getCause() instanceof DMLRuntimeException); + } + Assert.assertEquals(0, _producer.getUsedMemory()); + } + + private static IndexedMatrixValue tile(int index, double value) { + return new IndexedMatrixValue(new MatrixIndexes(index + 1L, 1), new MatrixBlock(4, 4, value)); + } + + private static MaterializedStore.AccessPattern sequentialPattern(int to) { + return new MaterializedStore.AccessPattern() { + private int _next; + + @Override + public boolean hasNext() { + return _next < to; + } + + @Override + public int next() { + return _next++; + } + + @Override + public boolean needs(int index) { + return true; + } + + @Override + public void consumed(int index) { + } + }; + } +} From 90d8e4e8729143a6ecbf46e5939af2dfa55b486e Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:33:55 +0200 Subject: [PATCH 080/132] [OOC] Add StateTable --- .../sysds/runtime/ooc/cache/OOCFuture.java | 35 +- .../sysds/runtime/ooc/store/StateTable.java | 414 ++++++++++++++++++ .../sysds/runtime/ooc/store/StoreLease.java | 4 +- .../runtime/ooc/util/StateTableUtils.java | 102 +++++ .../component/ooc/StateTableUtilsTest.java | 143 ++++++ 5 files changed, 693 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/store/StateTable.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/util/StateTableUtils.java create mode 100644 src/test/java/org/apache/sysds/test/component/ooc/StateTableUtilsTest.java diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java index 491fefaad87..d6796e35a87 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java @@ -111,6 +111,36 @@ public T get(long timeout, TimeUnit unit) throws InterruptedException, Execution } } + public OOCFuture thenCompose(Function> mapper) { + OOCFuture result = new OOCFuture<>(); + whenComplete((value, error) -> { + if(error != null) { + result.completeExceptionally(error); + return; + } + + final OOCFuture next; + try { + next = mapper.apply(value); + if(next == null) + throw new NullPointerException("thenCompose mapper returned null"); + } + catch(Throwable t) { + result.completeExceptionally(t); + return; + } + + next.whenComplete((nextValue, nextError) -> { + if(nextError != null) + result.completeExceptionally(nextError); + else + result.complete(nextValue); + }); + }); + + return result; + } + private void subscribe(Function mapper, Consumer action, BiConsumer completion) { T value; @@ -167,7 +197,6 @@ else if(resultError == null) action.accept(result); } catch(Throwable ignored) { - // Subscribers are independent; one failed callback must not prevent the remaining notifications. } } @@ -181,8 +210,8 @@ private static final class Subscriber { private Subscriber(Function mapper, Consumer action, BiConsumer completion, Subscriber next) { this.mapper = mapper; - this.action = (Consumer)action; - this.completion = (BiConsumer)(BiConsumer)completion; + this.action = (Consumer) action; + this.completion = (BiConsumer) completion; this.next = next; } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/StateTable.java b/src/main/java/org/apache/sysds/runtime/ooc/store/StateTable.java new file mode 100644 index 00000000000..80eb57dcfe4 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/StateTable.java @@ -0,0 +1,414 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.store; + +import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.BlockKey; +import org.apache.sysds.runtime.ooc.cache.OOCCache; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; +import org.apache.sysds.runtime.ooc.memory.ManagedPayload; +import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; +import org.apache.sysds.runtime.ooc.util.OOCUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicIntegerArray; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; +import java.util.function.IntToLongFunction; + +public final class StateTable implements AutoCloseable { + private static final int INITIAL_SLOTS = 64; + + private final OOCCache _cache; + private final long _streamId; + private final AtomicLong _nextGeneration = new AtomicLong(); + private final CopyOnWriteArrayList _evictionPolicies = new CopyOnWriteArrayList<>(); + private final AtomicBoolean _evictionPolicyInstalled = new AtomicBoolean(false); + private Slot[] _slots; + private volatile AtomicIntegerArray _generationSlots; + private volatile boolean _closed; + + public StateTable(OOCCache cache, long streamId) { + this(cache, streamId, INITIAL_SLOTS); + } + + public StateTable(OOCCache cache, long streamId, int numSlots) { + int capacity = Math.max(1, numSlots); + _cache = cache; + _streamId = streamId; + _generationSlots = new AtomicIntegerArray(capacity); + _slots = new Slot[capacity]; + } + + public void addEvictionPolicy(IntToLongFunction slotPolicy) { + _evictionPolicies.add(slotPolicy); + if(_evictionPolicyInstalled.compareAndSet(false, true)) + _cache.addEvictionPolicy(_streamId, this::scoreTableEntry); + } + + public void put(int index, ManagedPayload payload) { + putSlot(index, slot -> finalizeOwnedPut(index, slot, payload)); + } + + public void putReference(int index, BlockEntry pinned) { + checkPinned(pinned); + putSlot(index, slot -> finalizeReferencePut(index, slot, pinned)); + } + + private void putSlot(int index, Consumer finalizer) { + Slot slot; + synchronized(this) { + checkOpen(); + ensureCapacity(index); + if(_slots[index] != null) + throw new IllegalStateException("State table slot " + index + " is already occupied."); + slot = new Slot(); + _slots[index] = slot; + } + finalizer.accept(slot); + } + + public OOCFuture> putOrTake(int index, ManagedPayload payload, MemoryAllowance leaseAllowance) { + return putSlotOrTake(index, leaseAllowance, slot -> finalizeOwnedPut(index, slot, payload)); + } + + public OOCFuture> putReferenceOrTake(int index, BlockEntry pinned, MemoryAllowance leaseAllowance) { + checkPinned(pinned); + return putSlotOrTake(index, leaseAllowance, slot -> finalizeReferencePut(index, slot, pinned)); + } + + private OOCFuture> putSlotOrTake(int index, MemoryAllowance leaseAllowance, + Consumer finalizer) { + Slot putting = null; + Slot taken = null; + OOCFuture waitFor = null; + synchronized(this) { + checkOpen(); + ensureCapacity(index); + Slot existing = _slots[index]; + if(existing == null) { + putting = new Slot(); + _slots[index] = putting; + } + else if(existing._putFuture == null) { + _slots[index] = null; + taken = existing; + } + else { + waitFor = existing._putFuture; + } + } + if(putting != null) { + finalizer.accept(putting); + return OOCFuture.completed(null); + } + if(taken != null) + return pinTaken(taken, leaseAllowance); + return waitFor.thenCompose(ignored -> putSlotOrTake(index, leaseAllowance, finalizer)); + } + + public OOCFuture> take(int index, MemoryAllowance leaseAllowance) { + Slot taken = null; + OOCFuture waitFor = null; + synchronized(this) { + checkOpen(); + if(index < 0 || index >= _slots.length) + return OOCFuture.completed(null); + Slot existing = _slots[index]; + if(existing == null) + return OOCFuture.completed(null); + if(existing._putFuture == null) { + _slots[index] = null; + taken = existing; + } + else { + waitFor = existing._putFuture; + } + } + if(taken != null) + return pinTaken(taken, leaseAllowance); + return waitFor.thenCompose(ignored -> take(index, leaseAllowance)); + } + + public OOCFuture> acquire(int index, MemoryAllowance leaseAllowance) { + BlockKey key; + synchronized(this) { + checkOpen(); + if(index < 0 || index >= _slots.length) + return OOCFuture.completed(null); + Slot slot = _slots[index]; + if(slot == null || slot._putFuture != null) + return OOCFuture.completed(null); + key = slot._key; + } + OOCFuture pinned = OOCUtils.pinAdmitted(_cache, key.getStreamId(), key.getSequenceNumber(), + leaseAllowance, () -> _closed); + OOCFuture> result = new OOCFuture<>(); + pinned.whenComplete((entry, error) -> { + if(error != null) + result.completeExceptionally(error); + else + result.complete( + entry == null ? null : new StoreLease<>(entry, () -> _cache.unpin(entry, leaseAllowance))); + }); + return result; + } + + public StoreLease peek(int index, MemoryAllowance leaseAllowance) { + BlockKey key; + synchronized(this) { + checkOpen(); + if(index < 0 || index >= _slots.length) + return null; + Slot slot = _slots[index]; + if(slot == null || slot._putFuture != null) + return null; + key = slot._key; + } + BlockEntry entry = _cache.pinIfLive(key.getStreamId(), key.getSequenceNumber(), leaseAllowance); + return entry == null ? null : new StoreLease<>(entry, () -> _cache.unpin(entry, leaseAllowance)); + } + + public void clear(int index) { + Slot removed = null; + synchronized(this) { + if(index < 0 || index >= _slots.length) + return; + Slot slot = _slots[index]; + if(slot == null) + return; + _slots[index] = null; + if(slot._putFuture == null) + removed = slot; + else + slot._cleared = true; + } + if(removed != null) + releaseSlot(removed); + } + + @Override + public void close() { + List toRelease = new ArrayList<>(); + synchronized(this) { + if(_closed) + return; + _closed = true; + for(int i = 0; i < _slots.length; i++) { + Slot slot = _slots[i]; + if(slot == null) + continue; + _slots[i] = null; + if(slot._putFuture == null) + toRelease.add(slot); + else + slot._cleared = true; + } + } + for(Slot slot : toRelease) + releaseSlot(slot); + } + + private void finalizeOwnedPut(int index, Slot slot, ManagedPayload payload) { + BlockKey key = new BlockKey(_streamId, _nextGeneration.getAndIncrement()); + BlockEntry entry; + try { + payload.transfer(); + } + catch(RuntimeException ex) { + failPut(index, slot, ex); + throw ex; + } + try { + entry = _cache.putPinned(key.getStreamId(), key.getSequenceNumber(), payload.value(), payload.bytes(), + payload.owner()); + } + catch(RuntimeException ex) { + if(payload.bytes() > 0) + payload.owner().release(payload.bytes()); + failPut(index, slot, ex); + throw ex; + } + boolean cleared; + OOCFuture putFuture; + synchronized(this) { + slot._key = key; + slot._tableOwnedKey = true; + int generation = blockIndex(key.getSequenceNumber()); + ensureGenerationCapacity(generation); + _generationSlots.set(generation, index + 1); + cleared = slot._cleared; + putFuture = slot._putFuture; + slot._putFuture = null; + } + _cache.unpin(entry, payload.owner()); + if(cleared) + releaseSlot(slot); + putFuture.complete(null); + } + + private void finalizeReferencePut(int index, Slot slot, BlockEntry pinned) { + try { + _cache.reference(pinned); + } + catch(RuntimeException ex) { + failPut(index, slot, ex); + throw ex; + } + + boolean cleared; + OOCFuture putFuture; + synchronized(this) { + slot._key = pinned.getKey(); + slot._tableOwnedKey = false; + cleared = slot._cleared; + putFuture = slot._putFuture; + slot._putFuture = null; + } + if(cleared) + _cache.dereference(pinned.getKey()); + putFuture.complete(null); + } + + private void failPut(int index, Slot slot, RuntimeException ex) { + OOCFuture putFuture; + synchronized(this) { + if(index < _slots.length && _slots[index] == slot) + _slots[index] = null; + putFuture = slot._putFuture; + slot._putFuture = null; + } + if(putFuture != null) + putFuture.completeExceptionally(ex); + } + + private OOCFuture> pinTaken(Slot slot, MemoryAllowance leaseAllowance) { + OOCFuture pinned = OOCUtils.pinAdmitted(_cache, slot._key.getStreamId(), + slot._key.getSequenceNumber(), leaseAllowance, () -> _closed); + OOCFuture> result = new OOCFuture<>(); + pinned.whenComplete((entry, error) -> { + Throwable completionError = error; + try { + releaseSlot(slot); + } + catch(RuntimeException releaseError) { + if(completionError == null) + completionError = releaseError; + } + if(completionError == null && entry == null) + completionError = new IllegalStateException("State table closed while a take was pending."); + if(completionError != null) { + if(entry != null) { + try { + _cache.unpin(entry, leaseAllowance); + } + catch(RuntimeException ignored) { + } + } + result.completeExceptionally(completionError); + return; + } + result.complete(new StoreLease<>(entry, () -> _cache.unpin(entry, leaseAllowance))); + }); + return result; + } + + private void releaseSlot(Slot slot) { + if(slot._tableOwnedKey) { + int generation = blockIndex(slot._key.getSequenceNumber()); + AtomicIntegerArray slots = _generationSlots; + if(generation < slots.length()) + slots.set(generation, 0); + } + _cache.dereference(slot._key); + } + + private long scoreTableEntry(long generation) { + int index = blockIndex(generation); + AtomicIntegerArray slots = _generationSlots; + if(index >= slots.length()) + return Long.MAX_VALUE; + int encodedSlot = slots.get(index); + if(encodedSlot == 0) + return Long.MAX_VALUE; + int slot = encodedSlot - 1; + long score = Long.MAX_VALUE; + for(IntToLongFunction policy : _evictionPolicies) + score = Math.min(score, policy.applyAsLong(slot)); + return score; + } + + private void ensureGenerationCapacity(int index) { + AtomicIntegerArray slots = _generationSlots; + if(index < slots.length()) + return; + int newLength = slots.length(); + while(index >= newLength) { + if(newLength > Integer.MAX_VALUE / 2) + throw new IllegalStateException("State table generation map capacity overflow"); + newLength *= 2; + } + AtomicIntegerArray grown = new AtomicIntegerArray(newLength); + for(int i = 0; i < slots.length(); i++) + grown.set(i, slots.get(i)); + _generationSlots = grown; + } + + private static int blockIndex(long sequenceNumber) { + if(sequenceNumber < 0 || sequenceNumber > Integer.MAX_VALUE) + throw new IndexOutOfBoundsException("Invalid block index: " + sequenceNumber); + return (int) sequenceNumber; + } + + private void checkOpen() { + if(_closed) + throw new IllegalStateException("State table is closed."); + } + + private static void checkPinned(BlockEntry pinned) { + if(!pinned.isPinned()) + throw new IllegalArgumentException( + "Reference install requires the supplied entry to be pinned: " + pinned.getKey()); + } + + private void ensureCapacity(int index) { + if(index < 0) + throw new IndexOutOfBoundsException("Invalid slot index: " + index); + if(index < _slots.length) + return; + int newLength = _slots.length; + while(index >= newLength) + newLength *= 2; + Slot[] grown = new Slot[newLength]; + System.arraycopy(_slots, 0, grown, 0, _slots.length); + _slots = grown; + } + + private static final class Slot { + private boolean _cleared; + private boolean _tableOwnedKey; + private BlockKey _key; + private OOCFuture _putFuture = new OOCFuture<>(); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/StoreLease.java b/src/main/java/org/apache/sysds/runtime/ooc/store/StoreLease.java index 814c95cb56a..c9d4ca6f8cd 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/store/StoreLease.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/StoreLease.java @@ -31,11 +31,11 @@ public final class StoreLease implements AutoCloseabl private final AtomicInteger _shared; private boolean _open; - StoreLease(BlockEntry entry, Runnable releaser) { + public StoreLease(BlockEntry entry, Runnable releaser) { this(null, entry, releaser, new AtomicInteger(1)); } - StoreLease(T value, Runnable releaser) { + public StoreLease(T value, Runnable releaser) { this(value, null, releaser, new AtomicInteger(1)); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/StateTableUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/StateTableUtils.java new file mode 100644 index 00000000000..a6aecd40c71 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/StateTableUtils.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.util; + +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; +import org.apache.sysds.runtime.ooc.memory.ManagedPayload; +import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; +import org.apache.sysds.runtime.ooc.store.MaterializedCallback; +import org.apache.sysds.runtime.ooc.store.StateTable; +import org.apache.sysds.runtime.ooc.store.StoreLease; + +public final class StateTableUtils { + public static OOCFuture putOrTake(StateTable table, int slot, + OOCStream.QueueCallback tile, MemoryAllowance allowance) { + if(tile instanceof MaterializedCallback pinned && pinned.pinnedEntry() != null) + return putReferenceOrTake(table, slot, pinned, allowance); + ManagedPayload payload; + if(tile instanceof InMemoryQueueCallback managed && managed.getManagedBytes() > 0) { + payload = managed.extractManagedPayload(); + managed.close(); + } + else { + IndexedMatrixValue value = tile.get(); + long bytes = ((MatrixBlock) value.getValue()).getExactSerializedSize(); + allowance.reserveBlocking(bytes); + payload = new ManagedPayload<>(value, bytes, allowance); + tile.close(); + } + OOCFuture result = new OOCFuture<>(); + OOCFuture> matched; + try { + matched = table.putOrTake(slot, payload, allowance); + } + catch(RuntimeException ex) { + payload.release(); + return OOCFuture.failed(ex); + } + matched.whenComplete((lease, error) -> { + if(error != null) { + payload.release(); + result.completeExceptionally(error); + } + else if(lease == null) + result.complete(null); + else + result.complete(new Match(new MaterializedCallback(new StoreLease<>(payload.value(), payload::release)), + new MaterializedCallback(lease))); + }); + return result; + } + + private static OOCFuture putReferenceOrTake(StateTable table, int slot, + MaterializedCallback pinned, MemoryAllowance allowance) { + OOCFuture result = new OOCFuture<>(); + OOCFuture> matched; + try { + matched = table.putReferenceOrTake(slot, pinned.pinnedEntry(), allowance); + } + catch(RuntimeException ex) { + pinned.close(); + return OOCFuture.failed(ex); + } + matched.whenComplete((lease, error) -> { + if(error != null) { + pinned.close(); + result.completeExceptionally(error); + } + else if(lease == null) { + pinned.close(); + result.complete(null); + } + else + result.complete(new Match(pinned, new MaterializedCallback(lease))); + }); + return result; + } + + public record Match(OOCStream.QueueCallback left, + OOCStream.QueueCallback right) { + } +} diff --git a/src/test/java/org/apache/sysds/test/component/ooc/StateTableUtilsTest.java b/src/test/java/org/apache/sysds/test/component/ooc/StateTableUtilsTest.java new file mode 100644 index 00000000000..7debcd04e3a --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/ooc/StateTableUtilsTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.ooc; + +import java.util.concurrent.TimeUnit; + +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.ooc.cache.OOCCacheImpl; +import org.apache.sysds.runtime.ooc.memory.GlobalMemoryBroker; +import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; +import org.apache.sysds.runtime.ooc.memory.ManagedPayload; +import org.apache.sysds.runtime.ooc.memory.SyncMemoryAllowance; +import org.apache.sysds.runtime.ooc.store.MaterializedCallback; +import org.apache.sysds.runtime.ooc.store.StateTable; +import org.apache.sysds.runtime.ooc.store.StoreLease; +import org.apache.sysds.runtime.ooc.util.StateTableUtils; +import org.apache.sysds.test.component.ooc.cache.OOCCacheTestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class StateTableUtilsTest { + private static final long MEMORY_LIMIT = 100_000_000; + private static final long WAIT_SECONDS = 10; + private static final long TILE_BYTES = new MatrixBlock(4, 4, 1.0).getExactSerializedSize(); + + private SyncMemoryAllowance _producer; + private SyncMemoryAllowance _reader; + private OOCCacheImpl _cache; + private StateTable _source; + private StateTable _table; + + @Before + public void setUp() { + GlobalMemoryBroker broker = new GlobalMemoryBroker(1_000_000_000); + _producer = new SyncMemoryAllowance(broker); + _reader = new SyncMemoryAllowance(broker); + _producer.setTargetMemory(MEMORY_LIMIT); + _reader.setTargetMemory(MEMORY_LIMIT); + _cache = new OOCCacheImpl(new OOCCacheTestUtils.RecordingOOCIOHandler(), MEMORY_LIMIT, MEMORY_LIMIT); + _source = new StateTable<>(_cache, 1); + _table = new StateTable<>(_cache, 2); + } + + @After + public void tearDown() { + _source.close(); + _table.close(); + _cache.shutdown(); + _producer.destroy(); + _reader.destroy(); + } + + @Test + public void testCallbackPutOrTake() throws Exception { + _producer.reserveBlocking(TILE_BYTES); + _source.put(0, new ManagedPayload<>(tile(1.0), TILE_BYTES, _producer)); + StoreLease pinned = _source.peek(0, _reader); + Assert.assertNotNull(pinned); + Assert.assertNull(StateTableUtils.putOrTake(_table, 0, new MaterializedCallback(pinned), _reader) + .get(WAIT_SECONDS, TimeUnit.SECONDS)); + Assert.assertEquals(0, _reader.getUsedMemory()); + + _producer.reserveBlocking(TILE_BYTES); + StateTableUtils.Match referenced = StateTableUtils + .putOrTake(_table, 0, new InMemoryQueueCallback(tile(2.0), null, _producer, TILE_BYTES), _reader) + .get(WAIT_SECONDS, TimeUnit.SECONDS); + Assert.assertNotNull(referenced); + try(OOCStream.QueueCallback left = referenced.left(); + OOCStream.QueueCallback right = referenced.right()) { + Assert.assertEquals(2.0, left.get().getValue().get(0, 0), 0.0); + Assert.assertEquals(1.0, right.get().getValue().get(0, 0), 0.0); + } + + _producer.reserveBlocking(TILE_BYTES); + Assert.assertNull(StateTableUtils + .putOrTake(_table, 1, new InMemoryQueueCallback(tile(3.0), null, _producer, TILE_BYTES), _reader) + .get(WAIT_SECONDS, TimeUnit.SECONDS)); + StateTableUtils.Match copied = StateTableUtils + .putOrTake(_table, 1, new OOCStream.SimpleQueueCallback<>(tile(4.0), null), _reader) + .get(WAIT_SECONDS, TimeUnit.SECONDS); + Assert.assertNotNull(copied); + try(OOCStream.QueueCallback own = copied.left(); + OOCStream.QueueCallback partner = copied.right()) { + Assert.assertEquals(4.0, own.get().getValue().get(0, 0), 0.0); + Assert.assertEquals(3.0, partner.get().getValue().get(0, 0), 0.0); + } + + Assert.assertEquals(0, _producer.getUsedMemory()); + Assert.assertEquals(0, _reader.getUsedMemory()); + _source.close(); + _table.close(); + OOCCacheTestUtils.await(() -> _cache.getOwnedCacheSize() == 0, WAIT_SECONDS); + } + + @Test + public void testStateTableLifecycle() throws Exception { + _producer.reserveBlocking(TILE_BYTES); + _table.put(0, new ManagedPayload<>(tile(5.0), TILE_BYTES, _producer)); + try(StoreLease lease = _table.acquire(0, _reader).get(WAIT_SECONDS, TimeUnit.SECONDS)) { + Assert.assertNotNull(lease); + Assert.assertEquals(5.0, lease.value().getValue().get(0, 0), 0.0); + } + try(StoreLease lease = _table.take(0, _reader).get(WAIT_SECONDS, TimeUnit.SECONDS)) { + Assert.assertNotNull(lease); + Assert.assertEquals(5.0, lease.value().getValue().get(0, 0), 0.0); + } + Assert.assertNull(_table.take(0, _reader).get(WAIT_SECONDS, TimeUnit.SECONDS)); + + _producer.reserveBlocking(TILE_BYTES); + _table.put(1, new ManagedPayload<>(tile(6.0), TILE_BYTES, _producer)); + _table.clear(1); + Assert.assertNull(_table.take(1, _reader).get(WAIT_SECONDS, TimeUnit.SECONDS)); + Assert.assertEquals(0, _producer.getUsedMemory()); + Assert.assertEquals(0, _reader.getUsedMemory()); + OOCCacheTestUtils.await(() -> _cache.getOwnedCacheSize() == 0, WAIT_SECONDS); + } + + private static IndexedMatrixValue tile(double value) { + return new IndexedMatrixValue(new MatrixIndexes(1, 1), new MatrixBlock(4, 4, value)); + } +} From e9e7d1e930e4df8f3a327455beb2d62a48542604 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Fri, 17 Jul 2026 10:25:21 +0200 Subject: [PATCH 081/132] [SYSTEMDS-3891] OOC Memory Allowance/Broker Improvements --- .../ooc/memory/GlobalMemoryBroker.java | 162 ++++++++++-------- .../runtime/ooc/memory/MemoryAllowance.java | 4 + .../runtime/ooc/memory/MemoryBroker.java | 6 + .../ooc/memory/SyncMemoryAllowance.java | 86 ++++++---- .../ooc/memory/OOCMemoryAllowanceTest.java | 54 ++++++ 5 files changed, 211 insertions(+), 101 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/ooc/memory/GlobalMemoryBroker.java b/src/main/java/org/apache/sysds/runtime/ooc/memory/GlobalMemoryBroker.java index 6009182f156..f7ad7b28577 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/memory/GlobalMemoryBroker.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/memory/GlobalMemoryBroker.java @@ -20,10 +20,17 @@ package org.apache.sysds.runtime.ooc.memory; import java.util.ArrayList; -import java.util.LinkedList; import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; public class GlobalMemoryBroker implements MemoryBroker { + private static final long RECLAIM_RETRY_DELAY_MS = 5; + private static final double RECLAIM_PRESSURE = 0.85; + private static final ScheduledThreadPoolExecutor RECLAIM_EXECUTOR = createReclaimExecutor(); + private enum BrokerMode { RELAXED, STRICT } @@ -35,18 +42,28 @@ public static GlobalMemoryBroker get() { } private final long _allowedBytes; - private final List _allowances; - private final LinkedList _overconsumers; + private final CopyOnWriteArrayList _allowances; + private final AtomicBoolean _reclaimRunning; private long _usedBytes; private BrokerMode _brokerMode; private record TargetUpdate(MemoryAllowance _allowance, long _target) {} + private static ScheduledThreadPoolExecutor createReclaimExecutor() { + ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1, runnable -> { + Thread thread = new Thread(runnable, "ooc-memory-broker-reclaimer"); + thread.setDaemon(true); + return thread; + }); + executor.setRemoveOnCancelPolicy(true); + return executor; + } + public GlobalMemoryBroker(long allowedBytes) { _allowedBytes = allowedBytes; _usedBytes = 0; - _allowances = new ArrayList<>(); - _overconsumers = new LinkedList<>(); + _allowances = new CopyOnWriteArrayList<>(); + _reclaimRunning = new AtomicBoolean(false); } @Override @@ -56,29 +73,11 @@ public long requestMemory(MemoryAllowance allowance, long minSize, long maxSize) synchronized(this) { if(minSize < 0 || maxSize < minSize) throw new IllegalArgumentException(); - long share = getEqualShare(); long free = _allowedBytes - _usedBytes; - if(free < minSize) { - if(allowance.getGrantedMemory() > share && allowance.getTargetMemory() > allowance.getGrantedMemory()) - updates = List.of(new TargetUpdate(allowance, allowance.getUsedMemory())); - else { - MemoryAllowance largestConsumer = findAndRemoveLargestConsumer(); - if(largestConsumer != null) { - long newTarget = (long) (largestConsumer.getGrantedMemory() * 0.8); - if(newTarget <= share) - newTarget = share; - else - addOverconsumer(largestConsumer); - updates = List.of(new TargetUpdate(largestConsumer, newTarget)); - } - } - } - else { + if(free >= minSize) { allow = Math.min(free, maxSize); _usedBytes += allow; updates = rebalance(false); - if(allowance.getGrantedMemory() <= share && allowance.getGrantedMemory() + allow > share) - addOverconsumer(allowance); } } if(updates != null) @@ -86,46 +85,29 @@ public long requestMemory(MemoryAllowance allowance, long minSize, long maxSize) return allow; } - private MemoryAllowance findAndRemoveLargestConsumer() { - long largest = Long.MIN_VALUE; - MemoryAllowance allowance = null; - for(MemoryAllowance largestConsumer : _overconsumers) { - if(largestConsumer.getGrantedMemory() > largest) { - largest = largestConsumer.getGrantedMemory(); - allowance = largestConsumer; - } - } - _overconsumers.remove(allowance); - return allowance; - } - @Override public void freeMemory(MemoryAllowance allowance, long freedMemory) { - List updates = null; + List updates; synchronized(this) { if(freedMemory < 0) throw new IllegalArgumentException(); _usedBytes -= freedMemory; - if(allowance.isShutdown()) - updates = rebalance(false); - long share = getEqualShare(); - if(allowance.getGrantedMemory() <= share && allowance.getGrantedMemory() + freedMemory > share) - _overconsumers.remove(allowance); - else if(allowance.getGrantedMemory() <= allowance.getTargetMemory() && allowance.getGrantedMemory() > share) - addOverconsumer(allowance); + updates = rebalanceAfterFree(); } if(updates != null) applyTargetUpdates(updates); + if(freedMemory > 0) + notifyReservationWaiters(); } @Override public void shutdownAllowance(MemoryAllowance allowance) { List updates; synchronized(this) { - _overconsumers.remove(allowance); updates = rebalance(true); } applyTargetUpdates(updates); + notifyReservationWaiters(); } @Override @@ -135,11 +117,11 @@ public void destroyAllowance(MemoryAllowance allowance, long freedMemory) { if(freedMemory < 0) throw new IllegalArgumentException(); _allowances.remove(allowance); - _overconsumers.remove(allowance); _usedBytes -= freedMemory; updates = rebalance(true); } applyTargetUpdates(updates); + notifyReservationWaiters(); } @Override @@ -148,6 +130,55 @@ public synchronized void attachAllowance(MemoryAllowance allowance) { allowance.setTargetMemory(_allowedBytes); } + @Override + public void reservationBlocked(MemoryAllowance allowance, long bytes) { + if(_reclaimRunning.compareAndSet(false, true)) + RECLAIM_EXECUTOR.execute(this::runReclaim); + } + + private void runReclaim() { + try { + long reclaimed = 0; + for(MemoryAllowance allowance : _allowances) + if(!allowance.isShutdown()) + reclaimed += allowance.reclaimUnused(); + if(reclaimed == 0) + return; + + List updates; + synchronized(this) { + _usedBytes = Math.max(0, _usedBytes - reclaimed); + updates = rebalanceAfterFree(); + } + if(updates != null) + applyTargetUpdates(updates); + notifyReservationWaiters(); + } + finally { + if(shouldRetryReclaim()) + RECLAIM_EXECUTOR.schedule(this::runReclaim, RECLAIM_RETRY_DELAY_MS, TimeUnit.MILLISECONDS); + else { + _reclaimRunning.set(false); + if(shouldRetryReclaim() && _reclaimRunning.compareAndSet(false, true)) + RECLAIM_EXECUTOR.execute(this::runReclaim); + } + } + } + + private boolean shouldRetryReclaim() { + if(!hasReclaimPressure()) + return false; + for(MemoryAllowance allowance : _allowances) { + if(allowance instanceof SyncMemoryAllowance sync && sync.hasReservationWaiters()) + return true; + } + return false; + } + + private synchronized boolean hasReclaimPressure() { + return _usedBytes >= _allowedBytes * RECLAIM_PRESSURE; + } + private List rebalance(boolean force) { long free = _allowedBytes - _usedBytes; if(force) @@ -158,6 +189,13 @@ private List rebalance(boolean force) { return switchBrokerMode(BrokerMode.STRICT); } + private List rebalanceAfterFree() { + long free = _allowedBytes - _usedBytes; + if(_brokerMode == BrokerMode.RELAXED && free > _allowedBytes / 5) + return rebalanceToRelaxed(); + return rebalance(false); + } + private List switchBrokerMode(BrokerMode newMode) { if(newMode == _brokerMode) return null; @@ -181,7 +219,6 @@ private List rebalanceToStrict() { Math.min(allowance.getTargetMemory(), share + (long) ((allowance.getUsedMemory() - share) * 0.9)))); } } - refreshOverconsumers(updates); return updates; } @@ -193,34 +230,21 @@ private List rebalanceToRelaxed() { continue; updates.add(new TargetUpdate(allowance, allowance.getGrantedMemory() + free)); } - refreshOverconsumers(updates); return updates; } private long getEqualShare() { - return _allowances.isEmpty() ? _allowedBytes : _allowedBytes / _allowances.size(); - } - - private void addOverconsumer(MemoryAllowance allowance) { - if(!_overconsumers.contains(allowance)) - _overconsumers.add(allowance); + int active = 0; + for(MemoryAllowance allowance : _allowances) + if(!allowance.isShutdown()) + active++; + return active == 0 ? _allowedBytes : _allowedBytes / active; } - private void refreshOverconsumers(List updates) { - _overconsumers.clear(); - long share = getEqualShare(); + private void notifyReservationWaiters() { for(MemoryAllowance allowance : _allowances) { - if(allowance.isShutdown()) - continue; - long target = allowance.getTargetMemory(); - for(TargetUpdate update : updates) { - if(update._allowance == allowance) { - target = update._target; - break; - } - } - if(allowance.getGrantedMemory() > share && allowance.getGrantedMemory() <= target) - _overconsumers.add(allowance); + if(allowance instanceof SyncMemoryAllowance sync) + sync.onBrokerMemoryAvailable(); } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/memory/MemoryAllowance.java b/src/main/java/org/apache/sysds/runtime/ooc/memory/MemoryAllowance.java index b2db5ea7533..10a8bbe7d69 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/memory/MemoryAllowance.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/memory/MemoryAllowance.java @@ -53,4 +53,8 @@ default long getFreeMemory() { default boolean isUnderPressure() { return getGrantedMemory() > getTargetMemory(); } + + default long reclaimUnused() { + return 0; + } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/memory/MemoryBroker.java b/src/main/java/org/apache/sysds/runtime/ooc/memory/MemoryBroker.java index fb4d6ae182d..2c82cc156da 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/memory/MemoryBroker.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/memory/MemoryBroker.java @@ -21,8 +21,14 @@ public interface MemoryBroker { long requestMemory(MemoryAllowance allowance, long minSize, long maxSize); + void freeMemory(MemoryAllowance allowance, long freedMemory); + void shutdownAllowance(MemoryAllowance allowance); + void destroyAllowance(MemoryAllowance allowance, long freedMemory); + void attachAllowance(MemoryAllowance allowance); + + void reservationBlocked(MemoryAllowance allowance, long bytes); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/memory/SyncMemoryAllowance.java b/src/main/java/org/apache/sysds/runtime/ooc/memory/SyncMemoryAllowance.java index 2c4a1b7a0fa..900549819a0 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/memory/SyncMemoryAllowance.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/memory/SyncMemoryAllowance.java @@ -22,7 +22,8 @@ import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.ooc.cache.OOCFuture; -import java.util.ArrayDeque; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; public class SyncMemoryAllowance implements MemoryAllowance { @@ -36,7 +37,7 @@ public class SyncMemoryAllowance implements MemoryAllowance { protected volatile long _targetBytes; protected volatile boolean _shutdown; protected volatile boolean _destroyed; - private final ArrayDeque _reservationWaiters; + private final Queue _reservationWaiters; private boolean _drainingReservationWaiters; private boolean _reservationDrainRequested; @@ -62,7 +63,7 @@ public SyncMemoryAllowance(MemoryBroker broker, long consumptionLimit, long mini _targetBytes = 0; _shutdown = false; _destroyed = false; - _reservationWaiters = new ArrayDeque<>(); + _reservationWaiters = new ConcurrentLinkedQueue<>(); _drainingReservationWaiters = false; _reservationDrainRequested = false; broker.attachAllowance(this); @@ -70,12 +71,16 @@ public SyncMemoryAllowance(MemoryBroker broker, long consumptionLimit, long mini @Override public boolean tryReserve(long bytes) { + if(bytes < 0) + throw new IllegalArgumentException("Cannot reserve negative bytes: " + bytes); + if(bytes > _consumptionLimit) + throw new IllegalArgumentException("Cannot reserve more memory than the consumption limit"); long minRequest; long maxRequest; synchronized(this) { if(_shutdown || _destroyed) return false; - if(_usedBytes + bytes <= _grantedBytes) { + if(_usedBytes + bytes <= _grantedBytes && _usedBytes + bytes <= _targetBytes) { _usedBytes += bytes; return true; } @@ -85,9 +90,6 @@ public boolean tryReserve(long bytes) { maxRequest = Math.max(minRequest, Math.max(_grantedBytes, bytes) * 2); } - if(bytes > _consumptionLimit) - throw new IllegalArgumentException("Cannot reserve more memory than the consumption limit"); - long granted = _broker.requestMemory(this, minRequest, maxRequest); long refund = 0; boolean success = false; @@ -135,17 +137,20 @@ public OOCFuture reserveAsync(long bytes) { if(bytes > _consumptionLimit) return OOCFuture .failed(new IllegalArgumentException("Cannot reserve more memory than the consumption limit")); - if(tryReserve(bytes)) + if(_shutdown || _destroyed) + return OOCFuture.failed(new IllegalStateException("Cannot reserve memory on closed allowance.")); + if(_reservationWaiters.isEmpty() && tryReserve(bytes)) return OOCFuture.completed(null); OOCFuture future = new OOCFuture<>(); - synchronized(this) { - if(_shutdown || _destroyed) { - future.completeExceptionally(new IllegalStateException("Cannot reserve memory on closed allowance.")); - return future; - } - _reservationWaiters.addLast(new ReservationWaiter(bytes, future)); + ReservationWaiter waiter = new ReservationWaiter(bytes, future); + _reservationWaiters.add(waiter); + if((_shutdown || _destroyed) && _reservationWaiters.remove(waiter)) { + future.completeExceptionally(new IllegalStateException("Cannot reserve memory on closed allowance.")); + return future; } requestReservationDrain(); + if(!future.isDone()) + _broker.reservationBlocked(this, bytes); return future; } @@ -218,6 +223,8 @@ public long getTargetMemory() { @Override public void setTargetMemory(long targetMemory) { + if(targetMemory < 0) + throw new IllegalArgumentException("Target memory must not be negative: " + targetMemory); long freedMemory = 0; boolean drainWaiters = false; synchronized(this) { @@ -238,12 +245,21 @@ public void setTargetMemory(long targetMemory) { requestReservationDrain(); } + @Override + public synchronized long reclaimUnused() { + if(_shutdown || _destroyed || _grantedBytes <= _usedBytes) + return 0; + long reclaimed = _grantedBytes - _usedBytes; + _grantedBytes = _usedBytes; + notifyAll(); + return reclaimed; + } + @Override public void shutdown() { long freedMemory = 0; long destroyFreedMemory = 0; boolean destroy = false; - ArrayDeque waiters; synchronized(this) { if(_shutdown || _destroyed) return; @@ -259,8 +275,6 @@ public void shutdown() { else { freedMemory = oldGrantedBytes - _grantedBytes; } - waiters = new ArrayDeque<>(_reservationWaiters); - _reservationWaiters.clear(); notifyAll(); } _broker.shutdownAllowance(this); @@ -269,8 +283,9 @@ public void shutdown() { else if(freedMemory > 0) _broker.freeMemory(this, freedMemory); IllegalStateException ex = new IllegalStateException("Cannot reserve memory on closed allowance."); - while(!waiters.isEmpty()) - waiters.removeFirst().future.completeExceptionally(ex); + ReservationWaiter waiter; + while((waiter = _reservationWaiters.poll()) != null) + waiter.future.completeExceptionally(ex); } @Override @@ -278,6 +293,20 @@ public boolean isShutdown() { return _shutdown || _destroyed; } + void onBrokerMemoryAvailable() { + boolean drainWaiters; + synchronized(this) { + drainWaiters = !_reservationWaiters.isEmpty() && !_shutdown && !_destroyed; + notifyAll(); + } + if(drainWaiters) + requestReservationDrain(); + } + + boolean hasReservationWaiters() { + return !_reservationWaiters.isEmpty(); + } + private void requestReservationDrain() { synchronized(this) { _reservationDrainRequested = true; @@ -309,14 +338,11 @@ private void requestReservationDrain() { private void drainReservationWaitersOnce() { while(true) { - ReservationWaiter waiter; - synchronized(this) { - if(_shutdown || _destroyed) - return; - waiter = _reservationWaiters.peekFirst(); - if(waiter == null) - return; - } + if(_shutdown || _destroyed) + return; + ReservationWaiter waiter = _reservationWaiters.peek(); + if(waiter == null) + return; boolean admitted; try { admitted = tryReserve(waiter.bytes); @@ -335,11 +361,7 @@ private void drainReservationWaitersOnce() { } } - private synchronized boolean removeReservationWaiter(ReservationWaiter waiter) { - if(_reservationWaiters.peekFirst() == waiter) { - _reservationWaiters.removeFirst(); - return true; - } + private boolean removeReservationWaiter(ReservationWaiter waiter) { return _reservationWaiters.remove(waiter); } diff --git a/src/test/java/org/apache/sysds/test/component/ooc/memory/OOCMemoryAllowanceTest.java b/src/test/java/org/apache/sysds/test/component/ooc/memory/OOCMemoryAllowanceTest.java index 6ae3cd62ed8..979498502d1 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/memory/OOCMemoryAllowanceTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/memory/OOCMemoryAllowanceTest.java @@ -30,6 +30,7 @@ import org.apache.sysds.runtime.matrix.operators.BinaryOperator; import org.apache.sysds.runtime.matrix.operators.RightScalarOperator; import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; import org.apache.sysds.runtime.ooc.memory.CachedAllowance; import org.apache.sysds.runtime.ooc.memory.GlobalMemoryBroker; import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; @@ -45,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; import java.util.function.Function; @@ -62,6 +64,54 @@ public void testWorstCase() { test(false, 0, 1); } + @Test + public void testBlockedReservationReclaims() throws Exception { + GlobalMemoryBroker broker = new GlobalMemoryBroker(100); + SyncMemoryAllowance holder = new SyncMemoryAllowance(broker); + SyncMemoryAllowance waiter = new SyncMemoryAllowance(broker); + try { + holder.reserveBlocking(100); + OOCFuture reservation = waiter.reserveAsync(50); + Assert.assertFalse(reservation.isDone()); + + holder.release(50); + reservation.get(10, TimeUnit.SECONDS); + Assert.assertEquals(50, holder.getGrantedMemory()); + Assert.assertEquals(50, waiter.getUsedMemory()); + } + finally { + holder.release(holder.getUsedMemory()); + waiter.release(waiter.getUsedMemory()); + holder.destroy(); + waiter.destroy(); + } + } + + @Test + public void testReservationWaiters() throws Exception { + CoordinatedBroker broker = new CoordinatedBroker(new GlobalMemoryBroker(100)); + SyncMemoryAllowance allowance = new SyncMemoryAllowance(broker); + try { + allowance.reserveBlocking(100); + OOCFuture first = allowance.reserveAsync(60); + Assert.assertFalse(first.isDone()); + + allowance.release(20); + OOCFuture second = allowance.reserveAsync(20); + Assert.assertFalse(second.isDone()); + + allowance.release(60); + first.get(10, TimeUnit.SECONDS); + second.get(10, TimeUnit.SECONDS); + Assert.assertEquals(100, allowance.getUsedMemory()); + } + finally { + allowance.release(allowance.getUsedMemory()); + allowance.destroy(); + broker.destroy(); + } + } + public void test(boolean optimal, int nWarmup, int nMeasure) { //DMLScript.OOC_STATISTICS = true; long millis; @@ -363,6 +413,10 @@ public void attachAllowance(MemoryAllowance allowance) { applyTargetUpdates(updates); } + @Override + public void reservationBlocked(MemoryAllowance allowance, long bytes) { + } + @Override public long requestMemory(MemoryAllowance allowance, long minSize, long maxSize) { if(!_credits.containsKey(allowance)) From 24b70f7aaaac6675b066572c1c54ef37e4d6c214 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:02:04 +0200 Subject: [PATCH 082/132] [SYSTEMDS-3891] Remove Unused OOC Messaging Features --- .../ooc/AggregateTernaryOOCInstruction.java | 14 --- .../ooc/AggregateUnaryOOCInstruction.java | 16 --- .../ooc/BinaryOOCInstruction.java | 8 -- .../instructions/ooc/CachingStream.java | 78 -------------- .../ooc/MatrixIndexingOOCInstruction.java | 24 ----- .../instructions/ooc/OOCStreamable.java | 23 ---- .../ParameterizedBuiltinOOCInstruction.java | 6 -- .../instructions/ooc/PlaybackStream.java | 73 ------------- .../instructions/ooc/ReorgOOCInstruction.java | 6 -- .../ooc/SubscribableTaskQueue.java | 102 ------------------ .../ooc/TernaryOOCInstruction.java | 11 -- .../instructions/ooc/UnaryOOCInstruction.java | 4 - .../runtime/ooc/stream/FilteredOOCStream.java | 48 --------- .../runtime/ooc/stream/MergedOOCStream.java | 53 --------- .../runtime/ooc/stream/SourceOOCStream.java | 8 -- .../ooc/stream/SourceOOCStreamable.java | 50 --------- .../ooc/stream/SplittingOOCStream.java | 49 --------- .../runtime/ooc/stream/SubOOCStream.java | 50 --------- .../message/OOCGetStreamTypeMessage.java | 59 ---------- .../ooc/stream/message/OOCStreamMessage.java | 38 ------- 20 files changed, 720 deletions(-) delete mode 100644 src/main/java/org/apache/sysds/runtime/ooc/stream/message/OOCGetStreamTypeMessage.java delete mode 100644 src/main/java/org/apache/sysds/runtime/ooc/stream/message/OOCStreamMessage.java diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/AggregateTernaryOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/AggregateTernaryOOCInstruction.java index 2573ede14e4..3e3fafba7b8 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/AggregateTernaryOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/AggregateTernaryOOCInstruction.java @@ -39,7 +39,6 @@ import org.apache.sysds.runtime.matrix.operators.AggregateTernaryOperator; import org.apache.sysds.runtime.matrix.operators.Operator; import org.apache.sysds.runtime.meta.DataCharacteristics; -import org.apache.sysds.runtime.util.IndexRange; import java.util.ArrayList; import java.util.HashMap; @@ -155,19 +154,6 @@ private void processReduceRow(ExecutionContext ec, AggregateTernaryOperator abOp if(qIn3 != null) streams.add(qIn3); - for (OOCStream stream : streams) - stream.setDownstreamMessageRelay(qOut::messageDownstream); - - qOut.setUpstreamMessageRelay(msg -> - streams.forEach(stream -> stream.messageUpstream(streams.size() > 1 ? msg.split() : msg))); - - qOut.setIXTransform((downstream, range) -> { - if (downstream) - return new IndexRange(1, 1, range.colStart, range.colEnd); - else - return new IndexRange(1, dc.getRows(), range.colStart, range.colEnd); - }); - CompletableFuture fut = joinOOC(streams, qMid, blocks -> { MatrixBlock b1 = (MatrixBlock) blocks.get(0).getValue(); MatrixBlock b2 = (MatrixBlock) blocks.get(1).getValue(); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/AggregateUnaryOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/AggregateUnaryOOCInstruction.java index 38b228ccb1d..ac4e9bac919 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/AggregateUnaryOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/AggregateUnaryOOCInstruction.java @@ -35,7 +35,6 @@ import org.apache.sysds.runtime.matrix.operators.AggregateUnaryOperator; import org.apache.sysds.runtime.matrix.operators.Operator; import org.apache.sysds.runtime.meta.DataCharacteristics; -import org.apache.sysds.runtime.util.IndexRange; import java.util.HashMap; @@ -91,21 +90,6 @@ public void processInstruction( ExecutionContext ec ) { ec.getMatrixObject(output).setStreamHandle(qOut); - qIn.setDownstreamMessageRelay(qOut::messageDownstream); - qOut.setUpstreamMessageRelay(qIn::messageUpstream); - qOut.setIXTransform((downstream, range) -> { - if (downstream) { - if (aggun.isRowAggregate()) - return new IndexRange(range.rowStart, range.rowEnd, 1, 1); - else - return new IndexRange(1, 1, range.colStart, range.colEnd); - } - if (aggun.isRowAggregate()) - return new IndexRange(range.rowStart, range.rowEnd, 1, min.getNumColumns() - 1); - else - return new IndexRange(1, min.getNumRows() - 1, range.colStart, range.colEnd); - }); - // per-block aggregation (parallel map) mapOOC(qIn, qLocal, tmp -> { MatrixIndexes midx = aggun.isRowAggregate() ? diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/BinaryOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/BinaryOOCInstruction.java index 1352e7ff9c7..8a5f7cf49a7 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/BinaryOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/BinaryOOCInstruction.java @@ -67,12 +67,6 @@ protected void processMatrixMatrixInstruction(ExecutionContext ec) { OOCStream qIn2 = m2.getStreamHandle(); OOCStream qOut = new SubscribableTaskQueue<>(); ec.getMatrixObject(output).setStreamHandle(qOut); - qIn1.setDownstreamMessageRelay(qOut::messageDownstream); - qIn2.setDownstreamMessageRelay(qOut::messageDownstream); - qOut.setUpstreamMessageRelay(msg -> { - qIn1.messageUpstream(msg.split()); - qIn2.messageUpstream(msg.split()); - }); final boolean known1 = (m1.getNumRows() >= 0 && m1.getNumColumns() >= 0); final boolean known2 = (m2.getNumRows() >= 0 && m2.getNumColumns() >= 0); @@ -152,8 +146,6 @@ protected void processScalarMatrixInstruction(ExecutionContext ec) { OOCStream qIn = min.getStreamHandle(); OOCStream qOut = createWritableStream(); ec.getMatrixObject(output).setStreamHandle(qOut); - qIn.setDownstreamMessageRelay(qOut::messageDownstream); - qOut.setUpstreamMessageRelay(qIn::messageUpstream); mapOOC(qIn, qOut, tmp -> { IndexedMatrixValue tmpOut = new IndexedMatrixValue(); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java index abf1efde9c2..ff4362fe285 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java @@ -31,10 +31,7 @@ import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; import org.apache.sysds.runtime.ooc.stream.SourceOOCStream; -import org.apache.sysds.runtime.ooc.stream.message.OOCGetStreamTypeMessage; -import org.apache.sysds.runtime.ooc.stream.message.OOCStreamMessage; import org.apache.sysds.runtime.ooc.util.OOCUtils; -import org.apache.sysds.runtime.util.IndexRange; import shaded.parquet.it.unimi.dsi.fastutil.ints.IntArrayList; import java.util.ArrayList; @@ -43,9 +40,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; -import java.util.function.BiFunction; import java.util.function.Consumer; /** @@ -74,7 +69,6 @@ public class CachingStream implements OOCStreamable { private int _numBlocks = 0; private Consumer>[] _subscribers; - private CopyOnWriteArrayList> _downstreamRelays; // state flags private boolean _cacheInProgress = true; // caching in progress, in the first pass. @@ -92,7 +86,6 @@ public CachingStream(OOCStream source) { public CachingStream(OOCStream source, long streamId) { _source = source; - _source.setDownstreamMessageRelay(this::messageDownstream); _streamId = streamId; if(OOCWatchdog.WATCH) { _watchdogId = "CS-" + hashCode(); @@ -100,7 +93,6 @@ public CachingStream(OOCStream source, long streamId) { OOCWatchdog.registerOpen(_watchdogId, toString(), getCtxMsg(), this); } activateIndexing(); - _downstreamRelays = null; source.setSubscriber(tmp -> { try(tmp) { int blk; @@ -623,76 +615,6 @@ public void setData(CacheableData data) { _source.setData(data); } - @Override - public void messageUpstream(OOCStreamMessage msg) { - if (msg.isCancelled()) - return; - if(msg instanceof OOCGetStreamTypeMessage) { - ((OOCGetStreamTypeMessage) msg).setCachedType(); - activateIndexing(); - return; - } - - _source.messageUpstream(msg); - } - - @Override - public void messageDownstream(OOCStreamMessage msg) { - CopyOnWriteArrayList> relays = _downstreamRelays; - if (relays != null) { - for (Consumer relay : relays) { - if (msg.isCancelled()) - break; - relay.accept(msg); - } - } - } - - @Override - public void setUpstreamMessageRelay(Consumer relay) { - throw new UnsupportedOperationException(); - } - - @Override - public void setDownstreamMessageRelay(Consumer relay) { - addDownstreamMessageRelay(relay); - } - - @Override - public void addUpstreamMessageRelay(Consumer relay) { - throw new UnsupportedOperationException(); - } - - @Override - public void addDownstreamMessageRelay(Consumer relay) { - if (relay == null) - throw new IllegalArgumentException("Cannot set downstream relay to null"); - CopyOnWriteArrayList> relays = _downstreamRelays; - if (relays == null) { - synchronized(this) { - if (_downstreamRelays == null) - _downstreamRelays = new CopyOnWriteArrayList<>(); - relays = _downstreamRelays; - } - } - relays.add(0, relay); - } - - @Override - public void clearUpstreamMessageRelays() { - // No upstream relays supported - } - - @Override - public void clearDownstreamMessageRelays() { - _downstreamRelays = null; - } - - @Override - public void setIXTransform(BiFunction transform) { - throw new UnsupportedOperationException(); - } - @SuppressWarnings("unchecked") public void setSubscriber(Consumer> subscriber, boolean incrConsumers) { if(_deletable) diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/MatrixIndexingOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/MatrixIndexingOOCInstruction.java index 83d4ba58fd2..90e79d2373f 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/MatrixIndexingOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/MatrixIndexingOOCInstruction.java @@ -123,30 +123,6 @@ public void processInstruction(ExecutionContext ec) { addOutStream(qOut); mOut.setStreamHandle(qOut); - qIn.setDownstreamMessageRelay(qOut::messageDownstream); - qOut.setUpstreamMessageRelay(qIn::messageUpstream); - qOut.setIXTransform((downstream, range) -> { - if(downstream) { - long rs = range.rowStart - ix.rowStart + 1; - long re = range.rowEnd - ix.rowStart + 1; - long cs = range.colStart - ix.colStart + 1; - long ce = range.colEnd - ix.colStart + 1; - // TODO What happens if range is out of bounds? - rs = Math.max(1, rs); - cs = Math.max(1, cs); - re = Math.min(ix.rowSpan(), re); - ce = Math.min(ix.colSpan(), ce); - return new IndexRange(rs, re, cs, ce); - } - else { - long rs = range.rowStart + ix.rowStart; - long re = range.rowEnd + ix.rowStart; - long cs = range.colStart + ix.colStart; - long ce = range.colEnd + ix.colStart; - return new IndexRange(rs, re, cs, ce); - } - }); - if(firstBlockRow == lastBlockRow && firstBlockCol == lastBlockCol) { MatrixIndexes srcBlock = new MatrixIndexes(firstBlockRow + 1, firstBlockCol + 1); OOCStream filteredStream = new FilteredOOCStream<>(qIn, diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStreamable.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStreamable.java index 4f212f544b2..75ccdee3ee1 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStreamable.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStreamable.java @@ -21,11 +21,6 @@ import org.apache.sysds.runtime.controlprogram.caching.CacheableData; import org.apache.sysds.runtime.meta.DataCharacteristics; -import org.apache.sysds.runtime.ooc.stream.message.OOCStreamMessage; -import org.apache.sysds.runtime.util.IndexRange; - -import java.util.function.BiFunction; -import java.util.function.Consumer; public interface OOCStreamable { OOCStream getReadStream(); @@ -43,22 +38,4 @@ public interface OOCStreamable { CacheableData getData(); void setData(CacheableData data); - - void messageUpstream(OOCStreamMessage msg); - - void messageDownstream(OOCStreamMessage msg); - - void setUpstreamMessageRelay(Consumer relay); - - void setDownstreamMessageRelay(Consumer relay); - - void addUpstreamMessageRelay(Consumer relay); - - void addDownstreamMessageRelay(Consumer relay); - - void clearUpstreamMessageRelays(); - - void clearDownstreamMessageRelays(); - - void setIXTransform(BiFunction transform); } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ParameterizedBuiltinOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ParameterizedBuiltinOOCInstruction.java index 87b17f77192..2f71d0e4538 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ParameterizedBuiltinOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ParameterizedBuiltinOOCInstruction.java @@ -98,9 +98,6 @@ public void processInstruction(ExecutionContext ec) { double pattern = Double.parseDouble(params.get("pattern")); double replacement = Double.parseDouble(params.get("replacement")); - qIn.setDownstreamMessageRelay(qOut::messageDownstream); - qOut.setUpstreamMessageRelay(qIn::messageUpstream); - mapOOC(qIn, qOut, tmp -> new IndexedMatrixValue(tmp.getIndexes(), tmp.getValue().replaceOperations(new MatrixBlock(), pattern, replacement))); ec.getMatrixObject(output).setStreamHandle(qOut); @@ -156,9 +153,6 @@ else if(instOpcode.equalsIgnoreCase(Opcodes.REXPAND.toString())) { boolean ignore = Boolean.parseBoolean(params.get("ignore")); long blen = targetObj.getBlocksize(); - qIn.setDownstreamMessageRelay(qOut::messageDownstream); - qOut.setUpstreamMessageRelay(qIn::messageUpstream); - expandOOC(qIn, qOut, tmp -> { ArrayList out = new ArrayList<>(); LibMatrixReorg.rexpand(tmp, lmaxVal, dirRows, cast, ignore, blen, out); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/PlaybackStream.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/PlaybackStream.java index 3de438bf17b..7526b09f592 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/PlaybackStream.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/PlaybackStream.java @@ -23,14 +23,10 @@ import org.apache.sysds.runtime.controlprogram.caching.CacheableData; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.meta.DataCharacteristics; -import org.apache.sysds.runtime.ooc.stream.message.OOCStreamMessage; -import org.apache.sysds.runtime.util.IndexRange; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiFunction; import java.util.function.Consumer; public class PlaybackStream implements OOCStream { @@ -38,7 +34,6 @@ public class PlaybackStream implements OOCStream { private final AtomicInteger _streamIdx; private final AtomicBoolean _subscriberSet; private QueueCallback _lastDequeue; - private volatile CopyOnWriteArrayList> _downstreamRelays; public PlaybackStream(CachingStream streamCache) { this._streamCache = streamCache; @@ -123,27 +118,6 @@ public void setData(CacheableData data) { _streamCache.setData(data); } - @Override - public void messageUpstream(OOCStreamMessage msg) { - if(msg.isCancelled()) - return; - _streamCache.messageUpstream(msg); - } - - @Override - public void messageDownstream(OOCStreamMessage msg) { - if(msg.isCancelled()) - return; - CopyOnWriteArrayList> relays = _downstreamRelays; - if(relays != null) { - for(Consumer relay : relays) { - if(msg.isCancelled()) - break; - relay.accept(msg); - } - } - } - @Override public void setSubscriber(Consumer> subscriber) { if(!_subscriberSet.compareAndSet(false, true)) @@ -166,51 +140,4 @@ public boolean hasStreamCache() { public CachingStream getStreamCache() { return _streamCache; } - - @Override - public void setUpstreamMessageRelay(Consumer relay) { - throw new UnsupportedOperationException(); - } - - @Override - public void setDownstreamMessageRelay(Consumer relay) { - addDownstreamMessageRelay(relay); - } - - @Override - public void addUpstreamMessageRelay(Consumer relay) { - throw new UnsupportedOperationException(); - } - - @Override - public void addDownstreamMessageRelay(Consumer relay) { - if(relay == null) - throw new IllegalArgumentException("Cannot set downstream relay to null"); - CopyOnWriteArrayList> relays = _downstreamRelays; - if(relays == null) { - synchronized(this) { - if (_downstreamRelays == null) - _downstreamRelays = new CopyOnWriteArrayList<>(); - relays = _downstreamRelays; - } - } - relays.add(0, relay); - _streamCache.addDownstreamMessageRelay(relay); - } - - @Override - public void clearUpstreamMessageRelays() { - // No upstream relays supported - } - - @Override - public void clearDownstreamMessageRelays() { - _downstreamRelays = null; - _streamCache.clearDownstreamMessageRelays(); - } - - @Override - public void setIXTransform(BiFunction transform) { - throw new UnsupportedOperationException(); - } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java index 273d33341ab..94d896a3546 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java @@ -34,7 +34,6 @@ import org.apache.sysds.runtime.matrix.operators.Operator; import org.apache.sysds.runtime.matrix.operators.ReorgOperator; import org.apache.sysds.runtime.util.DataConverter; -import org.apache.sysds.runtime.util.IndexRange; public class ReorgOOCInstruction extends ComputationOOCInstruction { // sort-specific attributes (to enable variable attributes) @@ -110,11 +109,6 @@ public void processInstruction( ExecutionContext ec ) { OOCStream qOut = createWritableStream(); ec.getMatrixObject(output).setStreamHandle(qOut); - qIn.setDownstreamMessageRelay(qOut::messageDownstream); - qOut.setUpstreamMessageRelay(qIn::messageUpstream); - qOut.setIXTransform((downstream, range) -> - new IndexRange(range.colStart, range.colEnd, range.rowStart, range.rowEnd)); - // Transpose operation mapOOC(qIn, qOut, tmp -> { MatrixBlock inBlock = (MatrixBlock) tmp.getValue(); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/SubscribableTaskQueue.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/SubscribableTaskQueue.java index 67746b64257..04ce65e5725 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/SubscribableTaskQueue.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/SubscribableTaskQueue.java @@ -23,16 +23,11 @@ import org.apache.sysds.runtime.controlprogram.caching.CacheableData; import org.apache.sysds.runtime.controlprogram.parfor.LocalTaskQueue; import org.apache.sysds.runtime.meta.DataCharacteristics; -import org.apache.sysds.runtime.ooc.stream.message.OOCGetStreamTypeMessage; -import org.apache.sysds.runtime.ooc.stream.message.OOCStreamMessage; import org.apache.sysds.runtime.ooc.util.OOCUtils; -import org.apache.sysds.runtime.util.IndexRange; import java.util.LinkedList; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiFunction; import java.util.function.Consumer; public class SubscribableTaskQueue extends LocalTaskQueue> implements OOCStream { @@ -43,9 +38,6 @@ public class SubscribableTaskQueue extends LocalTaskQueue _lastDequeued = null; private CacheableData _cdata; private volatile Consumer> _subscriber = null; - private volatile CopyOnWriteArrayList> _upstreamMsgRelays = null; - private volatile CopyOnWriteArrayList> _downstreamMsgRelays = null; - private volatile BiFunction _ixTransform = null; private String _watchdogId; public SubscribableTaskQueue() { @@ -190,8 +182,6 @@ public synchronized void closeInput() { if(_closed.compareAndSet(false, true)) { super.closeInput(); onDeliveryFinished(); - _upstreamMsgRelays = null; - _downstreamMsgRelays = null; } else { throw new IllegalStateException("Multiple close input calls"); @@ -276,43 +266,6 @@ public OOCStream getWriteStream() { return this; } - @Override - public void messageUpstream(OOCStreamMessage msg) { - if(msg.isCancelled()) - return; - msg.addIXTransform(_ixTransform); - if(msg.isCancelled()) - return; - if(msg instanceof OOCGetStreamTypeMessage) { - if(_cdata != null) - ((OOCGetStreamTypeMessage) msg).setInMemoryType(); - return; - } - CopyOnWriteArrayList> relays = _upstreamMsgRelays; - if(relays != null) { - for(Consumer relay : relays) { - if(msg.isCancelled()) - break; - relay.accept(msg); - } - } - } - - @Override - public void messageDownstream(OOCStreamMessage msg) { - if(!msg.isCancelled()) - return; - msg.addIXTransform(_ixTransform); - CopyOnWriteArrayList> relays = _downstreamMsgRelays; - if(relays != null) { - for(Consumer relay : relays) { - if(msg.isCancelled()) - break; - relay.accept(msg); - } - } - } - @Override public boolean hasStreamCache() { return false; @@ -340,61 +293,6 @@ public void setData(CacheableData data) { _cdata = data; } - @Override - public void setUpstreamMessageRelay(Consumer relay) { - addUpstreamMessageRelay(relay); - } - - @Override - public void setDownstreamMessageRelay(Consumer relay) { - addDownstreamMessageRelay(relay); - } - - @Override - public void addUpstreamMessageRelay(Consumer relay) { - if(relay == null) - throw new IllegalArgumentException("Cannot set upstream relay to null"); - CopyOnWriteArrayList> relays = _upstreamMsgRelays; - if(relays == null) { - synchronized(this) { - if(_upstreamMsgRelays == null) - _upstreamMsgRelays = new CopyOnWriteArrayList<>(); - relays = _upstreamMsgRelays; - } - } - relays.add(0, relay); - } - - @Override - public void addDownstreamMessageRelay(Consumer relay) { - if(relay == null) - throw new IllegalArgumentException("Cannot set downstream relay to null"); - CopyOnWriteArrayList> relays = _downstreamMsgRelays; - if(relays == null) { - synchronized(this) { - if(_downstreamMsgRelays == null) - _downstreamMsgRelays = new CopyOnWriteArrayList<>(); - relays = _downstreamMsgRelays; - } - } - relays.add(0, relay); - } - - @Override - public void clearUpstreamMessageRelays() { - _upstreamMsgRelays = null; - } - - @Override - public void clearDownstreamMessageRelays() { - _downstreamMsgRelays = null; - } - - @Override - public void setIXTransform(BiFunction transform) { - _ixTransform = transform; - } - @Override public synchronized String toString() { return "STQ-" + hashCode(); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/TernaryOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/TernaryOOCInstruction.java index 6dfedfc1ff2..6036647cc7f 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/TernaryOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/TernaryOOCInstruction.java @@ -109,8 +109,6 @@ private void processSingleMatrixInstruction(ExecutionContext ec, int matrixPos) OOCStream qIn = mo.getStreamHandle(); OOCStream qOut = createWritableStream(); ec.getMatrixObject(output).setStreamHandle(qOut); - qIn.setDownstreamMessageRelay(qOut::messageDownstream); - qOut.setUpstreamMessageRelay(qIn::messageUpstream); mapOOC(qIn, qOut, tmp -> { IndexedMatrixValue outVal = new IndexedMatrixValue(); @@ -135,12 +133,6 @@ private void processTwoMatrixInstruction(ExecutionContext ec, int leftPos, int r OOCStream qOut = createWritableStream(); ec.getMatrixObject(output).setStreamHandle(qOut); - qOut.setUpstreamMessageRelay(msg -> { - leftStream.messageUpstream(msg.split()); - rightStream.messageUpstream(msg.split()); - }); - leftStream.setDownstreamMessageRelay(qOut::messageDownstream); - rightStream.setDownstreamMessageRelay(qOut::messageDownstream); joinOOC(leftStream, rightStream, qOut, (l, r) -> { IndexedMatrixValue outVal = new IndexedMatrixValue(); @@ -164,9 +156,6 @@ private void processThreeMatrixInstruction(ExecutionContext ec) { List> streams = List.of( m1.getStreamHandle(), m2.getStreamHandle(), m3.getStreamHandle()); - streams.forEach(s -> s.setDownstreamMessageRelay(qOut::messageDownstream)); - qOut.setUpstreamMessageRelay(msg -> streams.forEach(s -> s.messageUpstream(msg))); - joinOOC(streams, qOut, blocks -> { IndexedMatrixValue b1 = blocks.get(0); IndexedMatrixValue b2 = blocks.get(1); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/UnaryOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/UnaryOOCInstruction.java index a95cf0ec333..df2f50bf573 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/UnaryOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/UnaryOOCInstruction.java @@ -86,10 +86,6 @@ public void processInstruction( ExecutionContext ec ) { } ec.getMatrixObject(output).setStreamHandle(qOut); - if(!cumulative) { - qIn.setDownstreamMessageRelay(qOut::messageDownstream); - qOut.setUpstreamMessageRelay(qIn::messageUpstream); - } } private OOCStream processCumulativeUnaryInstruction(ExecutionContext ec, UnaryOperator uop, diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/FilteredOOCStream.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/FilteredOOCStream.java index f7f57744390..276f96680a6 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/FilteredOOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/FilteredOOCStream.java @@ -24,10 +24,7 @@ import org.apache.sysds.runtime.instructions.ooc.CachingStream; import org.apache.sysds.runtime.instructions.ooc.OOCStream; import org.apache.sysds.runtime.meta.DataCharacteristics; -import org.apache.sysds.runtime.ooc.stream.message.OOCStreamMessage; -import org.apache.sysds.runtime.util.IndexRange; -import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; @@ -150,49 +147,4 @@ public CacheableData getData() { public void setData(CacheableData data) { _data = data; } - - @Override - public void messageUpstream(OOCStreamMessage msg) { - _sourceStream.messageUpstream(msg); - } - - @Override - public void messageDownstream(OOCStreamMessage msg) { - _sourceStream.messageDownstream(msg); - } - - @Override - public void setUpstreamMessageRelay(Consumer relay) { - _sourceStream.setUpstreamMessageRelay(relay); - } - - @Override - public void setDownstreamMessageRelay(Consumer relay) { - _sourceStream.setDownstreamMessageRelay(relay); - } - - @Override - public void addUpstreamMessageRelay(Consumer relay) { - _sourceStream.addUpstreamMessageRelay(relay); - } - - @Override - public void addDownstreamMessageRelay(Consumer relay) { - _sourceStream.addDownstreamMessageRelay(relay); - } - - @Override - public void clearUpstreamMessageRelays() { - _sourceStream.clearUpstreamMessageRelays(); - } - - @Override - public void clearDownstreamMessageRelays() { - _sourceStream.clearDownstreamMessageRelays(); - } - - @Override - public void setIXTransform(BiFunction transform) { - _sourceStream.setIXTransform(transform); - } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/MergedOOCStream.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/MergedOOCStream.java index 51e38c8cce9..1c0f4977a27 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/MergedOOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/MergedOOCStream.java @@ -25,14 +25,11 @@ import org.apache.sysds.runtime.instructions.ooc.OOCStream; import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; import org.apache.sysds.runtime.meta.DataCharacteristics; -import org.apache.sysds.runtime.ooc.stream.message.OOCStreamMessage; -import org.apache.sysds.runtime.util.IndexRange; import java.util.Arrays; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiFunction; import java.util.function.Consumer; public class MergedOOCStream implements OOCStream { @@ -52,11 +49,6 @@ public MergedOOCStream(List> sources) { _failed = new AtomicBoolean(false); _sharedCache = findSharedCache(sources); - _taskQueue.setUpstreamMessageRelay(msg -> { - for(OOCStream source : _sources) - source.messageUpstream(msg); - }); - for(OOCStream source : _sources) { source.setSubscriber(cb -> { try { @@ -229,49 +221,4 @@ public CacheableData getData() { public void setData(CacheableData data) { _taskQueue.setData(data); } - - @Override - public void messageUpstream(OOCStreamMessage msg) { - _taskQueue.messageUpstream(msg); - } - - @Override - public void messageDownstream(OOCStreamMessage msg) { - _taskQueue.messageDownstream(msg); - } - - @Override - public void setUpstreamMessageRelay(Consumer relay) { - _taskQueue.setUpstreamMessageRelay(relay); - } - - @Override - public void setDownstreamMessageRelay(Consumer relay) { - _taskQueue.setDownstreamMessageRelay(relay); - } - - @Override - public void addUpstreamMessageRelay(Consumer relay) { - throw new UnsupportedOperationException(); - } - - @Override - public void addDownstreamMessageRelay(Consumer relay) { - _taskQueue.addDownstreamMessageRelay(relay); - } - - @Override - public void clearUpstreamMessageRelays() { - _taskQueue.clearUpstreamMessageRelays(); - } - - @Override - public void clearDownstreamMessageRelays() { - _taskQueue.clearDownstreamMessageRelays(); - } - - @Override - public void setIXTransform(BiFunction transform) { - _taskQueue.setIXTransform(transform); - } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/SourceOOCStream.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/SourceOOCStream.java index 0941cf0ea5c..3f1ef94f2a7 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/SourceOOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/SourceOOCStream.java @@ -26,7 +26,6 @@ import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; -import org.apache.sysds.runtime.ooc.stream.message.OOCStreamMessage; import org.apache.sysds.runtime.matrix.data.MatrixIndexes; import java.util.concurrent.ConcurrentHashMap; @@ -90,13 +89,6 @@ private void waitForBackpressure() { } } - @Override - public void messageUpstream(OOCStreamMessage msg) { - if(msg.isCancelled()) - return; - super.messageUpstream(msg); - } - public static class SourceGroupCallback implements OOCStream.GroupQueueCallback { private final List _data; private final OOCIOHandler.GroupSourceBlockDescriptor _descriptor; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/SourceOOCStreamable.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/SourceOOCStreamable.java index 4a5f018b2ac..4c9ca3683af 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/SourceOOCStreamable.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/SourceOOCStreamable.java @@ -25,11 +25,6 @@ import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.meta.DataCharacteristics; -import org.apache.sysds.runtime.ooc.stream.message.OOCStreamMessage; -import org.apache.sysds.runtime.util.IndexRange; - -import java.util.function.BiFunction; -import java.util.function.Consumer; public class SourceOOCStreamable implements OOCStreamable { private final CacheableData _data; @@ -77,49 +72,4 @@ public CacheableData getData() { public void setData(CacheableData data) { throw new UnsupportedOperationException(); } - - @Override - public void messageUpstream(OOCStreamMessage msg) { - - } - - @Override - public void messageDownstream(OOCStreamMessage msg) { - - } - - @Override - public void setUpstreamMessageRelay(Consumer relay) { - - } - - @Override - public void setDownstreamMessageRelay(Consumer relay) { - - } - - @Override - public void addUpstreamMessageRelay(Consumer relay) { - - } - - @Override - public void addDownstreamMessageRelay(Consumer relay) { - - } - - @Override - public void clearUpstreamMessageRelays() { - - } - - @Override - public void clearDownstreamMessageRelays() { - - } - - @Override - public void setIXTransform(BiFunction transform) { - - } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/SplittingOOCStream.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/SplittingOOCStream.java index b7fd9a1d1a4..7aef56e96ba 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/SplittingOOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/SplittingOOCStream.java @@ -24,10 +24,7 @@ import org.apache.sysds.runtime.instructions.ooc.CachingStream; import org.apache.sysds.runtime.instructions.ooc.OOCStream; import org.apache.sysds.runtime.meta.DataCharacteristics; -import org.apache.sysds.runtime.ooc.stream.message.OOCStreamMessage; -import org.apache.sysds.runtime.util.IndexRange; -import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; @@ -181,50 +178,4 @@ public CacheableData getData() { public void setData(CacheableData data) { throw new UnsupportedOperationException(); } - - @Override - public void messageUpstream(OOCStreamMessage msg) { - _sourceStream.messageUpstream(msg); - } - - @Override - public void messageDownstream(OOCStreamMessage msg) { - for(SubOOCStream sub : _subStreams) - sub.messageDownstream(msg); - } - - @Override - public void setUpstreamMessageRelay(Consumer relay) { - throw new UnsupportedOperationException(); - } - - @Override - public void setDownstreamMessageRelay(Consumer relay) { - throw new UnsupportedOperationException(); - } - - @Override - public void addUpstreamMessageRelay(Consumer relay) { - throw new UnsupportedOperationException(); - } - - @Override - public void addDownstreamMessageRelay(Consumer relay) { - throw new UnsupportedOperationException(); - } - - @Override - public void clearUpstreamMessageRelays() { - throw new UnsupportedOperationException(); - } - - @Override - public void clearDownstreamMessageRelays() { - throw new UnsupportedOperationException(); - } - - @Override - public void setIXTransform(BiFunction transform) { - throw new UnsupportedOperationException(); - } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/SubOOCStream.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/SubOOCStream.java index 9a231654c26..44f6542a4d3 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/SubOOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/SubOOCStream.java @@ -25,10 +25,7 @@ import org.apache.sysds.runtime.instructions.ooc.OOCStream; import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; import org.apache.sysds.runtime.meta.DataCharacteristics; -import org.apache.sysds.runtime.ooc.stream.message.OOCStreamMessage; -import org.apache.sysds.runtime.util.IndexRange; -import java.util.function.BiFunction; import java.util.function.Consumer; public class SubOOCStream implements OOCStream { @@ -39,7 +36,6 @@ public class SubOOCStream implements OOCStream { public SubOOCStream(OOCStream sourceStream) { _sourceStream = sourceStream; _taskQueue = new SubscribableTaskQueue<>(); - _taskQueue.setUpstreamMessageRelay(_sourceStream::messageUpstream); } @Override @@ -140,50 +136,4 @@ public CacheableData getData() { public void setData(CacheableData data) { _taskQueue.setData(data); } - - @Override - public void messageUpstream(OOCStreamMessage msg) { - _taskQueue.messageUpstream(msg); - } - - @Override - public void messageDownstream(OOCStreamMessage msg) { - _taskQueue.messageDownstream(msg); - } - - @Override - public void setUpstreamMessageRelay(Consumer relay) { - // Upstream is handled by source stream - throw new UnsupportedOperationException(); - } - - @Override - public void setDownstreamMessageRelay(Consumer relay) { - _taskQueue.setDownstreamMessageRelay(relay); - } - - @Override - public void addUpstreamMessageRelay(Consumer relay) { - throw new UnsupportedOperationException(); - } - - @Override - public void addDownstreamMessageRelay(Consumer relay) { - _taskQueue.addDownstreamMessageRelay(relay); - } - - @Override - public void clearUpstreamMessageRelays() { - _taskQueue.clearUpstreamMessageRelays(); - } - - @Override - public void clearDownstreamMessageRelays() { - _taskQueue.clearDownstreamMessageRelays(); - } - - @Override - public void setIXTransform(BiFunction transform) { - _taskQueue.setIXTransform(transform); - } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/message/OOCGetStreamTypeMessage.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/message/OOCGetStreamTypeMessage.java deleted file mode 100644 index 0866735d83f..00000000000 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/message/OOCGetStreamTypeMessage.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysds.runtime.ooc.stream.message; - -import org.apache.sysds.runtime.util.IndexRange; - -import java.util.function.BiFunction; - -public class OOCGetStreamTypeMessage implements OOCStreamMessage { - public static final byte STREAM_TYPE_UNKNOWN = 0; - public static final byte STREAM_TYPE_CACHED = 1; - public static final byte STREAM_TYPE_IN_MEMORY = 2; - - private byte _streamType; - - public OOCGetStreamTypeMessage() { - _streamType = 0; - } - - @Override - public void addIXTransform(BiFunction transform) {} - - public void setUnknownType() { - _streamType = STREAM_TYPE_UNKNOWN; - } - - public void setCachedType() { - _streamType = STREAM_TYPE_CACHED; - } - - public void setInMemoryType() { - _streamType = STREAM_TYPE_IN_MEMORY; - } - - public byte getStreamType() { - return _streamType; - } - - public boolean isRequestable() { - return _streamType == STREAM_TYPE_CACHED || _streamType == STREAM_TYPE_IN_MEMORY; - } -} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/message/OOCStreamMessage.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/message/OOCStreamMessage.java deleted file mode 100644 index 26459a74ffa..00000000000 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/message/OOCStreamMessage.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysds.runtime.ooc.stream.message; - -import org.apache.sysds.runtime.util.IndexRange; - -import java.util.function.BiFunction; - -public interface OOCStreamMessage { - default boolean isCancelled() { - return false; - } - - default void cancel() {} - - default OOCStreamMessage split() { - return this; - } - - void addIXTransform(BiFunction transform); -} From b120efb4aafdb051774e47809bb15c048e380ca3 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Sat, 18 Jul 2026 13:59:20 +0200 Subject: [PATCH 083/132] [SYSTEMDS-3891] Add StoreBackedStream --- .../runtime/ooc/store/CountingLiveness.java | 68 +++++++ .../ooc/store/MaterializedCallback.java | 16 +- .../ooc/store/SequentialAccessPattern.java | 64 ++++++ .../runtime/ooc/store/StoreBackedStream.java | 185 ++++++++++++++++++ .../component/ooc/MaterializedStoreTest.java | 115 +++++------ 5 files changed, 379 insertions(+), 69 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/store/CountingLiveness.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/store/SequentialAccessPattern.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/store/StoreBackedStream.java diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/CountingLiveness.java b/src/main/java/org/apache/sysds/runtime/ooc/store/CountingLiveness.java new file mode 100644 index 00000000000..04e86f90a84 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/CountingLiveness.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.store; + +import java.util.concurrent.atomic.AtomicIntegerArray; + +public final class CountingLiveness implements MaterializedStore.Liveness { + private final AtomicIntegerArray _remaining; + private final AtomicIntegerArray _reservable; + + public CountingLiveness(int size, int count) { + if(size < 0 || count < 0) + throw new IllegalArgumentException("Invalid args: size=" + size + ", count=" + count); + _remaining = new AtomicIntegerArray(size); + _reservable = new AtomicIntegerArray(size); + for(int i = 0; i < size; i++) { + _remaining.set(i, count); + _reservable.set(i, count); + } + } + + @Override + public boolean needs(int index) { + return index >= 0 && index < _remaining.length() && _remaining.get(index) > 0; + } + + @Override + public void consumed(int index) { + decrement(_remaining, index); + } + + @Override + public boolean reserve(int index) { + return index >= 0 && index < _reservable.length() && decrement(_reservable, index); + } + + @Override + public void unreserve(int index) { + _reservable.incrementAndGet(index); + } + + private static boolean decrement(AtomicIntegerArray counters, int index) { + while(true) { + int current = counters.get(index); + if(current <= 0) + return false; + if(counters.compareAndSet(index, current, current - 1)) + return true; + } + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedCallback.java b/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedCallback.java index 829e60b32ea..226dd5c2b68 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedCallback.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedCallback.java @@ -21,21 +21,21 @@ import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.instructions.ooc.OOCStream; -import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; import java.util.concurrent.atomic.AtomicReference; -public final class MaterializedCallback implements OOCStream.QueueCallback { - private final StoreLease _lease; +public final class MaterializedCallback implements OOCStream.QueueCallback { + private final StoreLease _lease; private final AtomicReference _failure; private boolean _closed; - public MaterializedCallback(StoreLease lease) { + public MaterializedCallback(StoreLease lease) { this(lease, new AtomicReference<>()); } - private MaterializedCallback(StoreLease lease, AtomicReference failure) { + private MaterializedCallback(StoreLease lease, AtomicReference failure) { _lease = lease; _failure = failure; } @@ -45,7 +45,7 @@ public BlockEntry pinnedEntry() { } @Override - public IndexedMatrixValue get() { + public T get() { DMLRuntimeException failure = _failure.get(); if(failure != null) throw failure; @@ -53,10 +53,10 @@ public IndexedMatrixValue get() { } @Override - public synchronized OOCStream.QueueCallback keepOpen() { + public synchronized OOCStream.QueueCallback keepOpen() { if(_closed) throw new IllegalStateException("Cannot keep open a closed callback"); - return new MaterializedCallback(_lease.retain(), _failure); + return new MaterializedCallback<>(_lease.retain(), _failure); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/SequentialAccessPattern.java b/src/main/java/org/apache/sysds/runtime/ooc/store/SequentialAccessPattern.java new file mode 100644 index 00000000000..18da6bc1452 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/SequentialAccessPattern.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.store; + +import org.apache.sysds.runtime.ooc.cache.collections.ConcurrentBitSet; + +public final class SequentialAccessPattern implements MaterializedStore.AccessPattern { + private final int _size; + private final ConcurrentBitSet _consumed; + private int _next; + private volatile int _consumedThrough; + + public SequentialAccessPattern(int size) { + if(size < 0) + throw new IllegalArgumentException("Size must not be negative: " + size); + _size = size; + _consumed = new ConcurrentBitSet(Math.max(1, size)); + _next = 0; + _consumedThrough = -1; + } + + @Override + public boolean hasNext() { + return _next < _size; + } + + @Override + public int next() { + if(!hasNext()) + throw new IllegalStateException("No remaining index"); + return _next++; + } + + @Override + public boolean needs(int index) { + return index >= 0 && index < _size && index > _consumedThrough && !_consumed.get(index); + } + + @Override + public synchronized void consumed(int index) { + if(index < 0 || index >= _size) + throw new IndexOutOfBoundsException("Invalid consumed index: " + index); + _consumed.set(index); + while(_consumedThrough + 1 < _size && _consumed.get(_consumedThrough + 1)) + _consumedThrough++; + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/StoreBackedStream.java b/src/main/java/org/apache/sysds/runtime/ooc/store/StoreBackedStream.java new file mode 100644 index 00000000000..7fad1105c5b --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/StoreBackedStream.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.store; + +import java.util.function.Consumer; + +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.controlprogram.caching.CacheableData; +import org.apache.sysds.runtime.instructions.ooc.CachingStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.meta.DataCharacteristics; +import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; + +public final class StoreBackedStream implements OOCStream { + private final OrderedMaterializedStoreReader _reader; + private volatile DMLRuntimeException _failure; + private boolean _subscriberSet; + private OOCStream.QueueCallback _lastDequeue; + private boolean _exhausted; + private CacheableData _data; + + public StoreBackedStream(OrderedMaterializedStoreReader reader) { + _reader = reader; + } + + @Override + public void enqueue(T value) { + throw new DMLRuntimeException("Cannot enqueue to a store-backed stream"); + } + + @Override + public void enqueue(QueueCallback callback) { + throw new DMLRuntimeException("Cannot enqueue to a store-backed stream"); + } + + @Override + public void closeInput() { + throw new DMLRuntimeException("Cannot close the input of a store-backed stream"); + } + + @Override + public synchronized T dequeue() { + QueueCallback callback = dequeueInternal(); + return callback == null ? null : callback.get(); + } + + @Override + public synchronized QueueCallback dequeueCB() { + return dequeueInternal(); + } + + private QueueCallback dequeueInternal() { + if(_subscriberSet) + throw new IllegalStateException("Cannot dequeue after setting a subscriber"); + if(_lastDequeue != null) { + _lastDequeue.close(); + _lastDequeue = null; + } + if(_failure != null) + throw _failure; + if(_exhausted) + return null; + try { + if(!_reader.hasNext()) { + _exhausted = true; + _reader.close(); + return null; + } + _lastDequeue = new MaterializedCallback<>(_reader.next()); + return _lastDequeue; + } + catch(InterruptedException e) { + Thread.currentThread().interrupt(); + throw recordFailure(e); + } + catch(RuntimeException e) { + throw recordFailure(e); + } + } + + @Override + public synchronized void setSubscriber(Consumer> subscriber) { + if(subscriber == null) + throw new IllegalArgumentException("Cannot set subscriber to null"); + if(_subscriberSet) + throw new IllegalStateException("Subscriber cannot be set multiple times"); + _subscriberSet = true; + Thread driver = new Thread(() -> drive(subscriber), "ooc-store-replay"); + driver.setDaemon(true); + driver.start(); + } + + private void drive(Consumer> subscriber) { + DMLRuntimeException failure = _failure; + if(failure != null) { + subscriber.accept(OOCStream.eos(failure)); + return; + } + try { + while(_reader.hasNext()) { + try(QueueCallback callback = new MaterializedCallback<>(_reader.next())) { + subscriber.accept(callback); + } + } + _reader.close(); + subscriber.accept(OOCStream.eos(null)); + } + catch(InterruptedException e) { + Thread.currentThread().interrupt(); + subscriber.accept(OOCStream.eos(recordFailure(e))); + } + catch(RuntimeException e) { + subscriber.accept(OOCStream.eos(recordFailure(e))); + } + } + + private synchronized DMLRuntimeException recordFailure(Exception error) { + if(_failure == null) + _failure = DMLRuntimeException.of(error); + _reader.close(); + return _failure; + } + + @Override + public void propagateFailure(DMLRuntimeException failure) { + recordFailure(failure); + } + + @Override + public OOCStream getReadStream() { + return this; + } + + @Override + public OOCStream getWriteStream() { + throw new UnsupportedOperationException("A store-backed stream has no write stream"); + } + + @Override + public boolean hasStreamCache() { + return false; + } + + @Override + public CachingStream getStreamCache() { + return null; + } + + @Override + public boolean isProcessed() { + return false; + } + + @Override + public DataCharacteristics getDataCharacteristics() { + return _data == null ? null : _data.getDataCharacteristics(); + } + + @Override + public CacheableData getData() { + return _data; + } + + @Override + public void setData(CacheableData data) { + _data = data; + } +} diff --git a/src/test/java/org/apache/sysds/test/component/ooc/MaterializedStoreTest.java b/src/test/java/org/apache/sysds/test/component/ooc/MaterializedStoreTest.java index d8ea0e3986f..14d26db3ae2 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/MaterializedStoreTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/MaterializedStoreTest.java @@ -20,6 +20,7 @@ package org.apache.sysds.test.component.ooc; import java.util.List; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -37,8 +38,11 @@ import org.apache.sysds.runtime.ooc.memory.SyncMemoryAllowance; import org.apache.sysds.runtime.ooc.store.IndexedMaterializedStoreReader; import org.apache.sysds.runtime.ooc.store.MaterializedStore; +import org.apache.sysds.runtime.ooc.store.CountingLiveness; import org.apache.sysds.runtime.ooc.store.OOCStreamMaterializer; import org.apache.sysds.runtime.ooc.store.OrderedMaterializedStoreReader; +import org.apache.sysds.runtime.ooc.store.SequentialAccessPattern; +import org.apache.sysds.runtime.ooc.store.StoreBackedStream; import org.apache.sysds.runtime.ooc.store.StoreLease; import org.apache.sysds.test.component.ooc.cache.OOCCacheTestUtils; import org.junit.After; @@ -96,32 +100,16 @@ public void testMaterializationReadersAndForgetting() throws Exception { Assert.assertEquals(0, _materializerAllowance.getUsedMemory()); Assert.assertEquals(3, _store.size()); - MaterializedStore.AccessPattern pattern = sequentialPattern(3); - boolean[] indexedConsumed = new boolean[3]; - MaterializedStore.Liveness liveness = new MaterializedStore.Liveness() { - @Override - public boolean needs(int index) { - return !indexedConsumed[index]; - } - - @Override - public void consumed(int index) { - indexedConsumed[index] = true; - } - }; - - OrderedMaterializedStoreReader ordered = _store.openReader(pattern, _readerAllowance, 2); - IndexedMaterializedStoreReader indexed = _store.openIndexedReader(liveness); + StoreBackedStream ordered = new StoreBackedStream<>( + _store.openReader(new SequentialAccessPattern(3), _readerAllowance, 2, false)); + IndexedMaterializedStoreReader indexed = _store + .openIndexedReader(new CountingLiveness(3, 1)); _store.sealReaders(); int index = 0; - while(ordered.hasNext()) { - try(StoreLease lease = ordered.next()) { - Assert.assertEquals(index + 1L, lease.value().getIndexes().getRowIndex()); - index++; - } - } - ordered.close(); + IndexedMatrixValue value; + while((value = ordered.dequeue()) != null) + Assert.assertEquals(++index, value.getIndexes().getRowIndex()); Assert.assertEquals(3, index); Assert.assertTrue(_cache.getOwnedCacheSize() > 0); @@ -151,7 +139,7 @@ public void testOrderedReaderRetries() throws Exception { _readerAllowance.destroy(); _readerAllowance = new SyncMemoryAllowance(_broker, TILE_BYTES); _readerAllowance.setTargetMemory(TILE_BYTES); - OrderedMaterializedStoreReader reader = _store.openReader(sequentialPattern(2), + OrderedMaterializedStoreReader reader = _store.openReader(new SequentialAccessPattern(2), _readerAllowance, 2, false); _store.sealReaders(); @@ -185,7 +173,7 @@ public void testSoftOrderingReturnsReadyRequestFirst() throws Exception { _readerAllowance.setTargetMemory(largeBytes); long heldBytes = largeBytes - TILE_BYTES; _readerAllowance.reserveBlocking(heldBytes); - OrderedMaterializedStoreReader reader = _store.openReader(sequentialPattern(2), + OrderedMaterializedStoreReader reader = _store.openReader(new SequentialAccessPattern(2), _readerAllowance, 2); _store.sealReaders(); @@ -211,19 +199,8 @@ public void testDirectRequests() throws Exception { materializer.accept(OOCStream.eos(null)); materializer.completion().get(WAIT_SECONDS, TimeUnit.SECONDS); - boolean[] needed = {true}; IndexedMaterializedStoreReader reader = _store - .openIndexedReader(new MaterializedStore.Liveness() { - @Override - public boolean needs(int index) { - return needed[index]; - } - - @Override - public void consumed(int index) { - needed[index] = false; - } - }); + .openIndexedReader(new CountingLiveness(1, 1)); _store.sealReaders(); try(StoreLease published = _store.requestPublished(0, _readerAllowance).get(WAIT_SECONDS, @@ -292,6 +269,46 @@ public void testLiveCallbackKeepsPublicationPinned() throws Exception { Assert.assertEquals(1, eos.get()); } + @Test + public void testStoreBackedStreamSubscriber() throws Exception { + OOCStreamMaterializer materializer = new OOCStreamMaterializer(_store, + indexes -> (int) indexes.getRowIndex() - 1, _materializerAllowance); + for(int i = 0; i < 2; i++) + materializer.accept(new OOCStream.SimpleQueueCallback<>(tile(i, i + 1.0), null)); + materializer.accept(OOCStream.eos(null)); + materializer.completion().get(WAIT_SECONDS, TimeUnit.SECONDS); + + StoreBackedStream stream = new StoreBackedStream<>( + _store.openReader(new SequentialAccessPattern(2), _readerAllowance, 1)); + _store.sealReaders(); + AtomicInteger count = new AtomicInteger(); + CountDownLatch complete = new CountDownLatch(1); + stream.setSubscriber(callback -> { + if(callback.isEos()) + complete.countDown(); + else + Assert.assertEquals(count.incrementAndGet(), callback.get().getIndexes().getRowIndex()); + }); + + Assert.assertTrue(complete.await(WAIT_SECONDS, TimeUnit.SECONDS)); + Assert.assertEquals(2, count.get()); + Assert.assertEquals(0, _readerAllowance.getUsedMemory()); + } + + @Test + public void testCountingLiveness() { + CountingLiveness liveness = new CountingLiveness(1, 2); + Assert.assertTrue(liveness.reserve(0)); + Assert.assertTrue(liveness.reserve(0)); + Assert.assertFalse(liveness.reserve(0)); + liveness.unreserve(0); + Assert.assertTrue(liveness.reserve(0)); + liveness.consumed(0); + Assert.assertTrue(liveness.needs(0)); + liveness.consumed(0); + Assert.assertFalse(liveness.needs(0)); + } + @Test public void testFailurePropagation() throws Exception { DMLRuntimeException sourceFailure = new DMLRuntimeException("injected failure"); @@ -332,28 +349,4 @@ private static IndexedMatrixValue tile(int index, double value) { return new IndexedMatrixValue(new MatrixIndexes(index + 1L, 1), new MatrixBlock(4, 4, value)); } - private static MaterializedStore.AccessPattern sequentialPattern(int to) { - return new MaterializedStore.AccessPattern() { - private int _next; - - @Override - public boolean hasNext() { - return _next < to; - } - - @Override - public int next() { - return _next++; - } - - @Override - public boolean needs(int index) { - return true; - } - - @Override - public void consumed(int index) { - } - }; - } } From 91f16cd03a778d5f97e7614defe58d6cad5937f9 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Sat, 18 Jul 2026 19:55:34 +0200 Subject: [PATCH 084/132] [SYSTEMDS-3891] Add AllocatedOOCStream and ReservationBudget --- .../runtime/ooc/memory/ReservationBudget.java | 130 ++++++++ .../ooc/stream/AllocatedOOCStream.java | 278 ++++++++++++++++++ .../ooc/memory/OOCMemoryAllowanceTest.java | 63 ++++ 3 files changed, 471 insertions(+) create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/memory/ReservationBudget.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/stream/AllocatedOOCStream.java diff --git a/src/main/java/org/apache/sysds/runtime/ooc/memory/ReservationBudget.java b/src/main/java/org/apache/sysds/runtime/ooc/memory/ReservationBudget.java new file mode 100644 index 00000000000..47e08202dfb --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/memory/ReservationBudget.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.memory; + +import org.apache.sysds.runtime.ooc.cache.OOCFuture; + +public final class ReservationBudget implements MemoryAllowance, AutoCloseable { + private final MemoryAllowance _parent; + private long _outstanding; + private long _available; + private boolean _closed; + + public ReservationBudget(MemoryAllowance parent, long bytes) { + if(parent == null) + throw new NullPointerException("parent"); + if(bytes < 0) + throw new IllegalArgumentException("Budget must not be negative: " + bytes); + _parent = parent; + _outstanding = bytes; + _available = bytes; + } + + @Override + public synchronized boolean tryReserve(long bytes) { + checkNonNegative(bytes); + if(bytes == 0) + return true; + if(_closed || _available < bytes) + return false; + _available -= bytes; + return true; + } + + @Override + public void reserveBlocking(long bytes) { + if(!tryReserve(bytes)) + throw insufficientBudget(bytes); + } + + @Override + public OOCFuture reserveAsync(long bytes) { + return tryReserve(bytes) ? OOCFuture.completed(null) : OOCFuture.failed(insufficientBudget(bytes)); + } + + @Override + public void release(long bytes) { + checkNonNegative(bytes); + if(bytes == 0) + return; + synchronized(this) { + long used = _outstanding - _available; + if(bytes > used) + throw new IllegalStateException("Cannot release " + bytes + " bytes from a budget using " + used); + _outstanding -= bytes; + } + _parent.release(bytes); + } + + @Override + public synchronized long getUsedMemory() { + return _outstanding - _available; + } + + @Override + public synchronized long getGrantedMemory() { + return _outstanding; + } + + @Override + public synchronized long getTargetMemory() { + return _outstanding; + } + + @Override + public void setTargetMemory(long targetMemory) { + throw new UnsupportedOperationException("Reservation budgets have a fixed target"); + } + + @Override + public void shutdown() { + close(); + } + + @Override + public synchronized boolean isShutdown() { + return _closed || _parent.isShutdown(); + } + + @Override + public void close() { + long released; + synchronized(this) { + if(_closed) + return; + _closed = true; + released = _available; + _available = 0; + _outstanding -= released; + } + if(released > 0) + _parent.release(released); + } + + private synchronized IllegalStateException insufficientBudget(long bytes) { + return new IllegalStateException( + "Cannot reserve " + bytes + " bytes from a budget with " + _available + " bytes available"); + } + + private static void checkNonNegative(long bytes) { + if(bytes < 0) + throw new IllegalArgumentException("Bytes must not be negative: " + bytes); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/AllocatedOOCStream.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/AllocatedOOCStream.java new file mode 100644 index 00000000000..6d3a7c5936f --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/AllocatedOOCStream.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.stream; + +import java.util.function.ToLongFunction; + +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; +import org.apache.sysds.runtime.ooc.memory.ReservationBudget; + +public final class AllocatedOOCStream extends SubscribableTaskQueue { + private final OOCStream _source; + private final MemoryAllowance _allowance; + private final ToLongFunction _reservationSize; + private volatile DMLRuntimeException _failure; + private int _pendingReservations; + private boolean _sourceComplete; + private boolean _outputClosed; + + public AllocatedOOCStream(OOCStream source, MemoryAllowance allowance, ToLongFunction reservationSize) { + _source = source; + _allowance = allowance; + _reservationSize = reservationSize; + setData(source.getData()); + source.setSubscriber(this::admit); + } + + public static ReservationBudget detachBudget(OOCStream.QueueCallback callback) { + return callback instanceof BudgetedQueueCallback budgeted ? budgeted.detachBudget() : null; + } + + private void admit(OOCStream.QueueCallback callback) { + if(callback.isFailure()) { + try(callback) { + callback.get(); + } + catch(DMLRuntimeException failure) { + fail(failure); + } + finishSource(); + return; + } + if(callback.isEos()) { + callback.close(); + finishSource(); + return; + } + if(_failure != null) { + callback.close(); + return; + } + try(callback) { + long bytes = _reservationSize.applyAsLong(callback.get()); + if(bytes < 0) + throw new IllegalArgumentException("Cannot reserve negative bytes: " + bytes); + if(bytes == 0) { + enqueueOwned(callback.keepOpen(), null); + return; + } + if(_allowance.tryReserve(bytes)) { + enqueueOwned(callback.keepOpen(), new ReservationBudget(_allowance, bytes)); + return; + } + retainUntilAllocated(callback, bytes); + } + catch(RuntimeException error) { + fail(DMLRuntimeException.of(error)); + } + } + + private void retainUntilAllocated(OOCStream.QueueCallback callback, long bytes) { + OOCStream.QueueCallback retained = callback.keepOpen(); + OOCFuture reservation; + synchronized(this) { + _pendingReservations++; + } + try { + reservation = _allowance.reserveAsync(bytes); + } + catch(RuntimeException error) { + try { + retained.close(); + } + finally { + releasePendingReservation(); + } + throw error; + } + reservation.whenComplete((ignored, error) -> { + try { + if(error != null) { + fail(DMLRuntimeException.of(error)); + retained.close(); + } + else if(_failure != null) { + _allowance.release(bytes); + retained.close(); + } + else + enqueueOwned(retained, new ReservationBudget(_allowance, bytes)); + } + catch(RuntimeException completionError) { + fail(DMLRuntimeException.of(completionError)); + } + finally { + releasePendingReservation(); + } + }); + } + + private void enqueueOwned(OOCStream.QueueCallback callback, ReservationBudget budget) { + OOCStream.QueueCallback output = budget == null ? callback : new BudgetedQueueCallback<>(callback, budget); + try { + enqueue(output); + } + catch(RuntimeException error) { + output.close(); + throw error; + } + } + + private boolean fail(DMLRuntimeException failure) { + synchronized(this) { + if(_failure != null) + return false; + _failure = failure; + } + super.propagateFailure(failure); + return true; + } + + private void releasePendingReservation() { + boolean close; + synchronized(this) { + if(_pendingReservations <= 0) + throw new IllegalStateException("Pending reservation count underflow"); + _pendingReservations--; + close = _sourceComplete && _pendingReservations == 0 && !_outputClosed; + if(close) + _outputClosed = true; + } + if(close) + closeInput(); + } + + private void finishSource() { + boolean close; + synchronized(this) { + if(_sourceComplete) + return; + _sourceComplete = true; + close = _pendingReservations == 0 && !_outputClosed; + if(close) + _outputClosed = true; + } + if(close) + closeInput(); + } + + @Override + public void propagateFailure(DMLRuntimeException failure) { + if(fail(failure)) + _source.propagateFailure(failure); + } + + private static final class BudgetedQueueCallback implements OOCStream.QueueCallback { + private final OOCStream.QueueCallback _callback; + private final BudgetedQueueCallback _budgetOwner; + private ReservationBudget _budget; + private int _budgetReferences; + private boolean _closed; + + private BudgetedQueueCallback(OOCStream.QueueCallback callback, ReservationBudget budget) { + _callback = callback; + _budgetOwner = this; + _budget = budget; + _budgetReferences = 1; + } + + private BudgetedQueueCallback(OOCStream.QueueCallback callback, BudgetedQueueCallback budgetOwner) { + _callback = callback; + _budgetOwner = budgetOwner; + } + + private synchronized ReservationBudget detachBudget() { + if(_closed) + throw new IllegalStateException("Cannot detach from a closed callback"); + return _budgetOwner.takeBudget(); + } + + private synchronized ReservationBudget takeBudget() { + ReservationBudget budget = _budget; + _budget = null; + return budget; + } + + @Override + public T get() { + return _callback.get(); + } + + @Override + public synchronized OOCStream.QueueCallback keepOpen() { + if(_closed) + throw new IllegalStateException("Cannot keep open a closed callback"); + OOCStream.QueueCallback retained = _callback.keepOpen(); + _budgetOwner.retainBudget(); + return new BudgetedQueueCallback<>(retained, _budgetOwner); + } + + private synchronized void retainBudget() { + _budgetReferences++; + } + + @Override + public void close() { + synchronized(this) { + if(_closed) + return; + _closed = true; + } + try { + _callback.close(); + } + finally { + _budgetOwner.releaseBudget(); + } + } + + private void releaseBudget() { + ReservationBudget budget = null; + synchronized(this) { + _budgetReferences--; + if(_budgetReferences == 0) { + budget = _budget; + _budget = null; + } + } + if(budget != null) + budget.close(); + } + + @Override + public void fail(DMLRuntimeException failure) { + _callback.fail(failure); + } + + @Override + public boolean isEos() { + return _callback.isEos(); + } + + @Override + public boolean isFailure() { + return _callback.isFailure(); + } + } +} diff --git a/src/test/java/org/apache/sysds/test/component/ooc/memory/OOCMemoryAllowanceTest.java b/src/test/java/org/apache/sysds/test/component/ooc/memory/OOCMemoryAllowanceTest.java index 979498502d1..c75458eef38 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/memory/OOCMemoryAllowanceTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/memory/OOCMemoryAllowanceTest.java @@ -19,6 +19,7 @@ package org.apache.sysds.test.component.ooc.memory; +import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.controlprogram.context.ExecutionContext; import org.apache.sysds.runtime.functionobjects.Plus; import org.apache.sysds.runtime.instructions.ooc.OOCInstruction; @@ -36,7 +37,9 @@ import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; import org.apache.sysds.runtime.ooc.memory.MemoryBroker; +import org.apache.sysds.runtime.ooc.memory.ReservationBudget; import org.apache.sysds.runtime.ooc.memory.SyncMemoryAllowance; +import org.apache.sysds.runtime.ooc.stream.AllocatedOOCStream; import org.junit.Assert; import org.junit.Test; import scala.Tuple3; @@ -112,6 +115,66 @@ public void testReservationWaiters() throws Exception { } } + @Test + public void testAllocatedStreamReservations() { + GlobalMemoryBroker broker = new GlobalMemoryBroker(100); + SyncMemoryAllowance allowance = new SyncMemoryAllowance(broker); + SubscribableTaskQueue source = new SubscribableTaskQueue<>(); + AllocatedOOCStream allocated = new AllocatedOOCStream<>(source, allowance, value -> 60); + try { + allowance.reserveBlocking(100); + source.enqueue(1); + Assert.assertEquals(100, allowance.getUsedMemory()); + + allowance.release(100); + OOCStream.QueueCallback first = allocated.dequeueCB(); + ReservationBudget budget = AllocatedOOCStream.detachBudget(first); + Assert.assertNotNull(budget); + first.close(); + Assert.assertEquals(60, allowance.getUsedMemory()); + budget.reserveBlocking(20); + budget.release(20); + Assert.assertEquals(40, allowance.getUsedMemory()); + budget.close(); + Assert.assertEquals(0, allowance.getUsedMemory()); + + source.enqueue(2); + OOCStream.QueueCallback second = allocated.dequeueCB(); + OOCStream.QueueCallback retained = second.keepOpen(); + second.close(); + Assert.assertEquals(60, allowance.getUsedMemory()); + retained.close(); + Assert.assertEquals(0, allowance.getUsedMemory()); + source.closeInput(); + Assert.assertNull(allocated.dequeueCB()); + } + finally { + if(allowance.getUsedMemory() > 0) + allowance.release(allowance.getUsedMemory()); + allowance.destroy(); + } + } + + @Test + public void testAllocatedStreamFailure() { + GlobalMemoryBroker broker = new GlobalMemoryBroker(100); + SyncMemoryAllowance allowance = new SyncMemoryAllowance(broker); + SubscribableTaskQueue source = new SubscribableTaskQueue<>(); + new AllocatedOOCStream<>(source, allowance, value -> 60); + try { + allowance.reserveBlocking(100); + source.enqueue(1); + source.propagateFailure(new DMLRuntimeException("injected failure")); + allowance.release(100); + Assert.assertEquals(0, allowance.getUsedMemory()); + } + finally { + if(allowance.getUsedMemory() > 0) + allowance.release(allowance.getUsedMemory()); + allowance.destroy(); + } + } + public void test(boolean optimal, int nWarmup, int nMeasure) { //DMLScript.OOC_STATISTICS = true; long millis; From ae34b95f8658c0a8bfd1dca96212d0aecdecb1b1 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Sun, 19 Jul 2026 20:53:46 +0200 Subject: [PATCH 085/132] [SYSTEMDS-3891] Add OOCInstructionUtils --- .../ooc/CSVReblockOOCInstruction.java | 5 +- .../ooc/DataGenOOCInstruction.java | 6 +- .../instructions/ooc/OOCInstruction.java | 322 ++---------------- .../runtime/ooc/stream/StreamContext.java | 52 ++- .../runtime/ooc/util/OOCInstructionUtils.java | 313 +++++++++++++++++ .../ooc/OOCInstructionUtilsTest.java | 110 ++++++ 6 files changed, 493 insertions(+), 315 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java create mode 100644 src/test/java/org/apache/sysds/test/component/ooc/OOCInstructionUtilsTest.java diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/CSVReblockOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/CSVReblockOOCInstruction.java index 4ac54ee3a57..2df1388c8ab 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/CSVReblockOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/CSVReblockOOCInstruction.java @@ -32,6 +32,7 @@ import org.apache.sysds.runtime.matrix.operators.Operator; import org.apache.sysds.runtime.meta.DataCharacteristics; import org.apache.sysds.runtime.ooc.stream.StreamContext; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; public class CSVReblockOOCInstruction extends ComputationOOCInstruction { private final int blen; @@ -74,14 +75,14 @@ public void processInstruction(ExecutionContext ec) { final long cols = mc.getCols(); final long nnz = mc.getNonZeros(); - submitOOCTask(() -> { + OOCInstructionUtils.submitOOCTask(() -> { try { reader.readMatrixAsStream(qOut, fileName, rows, cols, blen, nnz); } catch(Exception ex) { throw (ex instanceof DMLRuntimeException) ? (DMLRuntimeException) ex : new DMLRuntimeException(ex); } - }, new StreamContext().addOutStream(qOut)); + }, new StreamContext(_callerId, getExtendedOpcode()).addOutStream(qOut)); MatrixObject mout = ec.getMatrixObject(output); mout.setStreamHandle(qOut); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/DataGenOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/DataGenOOCInstruction.java index f44eb79dc98..348d6f6930c 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/DataGenOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/DataGenOOCInstruction.java @@ -38,6 +38,7 @@ import org.apache.sysds.runtime.matrix.data.RandomMatrixGenerator; import org.apache.sysds.runtime.matrix.operators.UnaryOperator; import org.apache.sysds.runtime.ooc.stream.StreamContext; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; import org.apache.sysds.runtime.util.UtilFunctions; public class DataGenOOCInstruction extends UnaryOOCInstruction { @@ -259,8 +260,7 @@ else if(method == Types.OpOpDG.SEQ) { final int maxK = (int) UtilFunctions.getSeqLength(lfrom, lto, lincr); final double finalLincr = lincr; - - submitOOCTask(() -> { + OOCInstructionUtils.submitOOCTask(() -> { int k = 0; double curFrom = lfrom; double curTo; @@ -286,7 +286,7 @@ else if(method == Types.OpOpDG.SEQ) { } qOut.closeInput(); - }, new StreamContext().addOutStream(qOut)); + }, new StreamContext(_callerId, getExtendedOpcode()).addOutStream(qOut)); } else throw new NotImplementedException(); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java index 80d71231646..805f8723486 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java @@ -41,8 +41,7 @@ import org.apache.sysds.runtime.ooc.stream.SplittingOOCStream; import org.apache.sysds.runtime.ooc.stream.StreamContext; import org.apache.sysds.runtime.ooc.stream.TaskContext; -import org.apache.sysds.runtime.util.CommonThreadPool; -import org.apache.sysds.utils.Statistics; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; import scala.Tuple2; import scala.Tuple4; import scala.Tuple5; @@ -60,22 +59,17 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.ForkJoinTask; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.LongAdder; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; public abstract class OOCInstruction extends Instruction { - public static final boolean ALLOW_PIPELINING = true; - public static final ExecutorService COMPUTE_EXECUTOR = CommonThreadPool.get(); - private static final AtomicInteger COMPUTE_IN_FLIGHT = new AtomicInteger(0); - private static final int COMPUTE_BACKPRESSURE_THRESHOLD = 100; + public static final boolean ALLOW_PIPELINING = OOCInstructionUtils.ALLOW_PIPELINING; + public static final ExecutorService COMPUTE_EXECUTOR = OOCInstructionUtils.COMPUTE_EXECUTOR; protected static final Log LOG = LogFactory.getLog(OOCInstruction.class.getName()); - private static final AtomicInteger nextStreamId = new AtomicInteger(0); private long nanoTime; public enum OOCType { @@ -86,7 +80,6 @@ public enum OOCType { protected final OOCInstruction.OOCType _ooctype; protected final boolean _requiresLabelUpdate; protected StreamContext _streamContext; - private LongAdder _localStatisticsAdder; public final int _callerId; protected OOCInstruction(OOCInstruction.OOCType type, String opcode, String istr) { @@ -101,17 +94,15 @@ protected OOCInstruction(OOCInstruction.OOCType type, Operator op, String opcode _requiresLabelUpdate = super.requiresLabelUpdate(); - if (DMLScript.STATISTICS) - _localStatisticsAdder = new LongAdder(); _callerId = DMLScript.OOC_LOG_EVENTS ? OOCEventLog.registerCaller(getExtendedOpcode() + "_" + hashCode()) : 0; } public static int getComputeInFlight() { - return COMPUTE_IN_FLIGHT.get(); + return OOCInstructionUtils.getComputeInFlight(); } public static int getComputeBackpressureThreshold() { - return COMPUTE_BACKPRESSURE_THRESHOLD; + return OOCInstructionUtils.getComputeBackpressureThreshold(); } @Override @@ -164,13 +155,13 @@ public void postprocessInstruction(ExecutionContext ec) { protected void addInStream(OOCStream... queue) { if(_streamContext == null) - _streamContext = new StreamContext(); + _streamContext = new StreamContext(_callerId, getExtendedOpcode()); _streamContext.addInStream(queue); } protected void addOutStream(OOCStream... queue) { if(_streamContext == null) - _streamContext = new StreamContext(); + _streamContext = new StreamContext(_callerId, getExtendedOpcode()); _streamContext.addOutStream(queue); } @@ -982,297 +973,30 @@ protected CompletableFuture scanOOC(OOCStream q }); } - protected CompletableFuture scanOOC(OOCStream qIn, OOCStream qOut, - Function seqFn, BiFunction scanner, - Function carryFn, long sequenceSize) { - return scanOOC(qIn, qOut, seqFn, (IndexedMatrixValue item, C carry) -> { - R out = scanner.apply(item, carry); - if(out == null) - throw new DMLRuntimeException("Ordered scan output must not be null."); - return new ScanStep<>(out, carryFn.apply(out)); - }, sequenceSize); - } - - protected CompletableFuture submitOOCTasks(final List> queues, BiConsumer> consumer) { + protected CompletableFuture submitOOCTasks(final List> queues, + BiConsumer> consumer) { return submitOOCTasks(queues, consumer, null, null); } - protected CompletableFuture submitOOCTasks(final List> queues, BiConsumer> consumer, BiFunction, Boolean> predicate, BiConsumer> onNotProcessed) { - addInStream(queues.toArray(OOCStream[]::new)); - if(!outStreamsDefined()) - throw new IllegalArgumentException("Explicit specification of all output streams is required before submitting tasks. If no output streams are present use addOutStream()."); - - final List activeTaskCtrs = new ArrayList<>(queues.size()); - final List> futures = new ArrayList<>(queues.size()); - - for(int i = 0; i < queues.size(); i++) { - activeTaskCtrs.add(new AtomicInteger(1)); - futures.add(new CompletableFuture<>()); - } - - final CompletableFuture globalFuture = CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new)); - final StreamContext streamContext = _streamContext.copy(); // Snapshot of the current stream context - if(streamContext == null || !streamContext.inStreamsDefined() || !streamContext.outStreamsDefined()) - throw new IllegalArgumentException("Explicit specification of all output streams is required before submitting tasks. If no output streams are present use addOutStream()."); - - int i = 0; - @SuppressWarnings("unused") - final int streamId = nextStreamId.getAndIncrement(); - - for (OOCStream queue : queues) { - final int k = i; - final AtomicInteger localTaskCtr = activeTaskCtrs.get(k); - final CompletableFuture localFuture = futures.get(k); - final AtomicBoolean closeRaceWatchdog = new AtomicBoolean(false); - - queue.setSubscriber(oocTask(callback -> { - long startTime = DMLScript.STATISTICS ? System.nanoTime() : 0; - try(callback) { - if(callback.isEos()) { - if(!closeRaceWatchdog.compareAndSet(false, true)) - throw new DMLRuntimeException( - "Race condition observed: NO_MORE_TASKS callback has been triggered more than once"); - - if(localTaskCtr.decrementAndGet() == 0) { - // Then we can run the finalization procedure already - localFuture.complete(null); - } - return; - } - - Consumer> process = cb -> { - if(predicate != null && !predicate.apply(k, cb)) { // Can get closed due to cancellation - if(onNotProcessed != null) - onNotProcessed.accept(k, cb); - return; - } - - if(localFuture.isDone()) { - if(onNotProcessed != null) - onNotProcessed.accept(k, cb); - return; - } - else { - localTaskCtr.incrementAndGet(); - } - - // The item needs to be pinned in memory to be accessible in the executor thread - final OOCStream.QueueCallback pinned = cb.keepOpen(); - - COMPUTE_IN_FLIGHT.incrementAndGet(); - try { - Runnable oocTask = oocTask(() -> { - long taskStartTime = DMLScript.STATISTICS || DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; - try(pinned) { - consumer.accept(k, pinned); - - if(localTaskCtr.decrementAndGet() == 0) { - TaskContext.defer(() -> localFuture.complete(null)); - } - } - finally { - COMPUTE_IN_FLIGHT.decrementAndGet(); - if (DMLScript.STATISTICS) { - _localStatisticsAdder.add(System.nanoTime() - taskStartTime); - if (globalFuture.isDone()) { - Statistics.maintainOOCHeavyHitter(getExtendedOpcode(), _localStatisticsAdder.sum()); - _localStatisticsAdder.reset(); - } - } - if (DMLScript.OOC_LOG_EVENTS) - OOCEventLog.onComputeEvent(_callerId, taskStartTime, System.nanoTime()); - } - }, localFuture, streamContext); - COMPUTE_EXECUTOR.submit(oocTask); - } - catch (Exception e) { - COMPUTE_IN_FLIGHT.decrementAndGet(); - throw e; - } - }; - - if(callback instanceof OOCStream.GroupQueueCallback) { - OOCStream.GroupQueueCallback group = (OOCStream.GroupQueueCallback) callback; - - if(localFuture.isDone()) { - for(int idx = 0; idx < group.size(); idx++) { - OOCStream.QueueCallback sub = group.getCallback(idx); - try(sub) { - if(onNotProcessed != null) - onNotProcessed.accept(k, sub); - } - } - return; - } - - localTaskCtr.incrementAndGet(); - final OOCStream.GroupQueueCallback pinnedGroup = - (OOCStream.GroupQueueCallback) group.keepOpen(); - - COMPUTE_IN_FLIGHT.incrementAndGet(); - try { - Runnable oocTask = oocTask(() -> { - long taskStartTime = DMLScript.STATISTICS || DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; - try(pinnedGroup) { - for(int idx = 0; idx < pinnedGroup.size(); idx++) { - OOCStream.QueueCallback sub = pinnedGroup.getCallback(idx); - try(sub) { - process.accept(sub); - } - } - - if(localTaskCtr.decrementAndGet() == 0) { - TaskContext.defer(() -> localFuture.complete(null)); - } - } - finally { - COMPUTE_IN_FLIGHT.decrementAndGet(); - if (DMLScript.STATISTICS) { - _localStatisticsAdder.add(System.nanoTime() - taskStartTime); - if (globalFuture.isDone()) { - Statistics.maintainOOCHeavyHitter(getExtendedOpcode(), _localStatisticsAdder.sum()); - _localStatisticsAdder.reset(); - } - } - if (DMLScript.OOC_LOG_EVENTS) - OOCEventLog.onComputeEvent(_callerId, taskStartTime, System.nanoTime()); - } - }, localFuture, streamContext); - COMPUTE_EXECUTOR.submit(oocTask); - } - catch (Exception e) { - COMPUTE_IN_FLIGHT.decrementAndGet(); - throw e; - } - } - else { - process.accept(callback); - } - - if(closeRaceWatchdog.get()) // Sanity check - throw new DMLRuntimeException("Race condition observed"); - } - catch(Throwable t) { - streamContext.failAll(DMLRuntimeException.of(t)); - throw t; - } - finally { - if (DMLScript.STATISTICS) { - _localStatisticsAdder.add(System.nanoTime() - startTime); - if (globalFuture.isDone()) { - Statistics.maintainOOCHeavyHitter(getExtendedOpcode(), _localStatisticsAdder.sum()); - _localStatisticsAdder.reset(); - } - } - } - }, null, streamContext)); - - i++; - } - - return globalFuture.handle((res, e) -> { - if (globalFuture.isCancelled() || globalFuture.isCompletedExceptionally()) { - futures.forEach(f -> { - if(!f.isDone()) { - if(globalFuture.isCancelled() || globalFuture.isCompletedExceptionally()) - f.cancel(true); - else - f.complete(null); - } - }); - } - - streamContext.clear(); - return null; - }); + protected CompletableFuture submitOOCTasks(final List> queues, + BiConsumer> consumer, + BiFunction, Boolean> predicate, + BiConsumer> onNotProcessed) { + if(_streamContext == null) + _streamContext = new StreamContext(_callerId, getExtendedOpcode()); + return OOCInstructionUtils.submitOOCTasks(queues, consumer, predicate, onNotProcessed, _streamContext); } - protected CompletableFuture submitOOCTasks(OOCStream queue, Consumer> consumer) { + protected CompletableFuture submitOOCTasks(OOCStream queue, + Consumer> consumer) { return submitOOCTasks(List.of(queue), (i, tmp) -> consumer.accept(tmp), null, null); } - protected CompletableFuture submitOOCTasks(OOCStream queue, Consumer> consumer, Function, Boolean> predicate, BiConsumer> onNotProcessed) { - return submitOOCTasks(List.of(queue), (i, tmp) -> consumer.accept(tmp), (i, tmp) -> predicate.apply(tmp), onNotProcessed); - } - - protected CompletableFuture submitOOCTask(Runnable r, StreamContext ctx) { - ExecutorService pool = CommonThreadPool.getDynamicPool(); - final CompletableFuture future = new CompletableFuture<>(); - try { - COMPUTE_IN_FLIGHT.incrementAndGet(); - pool.submit(oocTask(() -> { - long startTime = DMLScript.STATISTICS || DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; - try { - r.run(); - future.complete(null); - ctx.clear(); - if (DMLScript.STATISTICS) - Statistics.maintainOOCHeavyHitter(getExtendedOpcode(), System.nanoTime() - startTime); - if (DMLScript.OOC_LOG_EVENTS) - OOCEventLog.onComputeEvent(_callerId, startTime, System.nanoTime()); - } - finally { - COMPUTE_IN_FLIGHT.decrementAndGet(); - } - }, future, ctx)); - } - catch (Exception ex) { - COMPUTE_IN_FLIGHT.decrementAndGet(); - throw new DMLRuntimeException(ex); - } - - return future; - } - - private Runnable oocTask(Runnable r, CompletableFuture future, StreamContext ctx) { - return () -> { - boolean setContext = TaskContext.getContext() == null; - if(setContext) - TaskContext.setContext(new TaskContext()); - long startTime = DMLScript.STATISTICS ? System.nanoTime() : 0; - try { - r.run(); - if(setContext) { - while(TaskContext.runDeferred()) { - } - } - } - catch (Exception ex) { - DMLRuntimeException re = DMLRuntimeException.of(ex); - - ctx.failAll(re); - - if (future != null) - future.completeExceptionally(re); - - // Rethrow to ensure proper future handling - throw re; - } finally { - if(setContext) - TaskContext.clearContext(); - if (DMLScript.STATISTICS) - _localStatisticsAdder.add(System.nanoTime() - startTime); - } - }; - } - - private Consumer> oocTask(Consumer> c, CompletableFuture future, StreamContext ctx) { - return callback -> { - try { - c.accept(callback); - } - catch (Exception ex) { - DMLRuntimeException re = DMLRuntimeException.of(ex); - - ctx.failAll(re); - - if (future != null) - future.completeExceptionally(re); - - // Rethrow to ensure proper future handling - throw re; - } - }; + protected CompletableFuture submitOOCTasks(OOCStream queue, + Consumer> consumer, Function, Boolean> predicate, + BiConsumer> onNotProcessed) { + return submitOOCTasks(List.of(queue), (i, tmp) -> consumer.accept(tmp), (i, tmp) -> predicate.apply(tmp), + onNotProcessed); } /** diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/StreamContext.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/StreamContext.java index 9c9f2e3fc0e..4565d1768b7 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/StreamContext.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/StreamContext.java @@ -27,12 +27,38 @@ import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.LongAdder; public class StreamContext { + private final int _callerId; + private final String _opcode; + private final LongAdder _statistics; private Set> _inStreams; private Set> _outStreams; private DMLRuntimeException _failure; + public StreamContext() { + this(0, null); + } + + public StreamContext(int callerId, String opcode) { + _callerId = callerId; + _opcode = opcode; + _statistics = new LongAdder(); + } + + public int getCallerId() { + return _callerId; + } + + public String getExtendedOpcode() { + return _opcode; + } + + public LongAdder getLocalStatisticsLongAdder() { + return _statistics; + } + public boolean inStreamsDefined() { return _inStreams != null; } @@ -73,19 +99,23 @@ public void failAll(DMLRuntimeException e) { return; _failure = e; - for(OOCStream stream : _outStreams) { - try { - stream.propagateFailure(e); + if(_outStreams != null) + for(OOCStream stream : _outStreams) { + try { + stream.propagateFailure(e); + } + catch(Throwable ignored) { + } } - catch(Throwable ignored) {} - } - for(OOCStream stream : _inStreams) { - try { - stream.propagateFailure(e); + if(_inStreams != null) + for(OOCStream stream : _inStreams) { + try { + stream.propagateFailure(e); + } + catch(Throwable ignored) { + } } - catch(Throwable ignored) {} - } } public void clear() { @@ -94,7 +124,7 @@ public void clear() { } public StreamContext copy() { - StreamContext cpy = new StreamContext(); + StreamContext cpy = new StreamContext(_callerId, _opcode); cpy._inStreams = _inStreams; cpy._outStreams = _outStreams; return cpy; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java new file mode 100644 index 00000000000..75f6f9db2e5 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.util; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; + +import org.apache.sysds.api.DMLScript; +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.stats.OOCEventLog; +import org.apache.sysds.runtime.ooc.stream.StreamContext; +import org.apache.sysds.runtime.ooc.stream.TaskContext; +import org.apache.sysds.runtime.util.CommonThreadPool; +import org.apache.sysds.utils.Statistics; + +public final class OOCInstructionUtils { + public static final boolean ALLOW_PIPELINING = true; + public static final ExecutorService COMPUTE_EXECUTOR = CommonThreadPool.get(); + private static final AtomicInteger COMPUTE_IN_FLIGHT = new AtomicInteger(); + private static final int COMPUTE_BACKPRESSURE_THRESHOLD = 100; + + public static int getComputeInFlight() { + return COMPUTE_IN_FLIGHT.get(); + } + + public static int getComputeBackpressureThreshold() { + return COMPUTE_BACKPRESSURE_THRESHOLD; + } + + public static CompletableFuture submitOOCTasks(OOCStream queue, + Consumer> consumer, StreamContext context) { + return submitOOCTasks(List.of(queue), (i, callback) -> consumer.accept(callback), null, null, context); + } + + public static CompletableFuture submitOOCTasks(OOCStream queue, + Consumer> consumer, Function, Boolean> predicate, + BiConsumer> onNotProcessed, StreamContext context) { + return submitOOCTasks(List.of(queue), (i, callback) -> consumer.accept(callback), + (i, callback) -> predicate.apply(callback), onNotProcessed, context); + } + + public static CompletableFuture submitOOCTasks(List> queues, + BiConsumer> consumer, StreamContext context) { + return submitOOCTasks(queues, consumer, null, null, context); + } + + public static CompletableFuture submitOOCTasks(List> queues, + BiConsumer> consumer, + BiFunction, Boolean> predicate, + BiConsumer> onNotProcessed, StreamContext context) { + context.addInStream(queues.toArray(OOCStream[]::new)); + if(!context.outStreamsDefined()) + throw new IllegalArgumentException("Explicit specification of all output streams is required before " + + "submitting tasks. If no output streams are present use addOutStream()."); + + List activeTaskCounters = new ArrayList<>(queues.size()); + List> futures = new ArrayList<>(queues.size()); + for(int i = 0; i < queues.size(); i++) { + activeTaskCounters.add(new AtomicInteger(1)); + futures.add(new OOCFuture<>()); + } + + CompletableFuture globalFuture = new CompletableFuture<>(); + AtomicInteger remaining = new AtomicInteger(futures.size()); + if(futures.isEmpty()) + globalFuture.complete(null); + for(OOCFuture future : futures) + future.whenComplete((result, error) -> { + if(error != null) + globalFuture.completeExceptionally(error); + else if(remaining.decrementAndGet() == 0) + globalFuture.complete(null); + }); + StreamContext streamContext = context.copy(); + for(int i = 0; i < queues.size(); i++) + subscribe(queues.get(i), i, consumer, predicate, onNotProcessed, activeTaskCounters.get(i), futures.get(i), + globalFuture, streamContext); + + return globalFuture.handle((result, error) -> { + if(error != null) { + for(OOCFuture future : futures) + if(!future.isDone()) + future.completeExceptionally(error); + } + streamContext.clear(); + return null; + }); + } + + private static void subscribe(OOCStream queue, int streamIndex, + BiConsumer> consumer, + BiFunction, Boolean> predicate, + BiConsumer> onNotProcessed, AtomicInteger activeTaskCounter, + OOCFuture future, CompletableFuture globalFuture, StreamContext context) { + AtomicBoolean closed = new AtomicBoolean(); + queue.setSubscriber(guard(callback -> { + long startTime = DMLScript.STATISTICS ? System.nanoTime() : 0; + try(callback) { + if(callback.isEos()) { + if(!closed.compareAndSet(false, true)) + throw new DMLRuntimeException( + "Race condition observed: NO_MORE_TASKS callback has been triggered more than once"); + if(activeTaskCounter.decrementAndGet() == 0) + future.complete(null); + return; + } + + Consumer> process = item -> { + if(predicate != null && !predicate.apply(streamIndex, item)) { + if(onNotProcessed != null) + onNotProcessed.accept(streamIndex, item); + return; + } + if(future.isDone()) { + if(onNotProcessed != null) + onNotProcessed.accept(streamIndex, item); + return; + } + + activeTaskCounter.incrementAndGet(); + OOCStream.QueueCallback pinned = item.keepOpen(); + submit(() -> { + long taskStartTime = DMLScript.STATISTICS || DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; + try(pinned) { + consumer.accept(streamIndex, pinned); + if(activeTaskCounter.decrementAndGet() == 0) + TaskContext.defer(() -> future.complete(null)); + } + finally { + recordStatistics(context, globalFuture, taskStartTime); + recordEvent(context, taskStartTime); + } + }, future, context); + }; + + if(callback instanceof OOCStream.GroupQueueCallback) { + OOCStream.GroupQueueCallback group = (OOCStream.GroupQueueCallback) callback; + if(future.isDone()) { + for(int index = 0; index < group.size(); index++) { + try(OOCStream.QueueCallback item = group.getCallback(index)) { + if(onNotProcessed != null) + onNotProcessed.accept(streamIndex, item); + } + } + return; + } + + activeTaskCounter.incrementAndGet(); + OOCStream.GroupQueueCallback pinned = (OOCStream.GroupQueueCallback) group.keepOpen(); + submit(() -> { + long taskStartTime = DMLScript.STATISTICS || DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; + try(pinned) { + for(int index = 0; index < pinned.size(); index++) { + try(OOCStream.QueueCallback item = pinned.getCallback(index)) { + process.accept(item); + } + } + if(activeTaskCounter.decrementAndGet() == 0) + TaskContext.defer(() -> future.complete(null)); + } + finally { + recordStatistics(context, globalFuture, taskStartTime); + recordEvent(context, taskStartTime); + } + }, future, context); + } + else + process.accept(callback); + + if(closed.get()) + throw new DMLRuntimeException("Race condition observed"); + } + catch(RuntimeException error) { + context.failAll(DMLRuntimeException.of(error)); + throw error; + } + finally { + recordStatistics(context, globalFuture, startTime); + } + }, context)); + } + + public static OOCFuture submitOOCTask(Runnable task, StreamContext context) { + // May be blocking tasks, thus should not run on default executor pool + ExecutorService pool = CommonThreadPool.getDynamicPool(); + OOCFuture future = new OOCFuture<>(); + COMPUTE_IN_FLIGHT.incrementAndGet(); + try { + pool.submit(task(() -> { + long startTime = DMLScript.STATISTICS || DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; + try { + task.run(); + future.complete(null); + context.clear(); + if(DMLScript.STATISTICS && context.getExtendedOpcode() != null) + Statistics.maintainOOCHeavyHitter(context.getExtendedOpcode(), System.nanoTime() - startTime); + recordEvent(context, startTime); + } + finally { + COMPUTE_IN_FLIGHT.decrementAndGet(); + } + }, future, context)); + } + catch(RuntimeException error) { + COMPUTE_IN_FLIGHT.decrementAndGet(); + throw DMLRuntimeException.of(error); + } + return future; + } + + private static void submit(Runnable runnable, OOCFuture future, StreamContext context) { + COMPUTE_IN_FLIGHT.incrementAndGet(); + try { + COMPUTE_EXECUTOR.submit(task(() -> { + try { + runnable.run(); + } + finally { + COMPUTE_IN_FLIGHT.decrementAndGet(); + } + }, future, context)); + } + catch(RuntimeException error) { + COMPUTE_IN_FLIGHT.decrementAndGet(); + throw error; + } + } + + private static Runnable task(Runnable runnable, OOCFuture future, StreamContext context) { + return () -> { + boolean setContext = TaskContext.getContext() == null; + if(setContext) + TaskContext.setContext(new TaskContext()); + long startTime = DMLScript.STATISTICS ? System.nanoTime() : 0; + try { + runnable.run(); + if(setContext) { + while(TaskContext.runDeferred()) { + } + } + } + catch(RuntimeException error) { + DMLRuntimeException failure = DMLRuntimeException.of(error); + context.failAll(failure); + if(future != null) + future.completeExceptionally(failure); + throw failure; + } + finally { + if(setContext) + TaskContext.clearContext(); + if(DMLScript.STATISTICS) + context.getLocalStatisticsLongAdder().add(System.nanoTime() - startTime); + } + }; + } + + private static Consumer> guard(Consumer> consumer, + StreamContext context) { + return callback -> { + try { + consumer.accept(callback); + } + catch(RuntimeException error) { + DMLRuntimeException failure = DMLRuntimeException.of(error); + context.failAll(failure); + throw failure; + } + }; + } + + private static void recordStatistics(StreamContext context, CompletableFuture globalFuture, long startTime) { + if(!DMLScript.STATISTICS) + return; + context.getLocalStatisticsLongAdder().add(System.nanoTime() - startTime); + if(globalFuture.isDone() && context.getExtendedOpcode() != null) { + Statistics.maintainOOCHeavyHitter(context.getExtendedOpcode(), context.getLocalStatisticsLongAdder().sum()); + context.getLocalStatisticsLongAdder().reset(); + } + } + + private static void recordEvent(StreamContext context, long startTime) { + if(DMLScript.OOC_LOG_EVENTS && context.getCallerId() != 0) + OOCEventLog.onComputeEvent(context.getCallerId(), startTime, System.nanoTime()); + } +} diff --git a/src/test/java/org/apache/sysds/test/component/ooc/OOCInstructionUtilsTest.java b/src/test/java/org/apache/sysds/test/component/ooc/OOCInstructionUtilsTest.java new file mode 100644 index 00000000000..f4bab5d9813 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/ooc/OOCInstructionUtilsTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.ooc; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.store.MaterializedCallback; +import org.apache.sysds.runtime.ooc.store.StoreLease; +import org.apache.sysds.runtime.ooc.stream.StreamContext; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; +import org.junit.Assert; +import org.junit.Test; + +public class OOCInstructionUtilsTest { + @Test + public void testSubmitTasksClosesCallbacksAfterCompletion() throws Exception { + SubscribableTaskQueue source = new SubscribableTaskQueue<>(); + AtomicInteger processed = new AtomicInteger(); + AtomicInteger released = new AtomicInteger(); + CompletableFuture completion = OOCInstructionUtils.submitOOCTasks(source, callback -> { + Assert.assertEquals(1, callback.get().getIndexes().getRowIndex()); + processed.incrementAndGet(); + }, new StreamContext().addOutStream()); + + IndexedMatrixValue value = new IndexedMatrixValue(new MatrixIndexes(1, 1), new MatrixBlock(1, 1, 1.0)); + source.enqueue(new MaterializedCallback<>(new StoreLease<>(value, released::incrementAndGet))); + source.closeInput(); + completion.get(10, TimeUnit.SECONDS); + + Assert.assertEquals(1, processed.get()); + Assert.assertEquals(1, released.get()); + } + + @Test + public void testSubmitTasksWaitsForAllStreams() throws Exception { + SubscribableTaskQueue first = new SubscribableTaskQueue<>(); + SubscribableTaskQueue second = new SubscribableTaskQueue<>(); + AtomicInteger processed = new AtomicInteger(); + CompletableFuture completion = OOCInstructionUtils.submitOOCTasks(List.of(first, second), + (index, callback) -> processed.addAndGet(callback.get()), new StreamContext().addOutStream()); + + first.enqueue(1); + first.closeInput(); + second.enqueue(2); + Assert.assertFalse(completion.isDone()); + second.closeInput(); + completion.get(10, TimeUnit.SECONDS); + Assert.assertEquals(3, processed.get()); + } + + @Test + public void testSubmitTaskPropagatesFailure() throws Exception { + SubscribableTaskQueue output = new SubscribableTaskQueue<>(); + AtomicReference propagated = new AtomicReference<>(); + output.setSubscriber(callback -> { + try(callback) { + if(callback.isFailure()) { + try { + callback.get(); + } + catch(DMLRuntimeException failure) { + propagated.compareAndSet(null, failure); + } + } + } + }); + + OOCFuture completion = OOCInstructionUtils.submitOOCTask(() -> { + throw new DMLRuntimeException("injected failure"); + }, new StreamContext().addOutStream(output)); + try { + completion.get(10, TimeUnit.SECONDS); + Assert.fail("Expected task failure"); + } + catch(ExecutionException expected) { + Assert.assertTrue(expected.getCause() instanceof DMLRuntimeException); + } + output.closeInput(); + Assert.assertNotNull(propagated.get()); + Assert.assertEquals("injected failure", propagated.get().getMessage()); + } +} From 1e3ae353a1b1e1837b8cd8ea78c2bdde9dad94b5 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Mon, 20 Jul 2026 11:28:02 +0200 Subject: [PATCH 086/132] [SYSTEMDS-3891] Add OOC Primitives --- .../runtime/instructions/ooc/OOCStream.java | 7 ++ .../instructions/ooc/OOCStreamable.java | 9 ++ .../ooc/SubscribableTaskQueue.java | 14 +++ .../ooc/planning/OOCAccessPattern.java | 53 +++++++++++ .../runtime/ooc/primitives/OOCPrimitive.java | 92 +++++++++++++++++++ .../ooc/stream/AllocatedOOCStream.java | 6 ++ .../runtime/ooc/stream/FilteredOOCStream.java | 6 ++ .../runtime/ooc/stream/MergedOOCStream.java | 6 ++ .../ooc/stream/SplittingOOCStream.java | 6 ++ .../runtime/ooc/stream/SubOOCStream.java | 6 ++ .../test/component/ooc/OOCPrimitiveTest.java | 78 ++++++++++++++++ 11 files changed, 283 insertions(+) create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/planning/OOCAccessPattern.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java create mode 100644 src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStream.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStream.java index b4ffbbbaedb..49a41ce2365 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStream.java @@ -20,6 +20,7 @@ package org.apache.sysds.runtime.instructions.ooc; import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; import java.util.function.Consumer; @@ -48,6 +49,12 @@ static QueueCallback eos(DMLRuntimeException e) { */ void setSubscriber(Consumer> subscriber); + default void start() { + OOCPrimitive primitive = getPrimitive(); + if(primitive != null) + primitive.tryStartExecution(); + } + interface QueueCallback extends AutoCloseable { T get(); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStreamable.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStreamable.java index 75ccdee3ee1..0f087a7f55b 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStreamable.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStreamable.java @@ -21,6 +21,7 @@ import org.apache.sysds.runtime.controlprogram.caching.CacheableData; import org.apache.sysds.runtime.meta.DataCharacteristics; +import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; public interface OOCStreamable { OOCStream getReadStream(); @@ -38,4 +39,12 @@ public interface OOCStreamable { CacheableData getData(); void setData(CacheableData data); + + default OOCPrimitive getPrimitive() { + return null; + } + + default void assignPrimitive(OOCPrimitive primitive) { + throw new UnsupportedOperationException("Stream does not support primitive assignment"); + } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/SubscribableTaskQueue.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/SubscribableTaskQueue.java index 04ce65e5725..5400b6ba98f 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/SubscribableTaskQueue.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/SubscribableTaskQueue.java @@ -23,6 +23,7 @@ import org.apache.sysds.runtime.controlprogram.caching.CacheableData; import org.apache.sysds.runtime.controlprogram.parfor.LocalTaskQueue; import org.apache.sysds.runtime.meta.DataCharacteristics; +import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; import org.apache.sysds.runtime.ooc.util.OOCUtils; import java.util.LinkedList; @@ -37,6 +38,7 @@ public class SubscribableTaskQueue extends LocalTaskQueue _lastDequeued = null; private CacheableData _cdata; + private volatile OOCPrimitive _primitive; private volatile Consumer> _subscriber = null; private String _watchdogId; @@ -276,6 +278,18 @@ public CachingStream getStreamCache() { return null; } + @Override + public OOCPrimitive getPrimitive() { + return _primitive; + } + + @Override + public void assignPrimitive(OOCPrimitive primitive) { + if(_primitive != null) + throw new IllegalStateException("Primitive already assigned"); + _primitive = primitive; + } + @Override public DataCharacteristics getDataCharacteristics() { return _cdata == null ? null : _cdata.getDataCharacteristics(); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCAccessPattern.java b/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCAccessPattern.java new file mode 100644 index 00000000000..72a67fedfdf --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCAccessPattern.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.planning; + +public enum OOCAccessPattern { + ROW_MAJOR, COL_MAJOR, ANY, UNKNOWN, UNSET; + + public OOCAccessPattern fused(OOCAccessPattern other) { + return switch(this) { + case ANY -> other; + case UNKNOWN, UNSET -> this; + case ROW_MAJOR -> other == ROW_MAJOR || other == ANY ? this : UNKNOWN; + case COL_MAJOR -> other == COL_MAJOR || other == ANY ? this : UNKNOWN; + }; + } + + public OOCAccessPattern transposed() { + return switch(this) { + case ROW_MAJOR -> COL_MAJOR; + case COL_MAJOR -> ROW_MAJOR; + default -> this; + }; + } + + public OOCAccessPattern preferred(OOCAccessPattern preferred) { + return this == ANY || this == UNSET ? preferred : this; + } + + public boolean isPlannable() { + return this == ROW_MAJOR || this == COL_MAJOR || this == ANY; + } + + public boolean isUnset() { + return this == UNSET; + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java new file mode 100644 index 00000000000..67c3072de50 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.primitives; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; + +public abstract class OOCPrimitive { + private final List _children; + private final List _parents; + private final AtomicBoolean _executionStarted; + protected OOCAccessPattern _pattern; + + protected OOCPrimitive(List children) { + _parents = new ArrayList<>(); + List uniqueChildren = new ArrayList<>(children.size()); + for(OOCPrimitive child : children) { + if(containsIdentity(uniqueChildren, child)) + continue; + uniqueChildren.add(child); + child.addParent(this); + } + _children = List.copyOf(uniqueChildren); + _executionStarted = new AtomicBoolean(); + _pattern = OOCAccessPattern.UNSET; + } + + public final List getChildren() { + return _children; + } + + public final List getParents() { + return List.copyOf(_parents); + } + + private void addParent(OOCPrimitive parent) { + if(!containsIdentity(_parents, parent)) + _parents.add(parent); + } + + protected final void inferParentPatterns() { + for(OOCPrimitive parent : _parents) + if(parent._pattern.isUnset()) + parent.inferPatterns(); + } + + public final OOCAccessPattern getAccessPattern() { + return _pattern; + } + + public final boolean hasStartedExecution() { + return _executionStarted.get(); + } + + public final void tryStartExecution() { + if(_executionStarted.compareAndSet(false, true)) + startExecution(); + } + + private static boolean containsIdentity(List primitives, OOCPrimitive primitive) { + for(OOCPrimitive current : primitives) + if(current == primitive) + return true; + return false; + } + + protected abstract void startExecution(); + + public abstract void inferPatterns(); + + public abstract void requestPattern(OOCAccessPattern accessPattern); +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/AllocatedOOCStream.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/AllocatedOOCStream.java index 6d3a7c5936f..c940bdd47a6 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/AllocatedOOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/AllocatedOOCStream.java @@ -27,6 +27,7 @@ import org.apache.sysds.runtime.ooc.cache.OOCFuture; import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; import org.apache.sysds.runtime.ooc.memory.ReservationBudget; +import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; public final class AllocatedOOCStream extends SubscribableTaskQueue { private final OOCStream _source; @@ -49,6 +50,11 @@ public static ReservationBudget detachBudget(OOCStream.QueueCallback callback return callback instanceof BudgetedQueueCallback budgeted ? budgeted.detachBudget() : null; } + @Override + public OOCPrimitive getPrimitive() { + return _source.getPrimitive(); + } + private void admit(OOCStream.QueueCallback callback) { if(callback.isFailure()) { try(callback) { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/FilteredOOCStream.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/FilteredOOCStream.java index 276f96680a6..96eae6d46e0 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/FilteredOOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/FilteredOOCStream.java @@ -24,6 +24,7 @@ import org.apache.sysds.runtime.instructions.ooc.CachingStream; import org.apache.sysds.runtime.instructions.ooc.OOCStream; import org.apache.sysds.runtime.meta.DataCharacteristics; +import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; import java.util.function.Consumer; import java.util.function.Function; @@ -88,6 +89,11 @@ public CachingStream getStreamCache() { return _sourceStream.getStreamCache(); } + @Override + public OOCPrimitive getPrimitive() { + return _sourceStream.getPrimitive(); + } + @Override public void setSubscriber(Consumer> subscriber) { _sourceStream.setSubscriber(cb -> { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/MergedOOCStream.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/MergedOOCStream.java index 1c0f4977a27..dda930990be 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/MergedOOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/MergedOOCStream.java @@ -197,6 +197,12 @@ public OOCStream getReadStream() { return this; } + @Override + public void start() { + for(OOCStream source : _sources) + source.start(); + } + @Override public OOCStream getWriteStream() { throw new UnsupportedOperationException(); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/SplittingOOCStream.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/SplittingOOCStream.java index 7aef56e96ba..989cf82f009 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/SplittingOOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/SplittingOOCStream.java @@ -24,6 +24,7 @@ import org.apache.sysds.runtime.instructions.ooc.CachingStream; import org.apache.sysds.runtime.instructions.ooc.OOCStream; import org.apache.sysds.runtime.meta.DataCharacteristics; +import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; import java.util.function.Consumer; import java.util.function.Function; @@ -144,6 +145,11 @@ public CachingStream getStreamCache() { return _sourceStream.getStreamCache(); } + @Override + public OOCPrimitive getPrimitive() { + return _sourceStream.getPrimitive(); + } + @Override public void setSubscriber(Consumer> subscriber) { throw new UnsupportedOperationException(); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/SubOOCStream.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/SubOOCStream.java index 44f6542a4d3..653eb3ba223 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/SubOOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/SubOOCStream.java @@ -25,6 +25,7 @@ import org.apache.sysds.runtime.instructions.ooc.OOCStream; import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; import org.apache.sysds.runtime.meta.DataCharacteristics; +import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; import java.util.function.Consumer; @@ -86,6 +87,11 @@ public CachingStream getStreamCache() { return _sourceStream.getStreamCache(); } + @Override + public OOCPrimitive getPrimitive() { + return _sourceStream.getPrimitive(); + } + @Override public void setSubscriber(Consumer> subscriber) { _taskQueue.setSubscriber(cb -> { diff --git a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java new file mode 100644 index 00000000000..9b6e435ca60 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.component.ooc; + +import java.util.List; + +import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; +import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; +import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; +import org.apache.sysds.runtime.ooc.stream.FilteredOOCStream; +import org.junit.Assert; +import org.junit.Test; + +public class OOCPrimitiveTest { + @Test + public void testGraphPatternsAndExecution() { + TestPrimitive source = new TestPrimitive(List.of()); + TestPrimitive sink = new TestPrimitive(List.of(source, source)); + + Assert.assertEquals(List.of(source), sink.getChildren()); + Assert.assertEquals(List.of(sink), source.getParents()); + source.inferPatterns(); + Assert.assertEquals(OOCAccessPattern.ANY, source.getAccessPattern()); + Assert.assertEquals(OOCAccessPattern.ANY, sink.getAccessPattern()); + Assert.assertEquals(OOCAccessPattern.COL_MAJOR, OOCAccessPattern.ROW_MAJOR.transposed()); + Assert.assertEquals(OOCAccessPattern.UNKNOWN, OOCAccessPattern.ROW_MAJOR.fused(OOCAccessPattern.COL_MAJOR)); + + SubscribableTaskQueue stream = new SubscribableTaskQueue<>(); + stream.assignPrimitive(sink); + FilteredOOCStream filtered = new FilteredOOCStream<>(stream, ignored -> true); + Assert.assertSame(sink, filtered.getPrimitive()); + stream.start(); + filtered.start(); + Assert.assertTrue(sink.hasStartedExecution()); + Assert.assertEquals(1, sink._executions); + } + + private static final class TestPrimitive extends OOCPrimitive { + private int _executions; + + private TestPrimitive(List children) { + super(children); + } + + @Override + protected void startExecution() { + _executions++; + } + + @Override + public void inferPatterns() { + _pattern = OOCAccessPattern.ANY; + inferParentPatterns(); + } + + @Override + public void requestPattern(OOCAccessPattern accessPattern) { + _pattern = accessPattern; + } + } +} From 75c1e4895e7d78036edc6bac71fba865a734c6e1 Mon Sep 17 00:00:00 2001 From: MegaByteTron Date: Tue, 21 Jul 2026 11:23:09 +0200 Subject: [PATCH 087/132] [SYSTEMDS-3922] Fix distributed (SPARK/FED) quantile pick instruction Closes #2497 --- .../fed/QuantilePickFEDInstruction.java | 7 +-- .../spark/QuantilePickSPInstruction.java | 48 +++++++++++-------- .../runtime/matrix/data/MatrixBlock.java | 1 + .../functions/binary/matrix/QuantileTest.java | 32 +++++++++---- .../part2/FederatedQuantileTest.java | 11 +---- .../functions/binary/matrix/MedianBug.R | 34 +++++++++++++ .../functions/binary/matrix/MedianBug.dml | 25 ++++++++++ 7 files changed, 114 insertions(+), 44 deletions(-) create mode 100644 src/test/scripts/functions/binary/matrix/MedianBug.R create mode 100644 src/test/scripts/functions/binary/matrix/MedianBug.dml diff --git a/src/main/java/org/apache/sysds/runtime/instructions/fed/QuantilePickFEDInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/fed/QuantilePickFEDInstruction.java index e70f1dffa63..d16e1e16ff1 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/fed/QuantilePickFEDInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/fed/QuantilePickFEDInstruction.java @@ -234,7 +234,7 @@ public MatrixBlock getEquiHeightBins(ExecutionContext ec, int colID, double[ public void processRowQPick(ExecutionContext ec) { MatrixObject in = ec.getMatrixObject(input1); FederationMap fedMap = in.getFedMapping(); - boolean average = _type == OperationTypes.MEDIAN; + boolean average = _type == OperationTypes.MEDIAN || _type == OperationTypes.VALUEPICK; double[] quantiles = input2 != null ? (input2.isMatrix() ? ec.getMatrixInput(input2).getDenseBlockValues() : input2.isScalar() ? new double[] {ec.getScalarInput(input2).getDoubleValue()} : null) : @@ -749,16 +749,17 @@ protected ValuePick(long input, MatrixBlock quantiles) { super(new long[] {input}); _quantiles = quantiles; } + @Override public FederatedResponse execute(ExecutionContext ec, Data... data) { MatrixBlock mb = ((MatrixObject)data[0]).acquireReadAndRelease(); MatrixBlock picked; if (_quantiles.getLength() == 1) { return new FederatedResponse(FederatedResponse.ResponseType.SUCCESS, - new Object[] {mb.pickValue(_quantiles.get(0, 0))}); + new Object[] {mb.pickValue(_quantiles.get(0, 0), mb.getLength() % 2 == 0)}); } else { - picked = mb.pickValues(_quantiles, new MatrixBlock()); + picked = mb.pickValues(_quantiles, new MatrixBlock(), mb.getLength() % 2 == 0); return new FederatedResponse(FederatedResponse.ResponseType.SUCCESS, new Object[] {picked}); } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java index a30fbc456cb..75f84882478 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java @@ -115,12 +115,12 @@ public void processInstruction(ExecutionContext ec) { if( input2.isScalar() ) { ScalarObject quantile = ec.getScalarInput(input2); double[] wt = getWeightedQuantileSummary(in, mc, - new double[]{quantile.getDoubleValue()}); + new double[] {quantile.getDoubleValue()}, true); ec.setScalarOutput(output.getName(), new DoubleObject(wt[3])); } else { - double[] wt = getWeightedQuantileSummary(in, mc, DataConverter - .convertToDoubleVector(ec.getMatrixInput(input2.getName()))); + double[] wt = getWeightedQuantileSummary(in, mc, + DataConverter.convertToDoubleVector(ec.getMatrixInput(input2.getName())), true); ec.releaseMatrixInput(input2.getName()); int qlen = wt.length/3; MatrixBlock out = new MatrixBlock(qlen,1,false); @@ -130,20 +130,20 @@ public void processInstruction(ExecutionContext ec) { } break; } - + case MEDIAN: { - double[] wt = getWeightedQuantileSummary(in, mc, new double[]{0.5}); + double[] wt = getWeightedQuantileSummary(in, mc, new double[] {0.5}, true); ec.setScalarOutput(output.getName(), new DoubleObject(wt[3])); break; } - + case IQM: { - double[] wt = getWeightedQuantileSummary(in, mc, new double[]{0.25,0.75}); - long key25 = (long)Math.ceil(wt[1]); - long key75 = (long)Math.ceil(wt[2]); - JavaPairRDD out = in - .filter(new FilterFunction(key25+1,key75,mc.getBlocksize())) - .mapToPair(new ExtractAndSumFunction(key25+1, key75, mc.getBlocksize())); + double[] wt = getWeightedQuantileSummary(in, mc, new double[] {0.25, 0.75}, false); + long key25 = (long) Math.ceil(wt[1]); + long key75 = (long) Math.ceil(wt[2]); + JavaPairRDD out = in + .filter(new FilterFunction(key25 + 1, key75, mc.getBlocksize())) + .mapToPair(new ExtractAndSumFunction(key25 + 1, key75, mc.getBlocksize())); double sum = RDDAggregateUtils.sumStable(out).get(0, 0); double val = MatrixBlock.computeIQMCorrection( sum, wt[0], wt[3], wt[5], wt[4], wt[6]); @@ -165,10 +165,10 @@ public void processInstruction(ExecutionContext ec) { * @param quantiles one or more quantiles between 0 and 1. * @return a summary of weighted quantiles */ - private static double[] getWeightedQuantileSummary(JavaPairRDD w, DataCharacteristics mc, double[] quantiles) - { - double[] ret = new double[3*quantiles.length + 1]; - if( mc.getCols()==2 ) //weighted + private static double[] getWeightedQuantileSummary(JavaPairRDD w, + DataCharacteristics mc, double[] quantiles, boolean average) { + double[] ret = new double[3 * quantiles.length + 1]; + if(mc.getCols() == 2) // weighted { //sort blocks (values sorted but blocks and partitions are not) w = w.sortByKey(); @@ -213,11 +213,17 @@ private static double[] getWeightedQuantileSummary(JavaPairRDD data() { - return Arrays.asList(new Object[][] { - // {1000, 1, false}, - {128, 1, true}}); + return Arrays.asList(new Object[][] {{1000, 1, false}, {128, 1, true}}); } @Override @@ -71,19 +68,16 @@ public void setUp() { } @Test - @Ignore public void federatedQuantile1CP() { federatedQuartile(Types.ExecMode.SINGLE_NODE, TEST_NAME1, 0.25); } @Test - @Ignore public void federatedQuantile2CP() { federatedQuartile(Types.ExecMode.SINGLE_NODE, TEST_NAME1, 0.5); } @Test - @Ignore public void federatedQuantile3CP() { federatedQuartile(Types.ExecMode.SINGLE_NODE, TEST_NAME1, 0.75); } @@ -104,19 +98,16 @@ public void federatedQuantilesCP() { } @Test - @Ignore public void federatedQuantile1SP() { federatedQuartile(Types.ExecMode.SPARK, TEST_NAME1, 0.25); } @Test - @Ignore public void federatedQuantile2SP() { federatedQuartile(Types.ExecMode.SPARK, TEST_NAME1, 0.5); } @Test - @Ignore public void federatedQuantile3SP() { federatedQuartile(Types.ExecMode.SPARK, TEST_NAME1, 0.75); } diff --git a/src/test/scripts/functions/binary/matrix/MedianBug.R b/src/test/scripts/functions/binary/matrix/MedianBug.R new file mode 100644 index 00000000000..f868b652754 --- /dev/null +++ b/src/test/scripts/functions/binary/matrix/MedianBug.R @@ -0,0 +1,34 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +args <- commandArgs(TRUE) +options(digits=22) + +library("Matrix") + +A = as.matrix(c(1,5,7,10)) + +s = median(A); +m = as.matrix(s); + +writeMM(as(m, "CsparseMatrix"), paste(args[3], "R", sep="")); + + diff --git a/src/test/scripts/functions/binary/matrix/MedianBug.dml b/src/test/scripts/functions/binary/matrix/MedianBug.dml new file mode 100644 index 00000000000..21dbb15fe41 --- /dev/null +++ b/src/test/scripts/functions/binary/matrix/MedianBug.dml @@ -0,0 +1,25 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +A = as.matrix(list(1,5,7,10)); +s = median(A); +m = as.matrix(s); +write(m, $3, format="text"); From a84e8657312ea17ae93856244bd6308491387432 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:52:52 +0200 Subject: [PATCH 088/132] [SYSTEMDS-3891] Add OOC Planner and First Primitive Implementations --- .../runtime/instructions/ooc/OOCStream.java | 4 +- .../runtime/ooc/planning/OOCPlanner.java | 54 +++++++++++ .../ooc/primitives/MappingOOCPrimitive.java | 76 ++++++++++++++++ .../runtime/ooc/primitives/OOCPrimitive.java | 43 ++++++++- .../PlannableDataGenOOCPrimitive.java | 91 +++++++++++++++++++ .../ooc/primitives/TransposeOOCPrimitive.java | 76 ++++++++++++++++ .../runtime/ooc/util/OOCInstructionUtils.java | 60 ++++++++++++ .../sysds/runtime/ooc/util/OOCUtils.java | 64 +++++++++++++ .../test/component/ooc/OOCPrimitiveTest.java | 57 +++++++++++- 9 files changed, 516 insertions(+), 9 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/planning/OOCPlanner.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/primitives/MappingOOCPrimitive.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/primitives/PlannableDataGenOOCPrimitive.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/primitives/TransposeOOCPrimitive.java diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStream.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStream.java index 49a41ce2365..f5f44fdd3f0 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStream.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStream.java @@ -51,8 +51,8 @@ static QueueCallback eos(DMLRuntimeException e) { default void start() { OOCPrimitive primitive = getPrimitive(); - if(primitive != null) - primitive.tryStartExecution(); + if(primitive != null && !primitive.hasStartedExecution()) + primitive.start(); } interface QueueCallback extends AutoCloseable { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCPlanner.java b/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCPlanner.java new file mode 100644 index 00000000000..3db13c2691a --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCPlanner.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.planning; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Set; + +import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; + +public final class OOCPlanner { + public static void compile(OOCPrimitive root) { + List primitives = new ArrayList<>(); + collect(root, Collections.newSetFromMap(new IdentityHashMap<>()), primitives); + if(primitives.isEmpty()) + return; + + for(int i = primitives.size() - 1; i >= 0; i--) + if(primitives.get(i).getAccessPattern().isUnset()) + primitives.get(i).inferPatterns(); + if(root.getAccessPattern() == OOCAccessPattern.ANY || root.getAccessPattern().isUnset()) + root.requestPattern(OOCAccessPattern.ROW_MAJOR); + + for(OOCPrimitive primitive : primitives) + primitive.tryStartExecution(); + } + + private static void collect(OOCPrimitive primitive, Set visited, List result) { + if(primitive.hasStartedExecution() || !visited.add(primitive)) + return; + result.add(primitive); + for(OOCPrimitive child : primitive.getChildren()) + collect(child, visited, result); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/MappingOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/MappingOOCPrimitive.java new file mode 100644 index 00000000000..fe1285a63fa --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/MappingOOCPrimitive.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.primitives; + +import java.util.List; +import java.util.function.Function; + +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; +import org.apache.sysds.runtime.ooc.stream.StreamContext; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; + +public class MappingOOCPrimitive extends OOCPrimitive { + private final OOCStreamable _input; + private final OOCStreamable _output; + private final Function _operation; + + public MappingOOCPrimitive(OOCStreamable input, OOCStreamable output, + Function operation, StreamContext context) { + this(input.getPrimitive(), input, output, operation, context); + } + + private MappingOOCPrimitive(OOCPrimitive inputPrimitive, OOCStreamable input, + OOCStreamable output, Function operation, + StreamContext context) { + super(context, inputPrimitive == null ? List.of() : List.of(inputPrimitive)); + _input = input; + _output = output; + _operation = operation; + } + + @Override + protected void inferPatternsInternal() { + OOCAccessPattern inputPattern = getChildren().isEmpty() ? OOCAccessPattern.ANY : getChildren().get(0) + .getAccessPattern(); + _pattern = _pattern.preferred(inputPattern); + inferParentPatterns(); + } + + @Override + protected void requestPatternInternal(OOCAccessPattern accessPattern) { + _pattern = _pattern.preferred(accessPattern); + if(!getChildren().isEmpty()) + getChildren().get(0).requestPattern(accessPattern); + } + + @Override + protected void startExecution() { + OOCStream input = _input.getReadStream(); + OOCStream output = _output.getWriteStream(); + OOCInstructionUtils + .submitAdmittedOOCTasks(input, output, + value -> new IndexedMatrixValue(value.getIndexes(), _operation.apply(value)), _allowance, getContext()) + .thenRun(this::onComplete); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java index 67c3072de50..aacb35acdad 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java @@ -23,15 +23,24 @@ import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.sysds.runtime.ooc.memory.GlobalMemoryBroker; +import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; +import org.apache.sysds.runtime.ooc.memory.SyncMemoryAllowance; import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; +import org.apache.sysds.runtime.ooc.planning.OOCPlanner; +import org.apache.sysds.runtime.ooc.stream.StreamContext; public abstract class OOCPrimitive { + private final StreamContext _context; private final List _children; private final List _parents; + private final AtomicBoolean _started; private final AtomicBoolean _executionStarted; protected OOCAccessPattern _pattern; + protected MemoryAllowance _allowance; - protected OOCPrimitive(List children) { + protected OOCPrimitive(StreamContext context, List children) { + _context = context; _parents = new ArrayList<>(); List uniqueChildren = new ArrayList<>(children.size()); for(OOCPrimitive child : children) { @@ -41,10 +50,15 @@ protected OOCPrimitive(List children) { child.addParent(this); } _children = List.copyOf(uniqueChildren); + _started = new AtomicBoolean(); _executionStarted = new AtomicBoolean(); _pattern = OOCAccessPattern.UNSET; } + public final StreamContext getContext() { + return _context; + } + public final List getChildren() { return _children; } @@ -72,9 +86,30 @@ public final boolean hasStartedExecution() { return _executionStarted.get(); } + public final void start() { + if(_started.compareAndSet(false, true)) + OOCPlanner.compile(this); + } + public final void tryStartExecution() { - if(_executionStarted.compareAndSet(false, true)) + if(_executionStarted.compareAndSet(false, true)) { + _allowance = new SyncMemoryAllowance(GlobalMemoryBroker.get()); startExecution(); + } + } + + public final void onComplete() { + _allowance.shutdown(); + } + + public final void inferPatterns() { + if(!hasStartedExecution()) + inferPatternsInternal(); + } + + public final void requestPattern(OOCAccessPattern accessPattern) { + if(!hasStartedExecution() && _pattern != accessPattern) + requestPatternInternal(accessPattern); } private static boolean containsIdentity(List primitives, OOCPrimitive primitive) { @@ -86,7 +121,7 @@ private static boolean containsIdentity(List primitives, OOCPrimit protected abstract void startExecution(); - public abstract void inferPatterns(); + protected abstract void inferPatternsInternal(); - public abstract void requestPattern(OOCAccessPattern accessPattern); + protected abstract void requestPatternInternal(OOCAccessPattern accessPattern); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/PlannableDataGenOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/PlannableDataGenOOCPrimitive.java new file mode 100644 index 00000000000..83ffcd2acf4 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/PlannableDataGenOOCPrimitive.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.primitives; + +import java.util.List; +import java.util.function.Function; + +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; +import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.ooc.memory.ReservationBudget; +import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; +import org.apache.sysds.runtime.ooc.stream.AllocatedOOCStream; +import org.apache.sysds.runtime.ooc.stream.StreamContext; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; +import org.apache.sysds.runtime.ooc.util.OOCUtils; + +public class PlannableDataGenOOCPrimitive extends OOCPrimitive { + private final OOCStreamable _output; + private final Function _operation; + + public PlannableDataGenOOCPrimitive(OOCStreamable output, + Function operation, StreamContext context) { + super(context, List.of()); + _output = output; + _operation = operation; + } + + @Override + protected void inferPatternsInternal() { + if(_pattern.isUnset()) + _pattern = OOCAccessPattern.ANY; + inferParentPatterns(); + } + + @Override + protected void requestPatternInternal(OOCAccessPattern accessPattern) { + _pattern = accessPattern; + } + + @Override + protected void startExecution() { + OOCStream work = new SubscribableTaskQueue<>(); + OOCStream output = _output.getWriteStream(); + long outputBytes = OOCUtils.estimateOutputTileBytes(_output.getDataCharacteristics()); + AllocatedOOCStream admitted = new AllocatedOOCStream<>(work, _allowance, ignored -> outputBytes); + getContext().addOutStream(output); + OOCInstructionUtils.submitOOCTasks(admitted, callback -> { + ReservationBudget budget = AllocatedOOCStream.detachBudget(callback); + try { + MatrixIndexes indexes = callback.get(); + OOCUtils.enqueueExact(output, new IndexedMatrixValue(indexes, _operation.apply(indexes)), budget); + budget = null; + } + finally { + if(budget != null) + budget.close(); + } + }, getContext()).thenRun(output::closeInput).exceptionally(error -> { + output.propagateFailure(DMLRuntimeException.of(error)); + return null; + }).thenRun(this::onComplete); + + OOCInstructionUtils.submitOOCTask(() -> { + for(MatrixIndexes indexes : OOCUtils.getAccessPattern(_output.getDataCharacteristics(), _pattern)) + work.enqueue(indexes); + work.closeInput(); + }, new StreamContext().addOutStream(work)); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/TransposeOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/TransposeOOCPrimitive.java new file mode 100644 index 00000000000..938c753a197 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/TransposeOOCPrimitive.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.primitives; + +import java.util.List; +import java.util.function.Function; + +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; +import org.apache.sysds.runtime.ooc.stream.StreamContext; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; + +public class TransposeOOCPrimitive extends OOCPrimitive { + private final OOCStreamable _input; + private final OOCStreamable _output; + private final Function _operation; + + public TransposeOOCPrimitive(OOCStreamable input, OOCStreamable output, + Function operation, StreamContext context) { + this(input.getPrimitive(), input, output, operation, context); + } + + private TransposeOOCPrimitive(OOCPrimitive inputPrimitive, OOCStreamable input, + OOCStreamable output, Function operation, StreamContext context) { + super(context, inputPrimitive == null ? List.of() : List.of(inputPrimitive)); + _input = input; + _output = output; + _operation = operation; + } + + @Override + protected void inferPatternsInternal() { + _pattern = (getChildren().isEmpty() ? OOCAccessPattern.ANY : getChildren().get(0).getAccessPattern()) + .transposed(); + inferParentPatterns(); + } + + @Override + protected void requestPatternInternal(OOCAccessPattern accessPattern) { + _pattern = accessPattern; + if(!getChildren().isEmpty()) + getChildren().get(0).requestPattern(accessPattern.transposed()); + } + + @Override + protected void startExecution() { + OOCStream input = _input.getReadStream(); + OOCStream output = _output.getWriteStream(); + OOCInstructionUtils.submitAdmittedOOCTasks(input, output, value -> { + MatrixIndexes indexes = value.getIndexes(); + return new IndexedMatrixValue(new MatrixIndexes(indexes.getColumnIndex(), indexes.getRowIndex()), + _operation.apply((MatrixBlock) value.getValue())); + }, _allowance, getContext()).thenRun(this::onComplete); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java index 75f6f9db2e5..98bab492d53 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java @@ -33,8 +33,18 @@ import org.apache.sysds.api.DMLScript; import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.data.MatrixIndexes; import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; +import org.apache.sysds.runtime.ooc.memory.ReservationBudget; +import org.apache.sysds.runtime.ooc.primitives.MappingOOCPrimitive; +import org.apache.sysds.runtime.ooc.primitives.PlannableDataGenOOCPrimitive; +import org.apache.sysds.runtime.ooc.primitives.TransposeOOCPrimitive; import org.apache.sysds.runtime.ooc.stats.OOCEventLog; +import org.apache.sysds.runtime.ooc.stream.AllocatedOOCStream; import org.apache.sysds.runtime.ooc.stream.StreamContext; import org.apache.sysds.runtime.ooc.stream.TaskContext; import org.apache.sysds.runtime.util.CommonThreadPool; @@ -46,6 +56,31 @@ public final class OOCInstructionUtils { private static final AtomicInteger COMPUTE_IN_FLIGHT = new AtomicInteger(); private static final int COMPUTE_BACKPRESSURE_THRESHOLD = 100; + public static void dataGen(OOCStream output, Function operation, + StreamContext context) { + output.assignPrimitive(new PlannableDataGenOOCPrimitive(output, operation, context)); + } + + public static void equiMapBlock(OOCStreamable input, OOCStream output, + Function operation, StreamContext context) { + equiMap(input, output, value -> operation.apply((MatrixBlock) value.getValue()), context); + } + + public static void equiMap(OOCStreamable input, OOCStream output, + Function operation, StreamContext context) { + output.assignPrimitive(new MappingOOCPrimitive(input, output, operation, context)); + } + + public static void transposedMap(OOCStream input, OOCStream output, + Function operation, StreamContext context) { + output.assignPrimitive(new TransposeOOCPrimitive(input, output, operation, context)); + } + + public static void transpose(OOCStream input, OOCStream output, + StreamContext context) { + transposedMap(input, output, MatrixBlock::transpose, context); + } + public static int getComputeInFlight() { return COMPUTE_IN_FLIGHT.get(); } @@ -59,6 +94,31 @@ public static CompletableFuture submitOOCTasks(OOCStream queue, return submitOOCTasks(List.of(queue), (i, callback) -> consumer.accept(callback), null, null, context); } + public static CompletableFuture submitAdmittedOOCTasks(OOCStream in, + OOCStream out, Function operation, + MemoryAllowance allowance, StreamContext context) { + context.addOutStream(out); + long outputBytes = OOCUtils.estimateOutputTileBytes(out.getDataCharacteristics()); + AllocatedOOCStream admitted = new AllocatedOOCStream<>(in, allowance, + ignored -> outputBytes); + return submitOOCTasks(admitted, callback -> { + ReservationBudget budget = AllocatedOOCStream.detachBudget(callback); + try { + if(budget == null) + throw new DMLRuntimeException("Missing admitted output budget"); + OOCUtils.enqueueExact(out, operation.apply(callback.get()), budget); + budget = null; + } + finally { + if(budget != null) + budget.close(); + } + }, context).thenRun(out::closeInput).exceptionally(error -> { + out.propagateFailure(DMLRuntimeException.of(error)); + return null; + }); + } + public static CompletableFuture submitOOCTasks(OOCStream queue, Consumer> consumer, Function, Boolean> predicate, BiConsumer> onNotProcessed, StreamContext context) { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java index 74e6ed30a8f..7b8a8ae46b1 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java @@ -20,16 +20,23 @@ package org.apache.sysds.runtime.ooc.util; import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.runtime.meta.DataCharacteristics; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.ooc.cache.BlockEntry; import org.apache.sysds.runtime.ooc.cache.OOCCache; import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; +import org.apache.sysds.runtime.ooc.memory.ReservationBudget; +import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; import org.apache.sysds.runtime.util.IndexRange; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.function.BooleanSupplier; @@ -106,4 +113,61 @@ public static long getNumBlocks(DataCharacteristics dc) { } return -1; } + + public static Iterable getAccessPattern(DataCharacteristics dc, OOCAccessPattern pattern) { + long rows = dc.getRows() == 0 ? 0 : dc.getNumRowBlocks(); + long cols = dc.getCols() == 0 ? 0 : dc.getNumColBlocks(); + return getAccessPattern(rows, cols, pattern); + } + + public static Iterable getAccessPattern(long rows, long cols, OOCAccessPattern pattern) { + return () -> new Iterator<>() { + private final long _size = rows * cols; + private long _position; + + @Override + public boolean hasNext() { + return _position < _size; + } + + @Override + public MatrixIndexes next() { + long position = _position++; + return pattern == OOCAccessPattern.COL_MAJOR ? new MatrixIndexes(position % rows + 1, + position / rows + 1) : new MatrixIndexes(position / cols + 1, position % cols + 1); + } + }; + } + + public static long estimateOutputTileBytes(DataCharacteristics dc) { + if(dc == null || dc.getBlocksize() <= 0 || !dc.dimsKnown()) { + int blocksize = dc != null && dc.getBlocksize() > 0 ? dc.getBlocksize() : 1000; + return estimateMatrixBlockBytes(blocksize, blocksize); + } + return estimateMatrixBlockBytes(Math.min(dc.getBlocksize(), dc.getRows()), + Math.min(dc.getBlocksize(), dc.getCols())); + } + + private static long estimateMatrixBlockBytes(long rows, long cols) { + return Math.max(MatrixBlock.estimateSizeDenseInMemory(rows, cols), + MatrixBlock.estimateSizeSparseInMemory(rows, cols, 1.0)); + } + + public static void enqueueExact(OOCStream out, IndexedMatrixValue value, + ReservationBudget budget) { + long bytes = ((MatrixBlock) value.getValue()).getExactSerializedSize(); + OOCStream.QueueCallback callback = null; + try { + budget.reserveBlocking(bytes); + callback = new InMemoryQueueCallback(value, null, budget, bytes); + budget.close(); + out.enqueue(callback); + callback = null; + } + finally { + budget.close(); + if(callback != null) + callback.close(); + } + } } diff --git a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java index 9b6e435ca60..40abcf26348 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java @@ -19,12 +19,24 @@ package org.apache.sysds.test.component.ooc; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import org.apache.sysds.common.Types.FileFormat; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.meta.MatrixCharacteristics; +import org.apache.sysds.runtime.meta.MetaDataFormat; import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; import org.apache.sysds.runtime.ooc.stream.FilteredOOCStream; +import org.apache.sysds.runtime.ooc.stream.StreamContext; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; import org.junit.Assert; import org.junit.Test; @@ -50,28 +62,67 @@ public void testGraphPatternsAndExecution() { filtered.start(); Assert.assertTrue(sink.hasStartedExecution()); Assert.assertEquals(1, sink._executions); + Assert.assertEquals(1, source._executions); + Assert.assertEquals(OOCAccessPattern.ROW_MAJOR, sink.getAccessPattern()); + sink.inferPatterns(); + sink.requestPattern(OOCAccessPattern.COL_MAJOR); + Assert.assertEquals(OOCAccessPattern.ROW_MAJOR, sink.getAccessPattern()); + } + + @Test + public void testDataGenMapTransposePipeline() { + SubscribableTaskQueue generated = new SubscribableTaskQueue<>(); + SubscribableTaskQueue mapped = new SubscribableTaskQueue<>(); + SubscribableTaskQueue transposed = new SubscribableTaskQueue<>(); + generated.setData(new MatrixObject(ValueType.FP64, "/dev/null", + new MetaDataFormat(new MatrixCharacteristics(2, 3, 1), FileFormat.BINARY))); + mapped.setData(new MatrixObject(ValueType.FP64, "/dev/null", + new MetaDataFormat(new MatrixCharacteristics(2, 3, 1), FileFormat.BINARY))); + transposed.setData(new MatrixObject(ValueType.FP64, "/dev/null", + new MetaDataFormat(new MatrixCharacteristics(3, 2, 1), FileFormat.BINARY))); + + OOCInstructionUtils.dataGen(generated, + indexes -> new MatrixBlock(1, 1, (double) indexes.getRowIndex() * 10 + indexes.getColumnIndex()), + new StreamContext()); + OOCInstructionUtils.equiMapBlock(generated, mapped, input -> new MatrixBlock(1, 1, input.get(0, 0) + 1), + new StreamContext()); + OOCInstructionUtils.transpose(mapped, transposed, new StreamContext()); + + transposed.start(); + Map values = new HashMap<>(); + OOCStream.QueueCallback callback; + while((callback = transposed.dequeueCB()) != null) { + try(OOCStream.QueueCallback current = callback) { + IndexedMatrixValue value = current.get(); + values.put(value.getIndexes().getRowIndex() + "," + value.getIndexes().getColumnIndex(), + value.getValue().get(0, 0)); + } + } + Assert.assertEquals(Map.of("1,1", 12.0, "2,1", 13.0, "3,1", 14.0, "1,2", 22.0, "2,2", 23.0, "3,2", 24.0), + values); } private static final class TestPrimitive extends OOCPrimitive { private int _executions; private TestPrimitive(List children) { - super(children); + super(null, children); } @Override protected void startExecution() { _executions++; + onComplete(); } @Override - public void inferPatterns() { + protected void inferPatternsInternal() { _pattern = OOCAccessPattern.ANY; inferParentPatterns(); } @Override - public void requestPattern(OOCAccessPattern accessPattern) { + protected void requestPatternInternal(OOCAccessPattern accessPattern) { _pattern = accessPattern; } } From 43304d675a8fa24ba296aa53277a504b92460ef4 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Tue, 21 Jul 2026 18:46:33 +0200 Subject: [PATCH 089/132] [SYSTEMDS-3891] Wire First Reworked OOC Primitives --- .../controlprogram/caching/CacheableData.java | 4 +- .../controlprogram/caching/MatrixObject.java | 6 +- .../instructions/ooc/CachingStream.java | 13 +- .../ooc/CtableOOCInstruction.java | 6 + .../ooc/DataGenOOCInstruction.java | 38 +-- .../ooc/MatrixIndexingOOCInstruction.java | 5 + .../instructions/ooc/OOCInstruction.java | 297 +++++++++--------- .../instructions/ooc/PlaybackStream.java | 6 + .../instructions/ooc/ReorgOOCInstruction.java | 15 +- .../instructions/ooc/UnaryOOCInstruction.java | 13 +- .../runtime/ooc/cache/OOCCacheManager.java | 11 +- .../ooc/cache/io/OOCMatrixIOHandler.java | 82 +---- .../ooc/primitives/MappingOOCPrimitive.java | 14 +- .../ooc/primitives/TransposeOOCPrimitive.java | 13 +- .../runtime/ooc/util/OOCInstructionUtils.java | 8 +- .../test/component/ooc/OOCPrimitiveTest.java | 17 + 16 files changed, 259 insertions(+), 289 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java index e12d2caa5e3..ab27540fce7 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java @@ -635,7 +635,9 @@ && getRDDHandle() == null) ) { _requiresLocalWrite = false; } else if( hasStreamHandle() ) { - _data = readBlobFromStream( getStreamHandle() ); + OOCStream stream = getStreamHandle(); + stream.start(); + _data = readBlobFromStream(stream); } else if( getRDDHandle()==null || getRDDHandle().allowsShortCircuitRead() ) { if( DMLScript.STATISTICS ) diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java index 5430a7a6c32..28fa70f7741 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java @@ -637,8 +637,10 @@ protected long writeStreamToHDFS(String fname, String ofmt, int rep, FileFormatP MetaDataFormat iimd = (MetaDataFormat) _metaData; FileFormat fmt = (ofmt != null ? FileFormat.safeValueOf(ofmt) : iimd.getFileFormat()); MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(fmt, rep, fprop); - return writer.writeMatrixFromStream(fname, getStreamHandle(), - getNumRows(), getNumColumns(), ConfigurationManager.getBlocksize()); + OOCStream stream = getStreamHandle(); + stream.start(); + return writer.writeMatrixFromStream(fname, stream, getNumRows(), getNumColumns(), + ConfigurationManager.getBlocksize()); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java index ff4362fe285..dab3eca3a0f 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java @@ -30,6 +30,7 @@ import org.apache.sysds.runtime.ooc.cache.GroupedBlockKey; import org.apache.sysds.runtime.ooc.cache.io.OOCIOHandler; import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; +import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; import org.apache.sysds.runtime.ooc.stream.SourceOOCStream; import org.apache.sysds.runtime.ooc.util.OOCUtils; import shaded.parquet.it.unimi.dsi.fastutil.ints.IntArrayList; @@ -272,9 +273,10 @@ else if(tmp instanceof OOCCacheManager.CachedSubCallback cachedSub) { } } } - } catch (DMLRuntimeException e) { + } + catch(RuntimeException e) { // Propagate failure to subscribers - _failure = e; + _failure = DMLRuntimeException.of(e); synchronized (this) { notifyAll(); } @@ -595,6 +597,11 @@ public CachingStream getStreamCache() { return this; } + @Override + public OOCPrimitive getPrimitive() { + return _source.getPrimitive(); + } + @Override public boolean isProcessed() { return false; @@ -617,7 +624,7 @@ public void setData(CacheableData data) { @SuppressWarnings("unchecked") public void setSubscriber(Consumer> subscriber, boolean incrConsumers) { - if(_deletable) + if(_deletable && incrConsumers) throw new DMLRuntimeException("Cannot register a new subscriber on " + this + " because has been flagged for deletion"); if(_failure != null) throw _failure; diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/CtableOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/CtableOOCInstruction.java index 01fd348d101..f633cca82d4 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/CtableOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/CtableOOCInstruction.java @@ -106,6 +106,12 @@ public void processInstruction( ExecutionContext ec ) { } else cst3 = ec.getScalarInput(input3).getDoubleValue(); + qIn1.start(); + if(qIn2 != null) + qIn2.start(); + if(qIn3 != null) + qIn3.start(); + HashMap blocksIn2 = new HashMap<>(), blocksIn3 = new HashMap<>(); MatrixBlock block2, block3; diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/DataGenOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/DataGenOOCInstruction.java index 348d6f6930c..c5c2e299cbb 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/DataGenOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/DataGenOOCInstruction.java @@ -192,40 +192,32 @@ public void processInstruction(ExecutionContext ec) { long lcols = ec.getScalarInput(cols).getLongValue(); checkValidDimensions(lrows, lcols); - OOCStream qIn = createWritableStream(); - int nrb = (int)((lrows-1) / blen)+1; - int ncb = (int)((lcols-1) / blen)+1; - - for (int row = 0; row < nrb; row++) - for (int col = 0; col < ncb; col++) - qIn.enqueue(new MatrixIndexes(row+1, col+1)); - - qIn.closeInput(); - if(sparsity == 0.0 && lrows < Integer.MAX_VALUE && lcols < Integer.MAX_VALUE) { - mapOOC(qIn, qOut, idx -> { - long rlen = Math.min(blen, lrows - (idx.getRowIndex()-1) * blen); - long clen = Math.min(blen, lcols - (idx.getColumnIndex()-1) * blen); - return new IndexedMatrixValue(idx, new MatrixBlock((int)rlen, (int)clen, 0.0)); - }); + OOCInstructionUtils.dataGen(qOut, idx -> { + long rlen = Math.min(blen, lrows - (idx.getRowIndex() - 1) * blen); + long clen = Math.min(blen, lcols - (idx.getColumnIndex() - 1) * blen); + return new MatrixBlock((int) rlen, (int) clen, 0.0); + }, getContext()); return; } if(sparsity == 1.0 && minValue == maxValue) { - mapOOC(qIn, qOut, idx -> { - long rlen = Math.min(blen, lrows - (idx.getRowIndex()-1) * blen); - long clen = Math.min(blen, lcols - (idx.getColumnIndex()-1) * blen); - return new IndexedMatrixValue(idx, new MatrixBlock((int)rlen, (int)clen, minValue)); - }); + OOCInstructionUtils.dataGen(qOut, idx -> { + long rlen = Math.min(blen, lrows - (idx.getRowIndex() - 1) * blen); + long clen = Math.min(blen, lcols - (idx.getColumnIndex() - 1) * blen); + return new MatrixBlock((int) rlen, (int) clen, minValue); + }, getContext()); return; } Well1024a bigrand = LibMatrixDatagen.setupSeedsForRand(lSeed); + int nrb = (int) ((lrows - 1) / blen) + 1; + int ncb = (int) ((lcols - 1) / blen) + 1; int nb = nrb * ncb; long[] seeds = new long[nb]; for(int i = 0; i < nb; i++) seeds[i] = bigrand.nextLong(); - mapOOC(qIn, qOut, idx -> { + OOCInstructionUtils.dataGen(qOut, idx -> { long rlen = Math.min(blen, lrows - (idx.getRowIndex()-1) * blen); long clen = Math.min(blen, lcols - (idx.getColumnIndex()-1) * blen); @@ -242,8 +234,8 @@ public void processInstruction(ExecutionContext ec) { LibMatrixDatagen.genRandomNumbers(false, 0, 1, 0, 1, mout, getGenerator(rlen, clen), bSeed, null); mout.recomputeNonZeros(); - return new IndexedMatrixValue(idx, mout); - }); + return mout; + }, getContext()); } else if(method == Types.OpOpDG.SEQ) { double lfrom = ec.getScalarInput(seq_from).getDoubleValue(); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/MatrixIndexingOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/MatrixIndexingOOCInstruction.java index 90e79d2373f..33c9b680813 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/MatrixIndexingOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/MatrixIndexingOOCInstruction.java @@ -88,6 +88,7 @@ public void processInstruction(ExecutionContext ec) { Double scalarOut = null; IndexedMatrixValue tmp; + qIn.start(); while((tmp = qIn.dequeue()) != LocalTaskQueue.NO_MORE_TASKS) { if(tmp.getIndexes().getRowIndex() == firstBlockRow + 1 && tmp.getIndexes().getColumnIndex() == firstBlockCol + 1) { @@ -301,6 +302,7 @@ public void processInstruction(ExecutionContext ec) { qOut.propagateFailure(DMLRuntimeException.of(err)); return null; }); + qIn.start(); if(hasIntermediateStream) cachedStream.scheduleDeletion(); // We can immediately delete blocks after consumption @@ -356,6 +358,7 @@ else if(opcode.equalsIgnoreCase(Opcodes.LEFT_INDEX.toString())) { qOutRaw.closeInput(); return null; }); + qLhs.start(); return; } @@ -452,6 +455,8 @@ else if(opcode.equalsIgnoreCase(Opcodes.LEFT_INDEX.toString())) { qOutRaw.closeInput(); return null; }); + qLhs.start(); + qRhs.start(); } else throw new DMLRuntimeException( diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java index 805f8723486..b263b307e1d 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java @@ -153,6 +153,12 @@ public void postprocessInstruction(ExecutionContext ec) { OOCEventLog.onComputeEvent(_callerId, nanoTime, System.nanoTime()); } + protected StreamContext getContext() { + if(_streamContext == null) + _streamContext = new StreamContext(_callerId, getExtendedOpcode()); + return _streamContext; + } + protected void addInStream(OOCStream... queue) { if(_streamContext == null) _streamContext = new StreamContext(_callerId, getExtendedOpcode()); @@ -185,7 +191,11 @@ protected CompletableFuture filterOOC(OOCStream qIn, Consumer pr if (!inStreamsDefined() || !outStreamsDefined()) throw new NotImplementedException("filterOOC requires manual specification of all input and output streams for error propagation"); - return submitOOCTasks(qIn, c -> processor.accept(c.get()), p -> predicate.apply(p.get()), onNotProcessed != null ? (i, tmp) -> onNotProcessed.accept(tmp.get()) : null); + CompletableFuture future = submitOOCTasks(qIn, c -> processor.accept(c.get()), + p -> predicate.apply(p.get()), + onNotProcessed != null ? (i, tmp) -> onNotProcessed.accept(tmp.get()) : null); + qIn.start(); + return future; } protected OOCStream filteredOOCStream(OOCStream qIn, Function predicate) { @@ -222,27 +232,23 @@ protected CompletableFuture expandOOC(OOCStream qIn, OOCStream future = new CompletableFuture<>(); submitOOCTasks(qIn, tmp -> { - Collection out; - try(tmp) { - out = op.apply(tmp.get()); - } - if(!out.isEmpty()) { - deferredCtr.getAndIncrement(); - TaskContext.defer(() -> { - out.forEach(qOut::enqueue); - if(deferredCtr.decrementAndGet() == 0) - future.complete(null); - }); - } - }) - .thenRun(() -> { - if(deferredCtr.decrementAndGet() == 0) - future.complete(null); - }) - .exceptionally(err -> { - future.completeExceptionally(err); - return null; - }); + Collection out = op.apply(tmp.get()); + if(!out.isEmpty()) { + deferredCtr.getAndIncrement(); + TaskContext.defer(() -> { + out.forEach(qOut::enqueue); + if(deferredCtr.decrementAndGet() == 0) + future.complete(null); + }); + } + }).thenRun(() -> { + if(deferredCtr.decrementAndGet() == 0) + future.complete(null); + }).exceptionally(err -> { + future.completeExceptionally(err); + return null; + }); + qIn.start(); return future.thenRun(qOut::closeInput).exceptionally(err -> { DMLRuntimeException dmlErr = DMLRuntimeException.of(err); @@ -260,7 +266,7 @@ protected CompletableFuture mapOptionalOOC(OOCStream qIn, OOCStr Consumer> exec = tmp -> { Optional r; - try(tmp) { + try { r = optionalMapper.apply(tmp.get()); } catch(Exception e) { @@ -293,6 +299,7 @@ protected CompletableFuture mapOptionalOOC(OOCStream qIn, OOCStr future.completeExceptionally(err); return null; }); + qIn.start(); return future.thenRun(qOut::closeInput).exceptionally(err -> { DMLRuntimeException dmlErr = DMLRuntimeException.of(err); @@ -329,60 +336,61 @@ protected CompletableFuture broadcastJoinOOC(OOCStream rightReadStream = rightCached ? broadcast : rightCache.getReadStream(); CompletableFuture fut1 = submitOOCTasks(List.of(leftReadStream, rightReadStream), (i, tmp) -> { - try(tmp) { - P key = i == 0 ? onLeft.apply(tmp.get()) : onRight.apply(tmp.get()); + P key = i == 0 ? onLeft.apply(tmp.get()) : onRight.apply(tmp.get()); - if(i == 0) { // qIn stream - BroadcastedElement b; + if(i == 0) { // qIn stream + BroadcastedElement b; - synchronized(lock) { - b = availableBroadcastInput.get(key); + synchronized(lock) { + b = availableBroadcastInput.get(key); - if(b == null) { - availableLeftInput.compute(key, (k, v) -> { - if(v == null) - v = new ArrayList<>(); - v.add(tmp.get().getIndexes()); - return v; - }); - return; - } - } - - // Then items are present in cache - waitCtr.incrementAndGet(); - OOCCacheManager.requestManyBlocks( - List.of(leftCache.peekCachedBlockKey(tmp.get().getIndexes()), rightCache.peekCachedBlockKey(b.idx))) - .whenComplete((items, err) -> { - try { - broadcastingQueue.enqueue(new Tuple4<>(key, items.get(0).keepOpen(), items.get(1).keepOpen(), b)); - } finally { - items.forEach(OOCStream.QueueCallback::close); - } + if(b == null) { + availableLeftInput.compute(key, (k, v) -> { + if(v == null) + v = new ArrayList<>(); + v.add(tmp.get().getIndexes()); + return v; }); - } - else { // broadcast stream - BroadcastedElement b = new BroadcastedElement(tmp.get().getIndexes()); - List queued; - synchronized(lock) { - availableBroadcastInput.put(key, b); - queued = availableLeftInput.remove(key); + return; } + } - if(queued != null) { - for(MatrixIndexes idx : queued) { - waitCtr.incrementAndGet(); - - OOCCacheManager.requestManyBlocks( - List.of(leftCache.peekCachedBlockKey(idx), rightCache.peekCachedBlockKey(tmp.get().getIndexes()))) - .whenComplete((items, err) -> { - try{ - broadcastingQueue.enqueue(new Tuple4<>(key, items.get(0).keepOpen(), items.get(1).keepOpen(), b)); - } finally { - items.forEach(OOCStream.QueueCallback::close); - } - }); + // Then items are present in cache + waitCtr.incrementAndGet(); + OOCCacheManager.requestManyBlocks( + List.of(leftCache.peekCachedBlockKey(tmp.get().getIndexes()), rightCache.peekCachedBlockKey(b.idx))) + .whenComplete((items, err) -> { + try { + broadcastingQueue + .enqueue(new Tuple4<>(key, items.get(0).keepOpen(), items.get(1).keepOpen(), b)); + } + finally { + items.forEach(OOCStream.QueueCallback::close); } + }); + } + else { // broadcast stream + BroadcastedElement b = new BroadcastedElement(tmp.get().getIndexes()); + List queued; + synchronized(lock) { + availableBroadcastInput.put(key, b); + queued = availableLeftInput.remove(key); + } + + if(queued != null) { + for(MatrixIndexes idx : queued) { + waitCtr.incrementAndGet(); + + OOCCacheManager.requestManyBlocks(List.of(leftCache.peekCachedBlockKey(idx), + rightCache.peekCachedBlockKey(tmp.get().getIndexes()))).whenComplete((items, err) -> { + try { + broadcastingQueue.enqueue( + new Tuple4<>(key, items.get(0).keepOpen(), items.get(1).keepOpen(), b)); + } + finally { + items.forEach(OOCStream.QueueCallback::close); + } + }); } } } @@ -394,25 +402,24 @@ protected CompletableFuture broadcastJoinOOC(OOCStream fut2 = submitOOCTasks(List.of(broadcastingQueue), (i, tpl) -> { - try(tpl) { - final BroadcastedElement b = tpl.get()._4(); - final OOCStream.QueueCallback lValue = tpl.get()._2(); - final OOCStream.QueueCallback bValue = tpl.get()._3(); - - try(lValue; bValue) { - b.value = bValue.get(); - leftCache.incrProcessingCount(lValue.get().getIndexes(), 1); - qOut.enqueue(mapper.apply(lValue.get(), b)); - - if(b.tryRelease()) { - availableBroadcastInput.remove(tpl.get()._1()); - rightCache.incrProcessingCount(b.idx, 1); // Correct for incremented subscriber count to allow block deletion - } + final BroadcastedElement b = tpl.get()._4(); + final OOCStream.QueueCallback lValue = tpl.get()._2(); + final OOCStream.QueueCallback bValue = tpl.get()._3(); + + try(lValue; bValue) { + b.value = bValue.get(); + leftCache.incrProcessingCount(lValue.get().getIndexes(), 1); + qOut.enqueue(mapper.apply(lValue.get(), b)); + + if(b.tryRelease()) { + availableBroadcastInput.remove(tpl.get()._1()); + rightCache.incrProcessingCount(b.idx, 1); // Correct for incremented subscriber count to allow block + // deletion } - - if(waitCtr.decrementAndGet() == 0) - broadcastingQueue.closeInput(); } + + if(waitCtr.decrementAndGet() == 0) + broadcastingQueue.closeInput(); }); if(!qIn.hasStreamCache()) @@ -422,6 +429,8 @@ protected CompletableFuture broadcastJoinOOC(OOCStream fut = CompletableFuture.allOf(fut1, fut2); final StreamContext context = _streamContext.copy(); + qIn.start(); + broadcast.start(); return fut.thenRun(() -> { availableBroadcastInput.forEach((k, v) -> { rightCache.incrProcessingCount(v.idx, 1); @@ -458,50 +467,45 @@ protected CompletableFuture joinManyOOC(OOCStream leftReadStream = leftCached ? left : leftCache.getReadStream(); OOCStream rightReadStream = rightCached ? right : rightCache.getReadStream(); - CompletableFuture fut1 = submitOOCTasks(List.of(leftReadStream, rightReadStream), - (i, tmp) -> { - try(tmp) { - boolean leftItem = i == 0; - P key = (leftItem ? leftOn : rightOn).apply(tmp.get()); - Tuple2, List> tuple = joinMap.computeIfAbsent(key, - k -> new Tuple2<>(new ArrayList<>(releaseRightCount), new ArrayList<>(releaseLeftCount))); - BroadcastedElement b = new BroadcastedElement(tmp.get().getIndexes()); - List matches = leftItem ? tuple._2 : tuple._1; - List toInsert = leftItem ? tuple._1 : tuple._2; - int matchesSize; - boolean remove; - synchronized(tuple) { - toInsert.add(b); - matchesSize = matches.size(); - waitCtr.addAndGet(matchesSize); - remove = tuple._1.size() == releaseRightCount && tuple._2.size() == releaseLeftCount; - } - - // We have the guarantee that matches is append only so we don't need to synchronize for this - for(int mIdx = 0; mIdx < matchesSize; mIdx++) { - BroadcastedElement e = matches.get(mIdx); - OOCCacheManager.requestManyBlocks( - List.of(leftCache.peekCachedBlockKey(leftItem ? b.idx : e.idx), - rightCache.peekCachedBlockKey(leftItem ? e.idx : b.idx))).thenApply(joined -> { - try { - joinQueue.enqueue( - new Tuple5<>(key, joined.get(0).keepOpen(), joined.get(1).keepOpen(), - leftItem ? b : e, leftItem ? e : b)); - } - finally { - joined.forEach(OOCStream.QueueCallback::close); - } - return null; - }).exceptionally(t -> { - joinQueue.propagateFailure(DMLRuntimeException.of(t)); - return null; - }); - } + CompletableFuture fut1 = submitOOCTasks(List.of(leftReadStream, rightReadStream), (i, tmp) -> { + boolean leftItem = i == 0; + P key = (leftItem ? leftOn : rightOn).apply(tmp.get()); + Tuple2, List> tuple = joinMap.computeIfAbsent(key, + k -> new Tuple2<>(new ArrayList<>(releaseRightCount), new ArrayList<>(releaseLeftCount))); + BroadcastedElement b = new BroadcastedElement(tmp.get().getIndexes()); + List matches = leftItem ? tuple._2 : tuple._1; + List toInsert = leftItem ? tuple._1 : tuple._2; + int matchesSize; + boolean remove; + synchronized(tuple) { + toInsert.add(b); + matchesSize = matches.size(); + waitCtr.addAndGet(matchesSize); + remove = tuple._1.size() == releaseRightCount && tuple._2.size() == releaseLeftCount; + } - if(remove) - joinMap.remove(key); + // We have the guarantee that matches is append only so we don't need to synchronize for this + for(int mIdx = 0; mIdx < matchesSize; mIdx++) { + BroadcastedElement e = matches.get(mIdx); + OOCCacheManager.requestManyBlocks(List.of(leftCache.peekCachedBlockKey(leftItem ? b.idx : e.idx), + rightCache.peekCachedBlockKey(leftItem ? e.idx : b.idx))).thenApply(joined -> { + try { + joinQueue.enqueue(new Tuple5<>(key, joined.get(0).keepOpen(), joined.get(1).keepOpen(), + leftItem ? b : e, leftItem ? e : b)); + } + finally { + joined.forEach(OOCStream.QueueCallback::close); + } + return null; + }).exceptionally(t -> { + joinQueue.propagateFailure(DMLRuntimeException.of(t)); + return null; + }); } - }); + + if(remove) + joinMap.remove(key); + }); fut1 = fut1.thenApply(v -> { if(waitCtr.decrementAndGet() == 0) joinQueue.closeInput(); @@ -537,6 +541,8 @@ protected CompletableFuture joinManyOOC(OOCStream groupedReduceOOC(OOCStream CompletableFuture outFuture = new CompletableFuture<>(); CompletableFuture pipeFuture = pipeOOC(qIn, cb -> { - try(cb) { - Aggregator agg = aggregators.compute(cb.get().getIndexes(), (k, v) -> { - if(v == null) { - v = new Aggregator(reduce, emitCount); - busyCtr.incrementAndGet(); - v.getFuture().thenApply(imv -> { - qOut.enqueue(imv); - if(busyCtr.decrementAndGet() == 0) - outFuture.complete(null); - return null; - }) - .exceptionally(outFuture::completeExceptionally); - } - return v; - }); - agg.insert(cb.get()); - } + Aggregator agg = aggregators.compute(cb.get().getIndexes(), (k, v) -> { + if(v == null) { + v = new Aggregator(reduce, emitCount); + busyCtr.incrementAndGet(); + v.getFuture().thenApply(imv -> { + qOut.enqueue(imv); + if(busyCtr.decrementAndGet() == 0) + outFuture.complete(null); + return null; + }).exceptionally(outFuture::completeExceptionally); + } + return v; + }); + agg.insert(cb.get()); }); pipeFuture.thenRun(() -> { @@ -742,6 +746,7 @@ protected CompletableFuture groupedReduceOOC(OOCStream outFuture.complete(null); }); + qIn.start(); return outFuture.thenRun(qOut::closeInput); } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/PlaybackStream.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/PlaybackStream.java index 7526b09f592..167d7ba7aed 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/PlaybackStream.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/PlaybackStream.java @@ -23,6 +23,7 @@ import org.apache.sysds.runtime.controlprogram.caching.CacheableData; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.meta.DataCharacteristics; +import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; @@ -140,4 +141,9 @@ public boolean hasStreamCache() { public CachingStream getStreamCache() { return _streamCache; } + + @Override + public OOCPrimitive getPrimitive() { + return _streamCache.getPrimitive(); + } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java index 94d896a3546..40a677e5d71 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java @@ -30,9 +30,9 @@ import org.apache.sysds.runtime.instructions.cp.CPOperand; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.matrix.data.MatrixBlock; -import org.apache.sysds.runtime.matrix.data.MatrixIndexes; import org.apache.sysds.runtime.matrix.operators.Operator; import org.apache.sysds.runtime.matrix.operators.ReorgOperator; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; import org.apache.sysds.runtime.util.DataConverter; public class ReorgOOCInstruction extends ComputationOOCInstruction { @@ -105,19 +105,12 @@ public void processInstruction( ExecutionContext ec ) { ec.releaseMatrixInput(input1.getName()); ec.setMatrixOutput(output.getName(), soresBlock); } else if(r_op.fn instanceof SwapIndex) { - OOCStream qIn = min.getStreamHandle(); + OOCStreamable qIn = min.getStreamable(); OOCStream qOut = createWritableStream(); ec.getMatrixObject(output).setStreamHandle(qOut); - // Transpose operation - mapOOC(qIn, qOut, tmp -> { - MatrixBlock inBlock = (MatrixBlock) tmp.getValue(); - long oldRowIdx = tmp.getIndexes().getRowIndex(); - long oldColIdx = tmp.getIndexes().getColumnIndex(); - - MatrixBlock outBlock = inBlock.reorgOperations((ReorgOperator) _optr, new MatrixBlock(), -1, -1, -1); - return new IndexedMatrixValue(new MatrixIndexes(oldColIdx, oldRowIdx), outBlock); - }); + OOCInstructionUtils.transposedMap(qIn, qOut, + block -> block.reorgOperations((ReorgOperator) _optr, new MatrixBlock(), -1, -1, -1), getContext()); } } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/UnaryOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/UnaryOOCInstruction.java index df2f50bf573..8f7946ef701 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/UnaryOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/UnaryOOCInstruction.java @@ -35,6 +35,7 @@ import org.apache.sysds.runtime.matrix.data.MatrixIndexes; import org.apache.sysds.runtime.matrix.operators.UnaryOperator; import org.apache.sysds.runtime.meta.DataCharacteristics; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; public class UnaryOOCInstruction extends ComputationOOCInstruction { private UnaryOperator _uop = null; @@ -69,20 +70,17 @@ public void processInstruction( ExecutionContext ec ) { boolean cumSumProd = Builtin.isBuiltinCode(uop.fn, BuiltinCode.CUMSUMPROD); ec.getDataCharacteristics(output.getName()).set(min.getNumRows(), cumSumProd ? 1 : min.getNumColumns(), min.getBlocksize(), -1); - OOCStream qIn = min.getStreamHandle(); OOCStream qOut; boolean cumulative = isCumulativeUnary(uop); if(cumulative) { - qOut = processCumulativeUnaryInstruction(ec, uop, qIn); + qOut = processCumulativeUnaryInstruction(ec, uop, min.getStreamHandle()); } else { qOut = createWritableStream(); - mapOOC(qIn, qOut, tmp -> { - IndexedMatrixValue tmpOut = new IndexedMatrixValue(); - tmpOut.set(tmp.getIndexes(), tmp.getValue().unaryOperations(uop, new MatrixBlock())); - return tmpOut; - }); + OOCStreamable input = min.getStreamable(); + OOCInstructionUtils.equiMapBlock(input, qOut, block -> block.unaryOperations(uop, new MatrixBlock()), + getContext()); } ec.getMatrixObject(output).setStreamHandle(qOut); @@ -140,6 +138,7 @@ private OOCStream processCumulativeUnaryInstruction(Executio }); } + qIn.start(); return mergeOOCStreams(splitOutputs); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java index 78d12348fe5..07e1206a293 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java @@ -286,8 +286,12 @@ public static class CachedQueueCallback implements OOCStream.QueueCallback @SuppressWarnings("unchecked") CachedQueueCallback(BlockEntry result, DMLRuntimeException failure) { + this(result, (T) result.getData(), failure); + } + + private CachedQueueCallback(BlockEntry result, T data, DMLRuntimeException failure) { this._result = result; - this._data = (T)result.getData(); + this._data = data; this._failure = failure; this._pinned = new AtomicBoolean(true); } @@ -305,8 +309,11 @@ public T get() { public OOCStream.QueueCallback keepOpen() { if(!_pinned.get()) throw new IllegalStateException("Cannot keep open an already closed callback"); + T data = _data; + if(data == null) + throw new IllegalStateException("Cannot keep open an empty callback"); pin(_result); - return new CachedQueueCallback<>(_result, _failure); + return new CachedQueueCallback<>(_result, data, _failure); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java index bfe1565ab3a..e9487f7bd45 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java @@ -70,9 +70,6 @@ public class OOCMatrixIOHandler implements OOCIOHandler { private static final int READER_SIZE = 16; private static final long OVERFLOW = 8192 * 1024; private static final long MAX_PARTITION_SIZE = 8192 * 8192; - private static final long GROUP_TARGET_BYTES = 8L * 1024 * 1024; - private static final long GROUP_MAX_BYTES = 16L * 1024 * 1024; - private static final int GROUP_MAX_COUNT = 64; private static final long IDLE_FLUSH_MS = 1; private final String _spillDir; @@ -371,12 +368,6 @@ private void readSequenceFile(JobConf job, Path path, SourceReadRequest request, AtomicLong bytesRead, long byteLimit, Object budgetLock, ConcurrentLinkedDeque descriptors) throws IOException { MatrixIndexes key = new MatrixIndexes(); - List groupValues = new ArrayList<>(); - List groupDescs = new ArrayList<>(); - long groupBytes = 0; - long groupSerialized = 0; - long groupStart = -1; - long groupEnd = -1; try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { long pos = filePositions.get(fileIdx); @@ -409,50 +400,11 @@ else if (bytesRead.get() + blockSize > byteLimit) { SourceBlockDescriptor descriptor = new SourceBlockDescriptor(path.toString(), request.format, outIdx, recordStart, (int)(recordEnd - recordStart), blockSize); - boolean small = blockSize <= GROUP_TARGET_BYTES; - boolean contiguous = groupValues.isEmpty() || recordStart == groupEnd; - boolean canAdd = small - && contiguous - && groupValues.size() < GROUP_MAX_COUNT - && (groupBytes + (recordEnd - recordStart)) <= GROUP_MAX_BYTES; - - if (!canAdd && !groupValues.isEmpty()) { - flushSourceGroup(request, groupValues, groupDescs, groupStart, groupEnd, groupSerialized, - descriptors); - groupValues.clear(); - groupDescs.clear(); - groupBytes = 0; - groupSerialized = 0; - groupStart = -1; - groupEnd = -1; - } - - if (small) { - if (groupValues.isEmpty()) - groupStart = recordStart; - groupEnd = recordEnd; - groupValues.add(imv); - groupDescs.add(descriptor); - groupBytes += (recordEnd - recordStart); - groupSerialized += blockSize; - if (groupSerialized >= GROUP_TARGET_BYTES || groupBytes >= GROUP_MAX_BYTES || groupValues.size() >= GROUP_MAX_COUNT) { - flushSourceGroup(request, groupValues, groupDescs, groupStart, groupEnd, groupSerialized, - descriptors); - groupValues.clear(); - groupDescs.clear(); - groupBytes = 0; - groupSerialized = 0; - groupStart = -1; - groupEnd = -1; - } - } - else { - if (request.target instanceof SourceOOCStream src) - src.enqueue(imv, descriptor); - else - request.target.enqueue(imv); - descriptors.add(descriptor); - } + if(request.target instanceof SourceOOCStream src) + src.enqueue(imv, descriptor); + else + request.target.enqueue(imv); + descriptors.add(descriptor); filePositions.set(fileIdx, reader.getPosition()); if (DMLScript.OOC_LOG_EVENTS) { @@ -465,34 +417,11 @@ else if (bytesRead.get() + blockSize > byteLimit) { break; // Note that we knowingly go over limit, which could result in READER_SIZE*8MB overshoot } - if (!groupValues.isEmpty()) { - flushSourceGroup(request, groupValues, groupDescs, groupStart, groupEnd, groupSerialized, - descriptors); - } - if (!stop.get()) completed.set(fileIdx, 1); } } - private void flushSourceGroup(SourceReadRequest request, List values, - List descs, long start, long end, long totalSerialized, - ConcurrentLinkedDeque descriptors) { - if (values.isEmpty()) - return; - SourceBlockDescriptor first = descs.get(0); - OOCIOHandler.GroupSourceBlockDescriptor group = - new OOCIOHandler.GroupSourceBlockDescriptor(first.path, first.format, first.indexes, start, - (int)(end - start), totalSerialized, new ArrayList<>(descs)); - if (request.target instanceof SourceOOCStream src) - src.enqueueGroup(new ArrayList<>(values), group); - else { - for (IndexedMatrixValue v : values) - request.target.enqueue(v); - } - descriptors.addAll(group.blocks); - } - private void closeTarget(org.apache.sysds.runtime.instructions.ooc.OOCStream target, boolean close) { if(close) { try { @@ -504,7 +433,6 @@ private void closeTarget(org.apache.sysds.runtime.instructions.ooc.OOCStream _input; + private final OOCStream _input; private final OOCStreamable _output; private final Function _operation; public MappingOOCPrimitive(OOCStreamable input, OOCStreamable output, Function operation, StreamContext context) { - this(input.getPrimitive(), input, output, operation, context); + this(input.getReadStream(), output, operation, context); } - private MappingOOCPrimitive(OOCPrimitive inputPrimitive, OOCStreamable input, - OOCStreamable output, Function operation, - StreamContext context) { - super(context, inputPrimitive == null ? List.of() : List.of(inputPrimitive)); + private MappingOOCPrimitive(OOCStream input, OOCStreamable output, + Function operation, StreamContext context) { + super(context, input.getPrimitive() == null ? List.of() : List.of(input.getPrimitive())); _input = input; _output = output; _operation = operation; @@ -66,10 +65,9 @@ protected void requestPatternInternal(OOCAccessPattern accessPattern) { @Override protected void startExecution() { - OOCStream input = _input.getReadStream(); OOCStream output = _output.getWriteStream(); OOCInstructionUtils - .submitAdmittedOOCTasks(input, output, + .submitAdmittedOOCTasks(_input, output, value -> new IndexedMatrixValue(value.getIndexes(), _operation.apply(value)), _allowance, getContext()) .thenRun(this::onComplete); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/TransposeOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/TransposeOOCPrimitive.java index 938c753a197..1cbe504546f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/TransposeOOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/TransposeOOCPrimitive.java @@ -32,18 +32,18 @@ import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; public class TransposeOOCPrimitive extends OOCPrimitive { - private final OOCStreamable _input; + private final OOCStream _input; private final OOCStreamable _output; private final Function _operation; public TransposeOOCPrimitive(OOCStreamable input, OOCStreamable output, Function operation, StreamContext context) { - this(input.getPrimitive(), input, output, operation, context); + this(input.getReadStream(), output, operation, context); } - private TransposeOOCPrimitive(OOCPrimitive inputPrimitive, OOCStreamable input, - OOCStreamable output, Function operation, StreamContext context) { - super(context, inputPrimitive == null ? List.of() : List.of(inputPrimitive)); + private TransposeOOCPrimitive(OOCStream input, OOCStreamable output, + Function operation, StreamContext context) { + super(context, input.getPrimitive() == null ? List.of() : List.of(input.getPrimitive())); _input = input; _output = output; _operation = operation; @@ -65,9 +65,8 @@ protected void requestPatternInternal(OOCAccessPattern accessPattern) { @Override protected void startExecution() { - OOCStream input = _input.getReadStream(); OOCStream output = _output.getWriteStream(); - OOCInstructionUtils.submitAdmittedOOCTasks(input, output, value -> { + OOCInstructionUtils.submitAdmittedOOCTasks(_input, output, value -> { MatrixIndexes indexes = value.getIndexes(); return new IndexedMatrixValue(new MatrixIndexes(indexes.getColumnIndex(), indexes.getRowIndex()), _operation.apply((MatrixBlock) value.getValue())); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java index 98bab492d53..781cdd04ea2 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java @@ -71,12 +71,12 @@ public static void equiMap(OOCStreamable input, OOCStream input, OOCStream output, + public static void transposedMap(OOCStreamable input, OOCStream output, Function operation, StreamContext context) { output.assignPrimitive(new TransposeOOCPrimitive(input, output, operation, context)); } - public static void transpose(OOCStream input, OOCStream output, + public static void transpose(OOCStreamable input, OOCStream output, StreamContext context) { transposedMap(input, output, MatrixBlock::transpose, context); } @@ -191,8 +191,12 @@ private static void subscribe(OOCStream queue, int streamIndex, future.complete(null); return; } + if(callback.isFailure() && !(callback instanceof OOCStream.GroupQueueCallback)) + callback.get(); Consumer> process = item -> { + if(item.isFailure()) + item.get(); if(predicate != null && !predicate.apply(streamIndex, item)) { if(onNotProcessed != null) onNotProcessed.accept(streamIndex, item); diff --git a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java index 40abcf26348..0ce1e1ad38c 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java @@ -30,8 +30,10 @@ import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.data.MatrixIndexes; import org.apache.sysds.runtime.meta.MatrixCharacteristics; import org.apache.sysds.runtime.meta.MetaDataFormat; +import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; import org.apache.sysds.runtime.ooc.stream.FilteredOOCStream; @@ -41,6 +43,21 @@ import org.junit.Test; public class OOCPrimitiveTest { + @Test + public void testRetainForgottenCacheCallback() { + OOCCacheManager.reset(); + try(OOCStream.QueueCallback callback = OOCCacheManager.putAndPin(1, 1, + new IndexedMatrixValue(new MatrixIndexes(1, 1), new MatrixBlock(1, 1, 7d)))) { + OOCCacheManager.forget(1, 1); + try(OOCStream.QueueCallback retained = callback.keepOpen()) { + Assert.assertEquals(7, retained.get().getValue().get(0, 0), 0); + } + } + finally { + OOCCacheManager.reset(); + } + } + @Test public void testGraphPatternsAndExecution() { TestPrimitive source = new TestPrimitive(List.of()); From ce49a64a84af37d17f242ab617ca44e0bd5c02f8 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Wed, 22 Jul 2026 09:29:04 +0200 Subject: [PATCH 090/132] [SYSTEMDS-3891] Wire JoinOOCPrimitive (#2566) --- .../ooc/BinaryOOCInstruction.java | 17 +- .../runtime/ooc/cache/OOCCacheManager.java | 18 ++ .../ooc/primitives/JoinOOCPrimitive.java | 220 ++++++++++++++++++ .../runtime/ooc/util/OOCInstructionUtils.java | 7 + .../sysds/runtime/ooc/util/OOCUtils.java | 5 + .../test/component/ooc/OOCPrimitiveTest.java | 37 +++ 6 files changed, 296 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/primitives/JoinOOCPrimitive.java diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/BinaryOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/BinaryOOCInstruction.java index 8a5f7cf49a7..c252cd3d1ec 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/BinaryOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/BinaryOOCInstruction.java @@ -30,6 +30,7 @@ import org.apache.sysds.runtime.matrix.operators.BinaryOperator; import org.apache.sysds.runtime.matrix.operators.Operator; import org.apache.sysds.runtime.matrix.operators.ScalarOperator; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; public class BinaryOOCInstruction extends ComputationOOCInstruction { @@ -63,8 +64,6 @@ protected void processMatrixMatrixInstruction(ExecutionContext ec) { MatrixObject m1 = ec.getMatrixObject(input1); MatrixObject m2 = ec.getMatrixObject(input2); - OOCStream qIn1 = m1.getStreamHandle(); - OOCStream qIn2 = m2.getStreamHandle(); OOCStream qOut = new SubscribableTaskQueue<>(); ec.getMatrixObject(output).setStreamHandle(qOut); @@ -74,6 +73,8 @@ protected void processMatrixMatrixInstruction(ExecutionContext ec) { // If dimensions are unknown, we cannot safely detect broadcasting. // Fall back to strict key-based join and let downstream operators validate as needed. if(!known1 || !known2) { + OOCStream qIn1 = m1.getStreamHandle(); + OOCStream qIn2 = m2.getStreamHandle(); if(LOG.isWarnEnabled()) { LOG.warn("Falling back to key-wise OOC binary join for opcode '" + getOpcode() + "' due to unknown matrix dimensions: " + input1.getName() + "=" + m1.getNumRows() + "x" @@ -93,6 +94,8 @@ protected void processMatrixMatrixInstruction(ExecutionContext ec) { boolean isRowBroadcast = m1.getNumRows() > 1 && m2.getNumRows() == 1; if (isColBroadcast && !isRowBroadcast) { + OOCStream qIn1 = m1.getStreamHandle(); + OOCStream qIn2 = m2.getStreamHandle(); final long maxProcessesPerBroadcast = (m1.getNumColumns() + m1.getBlocksize() - 1) / m1.getBlocksize(); broadcastJoinOOC(qIn1, qIn2, qOut, (tmp1, b) -> { @@ -107,6 +110,8 @@ protected void processMatrixMatrixInstruction(ExecutionContext ec) { }, tmp -> tmp.getIndexes().getRowIndex()); } else if (isRowBroadcast && !isColBroadcast) { + OOCStream qIn1 = m1.getStreamHandle(); + OOCStream qIn2 = m2.getStreamHandle(); final long maxProcessesPerBroadcast = (m1.getNumRows() + m1.getBlocksize() - 1) / m1.getBlocksize(); broadcastJoinOOC(qIn1, qIn2, qOut, (tmp1, b) -> { @@ -126,12 +131,8 @@ else if (isRowBroadcast && !isColBroadcast) { + m1.getNumRows() + "x" + m1.getNumColumns() + " <=> " + m2.getNumRows() + "x" + m2.getNumColumns()); - joinOOC(qIn1, qIn2, qOut, (tmp1, tmp2) -> { - IndexedMatrixValue tmpOut = new IndexedMatrixValue(); - tmpOut.set(tmp1.getIndexes(), - tmp1.getValue().binaryOperations((BinaryOperator)_optr, tmp2.getValue(), tmpOut.getValue())); - return tmpOut; - }, IndexedMatrixValue::getIndexes); + OOCInstructionUtils.equiJoin(m1.getStreamable(), m2.getStreamable(), qOut, + (left, right) -> left.binaryOperations((BinaryOperator) _optr, right, new MatrixBlock()), getContext()); } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java index 07e1206a293..5cec3ae981d 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java @@ -30,6 +30,7 @@ import org.apache.sysds.runtime.ooc.cache.io.OOCMatrixIOHandler; import org.apache.sysds.runtime.ooc.cache.legacy.OOCCacheScheduler; import org.apache.sysds.runtime.ooc.cache.legacy.OOCLRUCacheScheduler; +import org.apache.sysds.runtime.ooc.cache.packed.OOCPackedCache; import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; import org.apache.sysds.runtime.ooc.stats.OOCEventLog; import org.apache.sysds.utils.Statistics; @@ -52,22 +53,27 @@ public class OOCCacheManager { private static final AtomicReference _ioHandler; private static final AtomicReference _scheduler; + private static final AtomicReference _globalCache; static { _evictionLimit = (long)(Runtime.getRuntime().maxMemory() * OOC_BUFFER_PERCENTAGE); _hardLimit = (long)(Runtime.getRuntime().maxMemory() * OOC_BUFFER_PERCENTAGE_HARD); _ioHandler = new AtomicReference<>(); _scheduler = new AtomicReference<>(); + _globalCache = new AtomicReference<>(); } public static void reset() { TeeOOCInstruction.reset(); OOCIOHandler ioHandler = _ioHandler.getAndSet(null); OOCCacheScheduler cacheScheduler = _scheduler.getAndSet(null); + OOCPackedCache globalCache = _globalCache.getAndSet(null); if (ioHandler != null) ioHandler.shutdown(); if (cacheScheduler != null) cacheScheduler.shutdown(); + if(globalCache != null) + globalCache.shutdown(); if (DMLScript.OOC_STATISTICS) Statistics.resetOOCEvictionStats(); @@ -118,6 +124,18 @@ public static OOCCacheScheduler getCacheIfInitialized() { return _scheduler.get(); } + public static OOCPackedCache getGlobalCache() { + while(true) { + OOCPackedCache cache = _globalCache.get(); + if(cache != null) + return cache; + cache = new OOCPackedCache(new OOCMatrixIOHandler(), _hardLimit, _evictionLimit); + if(_globalCache.compareAndSet(null, cache)) + return cache; + cache.shutdown(); + } + } + public static OOCIOHandler getIOHandler() { OOCIOHandler io = _ioHandler.get(); if(io != null) diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/JoinOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/JoinOOCPrimitive.java new file mode 100644 index 00000000000..99996eb7dac --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/JoinOOCPrimitive.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.primitives; + +import java.util.Objects; +import java.util.concurrent.ExecutionException; +import java.util.function.BiFunction; +import java.util.stream.Stream; + +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.instructions.ooc.CachingStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; +import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; +import org.apache.sysds.runtime.ooc.memory.ReservationBudget; +import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; +import org.apache.sysds.runtime.ooc.store.StateTable; +import org.apache.sysds.runtime.ooc.stream.StreamContext; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; +import org.apache.sysds.runtime.ooc.util.OOCUtils; +import org.apache.sysds.runtime.ooc.util.StateTableUtils; + +public class JoinOOCPrimitive extends OOCPrimitive { + private final OOCStream _left; + private final OOCStream _right; + private final OOCStreamable _output; + private final BiFunction _operation; + private StateTable _table; + + public JoinOOCPrimitive(OOCStreamable left, OOCStreamable right, + OOCStreamable output, BiFunction operation, + StreamContext context) { + this(left.getReadStream(), right.getReadStream(), output, operation, context); + } + + private JoinOOCPrimitive(OOCStream left, OOCStream right, + OOCStreamable output, BiFunction operation, + StreamContext context) { + super(context, Stream.of(left.getPrimitive(), right.getPrimitive()).filter(Objects::nonNull).toList()); + _left = left; + _right = right; + _output = output; + _operation = operation; + } + + @Override + protected void inferPatternsInternal() { + _pattern = OOCAccessPattern.ANY; + for(OOCPrimitive child : getChildren()) + _pattern = _pattern.fused(child.getAccessPattern()); + if(_pattern.isPlannable() && _pattern != OOCAccessPattern.ANY) + for(OOCPrimitive child : getChildren()) + child.requestPattern(_pattern); + inferParentPatterns(); + } + + @Override + protected void requestPatternInternal(OOCAccessPattern accessPattern) { + _pattern = accessPattern; + for(OOCPrimitive child : getChildren()) + child.requestPattern(accessPattern); + } + + @Override + protected void startExecution() { + _table = new StateTable<>(OOCCacheManager.getGlobalCache(), CachingStream._streamSeq.getNextID()); + OOCStream output = _output.getWriteStream(); + OOCStream matches = new SubscribableTaskQueue<>(); + long inputBytes = Math.max(OOCUtils.estimateOutputTileBytes(_left.getDataCharacteristics()), + OOCUtils.estimateOutputTileBytes(_right.getDataCharacteristics())); + long outputBytes = OOCUtils.estimateOutputTileBytes(_output.getDataCharacteristics()); + long taskBytes = outputBytes + 2 * inputBytes; + + getContext().addOutStream(output); + OOCInstructionUtils.submitOOCTasks(matches, callback -> { + try(JoinWork work = callback.get()) { + IndexedMatrixValue left = work._left.get(); + IndexedMatrixValue right = work._right.get(); + OOCUtils.enqueueExact(output, new IndexedMatrixValue(left.getIndexes(), + _operation.apply((MatrixBlock) left.getValue(), (MatrixBlock) right.getValue())), work._budget); + } + }, callback -> true, (index, callback) -> callback.get().close(), getContext()).thenRun(() -> { + try { + _table.close(); + onComplete(); + } + finally { + output.closeInput(); + } + }); + + OOCInstructionUtils.submitOOCTask(() -> drive(matches, taskBytes), new StreamContext().addOutStream(output)); + } + + private void drive(OOCStream matches, long taskBytes) { + long cols = _right.getDataCharacteristics().getNumColBlocks(); + int unmatched = 0; + try { + while(true) { + OOCStream.QueueCallback left = _left.dequeueCB(); + OOCStream.QueueCallback right = _right.dequeueCB(); + boolean leftEos = left == null || left.isEos(); + boolean rightEos = right == null || right.isEos(); + if(leftEos || rightEos) { + if(left != null) + left.close(); + if(right != null) + right.close(); + if(leftEos != rightEos) + throw new DMLRuntimeException("Join inputs contain a different number of blocks"); + break; + } + unmatched += accept(left, true, cols, taskBytes, matches); + unmatched += accept(right, false, cols, taskBytes, matches); + } + if(unmatched != 0) + throw new DMLRuntimeException("Join inputs contain " + unmatched + " unmatched blocks"); + } + finally { + matches.closeInput(); + } + } + + private int accept(OOCStream.QueueCallback callback, boolean left, long cols, long taskBytes, + OOCStream matches) { + if(callback == null) + return 0; + OOCStream.QueueCallback owned = null; + ReservationBudget budget = null; + try { + owned = callback.keepOpen(); + callback.close(); + callback = null; + budget = OOCUtils.reserveBudget(_allowance, taskBytes); + IndexedMatrixValue value = owned.get(); + long row = value.getIndexes().getRowIndex() - 1; + long col = value.getIndexes().getColumnIndex() - 1; + int slot = Math.toIntExact(row * cols + col); + OOCFuture future = StateTableUtils.putOrTake(_table, slot, owned, budget); + owned = null; + StateTableUtils.Match match = await(future); + if(match == null) + return 1; + JoinWork work = left ? new JoinWork(match.left(), match.right(), budget) : new JoinWork(match.right(), + match.left(), budget); + budget = null; + try { + matches.enqueue(work); + work = null; + } + finally { + if(work != null) + work.close(); + } + return -1; + } + finally { + if(callback != null) + callback.close(); + if(owned != null) + owned.close(); + if(budget != null) + budget.close(); + } + } + + private static StateTableUtils.Match await(OOCFuture future) { + try { + return future.get(); + } + catch(InterruptedException error) { + Thread.currentThread().interrupt(); + throw new DMLRuntimeException(error); + } + catch(ExecutionException error) { + throw DMLRuntimeException.of(error.getCause()); + } + } + + private static final class JoinWork implements AutoCloseable { + private final OOCStream.QueueCallback _left; + private final OOCStream.QueueCallback _right; + private final ReservationBudget _budget; + + private JoinWork(OOCStream.QueueCallback left, + OOCStream.QueueCallback right, ReservationBudget budget) { + _left = left; + _right = right; + _budget = budget; + } + + @Override + public void close() { + try(_left; _right; _budget) { + // Release + } + } + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java index 781cdd04ea2..773ef9da834 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java @@ -40,6 +40,7 @@ import org.apache.sysds.runtime.ooc.cache.OOCFuture; import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; import org.apache.sysds.runtime.ooc.memory.ReservationBudget; +import org.apache.sysds.runtime.ooc.primitives.JoinOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.MappingOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.PlannableDataGenOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.TransposeOOCPrimitive; @@ -81,6 +82,12 @@ public static void transpose(OOCStreamable input, OOCStream< transposedMap(input, output, MatrixBlock::transpose, context); } + public static void equiJoin(OOCStreamable left, OOCStreamable right, + OOCStream output, BiFunction operation, + StreamContext context) { + output.assignPrimitive(new JoinOOCPrimitive(left, right, output, operation, context)); + } + public static int getComputeInFlight() { return COMPUTE_IN_FLIGHT.get(); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java index 7b8a8ae46b1..5a98a05b78d 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java @@ -170,4 +170,9 @@ public static void enqueueExact(OOCStream out, IndexedMatrix callback.close(); } } + + public static ReservationBudget reserveBudget(MemoryAllowance allowance, long bytes) { + allowance.reserveBlocking(bytes); + return new ReservationBudget(allowance, bytes); + } } diff --git a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java index 0ce1e1ad38c..be6886ce1a6 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java @@ -26,6 +26,7 @@ import org.apache.sysds.common.Types.FileFormat; import org.apache.sysds.common.Types.ValueType; import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; +import org.apache.sysds.runtime.instructions.ooc.CachingStream; import org.apache.sysds.runtime.instructions.ooc.OOCStream; import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; @@ -119,6 +120,42 @@ public void testDataGenMapTransposePipeline() { values); } + @Test + public void testJoinOutOfOrder() { + SubscribableTaskQueue left = new SubscribableTaskQueue<>(); + SubscribableTaskQueue right = new SubscribableTaskQueue<>(); + SubscribableTaskQueue joined = new SubscribableTaskQueue<>(); + SubscribableTaskQueue addends = new SubscribableTaskQueue<>(); + SubscribableTaskQueue output = new SubscribableTaskQueue<>(); + for(SubscribableTaskQueue stream : List.of(left, right, joined, addends, output)) + stream.setData(new MatrixObject(ValueType.FP64, "/dev/null", + new MetaDataFormat(new MatrixCharacteristics(1, 2, 1), FileFormat.BINARY))); + CachingStream cachedLeft = new CachingStream(left); + left.enqueue(new IndexedMatrixValue(new MatrixIndexes(1, 1), new MatrixBlock(1, 1, 10d))); + left.enqueue(new IndexedMatrixValue(new MatrixIndexes(1, 2), new MatrixBlock(1, 1, 20d))); + right.enqueue(new IndexedMatrixValue(new MatrixIndexes(1, 2), new MatrixBlock(1, 1, 2d))); + right.enqueue(new IndexedMatrixValue(new MatrixIndexes(1, 1), new MatrixBlock(1, 1, 1d))); + addends.enqueue(new IndexedMatrixValue(new MatrixIndexes(1, 1), new MatrixBlock(1, 1, 100d))); + addends.enqueue(new IndexedMatrixValue(new MatrixIndexes(1, 2), new MatrixBlock(1, 1, 200d))); + left.closeInput(); + right.closeInput(); + addends.closeInput(); + OOCInstructionUtils.equiJoin(cachedLeft, right, joined, + (l, r) -> new MatrixBlock(1, 1, l.get(0, 0) + r.get(0, 0)), new StreamContext()); + OOCInstructionUtils.equiJoin(joined, addends, output, + (l, r) -> new MatrixBlock(1, 1, l.get(0, 0) + r.get(0, 0)), new StreamContext()); + + output.start(); + Map values = new HashMap<>(); + OOCStream.QueueCallback callback; + while((callback = output.dequeueCB()) != null) + try(OOCStream.QueueCallback current = callback) { + values.put(current.get().getIndexes().getColumnIndex(), current.get().getValue().get(0, 0)); + } + Assert.assertEquals(Map.of(1L, 111.0, 2L, 222.0), values); + cachedLeft.scheduleDeletion(); + } + private static final class TestPrimitive extends OOCPrimitive { private int _executions; From 0eb76ed9a4dc78afd91917d7c63cc03a665bda5d Mon Sep 17 00:00:00 2001 From: ywcb00 Date: Thu, 23 Jul 2026 12:00:12 +0200 Subject: [PATCH 091/132] [SYSTEMDS-3953] Add test case for array percentiles as argument for the quantile function NOTE: the tests address the corresponding jira issue and are ignored for now --- .../runtime/matrix/data/MatrixBlock.java | 8 ++--- .../functions/binary/matrix/QuantileTest.java | 26 ++++++++++++--- .../functions/binary/matrix/QuartileArray.R | 33 +++++++++++++++++++ .../functions/binary/matrix/QuartileArray.dml | 24 ++++++++++++++ 4 files changed, 82 insertions(+), 9 deletions(-) create mode 100644 src/test/scripts/functions/binary/matrix/QuartileArray.R create mode 100644 src/test/scripts/functions/binary/matrix/QuartileArray.dml diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java index 18bc30094c6..7525dab2f7f 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java @@ -4782,10 +4782,10 @@ public MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret, boolean av output=new MatrixBlock(qs.rlen, qs.clen, false); // resulting matrix is mostly likely be dense else output.reset(qs.rlen, qs.clen, false); - - for ( int i=0; i < qs.rlen; i++ ) { - // FIXME: [SYSTEMDS-3953] consider the average parameter here - output.set(i, 0, this.pickValue(qs.get(i,0)) ); + + for(int i = 0; i < qs.rlen; i++) { + // FIXME: include the average parameter here to fix SYSTEMDS-3953 + output.set(i, 0, this.pickValue(qs.get(i, 0))); } return output; diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java index 489b31080ef..810e2614e7c 100644 --- a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java +++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java @@ -21,6 +21,7 @@ import java.util.HashMap; +import org.junit.Ignore; import org.junit.Test; import org.apache.sysds.common.Types.ExecMode; import org.apache.sysds.common.Types.ExecType; @@ -36,6 +37,7 @@ public class QuantileTest extends AutomatedTestBase private final static String TEST_NAME3 = "IQM"; private final static String TEST_NAME4 = "QuantileBug"; private final static String TEST_NAME5 = "MedianBug"; + private final static String TEST_NAME6 = "QuartileArray"; private final static String TEST_DIR = "functions/binary/matrix/"; private final static String TEST_CLASS_DIR = TEST_DIR + QuantileTest.class.getSimpleName() + "/"; @@ -51,15 +53,17 @@ public void setUp() { TestUtils.clearAssertionInformation(); addTestConfiguration(TEST_NAME1, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "R" }) ); + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"})); addTestConfiguration(TEST_NAME2, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] { "R" }) ); + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"})); addTestConfiguration(TEST_NAME3, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] { "R" }) ); + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] {"R"})); addTestConfiguration(TEST_NAME4, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] { "R" }) ); + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] {"R"})); addTestConfiguration(TEST_NAME5, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] { "R" }) ); + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] {"R"})); + addTestConfiguration(TEST_NAME6, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] {"R"})); } @Test @@ -182,6 +186,18 @@ public void testMedianBugSP() { runQuantileTest(TEST_NAME5, -1, false, ExecType.SPARK); } + @Test + @Ignore // FIXME: fix SYSTEMDS-3953 + public void testQuartileArrayCP() { + runQuantileTest(TEST_NAME6, 0, false, ExecType.CP); + } + + @Test + @Ignore // FIXME: fix SYSTEMDS-3953 + public void testQuartileArraySP() { + runQuantileTest(TEST_NAME6, 0, false, ExecType.SPARK); + } + private void runQuantileTest( String TEST_NAME, double p, boolean sparse, ExecType et) { ExecMode platformOld = setExecMode(et); diff --git a/src/test/scripts/functions/binary/matrix/QuartileArray.R b/src/test/scripts/functions/binary/matrix/QuartileArray.R new file mode 100644 index 00000000000..3f4cff52597 --- /dev/null +++ b/src/test/scripts/functions/binary/matrix/QuartileArray.R @@ -0,0 +1,33 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +args <- commandArgs(TRUE) +options(digits=22) + +library("Matrix") + +A = as.matrix(c(1,5,7,10)) + +m = quantile(A, c(0.25, 0.5, 0.75)); + +writeMM(as(m, "CsparseMatrix"), paste(args[3], "R", sep="")); + + diff --git a/src/test/scripts/functions/binary/matrix/QuartileArray.dml b/src/test/scripts/functions/binary/matrix/QuartileArray.dml new file mode 100644 index 00000000000..a63ac8f0cbc --- /dev/null +++ b/src/test/scripts/functions/binary/matrix/QuartileArray.dml @@ -0,0 +1,24 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +A = as.matrix(list(1,5,7,10)); +m = quantile(A, as.matrix(list(0.25, 0.5, 0.75))); +write(m, $3, format="text"); From fe7f8f5c344537fad3bfb22780ec91597776e4a0 Mon Sep 17 00:00:00 2001 From: ywcb00 Date: Thu, 23 Jul 2026 14:18:30 +0200 Subject: [PATCH 092/132] [SYSTEMDS-2525] Fix sparsity rewrites by switching the MNC estimator to the metadata average estimator --- .../org/apache/sysds/hops/estim/MMNode.java | 11 ++- .../hops/rewrite/ProgramRewriteStatus.java | 12 --- ...riteMatrixMultChainOptimizationSparse.java | 83 +++++++----------- .../apache/sysds/test/AutomatedTestBase.java | 2 + .../RewriteMatrixMultChainOptSparseTest.java | 86 ++++++++++++++++--- 5 files changed, 121 insertions(+), 73 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/estim/MMNode.java b/src/main/java/org/apache/sysds/hops/estim/MMNode.java index 89c706fd87e..7483dba0223 100644 --- a/src/main/java/org/apache/sysds/hops/estim/MMNode.java +++ b/src/main/java/org/apache/sysds/hops/estim/MMNode.java @@ -47,6 +47,15 @@ public MMNode(MatrixBlock in) { _op = null; _misc = null; } + + public MMNode(DataCharacteristics mc) { + _m1 = null; + _m2 = null; + _data = null; + _mc = mc; + _op = null; + _misc = null; + } public MMNode(MMNode left, MMNode right, OpCode op, long[] misc) { _m1 = left; @@ -112,7 +121,7 @@ public MMNode getRight() { } public boolean isLeaf() { - return _data != null; + return _op == null; } public MatrixBlock getData() { diff --git a/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriteStatus.java b/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriteStatus.java index 0c86aab59db..089d65e509e 100644 --- a/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriteStatus.java +++ b/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriteStatus.java @@ -19,8 +19,6 @@ package org.apache.sysds.hops.rewrite; -import org.apache.sysds.runtime.controlprogram.LocalVariableMap; - public class ProgramRewriteStatus { //status of applied rewrites @@ -30,7 +28,6 @@ public class ProgramRewriteStatus //current context private boolean _inParforCtx = false; - private LocalVariableMap _vars = null; public ProgramRewriteStatus() { _rmBranches = false; @@ -38,11 +35,6 @@ public ProgramRewriteStatus() { _injectCheckpoints = false; } - public ProgramRewriteStatus(LocalVariableMap vars) { - this(); - _vars = vars; - } - public void setRemovedBranches(){ _rmBranches = true; } @@ -74,8 +66,4 @@ public void setInjectedCheckpoints(){ public boolean getInjectedCheckpoints(){ return _injectCheckpoints; } - - public LocalVariableMap getVariables() { - return _vars; - } } diff --git a/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java b/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java index 48b457f759d..80b71a1c902 100644 --- a/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java +++ b/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java @@ -23,17 +23,11 @@ import java.util.List; import org.apache.commons.lang3.mutable.MutableInt; -import org.apache.sysds.common.Types.OpOpData; import org.apache.sysds.hops.Hop; -import org.apache.sysds.hops.HopsException; +import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.hops.estim.MMNode; -import org.apache.sysds.hops.estim.EstimatorMatrixHistogram; -import org.apache.sysds.hops.estim.EstimatorMatrixHistogram.MatrixHistogram; +import org.apache.sysds.hops.estim.EstimatorBasicAvg; import org.apache.sysds.hops.estim.SparsityEstimator.OpCode; -import org.apache.sysds.runtime.controlprogram.LocalVariableMap; -import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; -import org.apache.sysds.runtime.instructions.cp.Data; -import org.apache.sysds.runtime.matrix.data.MatrixBlock; /** * Rule: Determine the optimal order of execution for a chain of @@ -53,9 +47,8 @@ protected void optimizeMMChain(Hop hop, List mmChain, List mmOperators double[] dimsArray = new double[mmChain.size() + 1]; boolean dimsKnown = getDimsArray( hop, mmChain, dimsArray ); MMNode[] sketchArray = new MMNode[mmChain.size() + 1]; - boolean inputsAvail = getInputMatrices(hop, mmChain, sketchArray, state); - - if( dimsKnown && inputsAvail ) { + boolean inputMetaAvail = getInputMatrixCharacteristics(hop, mmChain, sketchArray, state); + if(dimsKnown && inputMetaAvail) { // Step 3: clear the links among Hops within the identified chain clearLinksWithinChain ( hop, mmOperators ); @@ -66,7 +59,7 @@ protected void optimizeMMChain(Hop hop, List mmChain, List mmOperators int[][] split = mmChainDPSparse(dimsArray, sketchArray, mmChain.size()); // Step 5: Relink the hops using the optimal ordering (split[][]) found from DP. - LOG.trace("Optimal MM Chain: "); + LOG.trace("Optimal Sparse MM Chain:"); mmChainRelinkHops(mmOperators.get(0), 0, size - 1, mmChain, mmOperators, new MutableInt(1), split, 1); } } @@ -92,7 +85,7 @@ private static int[][] mmChainDPSparse(double[] dimArray, MMNode[] sketchArray, } //compute cost-optimal chains for increasing chain sizes - EstimatorMatrixHistogram estim = new EstimatorMatrixHistogram(true); + EstimatorBasicAvg estim = new EstimatorBasicAvg(); for( int l = 2; l <= size; l++ ) { // chain length for( int i = 0; i < size - l + 1; i++ ) { int j = i + l - 1; @@ -102,14 +95,14 @@ private static int[][] mmChainDPSparse(double[] dimArray, MMNode[] sketchArray, { //construct estimation nodes (w/ lazy propagation and memoization) MMNode tmp = new MMNode(dpMatrixS[i][k], dpMatrixS[k+1][j], OpCode.MM); - estim.estim(tmp, false); - MatrixHistogram lhs = (MatrixHistogram) dpMatrixS[i][k].getSynopsis(); - MatrixHistogram rhs = (MatrixHistogram) dpMatrixS[k+1][j].getSynopsis(); - - //recursive cost computation - double cost = dpMatrix[i][k] + dpMatrix[k + 1][j] - + dotProduct(lhs.getColCounts(), rhs.getRowCounts()); - + estim.estim(tmp); + + // recursive cost computation + double cost = dpMatrix[i][k] + dpMatrix[k + 1][j] + + OptimizerUtils.getSparsity(tmp.getLeft().getDataCharacteristics()) * + OptimizerUtils.getSparsity(tmp.getRight().getDataCharacteristics()) * + tmp.getLeft().getRows() * tmp.getLeft().getCols() * tmp.getRight().getCols(); + //prune suboptimal if( cost < dpMatrix[i][j] ) { dpMatrix[i][j] = cost; @@ -118,41 +111,31 @@ private static int[][] mmChainDPSparse(double[] dimArray, MMNode[] sketchArray, } } - if( LOG.isTraceEnabled() ){ - LOG.trace("mmchainopt [i="+(i+1)+",j="+(j+1)+"]: costs = "+dpMatrix[i][j]+", split = "+(split[i][j]+1)); - } + if(LOG.isTraceEnabled()) + LOG.trace("mmchainoptsparse [i=" + (i + 1) + ",j=" + (j + 1) + "]: costs = " + dpMatrix[i][j] + + ", split = " + (split[i][j] + 1)); } } return split; } - - private static boolean getInputMatrices(Hop hop, List chain, MMNode[] sketchArray, ProgramRewriteStatus state) { - boolean inputsAvail = true; - LocalVariableMap vars = state.getVariables(); - - for( int i=0; i chain, MMNode[] sketchArray, + ProgramRewriteStatus state) { + boolean inputMetaAvail = true; + + for(int counter = 0; counter < chain.size(); counter++) { + Hop currentHop = chain.get(counter); + inputMetaAvail &= currentHop.isMatrix(); + inputMetaAvail &= !currentHop.isFederated(); + inputMetaAvail &= (currentHop.getDataCharacteristics().getNonZeros() != -1); + if(inputMetaAvail) { + sketchArray[counter] = new MMNode(currentHop.getDataCharacteristics()); + } + else break; } - - return inputsAvail; - } - - private static MatrixBlock getMatrix(String name, LocalVariableMap vars) { - Data dat = vars.get(name); - if( !(dat instanceof MatrixObject) ) - throw new HopsException("Input '"+name+"' not a matrix: "+dat.getDataType()); - return ((MatrixObject)dat).acquireReadAndRelease(); - } - - private static double dotProduct(int[] h1cNnz, int[] h2rNnz) { - long fp = 0; - for( int j=0; j x.contains(str)); } diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java index 6f323b6aa99..d4d676dc0e2 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java @@ -19,6 +19,9 @@ package org.apache.sysds.test.functions.rewrite; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.LoggingEvent; import org.apache.sysds.common.Opcodes; import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.hops.recompile.Recompiler; @@ -26,26 +29,67 @@ import org.apache.sysds.test.AutomatedTestBase; import org.apache.sysds.test.TestConfiguration; import org.apache.sysds.test.TestUtils; +import org.apache.sysds.test.LoggingUtils; +import org.apache.sysds.test.LoggingUtils.TestAppender; + import org.junit.Assert; +import org.junit.runners.Parameterized; import org.junit.Test; +import org.junit.runner.RunWith; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; +import java.util.List; +import java.util.stream.DoubleStream; +import java.util.stream.Stream; +@RunWith(value = Parameterized.class) +@net.jcip.annotations.NotThreadSafe public class RewriteMatrixMultChainOptSparseTest extends AutomatedTestBase { private static final String TEST_NAME = "RewriteMatrixMultChainOpSparse"; private static final String TEST_DIR = "functions/rewrite/"; private static final String TEST_CLASS_DIR = TEST_DIR + RewriteMatrixMultChainOptSparseTest.class.getSimpleName() + "/"; + private static final String PACKAGE = "org.apache.sysds.hops.rewrite.HopRewriteRule"; + private static Level _oldLevel; + + @Parameterized.Parameter(0) + public int rows; + + @Parameterized.Parameter(1) + public int cols; + + @Parameterized.Parameter(2) + public double[] sparsities; - private static final int rows = 1000; - private static final int cols = 300; - private static final double eps = Math.pow(10, -10); + @Parameterized.Parameter(3) + public double eps; + + @Parameterized.Parameter(4) + public boolean tsmm; + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList(new Object[][] { + // {rows, cols, sparsities, eps, tsmm}, + {1000, 300, new double[] {0.10d, 0.10d}, Math.pow(10, -10), false}, + {5, 3, new double[] {0.1, 1}, Math.pow(10, -10), true},}); + } @Override public void setUp() { TestUtils.clearAssertionInformation(); addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"R"})); + _oldLevel = Logger.getLogger(PACKAGE).getLevel(); + Logger.getLogger(PACKAGE).setLevel(Level.TRACE); + } + + @Override + public void tearDown() { + super.tearDown(); + Logger.getLogger(PACKAGE).setLevel(_oldLevel); } @Test @@ -74,13 +118,19 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES = rewrites; OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES = rewrites; - double[][] X = getRandomMatrix(rows, cols, -1, 1, 0.10d, 7); - double[][] Y = getRandomMatrix(cols, 1, -1, 1, 0.10d, 3); - writeInputMatrixWithMTD("X", X, true); - writeInputMatrixWithMTD("Y", Y, true); + double[][] X = getRandomMatrix(rows, cols, -1, 1, sparsities[0], 7); + double[][] Y = getRandomMatrix(cols, 1, -1, 1, sparsities[1], 3); + long X_nnz = Stream.of(X).mapToLong(row -> DoubleStream.of(row).filter(val -> val != 0).count()).sum(); + long Y_nnz = Stream.of(Y).mapToLong(row -> DoubleStream.of(row).filter(val -> val != 0).count()).sum(); + writeInputMatrixWithMTD("X", X, X_nnz, true); + writeInputMatrixWithMTD("Y", Y, Y_nnz, true); + //execute tests + TestAppender appender = LoggingUtils.overwrite(); // capture log output runTest(true, false, null, -1); + List log_out = LoggingUtils.reinsert(appender); // revert the logger to print to stdout + runRScript(true); //compare matrices @@ -89,12 +139,28 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R"); if(rewrites) { - Assert.assertTrue(heavyHittersContainsSubString(Opcodes.MMCHAIN.toString()) || - heavyHittersContainsSubString("sp_mapmmchain")); + String delimiter = ";"; + String log_out_string = String.join(delimiter, + log_out.stream().map(l -> l.getMessage().toString()).toArray(String[]::new)); + Assert.assertTrue(log_out_string.contains("mmchainoptsparse")); + if(tsmm) { + Assert.assertTrue(log_out_string.contains("Optimal Sparse MM Chain:" + delimiter + "--(" + delimiter + + "----(" + delimiter + "------Hop parsertemp")); + Assert.assertTrue(heavyHittersContainsSubString(Opcodes.TSMM.toString()) || + heavyHittersContainsSubString("sp_tsmm")); + } + else { + Assert.assertTrue(log_out_string + .contains("Optimal Sparse MM Chain:" + delimiter + "--(" + delimiter + "----Hop parsertemp")); + Assert.assertTrue(heavyHittersContainsSubString(Opcodes.MMCHAIN.toString()) || + heavyHittersContainsSubString("sp_mapmmchain")); + } } else { + Assert.assertFalse( + log_out.stream().anyMatch(l -> l.getMessage().toString().contains("mmchainoptsparse"))); Assert.assertFalse(heavyHittersContainsSubString(Opcodes.MMCHAIN.toString()) || - heavyHittersContainsSubString("sp_mapmmchain")); + heavyHittersContainsSubString("sp_mapmmchain")); } } finally { From 460588b23c6ef245a252caae1a372fb44ecbb8f1 Mon Sep 17 00:00:00 2001 From: Janardhan Pulivarthi Date: Fri, 24 Jul 2026 16:13:23 +0000 Subject: [PATCH 093/132] [MINOR] Fix literal parsing issue in svn promote script --- dev/release/svn-staging-to-release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/release/svn-staging-to-release.sh b/dev/release/svn-staging-to-release.sh index e4e29ebaec1..04bfceb373d 100755 --- a/dev/release/svn-staging-to-release.sh +++ b/dev/release/svn-staging-to-release.sh @@ -74,7 +74,7 @@ cd svn-release-systemds if [[ $dry_run_flag != 1 ]]; then # This step prompts for the Apache Credentials - svn ci --username $ASF_USERNAME -m'Apache SystemDS $RELEASE_VERSION Released' --no-auth-cache \n + svn ci --username "$ASF_USERNAME" -m"Apache SystemDS $RELEASE_VERSION Released" --no-auth-cache [[ $? == 0 ]] && printf '\n Publishing to $RELEASE_LOCATION is complete!\n' else printf "\n==========\n" @@ -82,7 +82,7 @@ else printf "At $RELEASE_LOCATION \n" printf "\n==========\n" printf "You might want to manually check the files and run the following:\n" - printf "svn ci --username $ASF_USERNAME -m'Apache SystemDS $RELEASE_VERSION Released' --no-auth-cache \n" + printf "svn ci --username $ASF_USERNAME -m\"Apache SystemDS $RELEASE_VERSION Released\" --no-auth-cache\n" printf "\n==========\n" fi From 3c5414d596b6f34fafe58b801460cbed932d683c Mon Sep 17 00:00:00 2001 From: Jannik Lindemann Date: Fri, 24 Jul 2026 18:37:07 +0200 Subject: [PATCH 094/132] [MINOR] Downgrade Docker Action Majors --- .github/workflows/docker-release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index fb7ceea1cbe..1e369c535ee 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -46,7 +46,7 @@ jobs: # https://github.com/docker/metadata-action - name: Configure Docker metadata id: meta - uses: docker/metadata-action@v6 + uses: docker/metadata-action@v5 with: images: apache/systemds tags: ${{ github.event.inputs.version }},latest @@ -59,12 +59,12 @@ jobs: # https://github.com/docker/setup-buildx-action - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@v3 # https://github.com/docker/login-action - name: Login to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@v4 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -72,7 +72,7 @@ jobs: # https://github.com/docker/build-push-action - name: Build and push id: docker_build - uses: docker/build-push-action@v7 + uses: docker/build-push-action@v6 with: context: . file: ./docker/sysds.Dockerfile From 44099b30929677629e4bdd32b5f5a0a456493498 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann Date: Fri, 24 Jul 2026 18:39:46 +0200 Subject: [PATCH 095/132] Revert "[MINOR] Downgrade Docker Action Majors" This reverts commit f62cebd2fea23dd6b45ad86880bd123f6e11d3c0. --- .github/workflows/docker-release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 1e369c535ee..fb7ceea1cbe 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -46,7 +46,7 @@ jobs: # https://github.com/docker/metadata-action - name: Configure Docker metadata id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v6 with: images: apache/systemds tags: ${{ github.event.inputs.version }},latest @@ -59,12 +59,12 @@ jobs: # https://github.com/docker/setup-buildx-action - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 # https://github.com/docker/login-action - name: Login to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -72,7 +72,7 @@ jobs: # https://github.com/docker/build-push-action - name: Build and push id: docker_build - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: . file: ./docker/sysds.Dockerfile From 370b882d0fbbe84e3b18b6033911d76378d20505 Mon Sep 17 00:00:00 2001 From: Tuluyhan Sozen Date: Thu, 30 Jul 2026 16:09:54 +0200 Subject: [PATCH 096/132] [SYSTEMDS-3894] Add OOC covariance and extend TSMM support Closes #2529. Co-authored-by: Astha Shrestha Co-authored-by: AdityaPandey2612 --- .../sysds/runtime/functionobjects/COV.java | 2 + .../instructions/OOCInstructionParser.java | 5 +- .../ooc/CovarianceOOCInstruction.java | 123 +++++++++++++++ .../instructions/ooc/OOCInstruction.java | 2 +- .../instructions/ooc/TSMMOOCInstruction.java | 118 +++++++++++--- .../test/functions/ooc/CovarianceTest.java | 130 ++++++++++++++++ .../functions/ooc/CovarianceWeightsTest.java | 136 +++++++++++++++++ .../functions/ooc/TransposeSelfMMTest.java | 144 +++++++++++++----- src/test/scripts/functions/ooc/Covariance.dml | 28 ++++ .../functions/ooc/CovarianceWeights.dml | 29 ++++ src/test/scripts/functions/ooc/TSMM.dml | 2 +- src/test/scripts/functions/ooc/TSMMRight.dml | 28 ++++ 12 files changed, 688 insertions(+), 59 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/instructions/ooc/CovarianceOOCInstruction.java create mode 100644 src/test/java/org/apache/sysds/test/functions/ooc/CovarianceTest.java create mode 100644 src/test/java/org/apache/sysds/test/functions/ooc/CovarianceWeightsTest.java create mode 100644 src/test/scripts/functions/ooc/Covariance.dml create mode 100644 src/test/scripts/functions/ooc/CovarianceWeights.dml create mode 100644 src/test/scripts/functions/ooc/TSMMRight.dml diff --git a/src/main/java/org/apache/sysds/runtime/functionobjects/COV.java b/src/main/java/org/apache/sysds/runtime/functionobjects/COV.java index 836bf972ce6..36be4e8b589 100644 --- a/src/main/java/org/apache/sysds/runtime/functionobjects/COV.java +++ b/src/main/java/org/apache/sysds/runtime/functionobjects/COV.java @@ -63,6 +63,8 @@ private COV() { public Data execute(Data in1, double u, double v, double w2) { CmCovObject cov1=(CmCovObject) in1; + if(w2 == 0) + return cov1; if(cov1.isCOVAllZeros()) { cov1.w=w2; diff --git a/src/main/java/org/apache/sysds/runtime/instructions/OOCInstructionParser.java b/src/main/java/org/apache/sysds/runtime/instructions/OOCInstructionParser.java index 98a454283e2..f0050e86fda 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/OOCInstructionParser.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/OOCInstructionParser.java @@ -29,9 +29,10 @@ import org.apache.sysds.runtime.instructions.ooc.BinaryOOCInstruction; import org.apache.sysds.runtime.instructions.ooc.CSVReblockOOCInstruction; import org.apache.sysds.runtime.instructions.ooc.CentralMomentOOCInstruction; +import org.apache.sysds.runtime.instructions.ooc.CovarianceOOCInstruction; import org.apache.sysds.runtime.instructions.ooc.CtableOOCInstruction; -import org.apache.sysds.runtime.instructions.ooc.IndexingOOCInstruction; import org.apache.sysds.runtime.instructions.ooc.DataGenOOCInstruction; +import org.apache.sysds.runtime.instructions.ooc.IndexingOOCInstruction; import org.apache.sysds.runtime.instructions.ooc.OOCInstruction; import org.apache.sysds.runtime.instructions.ooc.ParameterizedBuiltinOOCInstruction; import org.apache.sysds.runtime.instructions.ooc.ReblockOOCInstruction; @@ -103,6 +104,8 @@ else if(parts.length == 4) return TeeOOCInstruction.parseInstruction(str); case CentralMoment: return CentralMomentOOCInstruction.parseInstruction(str); + case Covariance: + return CovarianceOOCInstruction.parseInstruction(str); case Ctable: return CtableOOCInstruction.parseInstruction(str); case ParameterizedBuiltin: diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/CovarianceOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/CovarianceOOCInstruction.java new file mode 100644 index 00000000000..1a2e7258944 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/CovarianceOOCInstruction.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.instructions.ooc; + +import java.util.List; + +import org.apache.sysds.common.Opcodes; +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; +import org.apache.sysds.runtime.controlprogram.context.ExecutionContext; +import org.apache.sysds.runtime.controlprogram.parfor.LocalTaskQueue; +import org.apache.sysds.runtime.functionobjects.COV; +import org.apache.sysds.runtime.instructions.InstructionUtils; +import org.apache.sysds.runtime.instructions.cp.CPOperand; +import org.apache.sysds.runtime.instructions.cp.CmCovObject; +import org.apache.sysds.runtime.instructions.cp.DoubleObject; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.operators.COVOperator; +import org.apache.sysds.runtime.meta.DataCharacteristics; + +public class CovarianceOOCInstruction extends ComputationOOCInstruction { + + private CovarianceOOCInstruction(COVOperator cov, CPOperand in1, CPOperand in2, CPOperand in3, CPOperand out, + String opcode, String str) { + super(OOCType.COV, cov, in1, in2, in3, out, opcode, str); + } + + public static CovarianceOOCInstruction parseInstruction(String str) { + String[] parts = InstructionUtils.getInstructionPartsWithValueType(str); + String opcode = parts[0]; + + if(!opcode.equalsIgnoreCase(Opcodes.COV.toString())) + throw new DMLRuntimeException("CovarianceOOCInstruction.parseInstruction():: Unknown opcode " + opcode); + + // the OOC instruction string matches the Spark format, + + COVOperator cov = new COVOperator(COV.getCOMFnObject()); + if(parts.length == 4) { // this is the case for unweighted cov.A.B.out + CPOperand in1 = new CPOperand(parts[1]); + CPOperand in2 = new CPOperand(parts[2]); + CPOperand out = new CPOperand(parts[3]); + return new CovarianceOOCInstruction(cov, in1, in2, null, out, opcode, str); + } + else if(parts.length == 5) {// this is the case for weighted cov.A.B.W.out + CPOperand in1 = new CPOperand(parts[1]); + CPOperand in2 = new CPOperand(parts[2]); + CPOperand in3 = new CPOperand(parts[3]); + CPOperand out = new CPOperand(parts[4]); + return new CovarianceOOCInstruction(cov, in1, in2, in3, out, opcode, str); + } + else { + throw new DMLRuntimeException("Invalid number of arguments in Instruction: " + str); + } + } + + @Override + public void processInstruction(ExecutionContext ec) { + COVOperator cov_op = (COVOperator) _optr; + + MatrixObject mo1 = ec.getMatrixObject(input1.getName()); + MatrixObject mo2 = ec.getMatrixObject(input2.getName()); + + OOCStream q1 = mo1.getStreamHandle(); + OOCStream q2 = mo2.getStreamHandle(); + + OOCStream covObjs = createWritableStream(); + + if(input3 == null) { + // unweighted covariance join the two tile streams by block index + joinOOC(q1, q2, covObjs, + (a, b) -> ((MatrixBlock) a.getValue()).covOperations(cov_op, (MatrixBlock) b.getValue()), + IndexedMatrixValue::getIndexes); + } + else { + // weighted covariance additionally join the weights tile stream + MatrixObject mo3 = ec.getMatrixObject(input3.getName()); + + DataCharacteristics dc1 = ec.getDataCharacteristics(input1.getName()); + DataCharacteristics dc2 = ec.getDataCharacteristics(input2.getName()); + DataCharacteristics dc3 = ec.getDataCharacteristics(input3.getName()); + if(dc1.getBlocksize() != dc2.getBlocksize() || dc1.getBlocksize() != dc3.getBlocksize()) + throw new DMLRuntimeException("Different block sizes are not yet supported"); + + OOCStream q3 = mo3.getStreamHandle(); + + joinOOC(List.of(q1, q2, q3), covObjs, + tiles -> ((MatrixBlock) tiles.get(0).getValue()).covOperations(cov_op, + (MatrixBlock) tiles.get(1).getValue(), (MatrixBlock) tiles.get(2).getValue()), + IndexedMatrixValue::getIndexes); + } + + try { + CmCovObject agg = covObjs.dequeue(); + CmCovObject next; + + while((next = covObjs.dequeue()) != LocalTaskQueue.NO_MORE_TASKS) + agg = (CmCovObject) cov_op.fn.execute(agg, next); + + ec.setScalarOutput(output.getName(), new DoubleObject(agg.getRequiredResult(cov_op))); + } + catch(Exception ex) { + throw new DMLRuntimeException(ex); + } + } +} diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java index b263b307e1d..e0dd905eb9f 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCInstruction.java @@ -74,7 +74,7 @@ public abstract class OOCInstruction extends Instruction { public enum OOCType { Reblock, Tee, Binary, Ternary, Unary, AggregateUnary, AggregateBinary, AggregateTernary, MAPMM, MMTSJ, - MAPMMCHAIN, Reorg, CM, Ctable, MatrixIndexing, ParameterizedBuiltin, Rand, Append, Quaternary, Reshape + MAPMMCHAIN, Reorg, CM, COV, Ctable, MatrixIndexing, ParameterizedBuiltin, Rand, Append, Quaternary, Reshape } protected final OOCInstruction.OOCType _ooctype; diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/TSMMOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/TSMMOOCInstruction.java index e0207940409..37b2ba93a77 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/TSMMOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/TSMMOOCInstruction.java @@ -19,9 +19,13 @@ package org.apache.sysds.runtime.instructions.ooc; +import java.util.List; +import java.util.concurrent.CompletableFuture; + import org.apache.sysds.common.Opcodes; import org.apache.sysds.lops.MMTSJ; import org.apache.sysds.lops.MMTSJ.MMTSJType; +import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; import org.apache.sysds.runtime.controlprogram.context.ExecutionContext; import org.apache.sysds.runtime.controlprogram.parfor.LocalTaskQueue; @@ -30,7 +34,9 @@ import org.apache.sysds.runtime.instructions.InstructionUtils; import org.apache.sysds.runtime.instructions.cp.CPOperand; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.LibMatrixReorg; import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.data.MatrixIndexes; import org.apache.sysds.runtime.matrix.operators.AggregateBinaryOperator; import org.apache.sysds.runtime.matrix.operators.AggregateOperator; import org.apache.sysds.runtime.matrix.operators.BinaryOperator; @@ -48,7 +54,7 @@ public static TSMMOOCInstruction parseInstruction(String str) { String[] parts = InstructionUtils.getInstructionPartsWithValueType(str); InstructionUtils.checkNumFields(parts, 3); String opcode = parts[0]; - CPOperand in1 = new CPOperand(parts[1]); // the large matrix (streamed), columns <= blocksize + CPOperand in1 = new CPOperand(parts[1]); // the large matrix (streamed) CPOperand out = new CPOperand(parts[2]); MMTSJ.MMTSJType mmtsjType = MMTSJ.MMTSJType.valueOf(parts[3]); @@ -59,34 +65,62 @@ public static TSMMOOCInstruction parseInstruction(String str) { } @Override - public void processInstruction( ExecutionContext ec ) { + public void processInstruction(ExecutionContext ec) { MatrixObject min = ec.getMatrixObject(input1); - int nRows = (int) min.getDataCharacteristics().getRows(); - int nCols = (int) min.getDataCharacteristics().getCols(); - int bLen = min.getDataCharacteristics().getBlocksize(); - - OOCStream qIn = min.getStreamHandle(); + int numRowBlocks = Math.toIntExact(min.getDataCharacteristics().getNumRowBlocks()); + int numColBlocks = Math.toIntExact(min.getDataCharacteristics().getNumColBlocks()); + if((_type.isLeft() && numColBlocks == 1) || (_type.isRight() && numRowBlocks == 1)) { + processSingleOutputTileInstruction(ec, min); + return; + } + + int blocksPerJoinGroup = _type.isLeft() ? numColBlocks : numRowBlocks; + int partialsPerOutput = _type.isLeft() ? numRowBlocks : numColBlocks; + + OOCStreamable inputStreamable = min.getStreamable(); + final boolean createdCache = !inputStreamable.hasStreamCache(); + final CachingStream inputCache = createdCache ? new CachingStream(min.getStreamHandle()) : inputStreamable + .getStreamCache(); + + OOCStream> groupedPartials = createWritableStream(); + OOCStream partials = createWritableStream(); + OOCStream out = createWritableStream(); + addOutStream(out); + ec.getMatrixObject(output).setStreamHandle(out); + + CompletableFuture joinFuture = joinManyOOC(inputCache.getReadStream(), inputCache.getReadStream(), + groupedPartials, this::createPartialOutputTiles, this::getJoinIndex, this::getJoinIndex, blocksPerJoinGroup, + blocksPerJoinGroup); + CompletableFuture expandFuture = expandOOC(groupedPartials, partials, values -> values); + BinaryOperator plus = InstructionUtils.parseBinaryOperator(Opcodes.PLUS.toString()); + CompletableFuture outFuture = groupedReduceOOC(partials, out, (left, right) -> { + MatrixBlock result = ((MatrixBlock) left.getValue()).binaryOperations(plus, right.getValue()); + left.setValue(result); + return left; + }, partialsPerOutput); - //validation check TODO extend compiler to not create OOC otherwise - if( (_type.isLeft() && nCols > bLen) - || (_type.isRight() && nRows > bLen) ) - { - throw new UnsupportedOperationException(); - } - - //int dim = _type.isLeft() ? nCols : nRows; + propagateFailuresToOutput(out, List.of(joinFuture, expandFuture, outFuture)); + + outFuture.whenComplete((result, error) -> { + if(createdCache) + inputCache.scheduleDeletion(); + }); + } + + private void processSingleOutputTileInstruction(ExecutionContext ec, MatrixObject min) { + OOCStream qIn = min.getStreamHandle(); + BinaryOperator plus = InstructionUtils.parseBinaryOperator(Opcodes.PLUS.toString()); MatrixBlock resultBlock = null; OOCStream tmpStream = createWritableStream(); - mapOOC(qIn, tmpStream, tmp -> ((MatrixBlock) tmp.getValue()) .transposeSelfMatrixMultOperations(new MatrixBlock(), _type)); MatrixBlock tmp; - while ((tmp = tmpStream.dequeue()) != LocalTaskQueue.NO_MORE_TASKS) { - if (resultBlock == null) + while((tmp = tmpStream.dequeue()) != LocalTaskQueue.NO_MORE_TASKS) { + if(resultBlock == null) resultBlock = tmp; else resultBlock.binaryOperationsInPlace(plus, tmp); @@ -94,4 +128,52 @@ public void processInstruction( ExecutionContext ec ) { ec.setMatrixOutput(output.getName(), resultBlock); } + + private long getJoinIndex(IndexedMatrixValue value) { + return _type.isLeft() ? value.getIndexes().getRowIndex() : value.getIndexes().getColumnIndex(); + } + + private long getOutputIndex(IndexedMatrixValue value) { + return _type.isLeft() ? value.getIndexes().getColumnIndex() : value.getIndexes().getRowIndex(); + } + + private List createPartialOutputTiles(IndexedMatrixValue left, IndexedMatrixValue right) { + long leftIndex = getOutputIndex(left); + long rightIndex = getOutputIndex(right); + if(leftIndex > rightIndex) + return List.of(); + + MatrixBlock leftBlock = (MatrixBlock) left.getValue(); + MatrixBlock rightBlock = (MatrixBlock) right.getValue(); + if(leftIndex == rightIndex) { + MatrixBlock diagonal = leftBlock.transposeSelfMatrixMultOperations(new MatrixBlock(), _type); + return List.of(new IndexedMatrixValue(new MatrixIndexes(leftIndex, rightIndex), diagonal)); + } + + MatrixBlock partial = multiplyOffDiagonal(leftBlock, rightBlock); + MatrixBlock mirror = LibMatrixReorg.transpose(partial); + return List.of(new IndexedMatrixValue(new MatrixIndexes(leftIndex, rightIndex), partial), + new IndexedMatrixValue(new MatrixIndexes(rightIndex, leftIndex), mirror)); + } + + private MatrixBlock multiplyOffDiagonal(MatrixBlock leftBlock, MatrixBlock rightBlock) { + if(_type.isLeft()) { + MatrixBlock leftTranspose = LibMatrixReorg.transpose(leftBlock); + return leftTranspose.aggregateBinaryOperations(leftTranspose, rightBlock, new MatrixBlock(), + (AggregateBinaryOperator) _optr); + } + + MatrixBlock rightTranspose = LibMatrixReorg.transpose(rightBlock); + return leftBlock.aggregateBinaryOperations(leftBlock, rightTranspose, new MatrixBlock(), + (AggregateBinaryOperator) _optr); + } + + private static void propagateFailuresToOutput(OOCStream out, List> futures) { + for(CompletableFuture future : futures) { + future.exceptionally(error -> { + out.propagateFailure(DMLRuntimeException.of(error)); + return null; + }); + } + } } diff --git a/src/test/java/org/apache/sysds/test/functions/ooc/CovarianceTest.java b/src/test/java/org/apache/sysds/test/functions/ooc/CovarianceTest.java new file mode 100644 index 00000000000..56404649ddc --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/ooc/CovarianceTest.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.ooc; + +import java.io.IOException; + +import org.apache.sysds.common.Opcodes; +import org.apache.sysds.common.Types; +import org.apache.sysds.runtime.instructions.Instruction; +import org.apache.sysds.runtime.io.MatrixWriter; +import org.apache.sysds.runtime.io.MatrixWriterFactory; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.meta.MatrixCharacteristics; +import org.apache.sysds.runtime.util.DataConverter; +import org.apache.sysds.runtime.util.HDFSTool; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +public class CovarianceTest extends AutomatedTestBase { + private final static String TEST_NAME = "Covariance"; + private final static String TEST_DIR = "functions/ooc/"; + private final static String TEST_CLASS_DIR = TEST_DIR + CovarianceTest.class.getSimpleName() + "/"; + private final static double eps = 1e-10; + + private final static String INPUT_A = "A"; + private final static String INPUT_B = "B"; + private final static String OUTPUT_CP = "R_CP"; + private final static String OUTPUT_OOC = "R_OOC"; + + private final static int rows = 1871; + private final static int cols = 1; + private final static int blocksize = 1000; + private final static int maxVal = 7; + + private final static double denseSparsity = 0.65; + private final static double sparseSparsity = 0.05; + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_NAME, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {OUTPUT_CP, OUTPUT_OOC})); + } + + @Test + public void testCovarianceDenseOOC() { + runCovarianceOOCCompareTest(false); + } + + @Test + public void testCovarianceSparseOOC() { + runCovarianceOOCCompareTest(true); + } + + private void runCovarianceOOCCompareTest(boolean sparse) { + Types.ExecMode platformOld = setExecMode(Types.ExecMode.SINGLE_NODE); + + try { + getAndLoadTestConfiguration(TEST_NAME); + + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + + double sparsity = sparse ? sparseSparsity : denseSparsity; + + double[][] A = getRandomMatrix(rows, cols, 1, maxVal, sparsity, 7); + double[][] B = getRandomMatrix(rows, cols, 1, maxVal, sparsity, 823); + + MatrixBlock ABlock = DataConverter.convertToMatrixBlock(A); + MatrixBlock BBlock = DataConverter.convertToMatrixBlock(B); + + writeBinaryMatrix(INPUT_A, ABlock, rows, cols, blocksize); + writeBinaryMatrix(INPUT_B, BBlock, rows, cols, blocksize); + + // Reference run: normal single-node CP execution. + programArgs = new String[] {"-args", input(INPUT_A), input(INPUT_B), output(OUTPUT_CP)}; + runTest(true, false, null, -1); + + // OOC run: compare the out-of-core covariance path against CP. + programArgs = new String[] {"-explain", "-stats", "-ooc", "-args", input(INPUT_A), input(INPUT_B), + output(OUTPUT_OOC)}; + runTest(true, false, null, -1); + + Assert.assertTrue("OOC wasn't used for covariance", + heavyHittersContainsString(Instruction.OOC_INST_PREFIX + Opcodes.COV)); + + MatrixBlock cpResult = DataConverter.readMatrixFromHDFS(output(OUTPUT_CP), Types.FileFormat.BINARY, 1, 1, + blocksize, 1); + + MatrixBlock oocResult = DataConverter.readMatrixFromHDFS(output(OUTPUT_OOC), Types.FileFormat.BINARY, 1, 1, + blocksize, 1); + + TestUtils.compareMatrices(cpResult, oocResult, eps); + } + catch(IOException ex) { + throw new RuntimeException(ex); + } + finally { + resetExecMode(platformOld); + } + } + + private void writeBinaryMatrix(String name, MatrixBlock mb, int rows, int cols, int blocksize) throws IOException { + MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(Types.FileFormat.BINARY); + writer.writeMatrixToHDFS(mb, input(name), rows, cols, blocksize, mb.getNonZeros()); + + HDFSTool.writeMetaDataFile(input(name + ".mtd"), Types.ValueType.FP64, + new MatrixCharacteristics(rows, cols, blocksize, mb.getNonZeros()), Types.FileFormat.BINARY); + } +} diff --git a/src/test/java/org/apache/sysds/test/functions/ooc/CovarianceWeightsTest.java b/src/test/java/org/apache/sysds/test/functions/ooc/CovarianceWeightsTest.java new file mode 100644 index 00000000000..03755ad7d20 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/ooc/CovarianceWeightsTest.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.ooc; + +import java.io.IOException; + +import org.apache.sysds.common.Opcodes; +import org.apache.sysds.common.Types; +import org.apache.sysds.runtime.instructions.Instruction; +import org.apache.sysds.runtime.io.MatrixWriter; +import org.apache.sysds.runtime.io.MatrixWriterFactory; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.meta.MatrixCharacteristics; +import org.apache.sysds.runtime.util.DataConverter; +import org.apache.sysds.runtime.util.HDFSTool; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +public class CovarianceWeightsTest extends AutomatedTestBase { + private final static String TEST_NAME = "CovarianceWeights"; + private final static String TEST_DIR = "functions/ooc/"; + private final static String TEST_CLASS_DIR = TEST_DIR + CovarianceWeightsTest.class.getSimpleName() + "/"; + private final static double eps = 1e-10; + + private final static String INPUT_A = "A"; + private final static String INPUT_B = "B"; + private final static String INPUT_W = "W"; + private final static String OUTPUT_CP = "R_CP"; + private final static String OUTPUT_OOC = "R_OOC"; + + private final static int rows = 1871; + private final static int cols = 1; + private final static int blocksize = 1000; + private final static int maxVal = 7; + + private final static double denseSparsity = 0.65; + private final static double sparseSparsity = 0.05; + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_NAME, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {OUTPUT_CP, OUTPUT_OOC})); + } + + @Test + public void testWeightedCovarianceDenseOOC() { + runWeightedCovarianceOOCCompareTest(false); + } + + @Test + public void testWeightedCovarianceSparseOOC() { + runWeightedCovarianceOOCCompareTest(true); + } + + private void runWeightedCovarianceOOCCompareTest(boolean sparse) { + Types.ExecMode platformOld = setExecMode(Types.ExecMode.SINGLE_NODE); + + try { + getAndLoadTestConfiguration(TEST_NAME); + + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + + double sparsity = sparse ? sparseSparsity : denseSparsity; + + double[][] A = getRandomMatrix(rows, cols, 1, maxVal, sparsity, 7); + double[][] B = getRandomMatrix(rows, cols, 1, maxVal, sparsity, 823); + + // Weights should be positive. Avoid zero/negative weights. + double[][] W = getRandomMatrix(rows, cols, 1, maxVal, sparsity, 1234); + + MatrixBlock ABlock = DataConverter.convertToMatrixBlock(A); + MatrixBlock BBlock = DataConverter.convertToMatrixBlock(B); + MatrixBlock WBlock = DataConverter.convertToMatrixBlock(W); + + writeBinaryMatrix(INPUT_A, ABlock, rows, cols, blocksize); + writeBinaryMatrix(INPUT_B, BBlock, rows, cols, blocksize); + writeBinaryMatrix(INPUT_W, WBlock, rows, cols, blocksize); + + // Reference run: normal single-node CP execution. + programArgs = new String[] {"-args", input(INPUT_A), input(INPUT_B), input(INPUT_W), output(OUTPUT_CP)}; + runTest(true, false, null, -1); + + // OOC run: compare the out-of-core weighted covariance path against CP. + programArgs = new String[] {"-explain", "-stats", "-ooc", "-args", input(INPUT_A), input(INPUT_B), + input(INPUT_W), output(OUTPUT_OOC)}; + runTest(true, false, null, -1); + + Assert.assertTrue("OOC wasn't used for weighted covariance", + heavyHittersContainsString(Instruction.OOC_INST_PREFIX + Opcodes.COV)); + + MatrixBlock cpResult = DataConverter.readMatrixFromHDFS(output(OUTPUT_CP), Types.FileFormat.BINARY, 1, 1, + blocksize, 1); + + MatrixBlock oocResult = DataConverter.readMatrixFromHDFS(output(OUTPUT_OOC), Types.FileFormat.BINARY, 1, 1, + blocksize, 1); + + TestUtils.compareMatrices(cpResult, oocResult, eps); + } + catch(IOException ex) { + throw new RuntimeException(ex); + } + finally { + resetExecMode(platformOld); + } + } + + private void writeBinaryMatrix(String name, MatrixBlock mb, int rows, int cols, int blocksize) throws IOException { + MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(Types.FileFormat.BINARY); + writer.writeMatrixToHDFS(mb, input(name), rows, cols, blocksize, mb.getNonZeros()); + + HDFSTool.writeMetaDataFile(input(name + ".mtd"), Types.ValueType.FP64, + new MatrixCharacteristics(rows, cols, blocksize, mb.getNonZeros()), Types.FileFormat.BINARY); + } +} diff --git a/src/test/java/org/apache/sysds/test/functions/ooc/TransposeSelfMMTest.java b/src/test/java/org/apache/sysds/test/functions/ooc/TransposeSelfMMTest.java index ed61038a716..6e49a8a7faa 100644 --- a/src/test/java/org/apache/sysds/test/functions/ooc/TransposeSelfMMTest.java +++ b/src/test/java/org/apache/sysds/test/functions/ooc/TransposeSelfMMTest.java @@ -21,7 +21,8 @@ import org.apache.sysds.common.Opcodes; import org.apache.sysds.common.Types; -import org.apache.sysds.lops.MMTSJ; +import org.apache.sysds.conf.DMLConfig; +import org.apache.sysds.lops.MMTSJ.MMTSJType; import org.apache.sysds.runtime.instructions.Instruction; import org.apache.sysds.runtime.io.MatrixWriter; import org.apache.sysds.runtime.io.MatrixWriterFactory; @@ -36,76 +37,107 @@ import org.junit.Test; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; public class TransposeSelfMMTest extends AutomatedTestBase { - private final static String TEST_NAME1 = "TSMM"; + private static final String TEST_NAME_LEFT = "TSMM"; + private static final String TEST_NAME_RIGHT = "TSMMRight"; private final static String TEST_DIR = "functions/ooc/"; private final static String TEST_CLASS_DIR = TEST_DIR + TransposeSelfMMTest.class.getSimpleName() + "/"; private final static double eps = 1e-8; private static final String INPUT_NAME = "X"; - private static final String OUTPUT_NAME = "res"; + private static final String OUTPUT_NAME_CP = "res_cp"; + private static final String OUTPUT_NAME_OOC = "res_ooc"; - private final static int rows = 2143; - private final static int cols = 123; + private static final int SINGLE_TILE_ROWS = 2143; + private static final int SINGLE_TILE_COLS = 123; + private static final int SINGLE_TILE_BLOCK_SIZE = 1000; + private static final int MULTI_TILE_ROWS = 1501; + private static final int MULTI_TILE_COLS = 1301; + private static final int MULTI_TILE_BLOCK_SIZE = 500; private final static double sparsity1 = 0.7; private final static double sparsity2 = 0.1; - private final int k = 1; @Override public void setUp() { TestUtils.clearAssertionInformation(); - TestConfiguration config = new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1); - addTestConfiguration(TEST_NAME1, config); + addTestConfiguration(TEST_NAME_LEFT, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME_LEFT)); + addTestConfiguration(TEST_NAME_RIGHT, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME_RIGHT)); } @Test - public void testTsmmDense() { - runTSMMTest(cols, false); + public void testTsmmLeftDenseSingleTile() { + runTSMMTest(MMTSJType.LEFT, SINGLE_TILE_ROWS, SINGLE_TILE_COLS, SINGLE_TILE_BLOCK_SIZE, false); } - + @Test - public void testTsmmSparse() { - runTSMMTest(cols, false); + public void testTsmmLeftSparseSingleTile() { + runTSMMTest(MMTSJType.LEFT, SINGLE_TILE_ROWS, SINGLE_TILE_COLS, SINGLE_TILE_BLOCK_SIZE, true); } - private void runTSMMTest(int cols, boolean sparse ) - { + @Test + public void testTsmmRightDenseSingleTile() { + runTSMMTest(MMTSJType.RIGHT, SINGLE_TILE_COLS, SINGLE_TILE_ROWS, SINGLE_TILE_BLOCK_SIZE, false); + } + + @Test + public void testTsmmLeftDenseMultiTile() { + runTSMMTest(MMTSJType.LEFT, MULTI_TILE_ROWS, MULTI_TILE_COLS, MULTI_TILE_BLOCK_SIZE, false); + } + + @Test + public void testTsmmLeftSparseMultiTile() { + runTSMMTest(MMTSJType.LEFT, MULTI_TILE_ROWS, MULTI_TILE_COLS, MULTI_TILE_BLOCK_SIZE, true); + } + + @Test + public void testTsmmRightDenseMultiTile() { + runTSMMTest(MMTSJType.RIGHT, MULTI_TILE_ROWS, MULTI_TILE_COLS, MULTI_TILE_BLOCK_SIZE, false); + } + + @Test + public void testTsmmRightSparseMultiTile() { + runTSMMTest(MMTSJType.RIGHT, MULTI_TILE_ROWS, MULTI_TILE_COLS, MULTI_TILE_BLOCK_SIZE, true); + } + + private void runTSMMTest(MMTSJType type, int rows, int cols, int blockSize, boolean sparse) { Types.ExecMode platformOld = setExecMode(Types.ExecMode.SINGLE_NODE); - try - { - getAndLoadTestConfiguration(TEST_NAME1); + try { + String testName = type.isLeft() ? TEST_NAME_LEFT : TEST_NAME_RIGHT; + getAndLoadTestConfiguration(testName); + setDefaultBlockSizeInConfig(blockSize); String HOME = SCRIPT_DIR + TEST_DIR; - fullDMLScriptName = HOME + TEST_NAME1 + ".dml"; - programArgs = new String[]{"-explain", "-stats", "-ooc", - "-args", input(INPUT_NAME), output(OUTPUT_NAME)}; + fullDMLScriptName = HOME + testName + ".dml"; - // 1. Generate the data in-memory as MatrixBlock objects double[][] A_data = getRandomMatrix(rows, cols, 0, 1, sparse?sparsity2:sparsity1, 10); - - // 2. Convert the double arrays to MatrixBlock objects MatrixBlock A_mb = DataConverter.convertToMatrixBlock(A_data); - - // 3. Create a binary matrix writer MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(Types.FileFormat.BINARY); - - // 4. Write matrix A to a binary SequenceFile - writer.writeMatrixToHDFS(A_mb, input(INPUT_NAME), rows, cols, 1000, A_mb.getNonZeros()); + writer.writeMatrixToHDFS(A_mb, input(INPUT_NAME), rows, cols, blockSize, A_mb.getNonZeros()); HDFSTool.writeMetaDataFile(input(INPUT_NAME + ".mtd"), Types.ValueType.FP64, - new MatrixCharacteristics(rows, cols, 1000, A_mb.getNonZeros()), Types.FileFormat.BINARY); + new MatrixCharacteristics(rows, cols, blockSize, A_mb.getNonZeros()), Types.FileFormat.BINARY); + + programArgs = new String[] {"-stats", "-args", input(INPUT_NAME), output(OUTPUT_NAME_CP)}; + runTest(true, false, null, -1); + programArgs = new String[] {"-explain", "-stats", "-ooc", "-args", input(INPUT_NAME), + output(OUTPUT_NAME_OOC)}; runTest(true, false, null, -1); - //check tsmm OOC Assert.assertTrue("OOC wasn't used for TSMM", heavyHittersContainsString(Instruction.OOC_INST_PREFIX + Opcodes.TSMM)); - - //compare results - MatrixBlock ret1 = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME), - Types.FileFormat.BINARY, cols, cols, 1000, cols*cols); - MatrixBlock ret2 = new MatrixBlock(rows, rows, false); - A_mb.transposeSelfMatrixMultOperations(ret2, MMTSJ.MMTSJType.LEFT, k); - TestUtils.compareMatrices(ret1, ret2, eps); + + MatrixCharacteristics meta = readDMLMetaDataFile(OUTPUT_NAME_OOC); + int outputDim = assertOutputMetadata(type, meta, rows, cols, blockSize); + assertDeepMultiTileOutput(meta); + + MatrixBlock actual = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME_OOC), Types.FileFormat.BINARY, + outputDim, outputDim, blockSize); + MatrixBlock expected = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME_CP), Types.FileFormat.BINARY, + outputDim, outputDim, blockSize); + TestUtils.compareMatrices(actual, expected, eps); + assertSymmetricOffDiagonal(actual, outputDim, blockSize); } catch (IOException e) { throw new RuntimeException(e); @@ -114,4 +146,40 @@ private void runTSMMTest(int cols, boolean sparse ) resetExecMode(platformOld); } } + + private static int assertOutputMetadata(MMTSJType type, MatrixCharacteristics meta, int inputRows, int inputCols, + int blockSize) { + int outputDim = type.isLeft() ? inputCols : inputRows; + Assert.assertEquals(type + " TSMM output row metadata", outputDim, meta.getRows()); + Assert.assertEquals(type + " TSMM output column metadata", outputDim, meta.getCols()); + Assert.assertEquals(type + " TSMM output blocksize metadata", blockSize, meta.getBlocksize()); + return outputDim; + } + + private static void assertSymmetricOffDiagonal(MatrixBlock actual, int outputDim, int blockSize) { + if(outputDim <= blockSize) + return; + + int[] rows = new int[] {0, Math.min(blockSize - 1, outputDim - 1)}; + int[] cols = new int[] {blockSize, outputDim - 1}; + for(int row : rows) + for(int col : cols) + Assert.assertEquals(actual.get(row, col), actual.get(col, row), eps); + } + + private static void assertDeepMultiTileOutput(MatrixCharacteristics meta) { + if(meta.getRows() <= meta.getBlocksize()) + return; + + Assert.assertTrue("Multi-tile TSMM tests should cover at least three output row blocks", + meta.getNumRowBlocks() >= 3); + Assert.assertTrue("Multi-tile TSMM tests should cover at least three output column blocks", + meta.getNumColBlocks() >= 3); + } + + private void setDefaultBlockSizeInConfig(int blockSize) throws IOException { + DMLConfig config = new DMLConfig(getCurConfigFile().getPath()); + config.setTextValue(DMLConfig.DEFAULT_BLOCK_SIZE, String.valueOf(blockSize)); + Files.write(getCurConfigFile().toPath(), config.serializeDMLConfig().getBytes(StandardCharsets.UTF_8)); + } } diff --git a/src/test/scripts/functions/ooc/Covariance.dml b/src/test/scripts/functions/ooc/Covariance.dml new file mode 100644 index 00000000000..675f35f43cf --- /dev/null +++ b/src/test/scripts/functions/ooc/Covariance.dml @@ -0,0 +1,28 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +A = read($1); +B = read($2); + +s = cov(A, B); +res = as.matrix(s); + +write(res, $3, format="binary"); diff --git a/src/test/scripts/functions/ooc/CovarianceWeights.dml b/src/test/scripts/functions/ooc/CovarianceWeights.dml new file mode 100644 index 00000000000..2c077a9699a --- /dev/null +++ b/src/test/scripts/functions/ooc/CovarianceWeights.dml @@ -0,0 +1,29 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +A = read($1); +B = read($2); +W = read($3); + +s = cov(A, B, W); +res = as.matrix(s); + +write(res, $4, format="binary"); diff --git a/src/test/scripts/functions/ooc/TSMM.dml b/src/test/scripts/functions/ooc/TSMM.dml index 432d2d9daab..2cea8f4226b 100644 --- a/src/test/scripts/functions/ooc/TSMM.dml +++ b/src/test/scripts/functions/ooc/TSMM.dml @@ -19,7 +19,7 @@ # #------------------------------------------------------------- -# Read input matrix and operator from command line args +# Read input matrix from command line args X = read($1); # Operation under test diff --git a/src/test/scripts/functions/ooc/TSMMRight.dml b/src/test/scripts/functions/ooc/TSMMRight.dml new file mode 100644 index 00000000000..37fd5d46b9d --- /dev/null +++ b/src/test/scripts/functions/ooc/TSMMRight.dml @@ -0,0 +1,28 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# Read input matrix from command line args +X = read($1); + +# Operation under test +res = X %*% t(X); + +write(res, $2, format="binary") From fd9c8454276fd3a5a8260ada7e8ad792a37c2b52 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Fri, 31 Jul 2026 20:31:48 +0200 Subject: [PATCH 097/132] [SYSTEMDS-3891] OOC Bugfixes and Improved Error Propagation --- .../ooc/SubscribableTaskQueue.java | 14 +++- .../ooc/memory/GlobalMemoryBroker.java | 6 ++ .../org/apache/sysds/utils/Statistics.java | 66 ++++++++++++++----- .../ooc/OOCInstructionUtilsTest.java | 16 ++++- 4 files changed, 82 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/SubscribableTaskQueue.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/SubscribableTaskQueue.java index 5400b6ba98f..9a449e8b331 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/SubscribableTaskQueue.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/SubscribableTaskQueue.java @@ -35,6 +35,7 @@ public class SubscribableTaskQueue extends LocalTaskQueue _lastDequeued = null; private CacheableData _cdata; @@ -146,6 +147,7 @@ public T dequeue() { _lastDequeued = deq; return deq.get(); } + _terminalDelivered.set(true); return null; } catch(InterruptedException e) { @@ -167,6 +169,8 @@ public OOCStream.QueueCallback dequeueCB() { onDeliveryFinished(); _lastDequeued = deq; } + else + _terminalDelivered.set(true); return deq == NO_MORE_TASKS ? null : deq; } catch(InterruptedException e) { @@ -239,8 +243,10 @@ private void onDeliveryFinished() { if(ctr == 0) { validateBlockCountOnClose(); Consumer> s = _subscriber; - if(s != null) + if(s != null) { s.accept(OOCStream.eos(_failure)); + _terminalDelivered.set(true); + } if(OOCWatchdog.WATCH) OOCWatchdog.registerClose(_watchdogId); @@ -250,12 +256,14 @@ private void onDeliveryFinished() { @Override public synchronized void propagateFailure(DMLRuntimeException re) { // Ignore late failures - if(_closed.get() && _availableCtr.get() == 0) + if(_terminalDelivered.get()) return; super.propagateFailure(re); Consumer> s = _subscriber; - if(s != null) + if(s != null) { s.accept(new SimpleQueueCallback<>(null, re)); + _terminalDelivered.set(true); + } } @Override diff --git a/src/main/java/org/apache/sysds/runtime/ooc/memory/GlobalMemoryBroker.java b/src/main/java/org/apache/sysds/runtime/ooc/memory/GlobalMemoryBroker.java index f7ad7b28577..a4e847deeef 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/memory/GlobalMemoryBroker.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/memory/GlobalMemoryBroker.java @@ -19,6 +19,8 @@ package org.apache.sysds.runtime.ooc.memory; +import org.apache.sysds.utils.Statistics; + import java.util.ArrayList; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; @@ -137,11 +139,14 @@ public void reservationBlocked(MemoryAllowance allowance, long bytes) { } private void runReclaim() { + Statistics.incrementOOCMemoryReclaimRun(); + long nanos = System.nanoTime(); try { long reclaimed = 0; for(MemoryAllowance allowance : _allowances) if(!allowance.isShutdown()) reclaimed += allowance.reclaimUnused(); + Statistics.accumulateOOCMemoryReclaimBytes(reclaimed); if(reclaimed == 0) return; @@ -155,6 +160,7 @@ private void runReclaim() { notifyReservationWaiters(); } finally { + Statistics.accumulateOOCMemoryReclaimTime(System.nanoTime() - nanos); if(shouldRetryReclaim()) RECLAIM_EXECUTOR.schedule(this::runReclaim, RECLAIM_RETRY_DELAY_MS, TimeUnit.MILLISECONDS); else { diff --git a/src/main/java/org/apache/sysds/utils/Statistics.java b/src/main/java/org/apache/sysds/utils/Statistics.java index 5102933911a..c257495c575 100644 --- a/src/main/java/org/apache/sysds/utils/Statistics.java +++ b/src/main/java/org/apache/sysds/utils/Statistics.java @@ -233,6 +233,9 @@ public Object getMeta(String key) { private static final LongAdder oocEvictionWriteCalls = new LongAdder(); private static final LongAdder oocEvictionWriteTimeNanos = new LongAdder(); private static final LongAdder oocEvictionWriteBytesSize = new LongAdder(); + private static final LongAdder oocMemoryReclaimRuns = new LongAdder(); + private static final LongAdder oocMemoryReclaimTime = new LongAdder(); + private static final LongAdder oocMemoryReclaimBytes = new LongAdder(); private static final AtomicLong oocStatsStartTime = new AtomicLong(System.nanoTime()); public static long getNoOfExecutedSPInst() { @@ -362,6 +365,9 @@ public static void resetOOCEvictionStats() { oocEvictionWriteCalls.reset(); oocEvictionWriteTimeNanos.reset(); oocEvictionWriteBytesSize.reset(); + oocMemoryReclaimRuns.reset(); + oocMemoryReclaimTime.reset(); + oocMemoryReclaimBytes.reset(); oocStatsStartTime.set(System.nanoTime()); } @@ -481,6 +487,18 @@ public static void accumulateOOCEvictionWriteBytes(long bytes) { oocEvictionWriteBytesSize.add(bytes); } + public static void incrementOOCMemoryReclaimRun() { + oocMemoryReclaimRuns.increment(); + } + + public static void accumulateOOCMemoryReclaimTime(long nanos) { + oocMemoryReclaimTime.add(nanos); + } + + public static void accumulateOOCMemoryReclaimBytes(long bytes) { + oocMemoryReclaimBytes.add(bytes); + } + public static String displayOOCEvictionStats() { long elapsedNanos = Math.max(1, System.nanoTime() - oocStatsStartTime.get()); double elapsedSeconds = elapsedNanos / 1e9; @@ -499,6 +517,9 @@ public static String displayOOCEvictionStats() { oocLoadFromDiskCalls.longValue(), oocLoadFromDiskTimeNanos.longValue() / 1e9, oocLoadFromDiskBytesSize.longValue() / 1e9)); sb.append(String.format(Locale.US, " evict writes:\t\t%d (time %.3f sec, %.3f GB)\n", oocEvictionWriteCalls.longValue(), oocEvictionWriteTimeNanos.longValue() / 1e9, oocEvictionWriteBytesSize.longValue() / 1e9)); + sb.append(String.format(Locale.US, " reclaim runs:\t\t%d (time %.3f sec, %.3f GB)\n", + oocMemoryReclaimRuns.longValue(), oocMemoryReclaimTime.longValue() / 1e9, + oocMemoryReclaimBytes.longValue() / 1e9)); return sb.toString(); } @@ -540,15 +561,15 @@ public static void reset() } public static void resetJITCompileTime(){ - jitCompileTime = -1 * getJITCompileTime(); + jitCompileTime = -1 * getCurrentJITCompileTime(); } public static void resetJVMgcTime(){ - jvmGCTime = -1 * getJVMgcTime(); + jvmGCTime = -1 * getCurrentJVMgcTime(); } public static void resetJVMgcCount(){ - jvmGCTime = -1 * getJVMgcCount(); + jvmGCCount = -1 * getCurrentJVMgcCount(); } public static void resetCPHeavyHitters(){ @@ -1117,38 +1138,53 @@ private static String byteCountToDisplaySize(double numBytes) { * @return JIT compile time */ public static long getJITCompileTime(){ - long ret = -1; //unsupported + long ret = getCurrentJITCompileTime(); + if(ret >= 0) + ret += jitCompileTime; // add from remote processes + return ret; + } + + private static long getCurrentJITCompileTime() { + long ret = -1; // unsupported CompilationMXBean cmx = ManagementFactory.getCompilationMXBean(); - if( cmx.isCompilationTimeMonitoringSupported() ) { + if(cmx.isCompilationTimeMonitoringSupported()) ret = cmx.getTotalCompilationTime(); - ret += jitCompileTime; //add from remote processes - } return ret; } public static long getJVMgcTime(){ - long ret = 0; + long ret = getCurrentJVMgcTime(); + if(ret > 0) + ret += jvmGCTime; + + return ret; + } + + private static long getCurrentJVMgcTime() { + long ret = 0; List gcxs = ManagementFactory.getGarbageCollectorMXBeans(); for( GarbageCollectorMXBean gcx : gcxs ) ret += gcx.getCollectionTime(); + return ret; + } + + public static long getJVMgcCount() { + long ret = getCurrentJVMgcCount(); if( ret>0 ) - ret += jvmGCTime; + ret += jvmGCCount; return ret; } - - public static long getJVMgcCount(){ - long ret = 0; + + private static long getCurrentJVMgcCount() { + long ret = 0; List gcxs = ManagementFactory.getGarbageCollectorMXBeans(); for( GarbageCollectorMXBean gcx : gcxs ) ret += gcx.getCollectionCount(); - if( ret>0 ) - ret += jvmGCCount; - return ret; } diff --git a/src/test/java/org/apache/sysds/test/component/ooc/OOCInstructionUtilsTest.java b/src/test/java/org/apache/sysds/test/component/ooc/OOCInstructionUtilsTest.java index f4bab5d9813..42ca04cbbb6 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/OOCInstructionUtilsTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/OOCInstructionUtilsTest.java @@ -26,11 +26,16 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import org.apache.sysds.common.Types.FileFormat; +import org.apache.sysds.common.Types.ValueType; import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.meta.MatrixCharacteristics; +import org.apache.sysds.runtime.meta.MetaDataFormat; import org.apache.sysds.runtime.ooc.cache.OOCFuture; import org.apache.sysds.runtime.ooc.store.MaterializedCallback; import org.apache.sysds.runtime.ooc.store.StoreLease; @@ -78,7 +83,9 @@ public void testSubmitTasksWaitsForAllStreams() throws Exception { @Test public void testSubmitTaskPropagatesFailure() throws Exception { - SubscribableTaskQueue output = new SubscribableTaskQueue<>(); + SubscribableTaskQueue output = new SubscribableTaskQueue<>(); + output.setData(new MatrixObject(ValueType.FP64, "/dev/null", + new MetaDataFormat(new MatrixCharacteristics(1, 1, 1), FileFormat.BINARY))); AtomicReference propagated = new AtomicReference<>(); output.setSubscriber(callback -> { try(callback) { @@ -92,6 +99,12 @@ public void testSubmitTaskPropagatesFailure() throws Exception { } } }); + try { + output.closeInput(); + Assert.fail("Expected block-count failure"); + } + catch(DMLRuntimeException expected) { + } OOCFuture completion = OOCInstructionUtils.submitOOCTask(() -> { throw new DMLRuntimeException("injected failure"); @@ -103,7 +116,6 @@ public void testSubmitTaskPropagatesFailure() throws Exception { catch(ExecutionException expected) { Assert.assertTrue(expected.getCause() instanceof DMLRuntimeException); } - output.closeInput(); Assert.assertNotNull(propagated.get()); Assert.assertEquals("injected failure", propagated.get().getMessage()); } From c833a3cf9d73232f337f1adebb13009076be2c0c Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:08:26 +0200 Subject: [PATCH 098/132] [SYSTEMDS-3891] OOC Add Materialized Planning Support --- .../instructions/ooc/CachingStream.java | 33 ++++- .../instructions/ooc/OOCStreamable.java | 10 ++ .../instructions/ooc/PlaybackStream.java | 7 +- .../runtime/ooc/planning/OOCPlanner.java | 29 ++++ .../runtime/ooc/planning/OOCStoreLayout.java | 36 +++++ .../ooc/primitives/MappingOOCPrimitive.java | 6 +- .../primitives/MaterializeOOCPrimitive.java | 117 +++++++++++++++ .../runtime/ooc/primitives/OOCPrimitive.java | 140 +++++++++++++++--- .../ooc/primitives/TransposeOOCPrimitive.java | 6 +- .../store/IndexedMaterializedStoreReader.java | 9 +- .../runtime/ooc/store/MaterializedStore.java | 53 ++++++- .../ooc/store/OOCStreamMaterializer.java | 13 +- .../store/OrderedMaterializedStoreReader.java | 9 +- .../sysds/runtime/ooc/store/StateTable.java | 33 +++-- .../sysds/runtime/ooc/store/StoreLease.java | 87 ++++++++--- .../ooc/stream/SourceOOCStreamable.java | 11 +- .../runtime/ooc/util/StateTableUtils.java | 11 +- .../ooc/OOCInstructionUtilsTest.java | 2 +- .../test/component/ooc/OOCPrimitiveTest.java | 82 +++++++++- .../component/ooc/StateTableUtilsTest.java | 10 +- 20 files changed, 613 insertions(+), 91 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/planning/OOCStoreLayout.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/primitives/MaterializeOOCPrimitive.java diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java index dab3eca3a0f..3b80175b73a 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/CachingStream.java @@ -79,6 +79,7 @@ public class CachingStream implements OOCStreamable { private boolean _deletable = false; private int _maxConsumptionCount = 0; + private int _lazyHandleReservations = 0; private String _watchdogId = null; public CachingStream(OOCStream source) { @@ -356,7 +357,7 @@ public synchronized void scheduleDeletion() { if (_deletable) return; // Deletion already scheduled - if (_cacheInProgress && _maxConsumptionCount == 0) + if(_cacheInProgress && _maxConsumptionCount == 0 && _lazyHandleReservations == 0) System.out.println("[WARN] Scheduling deletion for caching stream with no listeners: " + this); _deletable = true; @@ -370,6 +371,8 @@ public String toString() { } private synchronized void tryDeleteBlock(int i) { + if(_lazyHandleReservations > 0) + return; int cnt = _consumptionCounts.getInt(i); if (cnt > _maxConsumptionCount) throw new DMLRuntimeException("Cannot have more than " + _maxConsumptionCount + " consumptions."); @@ -582,6 +585,11 @@ public OOCStream getReadStream() { return new PlaybackStream(this); } + @Override + public OOCStream getReservedReadStream() { + return new PlaybackStream(this, consumeLazyHandleReservation()); + } + @Override public OOCStream getWriteStream() { return _source.getWriteStream(); @@ -721,6 +729,29 @@ public synchronized void incrSubscriberCount(int count) { _maxConsumptionCount += count; } + @Override + public synchronized void reserveLazyHandle() { + _lazyHandleReservations++; + } + + @Override + public synchronized void discardHandle() { + if(_lazyHandleReservations <= 0) + return; + _lazyHandleReservations--; + if(_deletable) + for(int i = 0; i < _consumptionCounts.size(); i++) + tryDeleteBlock(i); + } + + private synchronized boolean consumeLazyHandleReservation() { + if(_lazyHandleReservations <= 0) + return false; + _lazyHandleReservations--; + _maxConsumptionCount++; + return true; + } + /** * Artificially increase the processing count of a block. */ diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStreamable.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStreamable.java index 0f087a7f55b..10a4dc88174 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStreamable.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStreamable.java @@ -47,4 +47,14 @@ default OOCPrimitive getPrimitive() { default void assignPrimitive(OOCPrimitive primitive) { throw new UnsupportedOperationException("Stream does not support primitive assignment"); } + + default OOCStream getReservedReadStream() { + return getReadStream(); + } + + default void reserveLazyHandle() { + } + + default void discardHandle() { + } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/PlaybackStream.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/PlaybackStream.java index 167d7ba7aed..9b62405b460 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/PlaybackStream.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/PlaybackStream.java @@ -37,10 +37,15 @@ public class PlaybackStream implements OOCStream { private QueueCallback _lastDequeue; public PlaybackStream(CachingStream streamCache) { + this(streamCache, false); + } + + public PlaybackStream(CachingStream streamCache, boolean reserved) { this._streamCache = streamCache; this._streamIdx = new AtomicInteger(0); this._subscriberSet = new AtomicBoolean(false); - streamCache.incrSubscriberCount(1); + if(!reserved) + streamCache.incrSubscriberCount(1); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCPlanner.java b/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCPlanner.java index 3db13c2691a..38a5c609525 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCPlanner.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCPlanner.java @@ -25,10 +25,14 @@ import java.util.List; import java.util.Set; +import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.ooc.primitives.MaterializeOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; public final class OOCPlanner { public static void compile(OOCPrimitive root) { + injectMaterializations(root, Collections.newSetFromMap(new IdentityHashMap<>()), new IdentityHashMap<>()); List primitives = new ArrayList<>(); collect(root, Collections.newSetFromMap(new IdentityHashMap<>()), primitives); if(primitives.isEmpty()) @@ -44,6 +48,31 @@ public static void compile(OOCPrimitive root) { primitive.tryStartExecution(); } + @SuppressWarnings("unchecked") + private static void injectMaterializations(OOCPrimitive primitive, Set visited, + IdentityHashMap, MaterializeOOCPrimitive> boundaries) { + if(primitive.hasStartedExecution() || !visited.add(primitive)) + return; + for(OOCPrimitive.OOCMaterializedInputRequest request : primitive.requiredMaterializedInputs()) { + OOCStreamable input = (OOCStreamable) primitive + .getInput(request.inputIndex()); + MaterializeOOCPrimitive boundary = boundaries.compute(input, (k, v) -> { + if(v == null) { + MaterializeOOCPrimitive p = new MaterializeOOCPrimitive(input, request.layout(), + primitive.getContext()); + primitive.transferInputHandle(request.inputIndex()); + return p; + } + primitive.discardInputHandle(request.inputIndex()); + return v; + }); + boundary.registerRequest(request.expectedReaders()); + primitive.installMaterializedInput(request.inputIndex(), boundary); + } + for(OOCPrimitive child : primitive.getChildren()) + injectMaterializations(child, visited, boundaries); + } + private static void collect(OOCPrimitive primitive, Set visited, List result) { if(primitive.hasStartedExecution() || !visited.add(primitive)) return; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCStoreLayout.java b/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCStoreLayout.java new file mode 100644 index 00000000000..bafd5ec79f7 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCStoreLayout.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.planning; + +import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.meta.DataCharacteristics; + +public enum OOCStoreLayout { + ROW_MAJOR; + + public int linearize(MatrixIndexes indexes, DataCharacteristics characteristics) { + if(characteristics == null || !characteristics.dimsKnown() || characteristics.getBlocksize() <= 0) + throw new IllegalArgumentException("Materialized store layout requires known dimensions and block size."); + long columns = characteristics.getNumColBlocks(); + long index = Math.addExact(Math.multiplyExact(indexes.getRowIndex() - 1, columns), + indexes.getColumnIndex() - 1); + return Math.toIntExact(index); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/MappingOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/MappingOOCPrimitive.java index 345925cc83c..c4a96bc01dc 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/MappingOOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/MappingOOCPrimitive.java @@ -50,8 +50,8 @@ private MappingOOCPrimitive(OOCStream input, OOCStreamable c.requestPattern(accessPattern)); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/MaterializeOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/MaterializeOOCPrimitive.java new file mode 100644 index 00000000000..eaa9e8232c7 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/MaterializeOOCPrimitive.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.primitives; + +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.instructions.ooc.CachingStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; +import org.apache.sysds.runtime.ooc.planning.OOCStoreLayout; +import org.apache.sysds.runtime.ooc.store.MaterializedStore; +import org.apache.sysds.runtime.ooc.store.OOCStreamMaterializer; +import org.apache.sysds.runtime.ooc.stream.StreamContext; + +public final class MaterializeOOCPrimitive extends OOCPrimitive { + private final OOCStreamable _source; + private final OOCStoreLayout _layout; + private final OOCFuture> _store; + private final AtomicBoolean _finished; + private int _expectedReaders; + private int _consumers; + + public MaterializeOOCPrimitive(OOCStreamable source, OOCStoreLayout layout, + StreamContext context) { + super(context, source.getPrimitive() == null ? List.of() : List.of(source.getPrimitive())); + _source = source; + _layout = layout; + _store = new OOCFuture<>(); + _finished = new AtomicBoolean(); + } + + public synchronized void registerRequest(int expectedReaders) { + if(expectedReaders <= 0) + throw new IllegalArgumentException("Materialization request requires at least one reader."); + if(hasStartedExecution()) + throw new IllegalStateException("Cannot register a consumer after materialization started."); + _expectedReaders = Math.addExact(_expectedReaders, expectedReaders); + _consumers = Math.addExact(_consumers, 1); + } + + public OOCFuture> store() { + return _store; + } + + @Override + protected void inferPatternsInternal() { + _pattern = OOCAccessPattern.ROW_MAJOR; + for(OOCPrimitive child : getChildren()) + child.requestPattern(OOCAccessPattern.ROW_MAJOR); + inferParentPatterns(); + } + + @Override + protected void requestPatternInternal(OOCAccessPattern accessPattern) { + _pattern = OOCAccessPattern.ROW_MAJOR; + for(OOCPrimitive child : getChildren()) + child.requestPattern(OOCAccessPattern.ROW_MAJOR); + } + + @Override + protected void startExecution() { + try { + OOCStream source = _source.getReservedReadStream(); + MaterializedStore store = new MaterializedStore<>(OOCCacheManager.getGlobalCache(), + CachingStream._streamSeq.getNextID(), _expectedReaders, _consumers); + OOCStreamMaterializer materializer = new OOCStreamMaterializer(store, + indexes -> _layout.linearize(indexes, _source.getDataCharacteristics()), _allowance); + materializer.completion().whenComplete((ignored, error) -> { + if(error != null) + fail(error); + finish(); + }); + if(getContext() != null) + getContext().addInStream(source); + _store.complete(store); + materializer.attach(source); + } + catch(Throwable failure) { + _store.completeExceptionally(failure); + fail(failure); + finish(); + } + } + + private void fail(Throwable error) { + if(getContext() != null) + getContext().failAll(DMLRuntimeException.of(error)); + } + + private void finish() { + if(_finished.compareAndSet(false, true)) + onComplete(); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java index aacb35acdad..0bc20d54bd4 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java @@ -20,36 +20,55 @@ package org.apache.sysds.runtime.ooc.primitives; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Objects; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; import org.apache.sysds.runtime.ooc.memory.GlobalMemoryBroker; import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; import org.apache.sysds.runtime.ooc.memory.SyncMemoryAllowance; import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; import org.apache.sysds.runtime.ooc.planning.OOCPlanner; +import org.apache.sysds.runtime.ooc.planning.OOCStoreLayout; +import org.apache.sysds.runtime.ooc.store.MaterializedStore; import org.apache.sysds.runtime.ooc.stream.StreamContext; public abstract class OOCPrimitive { private final StreamContext _context; - private final List _children; - private final List _parents; + private final Set _children; + private final Set _parents; + private final List _inputs; private final AtomicBoolean _started; private final AtomicBoolean _executionStarted; protected OOCAccessPattern _pattern; protected MemoryAllowance _allowance; protected OOCPrimitive(StreamContext context, List children) { + this(context); + children.stream().filter(Objects::nonNull).forEach(child -> { + _children.add(child); + child._parents.add(this); + }); + } + + protected OOCPrimitive(StreamContext context, OOCStreamable... inputs) { + this(context); + for(OOCStreamable input : inputs) + _inputs.add(new InputSlot(input)); + rebuildInputChildren(); + } + + private OOCPrimitive(StreamContext context) { _context = context; - _parents = new ArrayList<>(); - List uniqueChildren = new ArrayList<>(children.size()); - for(OOCPrimitive child : children) { - if(containsIdentity(uniqueChildren, child)) - continue; - uniqueChildren.add(child); - child.addParent(this); - } - _children = List.copyOf(uniqueChildren); + _children = new HashSet<>(); + _parents = new HashSet<>(); + _inputs = new ArrayList<>(); _started = new AtomicBoolean(); _executionStarted = new AtomicBoolean(); _pattern = OOCAccessPattern.UNSET; @@ -59,17 +78,12 @@ public final StreamContext getContext() { return _context; } - public final List getChildren() { + public final Set getChildren() { return _children; } - public final List getParents() { - return List.copyOf(_parents); - } - - private void addParent(OOCPrimitive parent) { - if(!containsIdentity(_parents, parent)) - _parents.add(parent); + public final Set getParents() { + return _parents; } protected final void inferParentPatterns() { @@ -86,6 +100,59 @@ public final boolean hasStartedExecution() { return _executionStarted.get(); } + public List requiredMaterializedInputs() { + return List.of(); + } + + public final OOCStreamable getInput(int index) { + return _inputs.get(index)._source; + } + + public final OOCPrimitive getChildPrimitiveAt(int index) { + return _inputs.get(index)._primitive; + } + + public final void installMaterializedInput(int index, MaterializeOOCPrimitive boundary) { + if(hasStartedExecution()) + throw new IllegalStateException("Cannot replace an input after primitive execution started."); + InputSlot input = _inputs.get(index); + input._primitive = boundary; + rebuildInputChildren(); + } + + public final synchronized void transferInputHandle(int index) { + InputSlot input = _inputs.get(index); + if(!input._handleReserved) + throw new IllegalStateException("Input " + index + " no longer owns a lazy handle."); + input._handleReserved = false; + } + + public final void discardInputHandle(int index) { + OOCStreamable source; + synchronized(this) { + InputSlot input = _inputs.get(index); + if(!input._handleReserved) + return; + input._handleReserved = false; + source = input._source; + } + source.discardHandle(); + } + + @SuppressWarnings("unchecked") + protected final OOCStream getInputReadStream(int index) { + transferInputHandle(index); + return (OOCStream) _inputs.get(index)._source.getReservedReadStream(); + } + + protected final OOCFuture> getMaterializedInput(int index) { + OOCFuture> materialized = ((MaterializeOOCPrimitive) _inputs + .get(index)._primitive).store(); + if(materialized == null) + throw new IllegalStateException("Input " + index + " was not materialized by the planner."); + return materialized; + } + public final void start() { if(_started.compareAndSet(false, true)) OOCPlanner.compile(this); @@ -99,6 +166,8 @@ public final void tryStartExecution() { } public final void onComplete() { + for(int i = 0; i < _inputs.size(); i++) + discardInputHandle(i); _allowance.shutdown(); } @@ -112,11 +181,18 @@ public final void requestPattern(OOCAccessPattern accessPattern) { requestPatternInternal(accessPattern); } - private static boolean containsIdentity(List primitives, OOCPrimitive primitive) { - for(OOCPrimitive current : primitives) - if(current == primitive) - return true; - return false; + private void rebuildInputChildren() { + List next = new ArrayList<>(); + for(InputSlot input : _inputs) + if(input._primitive != null) + next.add(input._primitive); + for(OOCPrimitive child : _children) + if(!next.contains(child)) + child._parents.remove(this); + for(OOCPrimitive child : next) + child._parents.add(this); + _children.clear(); + _children.addAll(next); } protected abstract void startExecution(); @@ -124,4 +200,20 @@ private static boolean containsIdentity(List primitives, OOCPrimit protected abstract void inferPatternsInternal(); protected abstract void requestPatternInternal(OOCAccessPattern accessPattern); + + private static final class InputSlot { + private final OOCStreamable _source; + private OOCPrimitive _primitive; + private boolean _handleReserved; + + private InputSlot(OOCStreamable source) { + _source = source; + _primitive = source.getPrimitive(); + _handleReserved = true; + source.reserveLazyHandle(); + } + } + + public record OOCMaterializedInputRequest(int inputIndex, OOCStoreLayout layout, int expectedReaders) { + } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/TransposeOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/TransposeOOCPrimitive.java index 1cbe504546f..18b127bfd09 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/TransposeOOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/TransposeOOCPrimitive.java @@ -51,7 +51,7 @@ private TransposeOOCPrimitive(OOCStream input, OOCStreamable @Override protected void inferPatternsInternal() { - _pattern = (getChildren().isEmpty() ? OOCAccessPattern.ANY : getChildren().get(0).getAccessPattern()) + _pattern = (getChildren().isEmpty() ? OOCAccessPattern.ANY : getChildren().iterator().next().getAccessPattern()) .transposed(); inferParentPatterns(); } @@ -59,8 +59,8 @@ protected void inferPatternsInternal() { @Override protected void requestPatternInternal(OOCAccessPattern accessPattern) { _pattern = accessPattern; - if(!getChildren().isEmpty()) - getChildren().get(0).requestPattern(accessPattern.transposed()); + for(OOCPrimitive child : getChildren()) + child.requestPattern(accessPattern.transposed()); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/IndexedMaterializedStoreReader.java b/src/main/java/org/apache/sysds/runtime/ooc/store/IndexedMaterializedStoreReader.java index 3106705a34a..1c3a4c9005f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/store/IndexedMaterializedStoreReader.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/IndexedMaterializedStoreReader.java @@ -81,7 +81,7 @@ else if(entry == null) { result.complete(null); } else - result.complete(new StoreLease<>(entry, () -> release(index, entry, requestAllowance))); + result.complete(StoreLease.createAsync(entry, () -> release(index, entry, requestAllowance))); }); return result; } @@ -94,13 +94,14 @@ public StoreLease requestIfLive(int index, MemoryAllowance requestAllowance) _liveness.unreserve(index); return null; } - return new StoreLease<>(entry, () -> release(index, entry, requestAllowance)); + return StoreLease.createAsync(entry, () -> release(index, entry, requestAllowance)); } - private void release(int index, BlockEntry entry, MemoryAllowance requestAllowance) { - _cache.unpin(entry, requestAllowance); + private OOCFuture release(int index, BlockEntry entry, MemoryAllowance requestAllowance) { + OOCCache.UnpinHandle unpin = _cache.unpin(entry, requestAllowance); _liveness.consumed(index); _afterRelease.accept(index); + return unpin.getCompletionFuture(); } private void reserve(int index) { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedStore.java b/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedStore.java index d1ed01ffa3e..aab137d8298 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedStore.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedStore.java @@ -41,20 +41,38 @@ public final class MaterializedStore { private final BitSet _forgotten; private final AtomicInteger _published; private final AtomicInteger _publishedCount; + private final OOCFuture _completion; + private final OOCFuture _readersSealedFuture; + private final boolean _autoSealReaders; private volatile List _readers; private volatile int _completedSize; private volatile boolean _complete; private volatile boolean _readersSealed; private volatile boolean _closed; + private int _pendingReaders; + private int _consumers; public MaterializedStore(OOCCache cache, long streamId) { + this(cache, streamId, -1, 1); + } + + public MaterializedStore(OOCCache cache, long streamId, int expectedReaders, int consumers) { + if(expectedReaders == 0 || expectedReaders < -1) + throw new IllegalArgumentException("Expected reader count must be positive or disabled."); + if(consumers <= 0) + throw new IllegalArgumentException("Materialized store requires at least one consumer."); _cache = cache; _streamId = streamId; _registeredReaders = new ArrayList<>(); _forgotten = new BitSet(); _published = new AtomicInteger(); _publishedCount = new AtomicInteger(); + _completion = new OOCFuture<>(); + _readersSealedFuture = new OOCFuture<>(); + _autoSealReaders = expectedReaders > 0; + _pendingReaders = expectedReaders; + _consumers = consumers; _readers = Collections.emptyList(); } @@ -74,9 +92,10 @@ StoreLease publishPinnedLive(int index, T value, long bytes, MemoryAllowance } _publishedCount.incrementAndGet(); updatePublished(index + 1); - return new StoreLease<>(entry, () -> { - _cache.unpin(entry, allowance); + return StoreLease.createAsync(entry, () -> { + OOCCache.UnpinHandle unpin = _cache.unpin(entry, allowance); tryForget(index); + return unpin.getCompletionFuture(); }); } @@ -93,6 +112,19 @@ public synchronized void complete() { throw new IllegalStateException("Incomplete publication: " + _publishedCount.get() + " published items for logical range [0, " + _completedSize + ")"); _complete = true; + _completion.complete(null); + } + + void failMaterialization(Throwable error) { + _completion.completeExceptionally(error); + } + + public OOCFuture completion() { + return _completion; + } + + public OOCFuture readersSealed() { + return _readersSealedFuture; } public synchronized OrderedMaterializedStoreReader openReader(AccessPattern pattern, MemoryAllowance allowance, @@ -109,6 +141,7 @@ public synchronized OrderedMaterializedStoreReader openReader(AccessPattern p OrderedMaterializedStoreReader reader = new OrderedMaterializedStoreReader<>(_cache, _streamId, pattern, allowance, Math.max(1, maxPrefetch), softOrdering, this::forgetAfterReaderClose, this::tryForget); _registeredReaders.add(reader); + readerRegistered(); return reader; } @@ -120,6 +153,7 @@ public synchronized IndexedMaterializedStoreReader openIndexedReader(Liveness IndexedMaterializedStoreReader reader = new IndexedMaterializedStoreReader<>(_cache, _streamId, () -> _completedSize, liveness, this::forgetAfterReaderClose, this::tryForget); _registeredReaders.add(reader); + readerRegistered(); return reader; } @@ -136,7 +170,8 @@ public OOCFuture> requestPublished(int index, MemoryAllowance allo else if(entry == null) result.complete(null); else - result.complete(new StoreLease<>(entry, () -> _cache.unpin(entry, allowance))); + result.complete( + StoreLease.createAsync(entry, () -> _cache.unpin(entry, allowance).getCompletionFuture())); }); return result; } @@ -151,6 +186,16 @@ public synchronized void sealReaders() { int publishedSize = _complete ? _completedSize : _published.get(); for(int i = 0; i < publishedSize; i++) tryForget(i); + _readersSealedFuture.complete(null); + } + + private void readerRegistered() { + if(!_autoSealReaders) + return; + if(_pendingReaders <= 0) + throw new IllegalStateException("More materialized readers opened than declared."); + if(--_pendingReaders == 0) + sealReaders(); } public int size() { @@ -162,6 +207,8 @@ public void close() { synchronized(this) { if(_closed) return; + if(--_consumers > 0) + return; _closed = true; localReaders = _readersSealed ? _readers : new ArrayList<>(_registeredReaders); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/OOCStreamMaterializer.java b/src/main/java/org/apache/sysds/runtime/ooc/store/OOCStreamMaterializer.java index 2cede71f804..3462c2cd3af 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/store/OOCStreamMaterializer.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/OOCStreamMaterializer.java @@ -57,7 +57,14 @@ public OOCStreamMaterializer(MaterializedStore store, ToIntF } public void attach(OOCStream source) { - source.setSubscriber(this); + try { + source.setSubscriber(this); + } + catch(Throwable failure) { + DMLRuntimeException wrapped = DMLRuntimeException.of(failure); + fail(wrapped); + throw wrapped; + } } public OOCFuture completion() { @@ -106,7 +113,7 @@ private void publish(OOCStream.QueueCallback callback) { } try(lease) { for(Consumer> liveConsumer : _liveConsumers) { - try(OOCStream.QueueCallback alias = new MaterializedCallback(lease.retain())) { + try(OOCStream.QueueCallback alias = new MaterializedCallback<>(lease.retain())) { liveConsumer.accept(alias); } } @@ -120,6 +127,7 @@ private void finish() { _store.complete(); } catch(RuntimeException ex) { + _store.failMaterialization(ex); deliverEos(DMLRuntimeException.of(ex)); _completion.completeExceptionally(ex); return; @@ -131,6 +139,7 @@ private void finish() { private void fail(DMLRuntimeException failure) { if(!_done.compareAndSet(false, true)) return; + _store.failMaterialization(failure); deliverEos(failure); _completion.completeExceptionally(failure); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/OrderedMaterializedStoreReader.java b/src/main/java/org/apache/sysds/runtime/ooc/store/OrderedMaterializedStoreReader.java index 2dd048750c1..dae4ebaadd2 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/store/OrderedMaterializedStoreReader.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/OrderedMaterializedStoreReader.java @@ -140,13 +140,14 @@ public StoreLease next() throws InterruptedException { throw ex; } } - return new StoreLease<>(entry, () -> release(request._index, entry)); + return StoreLease.createAsync(entry, () -> release(request._index, entry)); } - public void release(int index, BlockEntry entry) { - _cache.unpin(entry, _allowance); + public OOCFuture release(int index, BlockEntry entry) { + OOCCache.UnpinHandle unpin = _cache.unpin(entry, _allowance); _pattern.consumed(index); _afterRelease.accept(index); + return unpin.getCompletionFuture(); } private void checkReady() { @@ -172,7 +173,7 @@ private StoreLease nextSoft() throws InterruptedException { if(entry == null) throw new IllegalStateException("Reader is closed"); fillSoft(); - return new StoreLease<>(entry, () -> release(request._index, entry)); + return StoreLease.createAsync(entry, () -> release(request._index, entry)); } private void fillStrict() { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/StateTable.java b/src/main/java/org/apache/sysds/runtime/ooc/store/StateTable.java index 80eb57dcfe4..b70a7f4988a 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/store/StateTable.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/StateTable.java @@ -63,8 +63,16 @@ public StateTable(OOCCache cache, long streamId, int numSlots) { public void addEvictionPolicy(IntToLongFunction slotPolicy) { _evictionPolicies.add(slotPolicy); - if(_evictionPolicyInstalled.compareAndSet(false, true)) + if(_evictionPolicyInstalled.compareAndSet(false, true)) { + synchronized(this) { + for(int index = 0; index < _slots.length; index++) { + Slot slot = _slots[index]; + if(slot != null && slot._putFuture == null && slot._tableOwnedKey) + registerGeneration(index, slot._key); + } + } _cache.addEvictionPolicy(_streamId, this::scoreTableEntry); + } } public void put(int index, ManagedPayload payload) { @@ -169,8 +177,8 @@ public OOCFuture> acquire(int index, MemoryAllowance leaseAllowanc if(error != null) result.completeExceptionally(error); else - result.complete( - entry == null ? null : new StoreLease<>(entry, () -> _cache.unpin(entry, leaseAllowance))); + result.complete(entry == null ? null : StoreLease.createAsync(entry, + () -> _cache.unpin(entry, leaseAllowance).getCompletionFuture())); }); return result; } @@ -187,7 +195,8 @@ public StoreLease peek(int index, MemoryAllowance leaseAllowance) { key = slot._key; } BlockEntry entry = _cache.pinIfLive(key.getStreamId(), key.getSequenceNumber(), leaseAllowance); - return entry == null ? null : new StoreLease<>(entry, () -> _cache.unpin(entry, leaseAllowance)); + return entry == null ? null : StoreLease.createAsync(entry, + () -> _cache.unpin(entry, leaseAllowance).getCompletionFuture()); } public void clear(int index) { @@ -255,9 +264,8 @@ private void finalizeOwnedPut(int index, Slot slot, ManagedPayload payload) { synchronized(this) { slot._key = key; slot._tableOwnedKey = true; - int generation = blockIndex(key.getSequenceNumber()); - ensureGenerationCapacity(generation); - _generationSlots.set(generation, index + 1); + if(_evictionPolicyInstalled.get()) + registerGeneration(index, key); cleared = slot._cleared; putFuture = slot._putFuture; slot._putFuture = null; @@ -329,13 +337,14 @@ private OOCFuture> pinTaken(Slot slot, MemoryAllowance leaseAllowa result.completeExceptionally(completionError); return; } - result.complete(new StoreLease<>(entry, () -> _cache.unpin(entry, leaseAllowance))); + result.complete( + StoreLease.createAsync(entry, () -> _cache.unpin(entry, leaseAllowance).getCompletionFuture())); }); return result; } private void releaseSlot(Slot slot) { - if(slot._tableOwnedKey) { + if(slot._tableOwnedKey && _evictionPolicyInstalled.get()) { int generation = blockIndex(slot._key.getSequenceNumber()); AtomicIntegerArray slots = _generationSlots; if(generation < slots.length()) @@ -344,6 +353,12 @@ private void releaseSlot(Slot slot) { _cache.dereference(slot._key); } + private void registerGeneration(int index, BlockKey key) { + int generation = blockIndex(key.getSequenceNumber()); + ensureGenerationCapacity(generation); + _generationSlots.set(generation, index + 1); + } + private long scoreTableEntry(long generation) { int index = blockIndex(generation); AtomicIntegerArray slots = _generationSlots; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/StoreLease.java b/src/main/java/org/apache/sysds/runtime/ooc/store/StoreLease.java index c9d4ca6f8cd..abc6d20079f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/store/StoreLease.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/StoreLease.java @@ -19,32 +19,43 @@ package org.apache.sysds.runtime.ooc.store; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + import org.apache.sysds.runtime.ooc.cache.BlockEntry; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; -import java.util.concurrent.atomic.AtomicInteger; - public final class StoreLease implements AutoCloseable { - private final Runnable _releaser; private final T _value; private final BlockEntry _entry; - private final AtomicInteger _shared; + private final SharedStoreLease _sharedLease; private boolean _open; - public StoreLease(BlockEntry entry, Runnable releaser) { - this(null, entry, releaser, new AtomicInteger(1)); + private StoreLease(T value, BlockEntry entry, SharedStoreLease release) { + _value = value; + _entry = entry; + _sharedLease = release; + _open = true; } - public StoreLease(T value, Runnable releaser) { - this(value, null, releaser, new AtomicInteger(1)); + public static StoreLease create(T value, Runnable releaser) { + return new StoreLease<>(value, null, new SharedStoreLease(() -> { + releaser.run(); + return OOCFuture.completed(null); + }, new AtomicInteger(1), new OOCFuture<>())); } - private StoreLease(T value, BlockEntry entry, Runnable releaser, AtomicInteger shared) { - _releaser = releaser; - _value = value; - _entry = entry; - _shared = shared; - _open = true; + public static StoreLease create(BlockEntry entry, Runnable releaser) { + return new StoreLease<>(null, entry, new SharedStoreLease(() -> { + releaser.run(); + return OOCFuture.completed(null); + }, new AtomicInteger(1), new OOCFuture<>())); + } + + public static StoreLease createAsync(BlockEntry entry, + Supplier> releaser) { + return new StoreLease<>(null, entry, new SharedStoreLease(releaser, new AtomicInteger(1), new OOCFuture<>())); } @SuppressWarnings("unchecked") @@ -63,16 +74,48 @@ synchronized BlockEntry entry() { public synchronized StoreLease retain() { if(!_open) throw new IllegalStateException("Lease is closed"); - _shared.incrementAndGet(); - return new StoreLease<>(_value, _entry, _releaser, _shared); + _sharedLease.references.incrementAndGet(); + return new StoreLease<>(_value, _entry, _sharedLease); + } + + public OOCFuture closeAsync() { + boolean release; + synchronized(this) { + if(!_open) + return _sharedLease.future; + _open = false; + release = _sharedLease.references.decrementAndGet() == 0; + } + if(release) { + OOCFuture released; + try { + released = _sharedLease.releaser.get(); + } + catch(Throwable error) { + _sharedLease.future.completeExceptionally(error); + return _sharedLease.future; + } + if(released == null) { + _sharedLease.future + .completeExceptionally(new NullPointerException("Asynchronous lease releaser returned null")); + return _sharedLease.future; + } + released.whenComplete((ignored, error) -> { + if(error == null) + _sharedLease.future.complete(null); + else + _sharedLease.future.completeExceptionally(error); + }); + } + return _sharedLease.future; } @Override - public synchronized void close() { - if(!_open) - return; - _open = false; - if(_shared.decrementAndGet() == 0) - _releaser.run(); + public void close() { + closeAsync(); + } + + private record SharedStoreLease(Supplier> releaser, AtomicInteger references, + OOCFuture future) { } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/stream/SourceOOCStreamable.java b/src/main/java/org/apache/sysds/runtime/ooc/stream/SourceOOCStreamable.java index 4c9ca3683af..f8672701880 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/stream/SourceOOCStreamable.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/stream/SourceOOCStreamable.java @@ -28,6 +28,7 @@ public class SourceOOCStreamable implements OOCStreamable { private final CacheableData _data; + private OOCStream _reservedReadStream; public SourceOOCStreamable(CacheableData data) { _data = data; @@ -35,12 +36,12 @@ public SourceOOCStreamable(CacheableData data) { @Override public OOCStream getReadStream() { - return _data.getStreamHandle(); + return _reservedReadStream != null ? _reservedReadStream : _data.getStreamHandle(); } @Override public OOCStream getWriteStream() { - return _data.getStreamHandle(); + return _reservedReadStream != null ? _reservedReadStream : _data.getStreamHandle(); } @Override @@ -72,4 +73,10 @@ public CacheableData getData() { public void setData(CacheableData data) { throw new UnsupportedOperationException(); } + + @Override + public synchronized void reserveLazyHandle() { + if(_reservedReadStream == null) + _reservedReadStream = _data.getStreamHandle(); + } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/StateTableUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/StateTableUtils.java index a6aecd40c71..641192c16a6 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/util/StateTableUtils.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/StateTableUtils.java @@ -33,7 +33,7 @@ public final class StateTableUtils { public static OOCFuture putOrTake(StateTable table, int slot, OOCStream.QueueCallback tile, MemoryAllowance allowance) { - if(tile instanceof MaterializedCallback pinned && pinned.pinnedEntry() != null) + if(tile instanceof MaterializedCallback pinned && pinned.pinnedEntry() != null) return putReferenceOrTake(table, slot, pinned, allowance); ManagedPayload payload; if(tile instanceof InMemoryQueueCallback managed && managed.getManagedBytes() > 0) { @@ -64,14 +64,15 @@ public static OOCFuture putOrTake(StateTable table, i else if(lease == null) result.complete(null); else - result.complete(new Match(new MaterializedCallback(new StoreLease<>(payload.value(), payload::release)), - new MaterializedCallback(lease))); + result.complete( + new Match(new MaterializedCallback<>(StoreLease.create(payload.value(), payload::release)), + new MaterializedCallback<>(lease))); }); return result; } private static OOCFuture putReferenceOrTake(StateTable table, int slot, - MaterializedCallback pinned, MemoryAllowance allowance) { + MaterializedCallback pinned, MemoryAllowance allowance) { OOCFuture result = new OOCFuture<>(); OOCFuture> matched; try { @@ -91,7 +92,7 @@ else if(lease == null) { result.complete(null); } else - result.complete(new Match(pinned, new MaterializedCallback(lease))); + result.complete(new Match(pinned, new MaterializedCallback<>(lease))); }); return result; } diff --git a/src/test/java/org/apache/sysds/test/component/ooc/OOCInstructionUtilsTest.java b/src/test/java/org/apache/sysds/test/component/ooc/OOCInstructionUtilsTest.java index 42ca04cbbb6..a889cc73e83 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/OOCInstructionUtilsTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/OOCInstructionUtilsTest.java @@ -56,7 +56,7 @@ public void testSubmitTasksClosesCallbacksAfterCompletion() throws Exception { }, new StreamContext().addOutStream()); IndexedMatrixValue value = new IndexedMatrixValue(new MatrixIndexes(1, 1), new MatrixBlock(1, 1, 1.0)); - source.enqueue(new MaterializedCallback<>(new StoreLease<>(value, released::incrementAndGet))); + source.enqueue(new MaterializedCallback<>(StoreLease.create(value, released::incrementAndGet))); source.closeInput(); completion.get(10, TimeUnit.SECONDS); diff --git a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java index be6886ce1a6..7a9c6ba1e51 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java @@ -22,12 +22,14 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.sysds.common.Types.FileFormat; import org.apache.sysds.common.Types.ValueType; import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; import org.apache.sysds.runtime.instructions.ooc.CachingStream; import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.matrix.data.MatrixBlock; @@ -36,7 +38,11 @@ import org.apache.sysds.runtime.meta.MetaDataFormat; import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; +import org.apache.sysds.runtime.ooc.planning.OOCStoreLayout; +import org.apache.sysds.runtime.ooc.primitives.MaterializeOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; +import org.apache.sysds.runtime.ooc.store.CountingLiveness; +import org.apache.sysds.runtime.ooc.store.IndexedMaterializedStoreReader; import org.apache.sysds.runtime.ooc.stream.FilteredOOCStream; import org.apache.sysds.runtime.ooc.stream.StreamContext; import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; @@ -64,8 +70,8 @@ public void testGraphPatternsAndExecution() { TestPrimitive source = new TestPrimitive(List.of()); TestPrimitive sink = new TestPrimitive(List.of(source, source)); - Assert.assertEquals(List.of(source), sink.getChildren()); - Assert.assertEquals(List.of(sink), source.getParents()); + Assert.assertEquals(Set.of(source), sink.getChildren()); + Assert.assertEquals(Set.of(sink), source.getParents()); source.inferPatterns(); Assert.assertEquals(OOCAccessPattern.ANY, source.getAccessPattern()); Assert.assertEquals(OOCAccessPattern.ANY, sink.getAccessPattern()); @@ -87,6 +93,30 @@ public void testGraphPatternsAndExecution() { Assert.assertEquals(OOCAccessPattern.ROW_MAJOR, sink.getAccessPattern()); } + @Test + public void testPlannerDoubleMaterialize() { + OOCCacheManager.reset(); + try { + SubscribableTaskQueue source = new SubscribableTaskQueue<>(); + source.setData(new MatrixObject(ValueType.FP64, "/dev/null", + new MetaDataFormat(new MatrixCharacteristics(0, 0, 1), FileFormat.BINARY))); + CachingStream cached = new CachingStream(source); + source.closeInput(); + MaterializingTestPrimitive sink = new MaterializingTestPrimitive(cached); + cached.getReadStream().setSubscriber(OOCStream.QueueCallback::close); + cached.scheduleDeletion(); + + sink.start(); + + Assert.assertEquals(1, sink.getChildren().size()); + Assert.assertTrue(sink.getChildPrimitiveAt(0) instanceof MaterializeOOCPrimitive); + Assert.assertEquals(1, sink._executions); + } + finally { + OOCCacheManager.reset(); + } + } + @Test public void testDataGenMapTransposePipeline() { SubscribableTaskQueue generated = new SubscribableTaskQueue<>(); @@ -156,6 +186,54 @@ public void testJoinOutOfOrder() { cachedLeft.scheduleDeletion(); } + private static final class MaterializingTestPrimitive extends OOCPrimitive { + private int _executions; + + private MaterializingTestPrimitive(OOCStreamable source) { + super(new StreamContext(), source, source); + } + + @Override + public List requiredMaterializedInputs() { + return List.of(new OOCMaterializedInputRequest(0, OOCStoreLayout.ROW_MAJOR, 1), + new OOCMaterializedInputRequest(1, OOCStoreLayout.ROW_MAJOR, 1)); + } + + @Override + protected void startExecution() { + getMaterializedInput(0).whenComplete((store, error) -> { + if(error != null) + return; + store.completion().whenComplete((ignored, completionError) -> { + if(completionError != null) + return; + IndexedMaterializedStoreReader first = store + .openIndexedReader(new CountingLiveness(0, 0)); + IndexedMaterializedStoreReader second = store + .openIndexedReader(new CountingLiveness(0, 0)); + Assert.assertTrue(store.readersSealed().isDone()); + first.close(); + second.close(); + store.close(); + store.close(); + _executions++; + onComplete(); + }); + }); + } + + @Override + protected void inferPatternsInternal() { + _pattern = OOCAccessPattern.ANY; + inferParentPatterns(); + } + + @Override + protected void requestPatternInternal(OOCAccessPattern accessPattern) { + _pattern = accessPattern; + } + } + private static final class TestPrimitive extends OOCPrimitive { private int _executions; diff --git a/src/test/java/org/apache/sysds/test/component/ooc/StateTableUtilsTest.java b/src/test/java/org/apache/sysds/test/component/ooc/StateTableUtilsTest.java index 7debcd04e3a..5ecbb070b48 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/StateTableUtilsTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/StateTableUtilsTest.java @@ -78,7 +78,7 @@ public void testCallbackPutOrTake() throws Exception { _source.put(0, new ManagedPayload<>(tile(1.0), TILE_BYTES, _producer)); StoreLease pinned = _source.peek(0, _reader); Assert.assertNotNull(pinned); - Assert.assertNull(StateTableUtils.putOrTake(_table, 0, new MaterializedCallback(pinned), _reader) + Assert.assertNull(StateTableUtils.putOrTake(_table, 0, new MaterializedCallback<>(pinned), _reader) .get(WAIT_SECONDS, TimeUnit.SECONDS)); Assert.assertEquals(0, _reader.getUsedMemory()); @@ -122,10 +122,10 @@ public void testStateTableLifecycle() throws Exception { Assert.assertNotNull(lease); Assert.assertEquals(5.0, lease.value().getValue().get(0, 0), 0.0); } - try(StoreLease lease = _table.take(0, _reader).get(WAIT_SECONDS, TimeUnit.SECONDS)) { - Assert.assertNotNull(lease); - Assert.assertEquals(5.0, lease.value().getValue().get(0, 0), 0.0); - } + StoreLease taken = _table.take(0, _reader).get(WAIT_SECONDS, TimeUnit.SECONDS); + Assert.assertNotNull(taken); + Assert.assertEquals(5.0, taken.value().getValue().get(0, 0), 0.0); + taken.closeAsync().get(WAIT_SECONDS, TimeUnit.SECONDS); Assert.assertNull(_table.take(0, _reader).get(WAIT_SECONDS, TimeUnit.SECONDS)); _producer.reserveBlocking(TILE_BYTES); From e61b9d430301a4a97fb3d9f40186328ded640daf Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:27:17 +0200 Subject: [PATCH 099/132] [SYSTEMDS-3891] Add Broadcast and Grouped Reduction Primitives --- .../instructions/ooc/MMultOOCInstruction.java | 30 ++ .../sysds/runtime/ooc/cache/OOCCache.java | 8 + .../ooc/cache/packed/OOCPackedCache.java | 8 + .../runtime/ooc/memory/ReservationBudget.java | 18 +- .../ooc/primitives/BroadcastOOCPrimitive.java | 318 ++++++++++++++++ .../primitives/GroupedReduceOOCPrimitive.java | 359 ++++++++++++++++++ .../runtime/ooc/util/OOCInstructionUtils.java | 18 + .../sysds/runtime/ooc/util/OOCUtils.java | 8 +- .../ooc/memory/OOCMemoryAllowanceTest.java | 9 + 9 files changed, 772 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/primitives/BroadcastOOCPrimitive.java create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/primitives/GroupedReduceOOCPrimitive.java diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/MMultOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/MMultOOCInstruction.java index 81f1102811d..d176a0c4184 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/MMultOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/MMultOOCInstruction.java @@ -33,6 +33,9 @@ import org.apache.sysds.runtime.matrix.operators.AggregateOperator; import org.apache.sysds.runtime.matrix.operators.BinaryOperator; import org.apache.sysds.runtime.matrix.operators.Operator; +import org.apache.sysds.runtime.meta.DataCharacteristics; +import org.apache.sysds.runtime.ooc.store.CountingLiveness; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; public class MMultOOCInstruction extends ComputationOOCInstruction { @@ -60,6 +63,33 @@ public void processInstruction( ExecutionContext ec ) { // 1. Identify the inputs MatrixObject min = ec.getMatrixObject(input1); // big matrix MatrixObject vin = ec.getMatrixObject(input2); // streamed vector + DataCharacteristics mdc = min.getDataCharacteristics(); + DataCharacteristics vdc = vin.getDataCharacteristics(); + + if(min != vin && mdc.getRows() > 0 && mdc.getCols() > 0 && vdc.getCols() > 0 && + mdc.getCols() == vdc.getRows() && vdc.getNumColBlocks() == 1) { + OOCStream partials = createWritableStream(); + OOCStream out = createWritableStream(); + partials.setData(min); + ec.getMatrixObject(output).setStreamHandle(out); + OOCInstructionUtils.indexedBroadcastMap(min.getStreamable(), vin.getStreamable(), partials, + left -> Math.toIntExact(left.getIndexes().getColumnIndex() - 1), + () -> new CountingLiveness(Math.toIntExact(vin.getDataCharacteristics().getNumRowBlocks()), + Math.toIntExact(min.getDataCharacteristics().getNumRowBlocks())), + (left, right) -> { + MatrixBlock leftBlock = (MatrixBlock) left.getValue(); + MatrixBlock rightBlock = (MatrixBlock) right.getValue(); + MatrixBlock partial = leftBlock.aggregateBinaryOperations(leftBlock, rightBlock, new MatrixBlock(), + (AggregateBinaryOperator) _optr); + MatrixIndexes indexes = left.getIndexes(); + return new IndexedMatrixValue(new MatrixIndexes(indexes.getRowIndex(), indexes.getColumnIndex()), + partial); + }, getContext()); + BinaryOperator plus = InstructionUtils.parseBinaryOperator(Opcodes.PLUS.toString()); + OOCInstructionUtils.rowGroupedReduce(partials, out, + (left, right) -> left.binaryOperations(plus, right, new MatrixBlock()), getContext()); + return; + } int emitLeftThreshold = (int)vin.getDataCharacteristics().getNumColBlocks(); int emitRightThreshold = (int)min.getDataCharacteristics().getNumRowBlocks(); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCache.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCache.java index aec88891595..8bbdb0e24de 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCache.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCache.java @@ -24,6 +24,14 @@ import java.util.function.LongUnaryOperator; public interface OOCCache { + /** + * Maximum bytes charged given the logical bytes of the requested entry. Use this method for reservation budget + * planning as logical byte size and pinned entry bytes may differ. + */ + default long maxPhysicalPinBytes(long logicalBytes) { + return logicalBytes; + } + /** * Pins an item backed by an allowance. A successful pin transfers memory ownership from the cache to the owner of * the allowance and guarantees data availability. While pinned, the bytes of the entry are not counted as diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/OOCPackedCache.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/OOCPackedCache.java index 9bbdcbeae5a..ad08c637bcf 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/OOCPackedCache.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/OOCPackedCache.java @@ -129,6 +129,14 @@ public OOCPackedCache(OOCCacheImpl physical, long packThresholdBytes, long packT }); } + @Override + public long maxPhysicalPinBytes(long logicalBytes) { + if(logicalBytes >= _packThresholdBytes) + return logicalBytes; + return _packTargetBytes > Long.MAX_VALUE - _packThresholdBytes ? Long.MAX_VALUE : _packTargetBytes + + _packThresholdBytes; + } + @Override public BlockEntry putPinned(long sId, long tId, Object data, long size, MemoryAllowance allowance) { if(size >= _packThresholdBytes) diff --git a/src/main/java/org/apache/sysds/runtime/ooc/memory/ReservationBudget.java b/src/main/java/org/apache/sysds/runtime/ooc/memory/ReservationBudget.java index 47e08202dfb..26bf4477f6e 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/memory/ReservationBudget.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/memory/ReservationBudget.java @@ -26,6 +26,7 @@ public final class ReservationBudget implements MemoryAllowance, AutoCloseable { private long _outstanding; private long _available; private boolean _closed; + private boolean _reusable; public ReservationBudget(MemoryAllowance parent, long bytes) { if(parent == null) @@ -37,6 +38,13 @@ public ReservationBudget(MemoryAllowance parent, long bytes) { _available = bytes; } + public synchronized ReservationBudget enableReuse() { + if(_closed || getUsedMemory() != 0) + throw new IllegalStateException("Budget reuse must be enabled before reserving memory."); + _reusable = true; + return this; + } + @Override public synchronized boolean tryReserve(long bytes) { checkNonNegative(bytes); @@ -64,13 +72,19 @@ public void release(long bytes) { checkNonNegative(bytes); if(bytes == 0) return; + boolean releaseParent; synchronized(this) { long used = _outstanding - _available; if(bytes > used) throw new IllegalStateException("Cannot release " + bytes + " bytes from a budget using " + used); - _outstanding -= bytes; + releaseParent = _closed || !_reusable; + if(releaseParent) + _outstanding -= bytes; + else + _available += bytes; } - _parent.release(bytes); + if(releaseParent) + _parent.release(bytes); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/BroadcastOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/BroadcastOOCPrimitive.java new file mode 100644 index 00000000000..e2a9be63c25 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/BroadcastOOCPrimitive.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.primitives; + +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; +import java.util.function.Supplier; +import java.util.function.ToIntFunction; + +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; +import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.memory.ReservationBudget; +import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; +import org.apache.sysds.runtime.ooc.planning.OOCStoreLayout; +import org.apache.sysds.runtime.ooc.store.IndexedMaterializedStoreReader; +import org.apache.sysds.runtime.ooc.store.MaterializedStore; +import org.apache.sysds.runtime.ooc.store.StoreLease; +import org.apache.sysds.runtime.ooc.stream.AllocatedOOCStream; +import org.apache.sysds.runtime.ooc.stream.StreamContext; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; +import org.apache.sysds.runtime.ooc.util.OOCUtils; + +public final class BroadcastOOCPrimitive extends OOCPrimitive { + private final OOCStreamable _broadcast; + private final OOCStreamable _output; + private final ToIntFunction _lookup; + private final Supplier _liveness; + private final BiFunction _operation; + private final AtomicBoolean _cleaned; + private final AtomicBoolean _failed; + private final AtomicBoolean _sourceComplete; + private final AtomicInteger _active; + private MaterializedStore _store; + private IndexedMaterializedStoreReader _reader; + private OOCStream _ready; + private OOCStream _outputStream; + + public BroadcastOOCPrimitive(OOCStreamable streamed, + OOCStreamable broadcast, OOCStreamable output, + ToIntFunction lookup, Supplier liveness, + BiFunction operation, StreamContext context) { + super(context, streamed, broadcast); + _broadcast = broadcast; + _output = output; + _lookup = lookup; + _liveness = liveness; + _operation = operation; + _cleaned = new AtomicBoolean(); + _failed = new AtomicBoolean(); + _sourceComplete = new AtomicBoolean(); + _active = new AtomicInteger(1); + } + + @Override + public List requiredMaterializedInputs() { + return List.of(new OOCMaterializedInputRequest(1, OOCStoreLayout.ROW_MAJOR, 1)); + } + + @Override + protected void inferPatternsInternal() { + _pattern = OOCAccessPattern.ROW_MAJOR; + for(OOCPrimitive child : getChildren()) + child.requestPattern(OOCAccessPattern.ROW_MAJOR); + inferParentPatterns(); + } + + @Override + protected void requestPatternInternal(OOCAccessPattern accessPattern) { + _pattern = OOCAccessPattern.ROW_MAJOR; + for(OOCPrimitive child : getChildren()) + child.requestPattern(OOCAccessPattern.ROW_MAJOR); + } + + @Override + protected void startExecution() { + _outputStream = _output.getWriteStream(); + _ready = new SubscribableTaskQueue<>(); + getContext().addOutStream(_outputStream, _ready); + OOCInstructionUtils.submitOOCTasks(_ready, callback -> process(callback.get()), getContext()) + .whenComplete((ignored, error) -> { + try { + if(error != null) + fail(error); + _outputStream.closeInput(); + } + catch(Throwable failure) { + fail(failure); + } + finally { + cleanup(); + } + }); + + getMaterializedInput(1).whenComplete((store, error) -> { + if(error != null) { + fail(error); + finishSource(); + return; + } + _store = store; + store.completion().whenComplete((ignored, completionError) -> { + if(completionError != null) { + fail(completionError); + finishSource(); + return; + } + try { + _reader = store.openIndexedReader(_liveness.get()); + startBroadcast(); + } + catch(Throwable failure) { + fail(failure); + finishSource(); + } + }); + }); + } + + private void startBroadcast() { + long broadcastLogical = OOCUtils.estimateFullTileBytes(_broadcast.getDataCharacteristics()); + long outputLogical = OOCUtils.estimateFullTileBytes(_output.getDataCharacteristics()); + long broadcastPin = OOCCacheManager.getGlobalCache().maxPhysicalPinBytes(broadcastLogical); + long taskBytes = broadcastPin * 2 + outputLogical * 2; + OOCStream streamed = getInputReadStream(0); + AllocatedOOCStream admitted = new AllocatedOOCStream<>(streamed, _allowance, + ignored -> taskBytes); + getContext().addInStream(streamed, admitted); + admitted.setSubscriber(this::accept); + } + + private void accept(OOCStream.QueueCallback callback) { + if(callback.isEos() || callback.isFailure()) { + try(callback) { + if(callback.isFailure()) + callback.get(); + } + catch(Throwable failure) { + fail(failure); + } + finishSource(); + return; + } + + ReservationBudget budget = null; + OOCStream.QueueCallback retained = null; + _active.incrementAndGet(); + try(callback) { + budget = AllocatedOOCStream.detachBudget(callback); + if(budget == null) + throw new DMLRuntimeException("Missing admitted broadcast task budget."); + IndexedMatrixValue streamed = callback.get(); + int lookup = _lookup.applyAsInt(streamed); + retained = callback.keepOpen(); + OOCFuture> requested = _reader.request(lookup, budget); + OOCStream.QueueCallback pendingStreamed = retained; + ReservationBudget pendingBudget = budget; + retained = null; + budget = null; + requested.whenComplete( + (broadcast, error) -> broadcastReady(pendingStreamed, broadcast, pendingBudget, lookup, error)); + } + catch(Throwable failure) { + fail(failure); + completeOne(); + } + finally { + if(retained != null) + retained.close(); + if(budget != null) + budget.close(); + } + } + + private void broadcastReady(OOCStream.QueueCallback streamed, + StoreLease broadcast, ReservationBudget budget, int lookup, Throwable error) { + if(error != null || broadcast == null) { + try { + streamed.close(); + if(broadcast != null) + broadcast.close(); + budget.close(); + } + finally { + fail(error != null ? error : new IllegalStateException("Missing broadcast tile " + lookup)); + completeOne(); + } + return; + } + BroadcastWork work = new BroadcastWork(streamed, broadcast, budget); + try { + _ready.enqueue(work); + } + catch(Throwable failure) { + work.close(); + fail(failure); + completeOne(); + } + } + + private void process(BroadcastWork work) { + ReservationBudget budget = work.takeBudget(); + try { + IndexedMatrixValue output = _operation.apply(work._streamed.get(), work._broadcast.value()); + OOCUtils.enqueueExact(_outputStream, output, budget); + budget = null; + } + catch(Throwable failure) { + fail(failure); + } + finally { + work.close(); + if(budget != null) + budget.close(); + completeOne(); + } + } + + private void finishSource() { + if(_sourceComplete.compareAndSet(false, true)) + completeOne(); + } + + private void completeOne() { + if(_active.decrementAndGet() != 0) + return; + try { + _ready.closeInput(); + } + catch(IllegalStateException ignored) { + // Failure propagation may already have closed the ready stream. + } + } + + private void fail(Throwable error) { + if(!_failed.compareAndSet(false, true)) + return; + DMLRuntimeException failure = DMLRuntimeException.of(error); + _outputStream.propagateFailure(failure); + getContext().failAll(failure); + } + + private void cleanup() { + if(!_cleaned.compareAndSet(false, true)) + return; + try { + if(_reader != null) + _reader.close(); + } + finally { + try { + if(_store != null) + _store.close(); + } + finally { + onComplete(); + } + } + } + + private static final class BroadcastWork implements AutoCloseable { + private OOCStream.QueueCallback _streamed; + private StoreLease _broadcast; + private ReservationBudget _budget; + + private BroadcastWork(OOCStream.QueueCallback streamed, + StoreLease broadcast, ReservationBudget budget) { + _streamed = streamed; + _broadcast = broadcast; + _budget = budget; + } + + private ReservationBudget takeBudget() { + ReservationBudget budget = _budget; + _budget = null; + return budget; + } + + @Override + public void close() { + if(_streamed != null) { + _streamed.close(); + _streamed = null; + } + if(_broadcast != null) { + _broadcast.close(); + _broadcast = null; + } + if(_budget != null) { + _budget.close(); + _budget = null; + } + } + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/GroupedReduceOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/GroupedReduceOOCPrimitive.java new file mode 100644 index 00000000000..ad126af0c71 --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/GroupedReduceOOCPrimitive.java @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.primitives; + +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; + +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.instructions.ooc.CachingStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; +import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.meta.DataCharacteristics; +import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.memory.ManagedPayload; +import org.apache.sysds.runtime.ooc.memory.ReservationBudget; +import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; +import org.apache.sysds.runtime.ooc.store.StateTable; +import org.apache.sysds.runtime.ooc.store.StoreLease; +import org.apache.sysds.runtime.ooc.stream.AllocatedOOCStream; +import org.apache.sysds.runtime.ooc.stream.StreamContext; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; +import org.apache.sysds.runtime.ooc.util.OOCUtils; + +public final class GroupedReduceOOCPrimitive extends OOCPrimitive { + private final OOCStream _input; + private final OOCStreamable _output; + private final BiFunction _merge; + private final AtomicBoolean _cleaned; + private final AtomicBoolean _failed; + private final AtomicBoolean _sourceComplete; + private final AtomicInteger _active; + private final AtomicInteger _finalizedGroups; + private StateTable _table; + private OOCStream _ready; + private OOCStream _outputStream; + private int _numGroups; + private int _groupSize; + + public GroupedReduceOOCPrimitive(OOCStreamable input, OOCStreamable output, + BiFunction merge, StreamContext context) { + this(input.getReadStream(), output, merge, context); + } + + private GroupedReduceOOCPrimitive(OOCStream input, OOCStreamable output, + BiFunction merge, StreamContext context) { + super(context, input.getPrimitive() == null ? List.of() : List.of(input.getPrimitive())); + _input = input; + _output = output; + _merge = merge; + _cleaned = new AtomicBoolean(); + _failed = new AtomicBoolean(); + _sourceComplete = new AtomicBoolean(); + _active = new AtomicInteger(1); + _finalizedGroups = new AtomicInteger(); + } + + @Override + protected void inferPatternsInternal() { + _pattern = OOCAccessPattern.ROW_MAJOR; + for(OOCPrimitive child : getChildren()) + child.requestPattern(OOCAccessPattern.ROW_MAJOR); + inferParentPatterns(); + } + + @Override + protected void requestPatternInternal(OOCAccessPattern accessPattern) { + _pattern = OOCAccessPattern.ROW_MAJOR; + for(OOCPrimitive child : getChildren()) + child.requestPattern(OOCAccessPattern.ROW_MAJOR); + } + + @Override + protected void startExecution() { + DataCharacteristics inputDc = _input.getDataCharacteristics(); + if(inputDc == null || !inputDc.dimsKnown() || inputDc.getBlocksize() <= 0) + throw new DMLRuntimeException("Grouped OOC reduction requires known input dimensions and block size."); + _numGroups = Math.toIntExact(inputDc.getNumRowBlocks()); + _groupSize = Math.toIntExact(inputDc.getNumColBlocks()); + _outputStream = _output.getWriteStream(); + _ready = new SubscribableTaskQueue<>(); + getContext().addInStream(_input).addOutStream(_outputStream, _ready); + _table = new StateTable<>(OOCCacheManager.getGlobalCache(), CachingStream._streamSeq.getNextID()); + + OOCInstructionUtils.submitOOCTasks(_ready, callback -> process(callback.get()), getContext()) + .whenComplete((ignored, error) -> { + try { + _outputStream.closeInput(); + } + catch(Throwable failure) { + fail(failure); + } + finally { + cleanup(); + } + }); + + long logicalBytes = Math.max(OOCUtils.estimateFullTileBytes(inputDc), + OOCUtils.estimateFullTileBytes(_output.getDataCharacteristics())); + long pinBytes = OOCCacheManager.getGlobalCache().maxPhysicalPinBytes(logicalBytes); + long taskBytes = pinBytes + logicalBytes * 2; + AllocatedOOCStream admitted = new AllocatedOOCStream<>(_input, _allowance, + ignored -> taskBytes); + getContext().addInStream(admitted); + admitted.setSubscriber(this::accept); + } + + private void accept(OOCStream.QueueCallback callback) { + if(callback.isEos() || callback.isFailure()) { + try(callback) { + if(callback.isFailure()) + callback.get(); + } + catch(Throwable failure) { + fail(failure); + } + finishSource(); + return; + } + + ReservationBudget budget = null; + ManagedPayload payload = null; + _active.incrementAndGet(); + try(callback) { + budget = AllocatedOOCStream.detachBudget(callback).enableReuse(); + IndexedMatrixValue input = callback.get(); + int group = Math.toIntExact(input.getIndexes().getRowIndex() - 1); + if(group < 0 || group >= _numGroups) + throw new DMLRuntimeException("Invalid grouped-reduce row block: " + (group + 1)); + IndexedMatrixValue value = new IndexedMatrixValue(new MatrixIndexes(group + 1L, 1), input.getValue()); + payload = payload(value, budget); + reduce(group, payload, budget); + payload = null; + budget = null; + } + catch(Throwable failure) { + fail(failure); + completeOne(); + } + finally { + if(payload != null) + payload.release(); + if(budget != null) + budget.close(); + } + } + + private void reduce(int group, ManagedPayload incoming, ReservationBudget budget) { + if(multiplicity(incoming.value()) == _groupSize) { + finalizeGroup(group, incoming, budget); + return; + } + OOCFuture> match; + try { + match = _table.putOrTake(group, incoming, budget); + } + catch(Throwable failure) { + incoming.release(); + budget.close(); + fail(failure); + completeOne(); + return; + } + match.whenComplete((existing, error) -> { + if(error != null) { + incoming.release(); + budget.close(); + fail(error); + completeOne(); + } + else if(existing == null) { + budget.close(); + completeOne(); + } + else { + MergeWork work = new MergeWork(group, incoming, existing, budget); + try { + _ready.enqueue(work); + } + catch(Throwable failure) { + work.close(); + fail(failure); + completeOne(); + } + } + }); + } + + private void process(MergeWork work) { + ReservationBudget budget = work.takeBudget(); + ManagedPayload merged = null; + OOCFuture released; + try { + IndexedMatrixValue left = work._existing.value(); + IndexedMatrixValue right = work._incoming.value(); + int count = Math.addExact(multiplicity(left), multiplicity(right)); + if(count > _groupSize) + throw new DMLRuntimeException("Too many partial tiles for grouped-reduce row " + (work._group + 1)); + MatrixBlock value = _merge.apply((MatrixBlock) left.getValue(), (MatrixBlock) right.getValue()); + merged = payload(new IndexedMatrixValue(new MatrixIndexes(work._group + 1L, count), value), budget); + work.releaseIncoming(); + released = work.closeExistingAsync(); + } + catch(Throwable failure) { + if(merged != null) + merged.release(); + work.close(); + budget.close(); + fail(failure); + completeOne(); + return; + } + + ManagedPayload next = merged; + released.whenComplete((ignored, error) -> { + if(error != null) { + next.release(); + budget.close(); + fail(error); + completeOne(); + } + else + reduce(work._group, next, budget); + }); + } + + private void finalizeGroup(int group, ManagedPayload payload, ReservationBudget budget) { + IndexedMatrixValue accumulated = payload.value(); + IndexedMatrixValue output = new IndexedMatrixValue(new MatrixIndexes(group + 1L, 1), accumulated.getValue()); + payload.release(); + try { + OOCUtils.enqueueExact(_outputStream, output, budget); + _finalizedGroups.incrementAndGet(); + } + catch(Throwable failure) { + budget.close(); + fail(failure); + } + completeOne(); + } + + private static ManagedPayload payload(IndexedMatrixValue value, ReservationBudget budget) { + long bytes = ((MatrixBlock) value.getValue()).getExactSerializedSize(); + budget.reserveBlocking(bytes); + return new ManagedPayload<>(value, bytes, budget); + } + + private static int multiplicity(IndexedMatrixValue value) { + return Math.toIntExact(value.getIndexes().getColumnIndex()); + } + + private void finishSource() { + if(_sourceComplete.compareAndSet(false, true)) + completeOne(); + } + + private void completeOne() { + int remaining = _active.decrementAndGet(); + if(remaining != 0) + return; + if(!_failed.get() && _finalizedGroups.get() != _numGroups) + fail(new DMLRuntimeException( + "Grouped reduction completed " + _finalizedGroups.get() + " of " + _numGroups + " row groups.")); + try { + _ready.closeInput(); + } + catch(IllegalStateException ignored) { + // Failure propagation may already have closed the ready stream. + } + } + + private void fail(Throwable error) { + if(!_failed.compareAndSet(false, true)) + return; + DMLRuntimeException failure = DMLRuntimeException.of(error); + _outputStream.propagateFailure(failure); + getContext().failAll(failure); + } + + private void cleanup() { + if(!_cleaned.compareAndSet(false, true)) + return; + try { + if(_table != null) + _table.close(); + } + finally { + onComplete(); + } + } + + private static final class MergeWork implements AutoCloseable { + private final int _group; + private ManagedPayload _incoming; + private StoreLease _existing; + private ReservationBudget _budget; + + private MergeWork(int group, ManagedPayload incoming, + StoreLease existing, ReservationBudget budget) { + _group = group; + _incoming = incoming; + _existing = existing; + _budget = budget; + } + + private ReservationBudget takeBudget() { + ReservationBudget budget = _budget; + _budget = null; + return budget; + } + + private void releaseIncoming() { + if(_incoming != null) { + _incoming.release(); + _incoming = null; + } + } + + private OOCFuture closeExistingAsync() { + StoreLease existing = _existing; + _existing = null; + return existing.closeAsync(); + } + + @Override + public void close() { + releaseIncoming(); + if(_existing != null) { + _existing.close(); + _existing = null; + } + if(_budget != null) + _budget.close(); + } + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java index 773ef9da834..f86c2360608 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java @@ -29,6 +29,8 @@ import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; +import java.util.function.ToIntFunction; import org.apache.sysds.api.DMLScript; import org.apache.sysds.runtime.DMLRuntimeException; @@ -40,11 +42,14 @@ import org.apache.sysds.runtime.ooc.cache.OOCFuture; import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; import org.apache.sysds.runtime.ooc.memory.ReservationBudget; +import org.apache.sysds.runtime.ooc.primitives.BroadcastOOCPrimitive; +import org.apache.sysds.runtime.ooc.primitives.GroupedReduceOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.JoinOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.MappingOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.PlannableDataGenOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.TransposeOOCPrimitive; import org.apache.sysds.runtime.ooc.stats.OOCEventLog; +import org.apache.sysds.runtime.ooc.store.MaterializedStore; import org.apache.sysds.runtime.ooc.stream.AllocatedOOCStream; import org.apache.sysds.runtime.ooc.stream.StreamContext; import org.apache.sysds.runtime.ooc.stream.TaskContext; @@ -88,6 +93,19 @@ public static void equiJoin(OOCStreamable left, OOCStreamabl output.assignPrimitive(new JoinOOCPrimitive(left, right, output, operation, context)); } + public static void indexedBroadcastMap(OOCStreamable streamed, + OOCStreamable broadcast, OOCStream output, + ToIntFunction lookup, Supplier liveness, + BiFunction operation, StreamContext context) { + output.assignPrimitive( + new BroadcastOOCPrimitive(streamed, broadcast, output, lookup, liveness, operation, context)); + } + + public static void rowGroupedReduce(OOCStreamable input, OOCStream output, + BiFunction merge, StreamContext context) { + output.assignPrimitive(new GroupedReduceOOCPrimitive(input, output, merge, context)); + } + public static int getComputeInFlight() { return COMPUTE_IN_FLIGHT.get(); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java index 5a98a05b78d..7981736753f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java @@ -141,13 +141,17 @@ public MatrixIndexes next() { public static long estimateOutputTileBytes(DataCharacteristics dc) { if(dc == null || dc.getBlocksize() <= 0 || !dc.dimsKnown()) { - int blocksize = dc != null && dc.getBlocksize() > 0 ? dc.getBlocksize() : 1000; - return estimateMatrixBlockBytes(blocksize, blocksize); + return estimateFullTileBytes(dc); } return estimateMatrixBlockBytes(Math.min(dc.getBlocksize(), dc.getRows()), Math.min(dc.getBlocksize(), dc.getCols())); } + public static long estimateFullTileBytes(DataCharacteristics dc) { + int blocksize = dc != null && dc.getBlocksize() > 0 ? dc.getBlocksize() : 1000; + return estimateMatrixBlockBytes(blocksize, blocksize); + } + private static long estimateMatrixBlockBytes(long rows, long cols) { return Math.max(MatrixBlock.estimateSizeDenseInMemory(rows, cols), MatrixBlock.estimateSizeSparseInMemory(rows, cols, 1.0)); diff --git a/src/test/java/org/apache/sysds/test/component/ooc/memory/OOCMemoryAllowanceTest.java b/src/test/java/org/apache/sysds/test/component/ooc/memory/OOCMemoryAllowanceTest.java index c75458eef38..8660252e634 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/memory/OOCMemoryAllowanceTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/memory/OOCMemoryAllowanceTest.java @@ -138,6 +138,15 @@ public void testAllocatedStreamReservations() { budget.close(); Assert.assertEquals(0, allowance.getUsedMemory()); + allowance.reserveBlocking(60); + ReservationBudget reusable = new ReservationBudget(allowance, 60).enableReuse(); + reusable.reserveBlocking(40); + reusable.release(40); + reusable.reserveBlocking(40); + reusable.release(40); + reusable.close(); + Assert.assertEquals(0, allowance.getUsedMemory()); + source.enqueue(2); OOCStream.QueueCallback second = allocated.dequeueCB(); OOCStream.QueueCallback retained = second.keepOpen(); From b08de578bd59fe15c65b8dcaf70686803d4d2dbe Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:19:30 +0200 Subject: [PATCH 100/132] [SYSTEMDS-3891] Wire OOC Materialized Stores --- .../controlprogram/ParForProgramBlock.java | 11 +- .../controlprogram/caching/CacheableData.java | 2 +- .../instructions/ooc/OOCStreamable.java | 7 + .../instructions/ooc/TeeOOCInstruction.java | 37 +- .../ooc/cache/packed/OOCPackedCache.java | 64 +++- .../runtime/ooc/cache/packed/PackBuilder.java | 2 +- .../primitives/MaterializeOOCPrimitive.java | 34 +- .../store/MaterializedStoreStreamable.java | 344 ++++++++++++++++++ .../ooc/store/OOCStreamMaterializer.java | 12 +- .../test/component/ooc/OOCPrimitiveTest.java | 36 ++ .../ooc/cache/OOCPackedCacheTest.java | 8 +- 11 files changed, 516 insertions(+), 41 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedStoreStreamable.java diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java index 793cf39ce69..26716507b51 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java @@ -91,6 +91,7 @@ import org.apache.sysds.runtime.instructions.cp.ScalarObject; import org.apache.sysds.runtime.instructions.cp.StringObject; import org.apache.sysds.runtime.instructions.cp.VariableCPInstruction; +import org.apache.sysds.runtime.instructions.ooc.TeeOOCInstruction; import org.apache.sysds.runtime.lineage.Lineage; import org.apache.sysds.runtime.lineage.LineageCacheConfig; import org.apache.sysds.runtime.lineage.LineageItem; @@ -730,6 +731,7 @@ private void executeLocalParFor( ExecutionContext ec, IntObject from, IntObject final LocalTaskQueue queue = new LocalTaskQueue<>(); final Thread[] threads = new Thread[_numThreads]; final LocalParWorker[] workers = new LocalParWorker[_numThreads]; + final Set resultVarNames = _resultVars.stream().map(v -> v._name).collect(Collectors.toSet()); @SuppressWarnings("unchecked") final HashMap[] workerBaselines = DMLScript.USE_OOC ? new HashMap[_numThreads] : null; try @@ -740,8 +742,11 @@ private void executeLocalParFor( ExecutionContext ec, IntObject from, IntObject workers[i] = createParallelWorker( _pwIDs[i], queue, ec, i); if(DMLScript.USE_OOC) { workerBaselines[i] = new HashMap<>(); - for(Map.Entry e : workers[i].getVariables().entrySet()) + for(Map.Entry e : workers[i].getVariables().entrySet()) { workerBaselines[i].put(e.getKey(), e.getValue()); + if(!resultVarNames.contains(e.getKey()) && e.getValue() instanceof MatrixObject matrix) + TeeOOCInstruction.incrRef(matrix.getStreamable(), 1); + } } threads[i] = new Thread( workers[i] , "PARFOR"); threads[i].setPriority(Thread.MAX_PRIORITY); @@ -785,8 +790,6 @@ private void executeLocalParFor( ExecutionContext ec, IntObject from, IntObject // Step 4) collecting results from each parallel worker //obtain results and cleanup other intermediates before result merge - Set resultVarNames = _resultVars.stream() - .map(v -> v._name).collect(Collectors.toSet()); LocalVariableMap [] localVariables = new LocalVariableMap [_numThreads]; for( int i=0; i<_numThreads; i++ ) { localVariables[i] = workers[i].getVariables(); @@ -796,6 +799,8 @@ private void executeLocalParFor( ExecutionContext ec, IntObject from, IntObject Data current = localVariables[i].get(var); if(current != null && current != workerBaselines[i].get(var)) VariableCPInstruction.processRmvarInstruction(workers[i].getExecutionContext(), var); + else if(current instanceof MatrixObject matrix) + TeeOOCInstruction.incrRef(matrix.getStreamable(), -1); } } } diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java index ab27540fce7..622f6fd00a2 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java @@ -491,7 +491,7 @@ public synchronized OOCStream getStreamHandle() { } OOCStream stream = _streamHandle.getReadStream(); - if(!stream.hasStreamCache()) + if(!_streamHandle.hasStreamCache() && !_streamHandle.hasMaterializedStore()) _streamHandle = null; // To ensure read once return stream; } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStreamable.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStreamable.java index 10a4dc88174..9f3c92dc749 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStreamable.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/OOCStreamable.java @@ -32,6 +32,13 @@ public interface OOCStreamable { CachingStream getStreamCache(); + default boolean hasMaterializedStore() { + return false; + } + + default void scheduleMaterializedStoreDeletion() { + } + boolean isProcessed(); DataCharacteristics getDataCharacteristics(); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/TeeOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/TeeOOCInstruction.java index 548e80df942..1a46185fd72 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/TeeOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/TeeOOCInstruction.java @@ -24,22 +24,24 @@ import org.apache.sysds.runtime.instructions.InstructionUtils; import org.apache.sysds.runtime.instructions.cp.CPOperand; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.ooc.store.MaterializedStoreStreamable; import java.util.concurrent.ConcurrentHashMap; public class TeeOOCInstruction extends ComputationOOCInstruction { - private static final ConcurrentHashMap refCtr = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap, Integer> refCtr = new ConcurrentHashMap<>(); public static void reset() { if (!refCtr.isEmpty()) { System.err.println("There are some dangling streams still in the cache: " + refCtr); - for(CachingStream cache : refCtr.keySet()) { + for(OOCStreamable stream : refCtr.keySet()) { try { - cache.scheduleDeletion(); + scheduleDeletion(stream); } catch(Exception ex) { - System.err.println("Failed to schedule deletion for dangling stream " + cache + ": " + ex.getMessage()); + System.err + .println("Failed to schedule deletion for dangling stream " + stream + ": " + ex.getMessage()); } } refCtr.clear(); @@ -50,19 +52,26 @@ public static void reset() { * Increments the reference counter of a stream by the set amount. */ public static void incrRef(OOCStreamable stream, int incr) { - if (!stream.hasStreamCache()) + if(!stream.hasStreamCache() && !stream.hasMaterializedStore()) return; - CachingStream cache = stream.getStreamCache(); + OOCStreamable handle = stream.hasStreamCache() ? stream.getStreamCache() : stream; - Integer ref = refCtr.compute(cache, (k, v) -> { + Integer ref = refCtr.compute(handle, (k, v) -> { if (v == null) v = 0; v += incr; return v <= 0 ? null : v; }); - if (ref == null) - cache.scheduleDeletion(); + if(ref == null) + scheduleDeletion(handle); + } + + private static void scheduleDeletion(OOCStreamable stream) { + if(stream.hasMaterializedStore()) + stream.scheduleMaterializedStoreDeletion(); + else + stream.getStreamCache().scheduleDeletion(); } protected TeeOOCInstruction(OOCType type, CPOperand in1, CPOperand out, String opcode, String istr) { @@ -82,15 +91,15 @@ public void processInstruction(ExecutionContext ec) { //get input stream MatrixObject min = ec.getMatrixObject(input1); OOCStreamable streamable = min.getStreamable(); - CachingStream handle; + OOCStreamable handle; - if(streamable.hasStreamCache()) { - handle = streamable.getStreamCache(); + if(streamable.hasStreamCache() || streamable.hasMaterializedStore()) { + handle = streamable.hasStreamCache() ? streamable.getStreamCache() : streamable; incrRef(handle, 1); } else { - // We also set the input stream handle - handle = new CachingStream(min.getStreamHandle()); + // The input and output matrix objects both retain the new reusable handle. + handle = new MaterializedStoreStreamable(min.getStreamHandle(), min); min.setStreamHandle(handle); incrRef(handle, 2); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/OOCPackedCache.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/OOCPackedCache.java index ad08c637bcf..bdc4d69f263 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/OOCPackedCache.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/OOCPackedCache.java @@ -43,6 +43,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.LockSupport; import java.util.function.LongUnaryOperator; +import java.util.function.Supplier; public final class OOCPackedCache implements OOCCache { private static final long PACKED_STREAM_ID = CachingStream._streamSeq.getNextID(); @@ -231,11 +232,8 @@ public OOCFuture pin(long sId, long tId, MemoryAllowance allowance) if(!(location instanceof SealedPackLocation packed)) return _physical.pin(sId, tId, allowance); - return packed.state().pin(_physical, allowance, false).map(physicalEntry -> { - if(physicalEntry == null) - return null; - return createLogicalPin(new BlockKey(sId, tId), packed); - }); + BlockKey key = new BlockKey(sId, tId); + return pinLogical(key, packed, () -> packed.state().pin(_physical, allowance, false)); } @Override @@ -248,11 +246,8 @@ public OOCFuture pinAdmitted(long sId, long tId, MemoryAllowance all if(!(location instanceof SealedPackLocation packed)) return _physical.pinAdmitted(sId, tId, allowance); - return packed.state().pinAdmitted(_physical, allowance).map(physicalEntry -> { - if(physicalEntry == null) - return null; - return createLogicalPin(new BlockKey(sId, tId), packed); - }); + BlockKey key = new BlockKey(sId, tId); + return pinLogical(key, packed, () -> packed.state().pinAdmitted(_physical, allowance)); } @Override @@ -265,9 +260,19 @@ public BlockEntry pinIfLive(long sId, long tId, MemoryAllowance allowance) { if(!(location instanceof SealedPackLocation packed)) return _physical.pinIfLive(sId, tId, allowance); - if(packed.state().pinIfLive(_physical, allowance) == null) - return null; - return createLogicalPin(new BlockKey(sId, tId), packed); + BlockKey key = new BlockKey(sId, tId); + packed.retain(); + try { + if(packed.state().pinIfLive(_physical, allowance) == null) { + releaseLocation(key, packed); + return null; + } + return createLogicalPin(key, packed); + } + catch(RuntimeException | Error error) { + releaseLocation(key, packed); + throw error; + } } @Override @@ -410,6 +415,7 @@ private UnpinHandle unpinPending(BlockEntry entry, PendingLogicalPin pin, Memory entry.unpin(); entry.setCacheMeta(null); PackedUnpinHandle handle = pin.builder().unpinProducer(entry, pin.slot(), allowance); + releasePendingPin(entry.getKey(), pin.builder(), pin.slot()); if(pin.builder().sealed && pin.builder().activePins == 0) pin.builder().transferProducerOwnership(_physical); scheduleSeal(pin.builder()); @@ -426,7 +432,9 @@ private UnpinHandle unpinPacked(BlockEntry entry, PackedLogicalPin pin, MemoryAl } entry.unpin(); entry.setCacheMeta(null); - return pin.location().state().unpin(this, _packReleaseDelayMs, allowance); + UnpinHandle handle = pin.location().state().unpin(this, _packReleaseDelayMs, allowance); + releaseLocation(entry.getKey(), pin.location()); + return handle; } void enqueueRelease(PackedPinState state) { @@ -492,6 +500,34 @@ private SealedPackLocation forceSeal(PendingPackLocation pending) { } } + private OOCFuture pinLogical(BlockKey key, SealedPackLocation location, + Supplier> pin) { + location.retain(); + OOCFuture physical; + try { + physical = pin.get(); + } + catch(RuntimeException | Error error) { + releaseLocation(key, location); + throw error; + } + physical.whenComplete((entry, error) -> { + if(entry == null || error != null) + releaseLocation(key, location); + }); + return physical.map(entry -> entry == null ? null : createLogicalPin(key, location)); + } + + private void releasePendingPin(BlockKey key, PackBuilder builder, int slot) { + if(builder.sealed) { + PackedCacheLocation location = getLocation(key.getStreamId(), key.getSequenceNumber()); + if(location instanceof SealedPackLocation packed) + releaseLocation(key, packed); + } + else if(builder.releaseSlot(slot) == 0) + clearLocation(key); + } + private static BlockEntry createLogicalPin(BlockKey logicalKey, SealedPackLocation location) { PackedBlock block = (PackedBlock) location.state().physicalEntry.getDataUnsafe(); Object data = block.values[location.slot()]; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackBuilder.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackBuilder.java index 9a5f998e17d..82f074394d4 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackBuilder.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackBuilder.java @@ -59,7 +59,7 @@ int append(long streamId, long tileId, Object value, long size) { tileIds[slot] = tileId; values[slot] = value; sizes[slot] = size; - refCounts[slot] = 1; + refCounts[slot] = 2; bytes += size; activePins++; return slot; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/MaterializeOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/MaterializeOOCPrimitive.java index eaa9e8232c7..e1002e98005 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/MaterializeOOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/MaterializeOOCPrimitive.java @@ -21,12 +21,16 @@ import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.ToIntFunction; import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.instructions.ooc.CachingStream; import org.apache.sysds.runtime.instructions.ooc.OOCStream; import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.meta.DataCharacteristics; import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; import org.apache.sysds.runtime.ooc.cache.OOCFuture; import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; @@ -40,19 +44,32 @@ public final class MaterializeOOCPrimitive extends OOCPrimitive { private final OOCStoreLayout _layout; private final OOCFuture> _store; private final AtomicBoolean _finished; + private final boolean _reusable; private int _expectedReaders; private int _consumers; public MaterializeOOCPrimitive(OOCStreamable source, OOCStoreLayout layout, StreamContext context) { + this(source, layout, context, false); + } + + private MaterializeOOCPrimitive(OOCStreamable source, OOCStoreLayout layout, + StreamContext context, boolean reusable) { super(context, source.getPrimitive() == null ? List.of() : List.of(source.getPrimitive())); _source = source; _layout = layout; _store = new OOCFuture<>(); _finished = new AtomicBoolean(); + _reusable = reusable; + } + + public static MaterializeOOCPrimitive reusable(OOCStreamable source) { + return new MaterializeOOCPrimitive(source, OOCStoreLayout.ROW_MAJOR, null, true); } public synchronized void registerRequest(int expectedReaders) { + if(_reusable) + throw new IllegalStateException("Reusable materialization registers readers dynamically."); if(expectedReaders <= 0) throw new IllegalArgumentException("Materialization request requires at least one reader."); if(hasStartedExecution()) @@ -84,10 +101,19 @@ protected void requestPatternInternal(OOCAccessPattern accessPattern) { protected void startExecution() { try { OOCStream source = _source.getReservedReadStream(); - MaterializedStore store = new MaterializedStore<>(OOCCacheManager.getGlobalCache(), - CachingStream._streamSeq.getNextID(), _expectedReaders, _consumers); - OOCStreamMaterializer materializer = new OOCStreamMaterializer(store, - indexes -> _layout.linearize(indexes, _source.getDataCharacteristics()), _allowance); + MaterializedStore store = _reusable ? new MaterializedStore<>( + OOCCacheManager.getGlobalCache(), + CachingStream._streamSeq.getNextID()) : new MaterializedStore<>(OOCCacheManager.getGlobalCache(), + CachingStream._streamSeq.getNextID(), _expectedReaders, _consumers); + DataCharacteristics characteristics = _source.getDataCharacteristics(); + AtomicInteger nextIndex = new AtomicInteger(); + ToIntFunction linearize; + if(_reusable && + (characteristics == null || !characteristics.dimsKnown() || characteristics.getBlocksize() <= 0)) + linearize = ignored -> nextIndex.getAndIncrement(); + else + linearize = indexes -> _layout.linearize(indexes, characteristics); + OOCStreamMaterializer materializer = new OOCStreamMaterializer(store, linearize, _allowance); materializer.completion().whenComplete((ignored, error) -> { if(error != null) fail(error); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedStoreStreamable.java b/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedStoreStreamable.java new file mode 100644 index 00000000000..5d3879de72e --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedStoreStreamable.java @@ -0,0 +1,344 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.store; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; + +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.controlprogram.caching.CacheableData; +import org.apache.sysds.runtime.instructions.ooc.CachingStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; +import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.meta.DataCharacteristics; +import org.apache.sysds.runtime.ooc.memory.GlobalMemoryBroker; +import org.apache.sysds.runtime.ooc.memory.SyncMemoryAllowance; +import org.apache.sysds.runtime.ooc.primitives.MaterializeOOCPrimitive; +import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; + +public final class MaterializedStoreStreamable implements OOCStreamable { + private static final int REPLAY_PREFETCH = 8; + + private final MaterializeOOCPrimitive _primitive; + private MaterializedStore _store; + private SyncMemoryAllowance _readerAllowance; + private CacheableData _data; + private boolean _deleteScheduled; + private boolean _materializationDone; + private boolean _readersSealed; + private boolean _closed; + private int _reservedReaders; + private int _pendingReaders; + private int _activeReaders; + + public MaterializedStoreStreamable(OOCStream source, CacheableData data) { + if(source == null) + throw new IllegalArgumentException("Materialized stream requires a source."); + _data = data; + _primitive = MaterializeOOCPrimitive.reusable(source); + _primitive.store().whenComplete((store, error) -> { + if(error != null) { + markMaterializationDone(); + return; + } + synchronized(this) { + _store = store; + } + store.completion().whenComplete((ignored, completionError) -> markMaterializationDone()); + tryFinalize(); + }); + } + + @Override + public OOCStream getReadStream() { + return createReader(false); + } + + @Override + public OOCStream getReservedReadStream() { + return createReader(true); + } + + private synchronized OOCStream createReader(boolean reserved) { + if(reserved && _reservedReaders > 0) + _reservedReaders--; + else if(_deleteScheduled) + throw new DMLRuntimeException("Cannot open a reader on a materialized stream scheduled for deletion."); + _pendingReaders++; + DeferredReader stream = new DeferredReader(this); + stream.setData(_data); + stream.assignPrimitive(_primitive); + return stream; + } + + private void openReader(DeferredReader output) { + _primitive.store().whenComplete((store, storeError) -> { + if(storeError != null) { + failPendingReader(output, storeError); + return; + } + store.completion().whenComplete((ignored, completionError) -> { + if(completionError != null) { + failPendingReader(output, completionError); + return; + } + OrderedMaterializedStoreReader reader = null; + try { + reader = store.openReader(new SequentialAccessPattern(store.size()), readerAllowance(), + REPLAY_PREFETCH); + synchronized(this) { + _pendingReaders--; + _activeReaders++; + } + tryFinalize(); + drive(output, reader); + } + catch(Throwable failure) { + if(reader == null) + failPendingReader(output, failure); + else { + reader.close(); + try { + output.propagateFailure(DMLRuntimeException.of(failure)); + } + finally { + finishReader(output); + } + } + } + }); + }); + } + + private void drive(DeferredReader output, OrderedMaterializedStoreReader reader) { + StoreBackedStream replay = new StoreBackedStream<>(reader); + replay.setData(_data); + replay.setSubscriber(callback -> { + if(callback.isFailure()) { + DMLRuntimeException failure; + try { + callback.get(); + failure = new DMLRuntimeException("Materialized replay failed."); + } + catch(Throwable error) { + failure = DMLRuntimeException.of(error); + } + try { + output.propagateFailure(failure); + } + finally { + finishReader(output); + } + } + else if(callback.isEos()) { + try { + output.closeInput(); + } + finally { + finishReader(output); + } + } + else { + OOCStream.QueueCallback retained = callback.keepOpen(); + try { + output.enqueue(retained); + } + catch(Throwable failure) { + retained.close(); + throw DMLRuntimeException.of(failure); + } + } + }); + } + + private void failPendingReader(DeferredReader output, Throwable error) { + if(!output.finish()) + return; + synchronized(this) { + _pendingReaders--; + } + try { + output.propagateFailure(DMLRuntimeException.of(error)); + } + finally { + tryFinalize(); + } + } + + private void finishReader(DeferredReader output) { + if(!output.finish()) + return; + synchronized(this) { + _activeReaders--; + } + tryFinalize(); + } + + private synchronized SyncMemoryAllowance readerAllowance() { + if(_readerAllowance == null) + _readerAllowance = new SyncMemoryAllowance(GlobalMemoryBroker.get()); + return _readerAllowance; + } + + private void markMaterializationDone() { + synchronized(this) { + _materializationDone = true; + } + tryFinalize(); + } + + @Override + public synchronized void reserveLazyHandle() { + if(_deleteScheduled) + throw new DMLRuntimeException("Cannot reserve a reader on a materialized stream scheduled for deletion."); + _reservedReaders++; + } + + @Override + public void discardHandle() { + synchronized(this) { + if(_reservedReaders <= 0) + return; + _reservedReaders--; + } + tryFinalize(); + } + + @Override + public void scheduleMaterializedStoreDeletion() { + synchronized(this) { + _deleteScheduled = true; + } + tryFinalize(); + } + + private void tryFinalize() { + MaterializedStore store; + SyncMemoryAllowance allowance = null; + boolean seal = false; + boolean close = false; + synchronized(this) { + store = _store; + if(!_deleteScheduled || _reservedReaders != 0 || _pendingReaders != 0) + return; + if(store != null && !_readersSealed) { + _readersSealed = true; + seal = true; + } + if(_materializationDone && _activeReaders == 0 && !_closed) { + _closed = true; + close = store != null; + allowance = _readerAllowance; + } + } + if(seal) + store.sealReaders(); + if(close) + store.close(); + if(allowance != null) + allowance.shutdown(); + } + + @Override + public boolean hasMaterializedStore() { + return true; + } + + @Override + public OOCStream getWriteStream() { + throw new UnsupportedOperationException("Materialized streams are read-only."); + } + + @Override + public boolean hasStreamCache() { + return false; + } + + @Override + public CachingStream getStreamCache() { + return null; + } + + @Override + public boolean isProcessed() { + return false; + } + + @Override + public synchronized DataCharacteristics getDataCharacteristics() { + return _data == null ? null : _data.getDataCharacteristics(); + } + + @Override + public synchronized CacheableData getData() { + return _data; + } + + @Override + public synchronized void setData(CacheableData data) { + _data = data; + } + + @Override + public OOCPrimitive getPrimitive() { + return _primitive; + } + + private static final class DeferredReader extends SubscribableTaskQueue { + private final MaterializedStoreStreamable _owner; + private final AtomicBoolean _activated; + private final AtomicBoolean _finished; + + private DeferredReader(MaterializedStoreStreamable owner) { + _owner = owner; + _activated = new AtomicBoolean(); + _finished = new AtomicBoolean(); + } + + @Override + public void setSubscriber(Consumer> subscriber) { + super.setSubscriber(subscriber); + activate(); + } + + @Override + public IndexedMatrixValue dequeue() { + activate(); + return super.dequeue(); + } + + @Override + public QueueCallback dequeueCB() { + activate(); + return super.dequeueCB(); + } + + private void activate() { + if(_activated.compareAndSet(false, true)) + _owner.openReader(this); + } + + private boolean finish() { + return _finished.compareAndSet(false, true); + } + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/OOCStreamMaterializer.java b/src/main/java/org/apache/sysds/runtime/ooc/store/OOCStreamMaterializer.java index 3462c2cd3af..1c85ea4f4b0 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/store/OOCStreamMaterializer.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/OOCStreamMaterializer.java @@ -91,7 +91,17 @@ public void accept(OOCStream.QueueCallback callback) { finish(); return; } - publish(callback); + if(callback instanceof OOCStream.GroupQueueCallback grouped) { + @SuppressWarnings("unchecked") + OOCStream.GroupQueueCallback group = (OOCStream.GroupQueueCallback) grouped; + for(int i = 0; i < group.size(); i++) { + try(OOCStream.QueueCallback item = group.getCallback(i)) { + publish(item); + } + } + } + else + publish(callback); } catch(RuntimeException ex) { fail(DMLRuntimeException.of(ex)); diff --git a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java index 7a9c6ba1e51..0fe0f2e01c3 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java @@ -43,6 +43,7 @@ import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; import org.apache.sysds.runtime.ooc.store.CountingLiveness; import org.apache.sysds.runtime.ooc.store.IndexedMaterializedStoreReader; +import org.apache.sysds.runtime.ooc.store.MaterializedStoreStreamable; import org.apache.sysds.runtime.ooc.stream.FilteredOOCStream; import org.apache.sysds.runtime.ooc.stream.StreamContext; import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; @@ -117,6 +118,41 @@ public void testPlannerDoubleMaterialize() { } } + @Test + public void testReusableMaterializedStream() { + OOCCacheManager.reset(); + try { + MatrixObject data = new MatrixObject(ValueType.FP64, "/dev/null", + new MetaDataFormat(new MatrixCharacteristics(1, 2, 1), FileFormat.BINARY)); + SubscribableTaskQueue source = new SubscribableTaskQueue<>(); + source.setData(data); + source.enqueue(new IndexedMatrixValue(new MatrixIndexes(1, 1), new MatrixBlock(1, 1, 3d))); + source.enqueue(new IndexedMatrixValue(new MatrixIndexes(1, 2), new MatrixBlock(1, 1, 4d))); + source.closeInput(); + + MaterializedStoreStreamable handle = new MaterializedStoreStreamable(source, data); + handle.reserveLazyHandle(); + handle.reserveLazyHandle(); + handle.scheduleMaterializedStoreDeletion(); + OOCStream first = handle.getReservedReadStream(); + OOCStream second = handle.getReservedReadStream(); + first.start(); + + for(OOCStream replay : List.of(first, second)) { + double sum = 0; + OOCStream.QueueCallback callback; + while((callback = replay.dequeueCB()) != null) + try(OOCStream.QueueCallback current = callback) { + sum += current.get().getValue().get(0, 0); + } + Assert.assertEquals(7, sum, 0); + } + } + finally { + OOCCacheManager.reset(); + } + } + @Test public void testDataGenMapTransposePipeline() { SubscribableTaskQueue generated = new SubscribableTaskQueue<>(); diff --git a/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCPackedCacheTest.java b/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCPackedCacheTest.java index 7a29f8781b4..3dbceb88d0a 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCPackedCacheTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/cache/OOCPackedCacheTest.java @@ -179,8 +179,8 @@ public void testReferenceAndDereferencePackedLocations() throws Exception { try { producer.reserveBlocking(BYTES); BlockEntry pending = cache.putPinned(STREAM_ID, 0, value(13.0), BYTES, producer); - Assert.assertEquals(2, cache.reference(pending)); - Assert.assertEquals(1, cache.dereference(pending)); + Assert.assertEquals(3, cache.reference(pending)); + Assert.assertEquals(2, cache.dereference(pending)); unpinAndFlush(cache, producer, new BlockEntry[] {pending}); awaitUsedMemory(producer, 0, WAIT_TIMEOUT_SEC); @@ -188,9 +188,11 @@ public void testReferenceAndDereferencePackedLocations() throws Exception { BlockEntry pinned = cache.pin(STREAM_ID, 0, reader).get(WAIT_TIMEOUT_SEC, TimeUnit.SECONDS); Assert.assertNotNull(pinned); Assert.assertEquals(13.0, scalar(pinned), 0.0); + Assert.assertEquals(3, cache.reference(pinned)); + Assert.assertEquals(2, cache.dereference(pinned)); + Assert.assertEquals(1, cache.dereference(new BlockKey(STREAM_ID, 0))); Assert.assertEquals(2, cache.reference(pinned)); Assert.assertEquals(1, cache.dereference(pinned)); - Assert.assertEquals(0, cache.dereference(new BlockKey(STREAM_ID, 0))); await(cache.unpin(pinned, reader), WAIT_TIMEOUT_SEC); awaitUsedMemory(reader, 0, WAIT_TIMEOUT_SEC); From be8c1507551fa16576f3c00193af32f484c3d1c8 Mon Sep 17 00:00:00 2001 From: Grigorii Turchenko Date: Tue, 4 Aug 2026 17:31:34 +0200 Subject: [PATCH 101/132] [SYSTEMDS-3955] Support SSL authentication for federated network communication --- .gitignore | 2 + conf/SystemDS-config.xml.template | 13 + pom.xml | 2 + scripts/tutorials/federated/conf/ssl.xml | 8 + .../java/org/apache/sysds/conf/DMLConfig.java | 7 + .../federated/FederatedData.java | 31 ++- .../federated/FederatedSSLUtil.java | 93 ++++++- .../federated/FederatedWorker.java | 83 +++---- .../federated/io/FederatedSSLTest.java | 232 +++++++++++++----- src/test/resources/cert/ca-cert.pem | 18 ++ src/test/resources/cert/localhost-cert.pem | 37 +++ src/test/resources/cert/localhost-key.pem | 28 +++ src/test/resources/cert/otherhost-cert.pem | 37 +++ src/test/resources/cert/otherhost-key.pem | 28 +++ src/test/resources/cert/untrusted-ca-cert.pem | 19 ++ .../io/config/OtherHostSSLConfig.xml | 29 +++ .../federated/io/config/SignedSSLConfig.xml | 29 +++ .../io/config/UntrustedSSLConfig.xml | 27 ++ .../federated/io/generate-certificates.sh | 72 ++++++ 19 files changed, 675 insertions(+), 120 deletions(-) create mode 100644 src/test/resources/cert/ca-cert.pem create mode 100644 src/test/resources/cert/localhost-cert.pem create mode 100644 src/test/resources/cert/localhost-key.pem create mode 100644 src/test/resources/cert/otherhost-cert.pem create mode 100644 src/test/resources/cert/otherhost-key.pem create mode 100644 src/test/resources/cert/untrusted-ca-cert.pem create mode 100644 src/test/scripts/functions/federated/io/config/OtherHostSSLConfig.xml create mode 100644 src/test/scripts/functions/federated/io/config/SignedSSLConfig.xml create mode 100644 src/test/scripts/functions/federated/io/config/UntrustedSSLConfig.xml create mode 100755 src/test/scripts/functions/federated/io/generate-certificates.sh diff --git a/.gitignore b/.gitignore index acb9435ac64..590413bb8f9 100644 --- a/.gitignore +++ b/.gitignore @@ -153,6 +153,8 @@ venv/* # resource optimization scripts/resource/output *.pem +# except the certificates of the federated SSL tests, which are checked in +!src/test/resources/cert/*.pem # docker tests docker/mountFolder/*.bin diff --git a/conf/SystemDS-config.xml.template b/conf/SystemDS-config.xml.template index 153dcb6ef2d..ced2352b4b8 100644 --- a/conf/SystemDS-config.xml.template +++ b/conf/SystemDS-config.xml.template @@ -146,6 +146,19 @@ none + + + + + + + + + + + 15 diff --git a/pom.xml b/pom.xml index be50b05a92c..2560a38b7b7 100644 --- a/pom.xml +++ b/pom.xml @@ -734,6 +734,8 @@ **/*.mtx **/*.mtd **/*.out + + src/test/resources/cert/*.pem **/__pycache__/** **/part-* **/*.keep diff --git a/scripts/tutorials/federated/conf/ssl.xml b/scripts/tutorials/federated/conf/ssl.xml index f375ba33fed..2a9cedd6658 100644 --- a/scripts/tutorials/federated/conf/ssl.xml +++ b/scripts/tutorials/federated/conf/ssl.xml @@ -18,4 +18,12 @@ --> true + + + /path/to/worker-cert.pem + /path/to/worker-key.pem + /path/to/ca-cert.pem \ No newline at end of file diff --git a/src/main/java/org/apache/sysds/conf/DMLConfig.java b/src/main/java/org/apache/sysds/conf/DMLConfig.java index d114ccf69b9..3a0829922a5 100644 --- a/src/main/java/org/apache/sysds/conf/DMLConfig.java +++ b/src/main/java/org/apache/sysds/conf/DMLConfig.java @@ -125,6 +125,9 @@ public class DMLConfig public static final String EVICTION_SHADOW_BUFFERSIZE = "sysds.gpu.eviction.shadow.bufferSize"; public static final String USE_SSL_FEDERATED_COMMUNICATION = "sysds.federated.ssl"; // boolean + public static final String FEDERATED_SSL_CERT = "sysds.federated.ssl.cert"; // Path to the worker X.509 certificate chain in PEM format, required if federated SSL is enabled + public static final String FEDERATED_SSL_KEY = "sysds.federated.ssl.key"; // Path to the worker private key in PKCS#8 PEM format, required if federated SSL is enabled + public static final String FEDERATED_SSL_TRUST = "sysds.federated.ssl.trust"; // Path to the trusted (CA) certificates in PEM format, required if federated SSL is enabled public static final String DEFAULT_FEDERATED_INITIALIZATION_TIMEOUT = "sysds.federated.initialization.timeout"; // int seconds public static final String FEDERATED_TIMEOUT = "sysds.federated.timeout"; // single request timeout default -1 to indicate infinite. public static final String FEDERATED_PLANNER = "sysds.federated.planner"; @@ -211,6 +214,9 @@ public class DMLConfig _defaultVals.put(GPU_RULE_BASED_PLACEMENT, "false"); _defaultVals.put(FLOATING_POINT_PRECISION, "double" ); _defaultVals.put(USE_SSL_FEDERATED_COMMUNICATION, "false"); + _defaultVals.put(FEDERATED_SSL_CERT, null); + _defaultVals.put(FEDERATED_SSL_KEY, null); + _defaultVals.put(FEDERATED_SSL_TRUST, null); _defaultVals.put(DEFAULT_FEDERATED_INITIALIZATION_TIMEOUT, "10"); _defaultVals.put(FEDERATED_TIMEOUT, "86400"); // default 1 day compute timeout. _defaultVals.put(FEDERATED_PLANNER, FederatedPlanner.RUNTIME.name()); @@ -475,6 +481,7 @@ public String getConfigInfo() { PRINT_GPU_MEMORY_INFO, AVAILABLE_GPUS, SYNCHRONIZE_GPU, EAGER_CUDA_FREE, GPU_RULE_BASED_PLACEMENT, FLOATING_POINT_PRECISION, GPU_EVICTION_POLICY, LOCAL_SPARK_NUM_THREADS, EVICTION_SHADOW_BUFFERSIZE, GPU_MEMORY_ALLOCATOR, GPU_MEMORY_UTILIZATION_FACTOR, USE_SSL_FEDERATED_COMMUNICATION, + FEDERATED_SSL_CERT, FEDERATED_SSL_KEY, FEDERATED_SSL_TRUST, DEFAULT_FEDERATED_INITIALIZATION_TIMEOUT, FEDERATED_TIMEOUT, FEDERATED_MONITOR_FREQUENCY, FEDERATED_COMPRESSION, ASYNC_PREFETCH, ASYNC_SPARK_BROADCAST, ASYNC_SPARK_CHECKPOINT, IO_COMPRESSION_CODEC }; diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java index 19277ba0843..3c6e64ada59 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java @@ -242,6 +242,8 @@ private static ChannelInitializer createChannel(InetSocketAddress DataRequestHandler handler) { final int timeout = ConfigurationManager.getFederatedTimeout(); final boolean ssl = ConfigurationManager.isFederatedSSL(); + if(ssl) + FederatedSSLUtil.SslConstructor(); return new ChannelInitializer<>() { @Override @@ -308,18 +310,43 @@ public synchronized static void createWorkGroup() { } private static class DataRequestHandler extends ChannelInboundHandlerAdapter { + // The promise is assigned by the requesting thread, while the channel events below are handled on the + // event loop, and the two orders are not guaranteed: a rejected SSL handshake already fails the channel + // while the requesting thread is still connecting. + private final Object _promLock = new Object(); private Promise _prom; + // A failure observed before the promise was assigned, e.g., a rejected SSL handshake + private Throwable _failure; public DataRequestHandler() { } public void setPromise(Promise prom) { - _prom = prom; + synchronized(_promLock) { + _prom = prom; + if(_failure != null) + _prom.tryFailure(_failure); + } } @Override public void channelRead(ChannelHandlerContext ctx, Object msg) { - _prom.setSuccess((FederatedResponse) msg); + synchronized(_promLock) { + _prom.setSuccess((FederatedResponse) msg); + } + ctx.close(); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + // fail the request instead of waiting for a response that never arrives + // covers a failed SSL handshake, e.g., if the worker certificate is not signed by a trusted authority. + synchronized(_promLock) { + if(_prom != null) + _prom.tryFailure(cause); + else + _failure = cause; + } ctx.close(); } diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedSSLUtil.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedSSLUtil.java index f1300ef0f4a..33a89a5a4f8 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedSSLUtil.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedSSLUtil.java @@ -19,19 +19,29 @@ package org.apache.sysds.runtime.controlprogram.federated; +import java.io.File; import java.net.InetSocketAddress; +import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLException; +import javax.net.ssl.SSLParameters; +import org.apache.log4j.Logger; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; import org.apache.sysds.runtime.DMLRuntimeException; import io.netty.channel.socket.SocketChannel; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.SslHandler; -import io.netty.handler.ssl.util.InsecureTrustManagerFactory; public class FederatedSSLUtil { + private static final Logger LOG = Logger.getLogger(FederatedSSLUtil.class); + + // The password of an encrypted worker private key is read from the environment instead of the configuration, + // so that it is not leaked when the configuration file is shared or published. + public static final String SSL_KEY_PASSWORD_ENV = "SYSTEMDS_FEDERATED_SSL_KEY_PASSWORD"; private FederatedSSLUtil(){ // private constructor. @@ -40,24 +50,93 @@ private FederatedSSLUtil(){ /** A Singleton constructed SSL context, that only is assigned if ssl is enabled. */ private static SslContextMan sslInstance = null; - protected static SslContextMan SslConstructor() { + protected synchronized static SslContextMan SslConstructor() { if(sslInstance == null) - return new SslContextMan(); - else - return sslInstance; + sslInstance = new SslContextMan(); + return sslInstance; + } + + // Drop the cached client side SSL context, so that the next connection is built from the current configuration. + // Only relevant if the configuration changes while the JVM is running, as it does in tests. + public synchronized static void resetClientContext() { + sslInstance = null; } protected static SslHandler createSSLHandler(SocketChannel ch, InetSocketAddress address) { - return SslConstructor().context.newHandler(ch.alloc(), address.getAddress().getHostAddress(), address.getPort()); + final SslContextMan man = SslConstructor(); + // prefer the configured host name over the resolved address, since certificates are issued for host names. + final String host = (address.getHostString() != null) ? address.getHostString() : address.getAddress() + .getHostAddress(); + final SslHandler handler = man.context.newHandler(ch.alloc(), host, address.getPort()); + + // the certificate of a worker has to be issued for the host it is contacted on, otherwise any worker + // with a trusted certificate could impersonate any other worker. + final SSLEngine engine = handler.engine(); + final SSLParameters params = engine.getSSLParameters(); + params.setEndpointIdentificationAlgorithm("HTTPS"); + engine.setSSLParameters(params); + + return handler; + } + + /** + * Construct the SSL context of a federated worker, based on the certificate and private key configured via + * {@link DMLConfig#FEDERATED_SSL_CERT} and {@link DMLConfig#FEDERATED_SSL_KEY}. Both are required, a worker that + * cannot be authenticated by the coordinator is not supported. If the private key is encrypted, its password is + * read from the {@link #SSL_KEY_PASSWORD_ENV} environment variable. + * + * @return The server side SSL context of the federated worker + */ + public static SslContext createServerContext() { + final DMLConfig conf = ConfigurationManager.getDMLConfig(); + final String certPath = conf.getTextValue(DMLConfig.FEDERATED_SSL_CERT); + final String keyPath = conf.getTextValue(DMLConfig.FEDERATED_SSL_KEY); + final String keyPassword = System.getenv(SSL_KEY_PASSWORD_ENV); + + if(!isSet(certPath) || !isSet(keyPath)) + throw new DMLRuntimeException("Federated SSL requires a signed certificate, configure the certificate " + + "chain in " + DMLConfig.FEDERATED_SSL_CERT + " and the matching private key in " + + DMLConfig.FEDERATED_SSL_KEY + "."); + + try { + LOG.info("Federated worker SSL using certificate: " + certPath); + return SslContextBuilder + .forServer(readableFile(certPath, DMLConfig.FEDERATED_SSL_CERT), + readableFile(keyPath, DMLConfig.FEDERATED_SSL_KEY), isSet(keyPassword) ? keyPassword : null) + .build(); + } + catch(SSLException e) { + throw new DMLRuntimeException("Static SSL setup failed for worker side", e); + } + } + + private static boolean isSet(String value) { + return value != null && !value.trim().isEmpty(); } + private static File readableFile(String path, String configName) { + final File f = new File(path.trim()); + if(!f.canRead()) + throw new DMLRuntimeException( + "Federated SSL file configured in " + configName + " is not a readable file: " + path); + return f; + } private static class SslContextMan { protected final SslContext context; private SslContextMan() { + final DMLConfig conf = ConfigurationManager.getDMLConfig(); + final String trustPath = conf.getTextValue(DMLConfig.FEDERATED_SSL_TRUST); + + if(!isSet(trustPath)) + throw new DMLRuntimeException("Federated SSL requires the certificates that are trusted to sign " + + "worker certificates, configure them in " + DMLConfig.FEDERATED_SSL_TRUST + "."); + try { - context = SslContextBuilder.forClient().trustManager(InsecureTrustManagerFactory.INSTANCE).build(); + LOG.debug("Federated SSL trusting certificates in: " + trustPath); + context = SslContextBuilder.forClient() + .trustManager(readableFile(trustPath, DMLConfig.FEDERATED_SSL_TRUST)).build(); } catch(SSLException e) { throw new DMLRuntimeException("Static SSL setup failed for client side", e); diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java index fc8989053bc..682cc8e3fff 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java @@ -20,20 +20,16 @@ package org.apache.sysds.runtime.controlprogram.federated; import java.io.Serializable; -import java.security.cert.CertificateException; import java.util.Optional; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import javax.net.ssl.SSLException; - import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.log4j.Logger; import org.apache.sysds.api.DMLScript; import org.apache.sysds.conf.ConfigurationManager; import org.apache.sysds.conf.DMLConfig; -import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.controlprogram.caching.CacheBlock; import org.apache.sysds.runtime.controlprogram.federated.compression.CompressionDecoderEndStatisticsHandler; import org.apache.sysds.runtime.controlprogram.federated.compression.CompressionDecoderStartStatisticsHandler; @@ -63,8 +59,6 @@ import io.netty.handler.codec.serialization.ObjectDecoder; import io.netty.handler.codec.serialization.ObjectEncoder; import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslContextBuilder; -import io.netty.handler.ssl.util.SelfSignedCertificate; import io.netty.util.concurrent.DefaultThreadFactory; @SuppressWarnings("deprecation") @@ -121,12 +115,16 @@ private void run() { LOG.info("Started Federated Worker at port: " + _port); f.channel().closeFuture().sync(); } - catch(Exception e) { + catch(InterruptedException e) { LOG.info("Federated worker interrupted"); - if(_debug) { - LOG.error(e.getMessage()); + if(_debug) + e.printStackTrace(); + } + catch(Exception e) { + // report why the worker stops, e.g., a missing certificate with ssl enabled, otherwise it exits silently + LOG.error("Federated worker stopped: " + e.getMessage()); + if(_debug) e.printStackTrace(); - } } finally { LOG.info("Federated Worker Shutting down."); @@ -192,47 +190,30 @@ protected void encode(ChannelHandlerContext ctx, Serializable msg, ByteBuf out) } private ChannelInitializer createChannel(boolean ssl) { - try { - // TODO add ability to use real ssl files, not self signed certificates. - final SelfSignedCertificate cert; - final SslContext cont2; - final boolean sslEnabled = ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.USE_SSL_FEDERATED_COMMUNICATION) || ssl; - - if(ssl) { - cert = new SelfSignedCertificate(); - cont2 = SslContextBuilder.forServer(cert.certificate(), cert.privateKey()).build(); - } - else { - cert = null; - cont2 = null; + final SslContext sslContext = ssl ? FederatedSSLUtil.createServerContext() : null; + + return new ChannelInitializer<>() { + @Override + public void initChannel(SocketChannel ch) { + final ChannelPipeline cp = ch.pipeline(); + if(sslContext != null) + cp.addLast(sslContext.newHandler(ch.alloc())); + + final Optional> compressionStrategy = FederationUtils + .compressionStrategy(); + cp.addLast("NetworkTrafficCounter", new NetworkTrafficCounter(FederatedStatistics::logWorkerTraffic)); + cp.addLast("CompressionDecodingStartStatistics", new CompressionDecoderStartStatisticsHandler()); + compressionStrategy.ifPresent(strategy -> cp.addLast("CompressionDecoder", strategy.left)); + cp.addLast("CompressionDecoderEndStatistics", new CompressionDecoderEndStatisticsHandler()); + cp.addLast("ObjectDecoder", new ObjectDecoder(Integer.MAX_VALUE, + ClassResolvers.weakCachingResolver(ClassLoader.getSystemClassLoader()))); + cp.addLast("CompressionEncodingEndStatistics", new CompressionEncoderEndStatisticsHandler()); + compressionStrategy.ifPresent(strategy -> cp.addLast("CompressionEncoder", strategy.right)); + cp.addLast("CompressionEncodingStartStatistics", new CompressionEncoderStartStatisticsHandler()); + cp.addLast("ObjectEncoder", new ObjectEncoder()); + cp.addLast(FederationUtils.decoder(), new FederatedResponseEncoder()); + cp.addLast(new FederatedWorkerHandler(_flt, _frc, _fan, networkTimer)); } - - return new ChannelInitializer<>() { - @Override - public void initChannel(SocketChannel ch) { - final ChannelPipeline cp = ch.pipeline(); - if(sslEnabled) - cp.addLast(cont2.newHandler(ch.alloc())); - - final Optional> compressionStrategy = FederationUtils.compressionStrategy(); - cp.addLast("NetworkTrafficCounter", new NetworkTrafficCounter(FederatedStatistics::logWorkerTraffic)); - cp.addLast("CompressionDecodingStartStatistics", new CompressionDecoderStartStatisticsHandler()); - compressionStrategy.ifPresent(strategy -> cp.addLast("CompressionDecoder", strategy.left)); - cp.addLast("CompressionDecoderEndStatistics", new CompressionDecoderEndStatisticsHandler()); - cp.addLast("ObjectDecoder", - new ObjectDecoder(Integer.MAX_VALUE, - ClassResolvers.weakCachingResolver(ClassLoader.getSystemClassLoader()))); - cp.addLast("CompressionEncodingEndStatistics", new CompressionEncoderEndStatisticsHandler()); - compressionStrategy.ifPresent(strategy -> cp.addLast("CompressionEncoder", strategy.right)); - cp.addLast("CompressionEncodingStartStatistics", new CompressionEncoderStartStatisticsHandler()); - cp.addLast("ObjectEncoder", new ObjectEncoder()); - cp.addLast(FederationUtils.decoder(), new FederatedResponseEncoder()); - cp.addLast(new FederatedWorkerHandler(_flt, _frc, _fan, networkTimer)); - } - }; - } - catch(CertificateException | SSLException e) { - throw new DMLRuntimeException("Failed creating channel SSL", e); - } + }; } } diff --git a/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedSSLTest.java b/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedSSLTest.java index 5f5c09e07cb..ac462e447ab 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedSSLTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedSSLTest.java @@ -18,132 +18,244 @@ */ package org.apache.sysds.test.functions.federated.io; - import java.io.File; -import java.util.Arrays; -import java.util.Collection; +import java.net.InetSocketAddress; +import java.security.cert.CertificateException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import javax.net.ssl.SSLException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.sysds.common.Types; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; import org.apache.sysds.runtime.controlprogram.federated.FederatedData; +import org.apache.sysds.runtime.controlprogram.federated.FederatedRequest; +import org.apache.sysds.runtime.controlprogram.federated.FederatedRequest.RequestType; +import org.apache.sysds.runtime.controlprogram.federated.FederatedResponse; +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.controlprogram.federated.FederatedSSLUtil; import org.apache.sysds.runtime.meta.MatrixCharacteristics; import org.apache.sysds.test.AutomatedTestBase; import org.apache.sysds.test.TestConfiguration; import org.apache.sysds.test.TestUtils; import org.apache.sysds.test.functions.federated.FederatedTestObjectConstructor; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -@RunWith(value = Parameterized.class) @net.jcip.annotations.NotThreadSafe public class FederatedSSLTest extends AutomatedTestBase { private static final Log LOG = LogFactory.getLog(FederatedSSLTest.class.getName()); - // This test use the same scripts as the Federated Reader tests, just with SSL enabled. + // These tests use the same scripts as the Federated Reader tests, just with SSL enabled. private final static String TEST_DIR = "functions/federated/io/"; private final static String TEST_NAME = "FederatedReaderTest"; private final static String TEST_CLASS_DIR = TEST_DIR + FederatedSSLTest.class.getSimpleName() + "/"; private final static int blocksize = 1024; - private final static File TEST_CONF_FILE = new File(SCRIPT_DIR + TEST_DIR + "SSLConfig.xml"); + private final static int rows = 10; + private final static int cols = 13; - @Parameterized.Parameter() - public int rows; - @Parameterized.Parameter(1) - public int cols; - @Parameterized.Parameter(2) - public boolean rowPartitioned; - @Parameterized.Parameter(3) - public int fedCount; + private final static String CONF_DIR = SCRIPT_DIR + TEST_DIR + "config/"; + // Certificate issued for localhost and signed by the authority the coordinator trusts + private final static File SIGNED_CONF = new File(CONF_DIR, "SignedSSLConfig.xml"); + // Coordinator trusting an authority unrelated to the one that signed the worker certificate + private final static File UNTRUSTED_CONF = new File(CONF_DIR, "UntrustedSSLConfig.xml"); + // Certificate signed by the trusted authority, but issued for another host + private final static File OTHER_HOST_CONF = new File(CONF_DIR, "OtherHostSSLConfig.xml"); + // SSL enabled without configuring any certificate, which is not supported + private final static File NO_CERT_CONF = new File(SCRIPT_DIR + TEST_DIR + "SSLConfig.xml"); + + private File confFile = SIGNED_CONF; + private File workerConfFile = null; @Override public void setUp() { TestUtils.clearAssertionInformation(); addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME)); + // The SSL context of the coordinator is cached for the JVM, so it should be cleared in between the tests + FederatedSSLUtil.resetClientContext(); + FederatedData.resetFederatedSites(); } - @Parameterized.Parameters - public static Collection data() { - // number of rows or cols has to be >= number of federated locations. - return Arrays.asList(new Object[][] {{10, 13, true, 2}}); + // A certificate signed by the trusted authority and issued for the contacted host is accepted. + @Test + public void federatedSinglenodeReadSigned() { + // the workers and the coordinator share one configuration, the coordinator trusts the signing authority + confFile = SIGNED_CONF; + federatedRead(Types.ExecMode.SINGLE_NODE); } + // A certificate signed by an authority the coordinator does not know is rejected. @Test - @Ignore - public void federatedSinglenodeRead() { - federatedRead(Types.ExecMode.SINGLE_NODE); + public void untrustedCertificateIsRejected() { + workerConfFile = SIGNED_CONF; + confFile = UNTRUSTED_CONF; + assertRequestFails(); + } + + // A certificate signed by the trusted authority, but issued for another host, is rejected as well. + @Test + public void certificateOfOtherHostIsRejected() { + confFile = OTHER_HOST_CONF; + assertRequestFails(); } - public void federatedRead(Types.ExecMode execMode) { - Types.ExecMode oldPlatform = setExecMode(execMode); + // A worker cannot serve SSL without a certificate, generating one is not supported. + @Test + public void workerCertificateIsRequired() throws Exception { + confFile = NO_CERT_CONF; + getAndLoadTestConfiguration(TEST_NAME); + ConfigurationManager.setGlobalConfig(new DMLConfig(getCurConfigFile().getPath())); + + try { + FederatedSSLUtil.createServerContext(); + Assert.fail("A worker without a configured certificate should not start SSL."); + } + catch(DMLRuntimeException e) { + Assert.assertTrue("Expected a hint at the certificate configuration but got: " + e.getMessage(), + e.getMessage().contains(DMLConfig.FEDERATED_SSL_CERT)); + } + } + + // A coordinator cannot connect without knowing which certificates to trust. + @Test + public void trustedCertificatesAreRequired() { + workerConfFile = SIGNED_CONF; + confFile = NO_CERT_CONF; + getAndLoadTestConfiguration(TEST_NAME); + fullDMLScriptName = ""; + final int port = getRandomAvailablePort(); + final Thread[] workers = startWorkers(port); + + try { + // the worker startup sets the global configuration in this JVM, therefore the coordinator + // configuration has to be applied after the workers are up. + ConfigurationManager.setGlobalConfig(new DMLConfig(getCurConfigFile().getPath())); + + FederatedData.executeFederatedOperation(new InetSocketAddress("localhost", port), + new FederatedRequest(RequestType.CLEAR)).get(FED_WORKER_WAIT, TimeUnit.MILLISECONDS); + Assert.fail("A coordinator without trusted certificates should not connect."); + } + catch(Exception e) { + Assert.assertTrue("Expected a hint at the trust configuration but got: " + e, + messageContains(e, DMLConfig.FEDERATED_SSL_TRUST)); + } + finally { + TestUtils.shutdownThreads(workers); + } + } + + // Read a federated matrix over SSL and compare against the same matrix read locally. + private void federatedRead(Types.ExecMode execMode) { + final Types.ExecMode oldPlatform = setExecMode(execMode); getAndLoadTestConfiguration(TEST_NAME); setOutputBuffering(true); - + // write input matrices - int halfRows = rows / 2; - long[][] begins = new long[][] {new long[] {0, 0}, new long[] {halfRows, 0}}; - long[][] ends = new long[][] {new long[] {halfRows, cols}, new long[] {rows, cols}}; + final int halfRows = rows / 2; + final long[][] begins = new long[][] {new long[] {0, 0}, new long[] {halfRows, 0}}; + final long[][] ends = new long[][] {new long[] {halfRows, cols}, new long[] {rows, cols}}; // We have two matrices handled by a single federated worker - double[][] X1 = getRandomMatrix(halfRows, cols, 0, 1, 1, 42); - double[][] X2 = getRandomMatrix(halfRows, cols, 0, 1, 1, 1340); + final double[][] X1 = getRandomMatrix(halfRows, cols, 0, 1, 1, 42); + final double[][] X2 = getRandomMatrix(halfRows, cols, 0, 1, 1, 1340); writeInputMatrixWithMTD("X1", X1, false, new MatrixCharacteristics(halfRows, cols, blocksize, halfRows * cols)); writeInputMatrixWithMTD("X2", X2, false, new MatrixCharacteristics(halfRows, cols, blocksize, halfRows * cols)); // empty script name because we don't execute any script, just start the worker fullDMLScriptName = ""; - int port1 = getRandomAvailablePort(); - int port2 = getRandomAvailablePort(); - Thread[] workers = startLocalFedWorkerThreads(new int[] {port1, port2}, null, FED_WORKER_WAIT); - String host = "localhost"; + final int port1 = getRandomAvailablePort(); + final int port2 = getRandomAvailablePort(); + final Thread[] workers = startWorkers(port1, port2); - try { - MatrixObject fed = FederatedTestObjectConstructor.constructFederatedInput( - rows, cols, blocksize, host, begins, ends, new int[] {port1, port2}, - new String[] {input("X1"), input("X2")}, input("X.json")); - //FIXME: reset avoids deadlock on reference script - //(because federated matrix creation added to federated sites - blocks on clear) - //However, there seems to be a regression regarding the SSL handling in general + final MatrixObject fed = FederatedTestObjectConstructor.constructFederatedInput(rows, cols, blocksize, + "localhost", begins, ends, new int[] {port1, port2}, new String[] {input("X1"), input("X2")}, + input("X.json")); + // FIXME: reset avoids deadlock on reference script + // (because federated matrix creation added to federated sites - blocks on clear) FederatedData.resetFederatedSites(); writeInputFederatedWithMTD("X.json", fed); + // Run reference dml script with normal matrix - fullDMLScriptName = SCRIPT_DIR + "functions/federated/io/" + TEST_NAME + (rowPartitioned ? "Row" : "Col") - + "2Reference.dml"; + fullDMLScriptName = SCRIPT_DIR + TEST_DIR + TEST_NAME + "Row2Reference.dml"; programArgs = new String[] {"-stats", "-args", input("X1"), input("X2")}; - String refOut = runTest(null).toString(); - + final String refOut = runTest(null).toString(); + // Run federated - fullDMLScriptName = SCRIPT_DIR + "functions/federated/io/" + TEST_NAME + ".dml"; + fullDMLScriptName = SCRIPT_DIR + TEST_DIR + TEST_NAME + ".dml"; programArgs = new String[] {"-stats", "-args", input("X.json")}; - String out = runTest(null).toString(); + final String out = runTest(null).toString(); Assert.assertTrue(heavyHittersContainsString("fed_uak+")); // Verify output - Assert.assertEquals(Double.parseDouble(refOut.split("\n")[0]), - Double.parseDouble(out.split("\n")[0]), 0.00001); + Assert.assertEquals(Double.parseDouble(refOut.split("\n")[0]), Double.parseDouble(out.split("\n")[0]), + 0.00001); } catch(Exception e) { e.printStackTrace(); - Assert.assertTrue(false); + Assert.fail("Federated read over SSL failed: " + e.getMessage()); } finally { resetExecMode(oldPlatform); + TestUtils.shutdownThreads(workers); + } + } + + // The coordinator rejects the certificate of the worker instead of getting an answer to its request. + private void assertRequestFails() { + getAndLoadTestConfiguration(TEST_NAME); + fullDMLScriptName = ""; + final int port = getRandomAvailablePort(); + final Thread[] workers = startWorkers(port); + + try { + // the worker startup sets the global configuration in this JVM, therefore the coordinator + // configuration has to be applied after the workers are up. + ConfigurationManager.setGlobalConfig(new DMLConfig(getCurConfigFile().getPath())); + + final Future f = FederatedData.executeFederatedOperation( + new InetSocketAddress("localhost", port), new FederatedRequest(RequestType.CLEAR)); + + f.get(FED_WORKER_WAIT, TimeUnit.MILLISECONDS); + Assert.fail("The request to a worker with a rejected certificate should not succeed."); } + catch(ExecutionException e) { + Assert.assertTrue("Expected an SSL failure but got: " + e.getCause(), isSSLFailure(e.getCause())); + } + catch(Exception e) { + e.printStackTrace(); + Assert.fail("Expected an SSL failure but got: " + e); + } + finally { + TestUtils.shutdownThreads(workers); + } + } + + // The workers have to be started with a configuration as well, otherwise they do not enable SSL. + private Thread[] startWorkers(int... ports) { + final File conf = (workerConfFile != null) ? workerConfFile : getCurConfigFile(); + return startLocalFedWorkerThreads(ports, new String[] {"-config", conf.getPath()}, FED_WORKER_WAIT); + } + + private static boolean isSSLFailure(Throwable t) { + for(Throwable c = t; c != null; c = c.getCause()) + if(c instanceof SSLException || c instanceof CertificateException) + return true; + return false; + } - TestUtils.shutdownThreads(workers); + private static boolean messageContains(Throwable t, String text) { + for(Throwable c = t; c != null; c = c.getCause()) + if(c.getMessage() != null && c.getMessage().contains(text)) + return true; + return false; } - /** - * Override default configuration with custom test configuration to ensure - * scratch space and local temporary directory locations are also updated. - */ @Override protected File getConfigTemplateFile() { - // Instrumentation in this test's output log to show custom configuration file used for template. - LOG.info("This test case overrides default configuration with " + TEST_CONF_FILE.getPath()); - return TEST_CONF_FILE; + return confFile; } } diff --git a/src/test/resources/cert/ca-cert.pem b/src/test/resources/cert/ca-cert.pem new file mode 100644 index 00000000000..5a6e3a18c2c --- /dev/null +++ b/src/test/resources/cert/ca-cert.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC7TCCAdWgAwIBAgIJAIxu3kp0I0VqMA0GCSqGSIb3DQEBCwUAMBsxGTAXBgNV +BAMTEFN5c3RlbURTIFRlc3QgQ0EwIBcNMjYwNzI5MTIyOTA4WhgPMjEyNjA3MDUx +MjI5MDhaMBsxGTAXBgNVBAMTEFN5c3RlbURTIFRlc3QgQ0EwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQCr0ehP5tRAlvTEPNj0nx5D0sJTQRkdMCs3VERt +dDRebaecIOZse5Uve4r5qKr/YyTnscFZvl4CeM04AKXljhcUPeea/e2eeQegh7IF +bCPWJXcbTFe4LzJ89asZZEqV7BN0IfDU9OoGKyPILF/1UJ79e8N89KoIiHZYFl9o +Lj4KlD3dWKRluWG6TWYxBYcnBt2D/cFXXaEa62rqoVp1hIQ/MF232V9eOBvYiCPs +IYnhPAIjkYz8LhYp02azeULGD3mrck08AFeGjLuXHBVWsFkcM0nb2stOenhkKj8k +MqbNVA8/iFPrrbkCLROffZnIa0JficXAR2dbe7eUXhHnCpt1AgMBAAGjMjAwMB0G +A1UdDgQWBBQFgrHcX+3rTCqFWbekJ4qIrH68izAPBgNVHRMBAf8EBTADAQH/MA0G +CSqGSIb3DQEBCwUAA4IBAQCfHThIA+XEdpP3a/ySBNS2yrsrKe1G6JDRdzMNQiQv +t1ymTQ1Dm7bDUo7L1fFGX/cMjCo3+2Q5owIGO4t9DYJ6cpSBAad1HLNVmQOTtIeD +/IXwCpODWe4ZL/zcuJcCbidLuEsy56fLNeuf2fkkWCaGL6ehpHsp09MFwgrU7xLa +MpPBchHsKmbx8QbpQBYpMSGKvS9uTtrRyEy6OKyyXJfsfR6GlJbVH1cgUN+eg0cp +PiCbrd6PyeobawteRrfJkCqKvh6/nvdSHXnkfmz47RMC2Obs41LX43Boh5UdHw65 +aJfjh7O8CCHALCWHkvlJOfbEI6s1fj79H98r1//ifY+8 +-----END CERTIFICATE----- diff --git a/src/test/resources/cert/localhost-cert.pem b/src/test/resources/cert/localhost-cert.pem new file mode 100644 index 00000000000..d85ca651bf3 --- /dev/null +++ b/src/test/resources/cert/localhost-cert.pem @@ -0,0 +1,37 @@ +-----BEGIN CERTIFICATE----- +MIIDETCCAfmgAwIBAgIIKPKZck8b4KYwDQYJKoZIhvcNAQELBQAwGzEZMBcGA1UE +AxMQU3lzdGVtRFMgVGVzdCBDQTAgFw0yNjA3MjkxMjI5MTBaGA8yMTI2MDcwNTEy +MjkxMFowFDESMBAGA1UEAxMJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEA5j66iS7szE2psgbLYeLxTlRdSeXhMroUVAhO0jVo59Jafc3S +3sdrOncf1IbUt5Tq8CCVp2dKmRH0TYnWVFsHj8aJzOz4zfTXjlFSbLMWNf64kLM7 +F0F8KxlInmXnMJNrBYDQe+fVw7tjx7eBhahpfLtti7cQZHF8mWirIi3N4fRcTUQZ +tLujsLtL/w8Jl6ef/MZ6zo29Rqg8SsgVtZSWz0m/pS8NfvrIY7I/2lbixaV9HlQD +fCapODmwo5wRSQFHg4PcykZhFUsqV6nOcCmqwfaMlFIhdVRTyvYYO5kDdgqvyOJX +NYwgHCEckHXiOywwfbZAPzhe5kRDOh8mRNxZrQIDAQABo14wXDAdBgNVHQ4EFgQU +bYjG7TKhIoelk+qGTt3UtoFiZ5AwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAAB +MB8GA1UdIwQYMBaAFAWCsdxf7etMKoVZt6QnioisfryLMA0GCSqGSIb3DQEBCwUA +A4IBAQBCkX9kwwZSt+N01Pa9QrlRYyqIJJaUTJXz4X8MxhZ8OrmjV87b0lUDf1wQ +0jlIRIenR/pl1438O/4LEIJb3tErajn8+QqThoDSrlQjhjH5qzef7zUf4f62GbHZ +g0Gbn8IVgYROTk/O72S23jAIsxLF9ZA/DQz3SBRbGynb8OEZQeBBps68YaStlVwk +g04UeycHNuZGV/hB9IgGAwJkH/lbY24CQ5ikTzOfLhHCDdGXEkybJgJNOnOg+ceM +pOfxzeb2zk3M4o66MA0aLMbM2amWvdUBopJKRfEwBC7erNVfZOsx1VnO+zSxgl9L +Ae/CfvTpWoT11H5N8Qop7PoMa70V +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIC7TCCAdWgAwIBAgIJAIxu3kp0I0VqMA0GCSqGSIb3DQEBCwUAMBsxGTAXBgNV +BAMTEFN5c3RlbURTIFRlc3QgQ0EwIBcNMjYwNzI5MTIyOTA4WhgPMjEyNjA3MDUx +MjI5MDhaMBsxGTAXBgNVBAMTEFN5c3RlbURTIFRlc3QgQ0EwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQCr0ehP5tRAlvTEPNj0nx5D0sJTQRkdMCs3VERt +dDRebaecIOZse5Uve4r5qKr/YyTnscFZvl4CeM04AKXljhcUPeea/e2eeQegh7IF +bCPWJXcbTFe4LzJ89asZZEqV7BN0IfDU9OoGKyPILF/1UJ79e8N89KoIiHZYFl9o +Lj4KlD3dWKRluWG6TWYxBYcnBt2D/cFXXaEa62rqoVp1hIQ/MF232V9eOBvYiCPs +IYnhPAIjkYz8LhYp02azeULGD3mrck08AFeGjLuXHBVWsFkcM0nb2stOenhkKj8k +MqbNVA8/iFPrrbkCLROffZnIa0JficXAR2dbe7eUXhHnCpt1AgMBAAGjMjAwMB0G +A1UdDgQWBBQFgrHcX+3rTCqFWbekJ4qIrH68izAPBgNVHRMBAf8EBTADAQH/MA0G +CSqGSIb3DQEBCwUAA4IBAQCfHThIA+XEdpP3a/ySBNS2yrsrKe1G6JDRdzMNQiQv +t1ymTQ1Dm7bDUo7L1fFGX/cMjCo3+2Q5owIGO4t9DYJ6cpSBAad1HLNVmQOTtIeD +/IXwCpODWe4ZL/zcuJcCbidLuEsy56fLNeuf2fkkWCaGL6ehpHsp09MFwgrU7xLa +MpPBchHsKmbx8QbpQBYpMSGKvS9uTtrRyEy6OKyyXJfsfR6GlJbVH1cgUN+eg0cp +PiCbrd6PyeobawteRrfJkCqKvh6/nvdSHXnkfmz47RMC2Obs41LX43Boh5UdHw65 +aJfjh7O8CCHALCWHkvlJOfbEI6s1fj79H98r1//ifY+8 +-----END CERTIFICATE----- diff --git a/src/test/resources/cert/localhost-key.pem b/src/test/resources/cert/localhost-key.pem new file mode 100644 index 00000000000..850a43a06a6 --- /dev/null +++ b/src/test/resources/cert/localhost-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDmPrqJLuzMTamy +Bsth4vFOVF1J5eEyuhRUCE7SNWjn0lp9zdLex2s6dx/UhtS3lOrwIJWnZ0qZEfRN +idZUWwePxonM7PjN9NeOUVJssxY1/riQszsXQXwrGUieZecwk2sFgNB759XDu2PH +t4GFqGl8u22LtxBkcXyZaKsiLc3h9FxNRBm0u6Owu0v/DwmXp5/8xnrOjb1GqDxK +yBW1lJbPSb+lLw1++shjsj/aVuLFpX0eVAN8Jqk4ObCjnBFJAUeDg9zKRmEVSypX +qc5wKarB9oyUUiF1VFPK9hg7mQN2Cq/I4lc1jCAcIRyQdeI7LDB9tkA/OF7mREM6 +HyZE3FmtAgMBAAECggEAHky1/pSiy/YSdV+ohy6248B9cFqkqqjLQQ3A1a/6qLtJ +dlHORMwIg+6mTTEbMDeUPVqEZz3UFtXCiSuw/XPnSFfvzXyH946XiV6RUsW0kBF/ +12cGyTYwcXmH0XSGmqFjzZsYlJ27R2FTLbasAFtb2nLN5TuHmDhJFeUs1Dgj5m6i +gl9gUjAMXqKoyibU/EQsfK7Jrb+++RIr12O0TmsOhsaklRWBHbS9Amq8XTKDCfnl +R08fpwn+sKAetA9q/rrx2oN1Jd9Hmvgii9BiCtT3L3RLwqKWv5EnsDRKQI1Y88cb +VZA3YTU5OskBFRCiJSbcPr48ejYNztLzpNLaBdyT0QKBgQDrMHLJU+6SFtyEsLqh +oadZEtJYhlVMtLg1SH7ro2CoRyaK3E8RPIE3i80pPri8Ah7Wy1bD6/Fn6dYE6a+W +VjyV+Uf7dIPpqgMIlsffcAri+PAY0dZmGu8/iLPauWQiteDK68b6PANGM5FZk/d+ +ccHR7mx0OKwV3HVVaeWhRuMd5QKBgQD6nkhs9fnq+mm2I9mTA1a8V/cP0XnxeTFR +RhC3bqOwf+Fyrnv6UvQEIIWH4TaFfz4RR8WjeJdGqQ+z+Qak4sVjealgJTNZ5F2u +YKrn4pIZDzIyyTI19+y8tp3EywQFEF1zKItfi2h0RiUOPsTo+vD0YYhcqC9oMbHE +ODBpUkFQKQKBgQCWSCYA2Z3nQa51J0yKPXZmp207XdMhqZTPj1xyi7oWrShGsNHh +LK1Q5gcZpNd8Y0p7bAEsPhbKlJPKHdyyDra2Ckzhs6ka5ST9FwPulXSPZgxdf7Al +HG7mRR7P04jV2Swj3hcODMz2zbrB55fM9zmnQFeSyCfF7FIZWwp9TIORtQKBgFed +/rQZSsZbxZln7yj2gdxW5IkjMv644AUJ+c4nYBLUonz1g2KAnc7Tj9txYR5K3egs +r2v3POv3LwY8iZYbseaVIiH633kN3bKZGSb4jxsztNkMfgFgK+PN9FpYn48lqYYZ +JqDAnEQKQeo5B55sHNFTR9kc83X56awv+LzZhPwBAoGALOuOGgxpXNWogMromKTp +zNJt3MmSaJttWIq+peCWq5rinNIwvolYEQIJF3KDmvq7NPHBStEa5LZUVLUEyRO+ +hpBa4vmWmM7BSieVrer7P60zFr3fKoPh8+6wEHkmSCBpHel2iffU2zIcKuqQzSqi +1h2tg7p4ZKqHjislPnaznMc= +-----END PRIVATE KEY----- diff --git a/src/test/resources/cert/otherhost-cert.pem b/src/test/resources/cert/otherhost-cert.pem new file mode 100644 index 00000000000..71f9287eed8 --- /dev/null +++ b/src/test/resources/cert/otherhost-cert.pem @@ -0,0 +1,37 @@ +-----BEGIN CERTIFICATE----- +MIIDGzCCAgOgAwIBAgIIOj85LBlEMC8wDQYJKoZIhvcNAQELBQAwGzEZMBcGA1UE +AxMQU3lzdGVtRFMgVGVzdCBDQTAgFw0yNjA3MjkxMjI5MTFaGA8yMTI2MDcwNTEy +MjkxMVowHDEaMBgGA1UEAxMRb3RoZXIuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQC9/pR8Tu2JQTGasc7yJrRqIjLF5KLj0ysUeEgi +lS0i3sC/hbe165ckbxiKH38+ipiU/h9L+Z5nuLFuQc8RW3ER9TPcvYnIdRtLpq1j +eUGhqAxj3ZgR1MPrpXNJGZRUgAWuGwCCRDFuHdFSWfVh3XF2hpxVIpWgP6dZ4vku +eB2M23d4xyIMCuZ124q56QeOc6mPR0OTGsjn5/H1JFOLNGoZ5xaUcMcIKVs2gosm +1opKI5x3Ty/v1Pdqemj3zBPp1asvzVCn0zF/pnV0MNKQWlrQMiyhWg51f6QNasei +KwpguLp8d1Lt1Q0yLyBoAsf8trK39j95vukyYT2abxLEwcK9AgMBAAGjYDBeMB0G +A1UdDgQWBBT8kGblKRiZZ5Jitaq5QFfXbAAdUTAcBgNVHREEFTATghFvdGhlci5l +eGFtcGxlLmNvbTAfBgNVHSMEGDAWgBQFgrHcX+3rTCqFWbekJ4qIrH68izANBgkq +hkiG9w0BAQsFAAOCAQEAFBOoWyq7NxMAatcgakb1ldOr7HImXBdfgno2B4Hi/R3e +NBUkXCdEoO0FBFDu2qcOgXqIv1d62RCTzgcWcmQM8Iyx6zVmgyIRB2QFIrgHd3F+ +ndO59vz5CsAW8lZ6yDtLTMvM8tjPZrHXgkqpUXGry0wTFBodrYuHTFpUeArmeJ74 +r1kGyrc0P5+ZVeo+PI6rGZRr6prne7sdNHROSLcPWtYyaSIROlEML7mAq1TbzCH1 +Dh+6F89G8mEQG/JPSB3/b3kPFQnm+4VgT1KajWOnT67gqUFI69iBh/DzdqERWkfB +ljUrfnZokHKEaoAZFH3I0B+3dmI2tg1YVzPKjfkA2g== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIC7TCCAdWgAwIBAgIJAIxu3kp0I0VqMA0GCSqGSIb3DQEBCwUAMBsxGTAXBgNV +BAMTEFN5c3RlbURTIFRlc3QgQ0EwIBcNMjYwNzI5MTIyOTA4WhgPMjEyNjA3MDUx +MjI5MDhaMBsxGTAXBgNVBAMTEFN5c3RlbURTIFRlc3QgQ0EwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQCr0ehP5tRAlvTEPNj0nx5D0sJTQRkdMCs3VERt +dDRebaecIOZse5Uve4r5qKr/YyTnscFZvl4CeM04AKXljhcUPeea/e2eeQegh7IF +bCPWJXcbTFe4LzJ89asZZEqV7BN0IfDU9OoGKyPILF/1UJ79e8N89KoIiHZYFl9o +Lj4KlD3dWKRluWG6TWYxBYcnBt2D/cFXXaEa62rqoVp1hIQ/MF232V9eOBvYiCPs +IYnhPAIjkYz8LhYp02azeULGD3mrck08AFeGjLuXHBVWsFkcM0nb2stOenhkKj8k +MqbNVA8/iFPrrbkCLROffZnIa0JficXAR2dbe7eUXhHnCpt1AgMBAAGjMjAwMB0G +A1UdDgQWBBQFgrHcX+3rTCqFWbekJ4qIrH68izAPBgNVHRMBAf8EBTADAQH/MA0G +CSqGSIb3DQEBCwUAA4IBAQCfHThIA+XEdpP3a/ySBNS2yrsrKe1G6JDRdzMNQiQv +t1ymTQ1Dm7bDUo7L1fFGX/cMjCo3+2Q5owIGO4t9DYJ6cpSBAad1HLNVmQOTtIeD +/IXwCpODWe4ZL/zcuJcCbidLuEsy56fLNeuf2fkkWCaGL6ehpHsp09MFwgrU7xLa +MpPBchHsKmbx8QbpQBYpMSGKvS9uTtrRyEy6OKyyXJfsfR6GlJbVH1cgUN+eg0cp +PiCbrd6PyeobawteRrfJkCqKvh6/nvdSHXnkfmz47RMC2Obs41LX43Boh5UdHw65 +aJfjh7O8CCHALCWHkvlJOfbEI6s1fj79H98r1//ifY+8 +-----END CERTIFICATE----- diff --git a/src/test/resources/cert/otherhost-key.pem b/src/test/resources/cert/otherhost-key.pem new file mode 100644 index 00000000000..d85091b9385 --- /dev/null +++ b/src/test/resources/cert/otherhost-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC9/pR8Tu2JQTGa +sc7yJrRqIjLF5KLj0ysUeEgilS0i3sC/hbe165ckbxiKH38+ipiU/h9L+Z5nuLFu +Qc8RW3ER9TPcvYnIdRtLpq1jeUGhqAxj3ZgR1MPrpXNJGZRUgAWuGwCCRDFuHdFS +WfVh3XF2hpxVIpWgP6dZ4vkueB2M23d4xyIMCuZ124q56QeOc6mPR0OTGsjn5/H1 +JFOLNGoZ5xaUcMcIKVs2gosm1opKI5x3Ty/v1Pdqemj3zBPp1asvzVCn0zF/pnV0 +MNKQWlrQMiyhWg51f6QNaseiKwpguLp8d1Lt1Q0yLyBoAsf8trK39j95vukyYT2a +bxLEwcK9AgMBAAECggEAF+o31AbWUPDHFub7OtFC49oepHixRTaTJV43jDzVQ96g +iesBsxEuwuwF/XrV+DAXYSe0lkpbFUiy8sMvVoq5So6gAtDLy1LsRuM5z3vXlkrS +Fm7x4Yqzx5FZl9GzsUg1DtOAxqThSPBRZQmEQNeQHOB4RKIYDeX9QWv3vBDr/Ur5 +qAwekZR8voIT/RJMdbLngNlmwEL6xWromOFz2GxF4Cb86F9/HFKg3rP7FjUIpR/Z +rlha4zN2mMexgDIHpvrng/sWkwC+sOexIK3lkHgp2nvIeD3GVXnmY5dOh0NDdoFL +3L5l+8lKaADq96kELL9Txf1snBrGtd8H0LO6UfEyEQKBgQC/wZMQm21COvz1zBaw +0hMCRQC1VFt2AYrQE8pmnv70hIdpAHvHWh+sDfrDN1zwZwcbHlFF4I1kZTF+mEsd +z4UDQUk/2wD/2QZ77jB5QaeuLBe16S6ULbIi67LVhWFAR5i5ILYjNmhRJtz0Ve5K +E4/oNLsNRkCIMmbJmZ7mRGjOjQKBgQD9pejNKylzghKd3KTGAVRhoOBiPC4SgGYd +sIjdI3GxbuWbfW4RNh5QtNvrFxPJOSiZRKIMjR+gG+wYMGzxIx5zyLrWXqDIf4zR +iJOMAw4uWN/1wrCzVbi7X5eu4IVG3cIkWNZSFnSGkKUOBTeiW9WZmjKG6bVMOfbJ +EsG6K4iQ8QKBgEV9mxQbn16vDdjtmxN9LdJWu0j7RyHesTVy1piV6gMmvAO7XyAB +cxThBA0W1SFx1MtpEz7lf5fwbB1ah25INAXX9PmlHhmZxpXG3d4zgtbFt9n+pRih +7rpk/CwQ6AtpZtlAF4FvSCKQmOYa9f32VOJrqZXH7b7ttP4+I62DARBJAoGAMZwL +hmVUvCTKo1mOWLPV3ypp+Iywrimyz0fB3Q6bpAp+mgTUTEV7dGmLQdXHpumpCSEl +WLMZZmVPrgN6q0clI5w0/syPQefAkRLXWOEYGvSDCTxE9y5i7TLrJeb/6jZhTF6b +vH5r2A3eWnmmwfiYNGy2STDYpsoHfJhQj6sIEOECgYAZx4mGB1MC7WcDtrZW2Z+M +OZlIAOgTtkVHJG1L0eESHAAngGWJeSE6ygOFlDUwYnL4jGH0TAEqGdd5NR/Iz0vE +M9kZn55PeNrxukri3tCTKyTVp0vsIUfhtUzL46rA2xDPn3rKnnaE8fAqGQl1Vqxz +ycVzx8wP/DN0GMKmualGJA== +-----END PRIVATE KEY----- diff --git a/src/test/resources/cert/untrusted-ca-cert.pem b/src/test/resources/cert/untrusted-ca-cert.pem new file mode 100644 index 00000000000..902ad24f8a6 --- /dev/null +++ b/src/test/resources/cert/untrusted-ca-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDADCCAeigAwIBAgIIJS61Qw8p3ygwDQYJKoZIhvcNAQELBQAwJTEjMCEGA1UE +AxMaU3lzdGVtRFMgVW50cnVzdGVkIFRlc3QgQ0EwIBcNMjYwNzI5MTIyOTA5WhgP +MjEyNjA3MDUxMjI5MDlaMCUxIzAhBgNVBAMTGlN5c3RlbURTIFVudHJ1c3RlZCBU +ZXN0IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAivWdwNG5FIPW +T5ADOF0DtQwvPd9MbMBBL+xSD60hKbSYArofirH8T3Z4MtDc58e8rWIZuTvy4qcZ +e2n+vAHZPyPWmHEjNsUGO1Vowd+QEIkzW0y+QTA2Z4KIAAAu1BO4o7mCnyMT34C+ +l3J9mK9TaBHm4vneYRHHuVMf70Y4cgBh9afgA9eCt3782SeSMTykLtvDvpBm3p2f +YcPB7BNkVqv31Rel4Ru4veY71hZ6hzbmyS0eRQiTY39T072Oa8GAdud8ahTyfbre +DcbK8MfKAIopCSVymSHa2LitQFdB1AAWCkZYDO8kXAvtRT5jeM6r4k4wTQ6cctGL +g/XVXJb8zwIDAQABozIwMDAdBgNVHQ4EFgQUPpgtLYjF0t4Ze1cnCKt0K+xbeLcw +DwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAQ+PeSvSVNTALOm11 +4W72YAOgfukTVVwnZGp3DKo1MSX9hW5vr/hpz+xknHoMdssscWWMtTvdXG2mimvM +Ryf+RHGNaWvTLb6GQFL9XWtNLEpQed/eUg8mKj/HUM7s4CqitWX9HN7RuL1sI20b +L+oFvb/q8wMlbzjCPQsqyuiJ8r2nWO63QSx6QMFj51/z38NbgW++pcB4cVUFh8Jp +HHvCDuwgCKKAGSkxq020QbU190VA1bUVWmd7B/gfVQvgVJiIxTwT9pbFKp+ImgE0 +Hmjt5kZiRB3op52rBGySfphnEKCYbGRtoS00ry2gZ60isG4OkyCk4spPVrXtdVzd +/pvgPQ== +-----END CERTIFICATE----- diff --git a/src/test/scripts/functions/federated/io/config/OtherHostSSLConfig.xml b/src/test/scripts/functions/federated/io/config/OtherHostSSLConfig.xml new file mode 100644 index 00000000000..8cf52db7d13 --- /dev/null +++ b/src/test/scripts/functions/federated/io/config/OtherHostSSLConfig.xml @@ -0,0 +1,29 @@ + + + + + 2 + true + src/test/resources/cert/otherhost-cert.pem + src/test/resources/cert/otherhost-key.pem + src/test/resources/cert/ca-cert.pem + 128 + diff --git a/src/test/scripts/functions/federated/io/config/SignedSSLConfig.xml b/src/test/scripts/functions/federated/io/config/SignedSSLConfig.xml new file mode 100644 index 00000000000..702f014450f --- /dev/null +++ b/src/test/scripts/functions/federated/io/config/SignedSSLConfig.xml @@ -0,0 +1,29 @@ + + + + + 2 + true + src/test/resources/cert/localhost-cert.pem + src/test/resources/cert/localhost-key.pem + src/test/resources/cert/ca-cert.pem + 128 + diff --git a/src/test/scripts/functions/federated/io/config/UntrustedSSLConfig.xml b/src/test/scripts/functions/federated/io/config/UntrustedSSLConfig.xml new file mode 100644 index 00000000000..6a2d1d4dae2 --- /dev/null +++ b/src/test/scripts/functions/federated/io/config/UntrustedSSLConfig.xml @@ -0,0 +1,27 @@ + + + + + 2 + true + src/test/resources/cert/untrusted-ca-cert.pem + 128 + diff --git a/src/test/scripts/functions/federated/io/generate-certificates.sh b/src/test/scripts/functions/federated/io/generate-certificates.sh new file mode 100755 index 00000000000..9ce4264d0d9 --- /dev/null +++ b/src/test/scripts/functions/federated/io/generate-certificates.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- +# +# Regenerates the certificates in src/test/resources/cert, used by the federated SSL tests. The generated +# files are checked in, so this script only has to be run if a certificate has to be replaced, for instance +# because it expired or because the used algorithms are no longer accepted. +# +# The certificates are only used by tests, they authenticate nothing outside of a test run. +# +# Usage: src/test/scripts/functions/federated/io/generate-certificates.sh + +set -euo pipefail +# from src/test/scripts/functions/federated/io up to src/test, then into the certificate directory +cd "$(dirname "$0")/../../../../resources/cert" + +PW=changeit +# ~100 years, the certificates are checked in and should not have to be replaced because of expiry +DAYS=36500 + +rm -f ./*.pem ./*.p12 ./*.csr + +# The authority the coordinator trusts, the basic constraint marks it as allowed to sign other certificates. +keytool -genkeypair -alias ca -dname "CN=SystemDS Test CA" -ext bc:c -keyalg RSA -keysize 2048 \ + -validity $DAYS -storetype PKCS12 -keystore ca.p12 -storepass $PW -keypass $PW +keytool -exportcert -alias ca -keystore ca.p12 -storepass $PW -rfc -file ca-cert.pem + +# A second authority, unrelated to the one above and unknown to the coordinator. +keytool -genkeypair -alias ca -dname "CN=SystemDS Untrusted Test CA" -ext bc:c -keyalg RSA -keysize 2048 \ + -validity $DAYS -storetype PKCS12 -keystore untrusted-ca.p12 -storepass $PW -keypass $PW +keytool -exportcert -alias ca -keystore untrusted-ca.p12 -storepass $PW -rfc -file untrusted-ca-cert.pem + +# Worker certificates signed by the first authority. The common name and the subject alternative names have +# to match the host the coordinator connects to, otherwise the host name verification rejects the worker. +gen_worker() { + local name=$1 dname=$2 san=$3 + keytool -genkeypair -alias worker -dname "$dname" -keyalg RSA -keysize 2048 -validity $DAYS \ + -storetype PKCS12 -keystore "$name.p12" -storepass $PW -keypass $PW + keytool -certreq -alias worker -keystore "$name.p12" -storepass $PW -file "$name.csr" + keytool -gencert -alias ca -keystore ca.p12 -storepass $PW -infile "$name.csr" \ + -outfile "$name-cert.pem" -rfc -validity $DAYS -ext "san=$san" + # the chain presented by the worker, leaf certificate first + cat ca-cert.pem >> "$name-cert.pem" + # the private key, unencrypted PKCS#8 as read by the worker + openssl pkcs12 -in "$name.p12" -nocerts -nodes -passin "pass:$PW" \ + | sed -n '/BEGIN PRIVATE KEY/,/END PRIVATE KEY/p' > "$name-key.pem" +} + +gen_worker localhost "CN=localhost" "dns:localhost,ip:127.0.0.1" +gen_worker otherhost "CN=other.example.com" "dns:other.example.com" + +rm -f ./*.p12 ./*.csr +echo "Generated:" +ls -1 ./*.pem From 391623c9a595ae5bd5db30e98cb8cb5bdc373068 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:39:02 +0200 Subject: [PATCH 102/132] [SYSTEMDS-3891] OOC Primitive Simplification and Streamable Injection Robustness --- .../runtime/ooc/planning/OOCPlanner.java | 16 ++++----- .../primitives/GroupedReduceOOCPrimitive.java | 15 +++----- .../ooc/primitives/JoinOOCPrimitive.java | 36 +++++++++---------- .../ooc/primitives/MappingOOCPrimitive.java | 22 +++++------- .../primitives/MaterializeOOCPrimitive.java | 5 ++- .../runtime/ooc/primitives/OOCPrimitive.java | 22 ++++++------ .../PlannableDataGenOOCPrimitive.java | 3 +- .../ooc/primitives/TransposeOOCPrimitive.java | 22 +++++------- .../store/MaterializedStoreStreamable.java | 4 +-- .../test/component/ooc/OOCPrimitiveTest.java | 3 +- 10 files changed, 62 insertions(+), 86 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCPlanner.java b/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCPlanner.java index 38a5c609525..03eea8dcf31 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCPlanner.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCPlanner.java @@ -56,16 +56,12 @@ private static void injectMaterializations(OOCPrimitive primitive, Set input = (OOCStreamable) primitive .getInput(request.inputIndex()); - MaterializeOOCPrimitive boundary = boundaries.compute(input, (k, v) -> { - if(v == null) { - MaterializeOOCPrimitive p = new MaterializeOOCPrimitive(input, request.layout(), - primitive.getContext()); - primitive.transferInputHandle(request.inputIndex()); - return p; - } - primitive.discardInputHandle(request.inputIndex()); - return v; - }); + MaterializeOOCPrimitive boundary = boundaries.get(input); + if(boundary == null) { + boundary = new MaterializeOOCPrimitive(input, request.layout(), primitive.getContext()); + boundaries.put(input, boundary); + } + primitive.discardInputHandle(request.inputIndex()); boundary.registerRequest(request.expectedReaders()); primitive.installMaterializedInput(request.inputIndex(), boundary); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/GroupedReduceOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/GroupedReduceOOCPrimitive.java index ad126af0c71..aa67497206e 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/GroupedReduceOOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/GroupedReduceOOCPrimitive.java @@ -19,7 +19,6 @@ package org.apache.sysds.runtime.ooc.primitives; -import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; @@ -46,7 +45,7 @@ import org.apache.sysds.runtime.ooc.util.OOCUtils; public final class GroupedReduceOOCPrimitive extends OOCPrimitive { - private final OOCStream _input; + private final OOCStreamable _input; private final OOCStreamable _output; private final BiFunction _merge; private final AtomicBoolean _cleaned; @@ -62,12 +61,7 @@ public final class GroupedReduceOOCPrimitive extends OOCPrimitive { public GroupedReduceOOCPrimitive(OOCStreamable input, OOCStreamable output, BiFunction merge, StreamContext context) { - this(input.getReadStream(), output, merge, context); - } - - private GroupedReduceOOCPrimitive(OOCStream input, OOCStreamable output, - BiFunction merge, StreamContext context) { - super(context, input.getPrimitive() == null ? List.of() : List.of(input.getPrimitive())); + super(context, input); _input = input; _output = output; _merge = merge; @@ -98,11 +92,12 @@ protected void startExecution() { DataCharacteristics inputDc = _input.getDataCharacteristics(); if(inputDc == null || !inputDc.dimsKnown() || inputDc.getBlocksize() <= 0) throw new DMLRuntimeException("Grouped OOC reduction requires known input dimensions and block size."); + OOCStream input = getInputReadStream(0); _numGroups = Math.toIntExact(inputDc.getNumRowBlocks()); _groupSize = Math.toIntExact(inputDc.getNumColBlocks()); _outputStream = _output.getWriteStream(); _ready = new SubscribableTaskQueue<>(); - getContext().addInStream(_input).addOutStream(_outputStream, _ready); + getContext().addInStream(input).addOutStream(_outputStream, _ready); _table = new StateTable<>(OOCCacheManager.getGlobalCache(), CachingStream._streamSeq.getNextID()); OOCInstructionUtils.submitOOCTasks(_ready, callback -> process(callback.get()), getContext()) @@ -122,7 +117,7 @@ protected void startExecution() { OOCUtils.estimateFullTileBytes(_output.getDataCharacteristics())); long pinBytes = OOCCacheManager.getGlobalCache().maxPhysicalPinBytes(logicalBytes); long taskBytes = pinBytes + logicalBytes * 2; - AllocatedOOCStream admitted = new AllocatedOOCStream<>(_input, _allowance, + AllocatedOOCStream admitted = new AllocatedOOCStream<>(input, _allowance, ignored -> taskBytes); getContext().addInStream(admitted); admitted.setSubscriber(this::accept); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/JoinOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/JoinOOCPrimitive.java index 99996eb7dac..a4ef2c30ced 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/JoinOOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/JoinOOCPrimitive.java @@ -19,10 +19,8 @@ package org.apache.sysds.runtime.ooc.primitives; -import java.util.Objects; import java.util.concurrent.ExecutionException; import java.util.function.BiFunction; -import java.util.stream.Stream; import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.instructions.ooc.CachingStream; @@ -42,8 +40,8 @@ import org.apache.sysds.runtime.ooc.util.StateTableUtils; public class JoinOOCPrimitive extends OOCPrimitive { - private final OOCStream _left; - private final OOCStream _right; + private final OOCStreamable _left; + private final OOCStreamable _right; private final OOCStreamable _output; private final BiFunction _operation; private StateTable _table; @@ -51,13 +49,7 @@ public class JoinOOCPrimitive extends OOCPrimitive { public JoinOOCPrimitive(OOCStreamable left, OOCStreamable right, OOCStreamable output, BiFunction operation, StreamContext context) { - this(left.getReadStream(), right.getReadStream(), output, operation, context); - } - - private JoinOOCPrimitive(OOCStream left, OOCStream right, - OOCStreamable output, BiFunction operation, - StreamContext context) { - super(context, Stream.of(left.getPrimitive(), right.getPrimitive()).filter(Objects::nonNull).toList()); + super(context, left, right); _left = left; _right = right; _output = output; @@ -84,6 +76,8 @@ protected void requestPatternInternal(OOCAccessPattern accessPattern) { @Override protected void startExecution() { + OOCStream left = getInputReadStream(0); + OOCStream right = getInputReadStream(1); _table = new StateTable<>(OOCCacheManager.getGlobalCache(), CachingStream._streamSeq.getNextID()); OOCStream output = _output.getWriteStream(); OOCStream matches = new SubscribableTaskQueue<>(); @@ -95,10 +89,12 @@ protected void startExecution() { getContext().addOutStream(output); OOCInstructionUtils.submitOOCTasks(matches, callback -> { try(JoinWork work = callback.get()) { - IndexedMatrixValue left = work._left.get(); - IndexedMatrixValue right = work._right.get(); - OOCUtils.enqueueExact(output, new IndexedMatrixValue(left.getIndexes(), - _operation.apply((MatrixBlock) left.getValue(), (MatrixBlock) right.getValue())), work._budget); + IndexedMatrixValue mleft = work._left.get(); + IndexedMatrixValue mright = work._right.get(); + OOCUtils.enqueueExact(output, + new IndexedMatrixValue(mleft.getIndexes(), + _operation.apply((MatrixBlock) mleft.getValue(), (MatrixBlock) mright.getValue())), + work._budget); } }, callback -> true, (index, callback) -> callback.get().close(), getContext()).thenRun(() -> { try { @@ -110,16 +106,18 @@ protected void startExecution() { } }); - OOCInstructionUtils.submitOOCTask(() -> drive(matches, taskBytes), new StreamContext().addOutStream(output)); + OOCInstructionUtils.submitOOCTask(() -> drive(left, right, matches, taskBytes), + new StreamContext().addOutStream(output)); } - private void drive(OOCStream matches, long taskBytes) { + private void drive(OOCStream leftInput, OOCStream rightInput, + OOCStream matches, long taskBytes) { long cols = _right.getDataCharacteristics().getNumColBlocks(); int unmatched = 0; try { while(true) { - OOCStream.QueueCallback left = _left.dequeueCB(); - OOCStream.QueueCallback right = _right.dequeueCB(); + OOCStream.QueueCallback left = leftInput.dequeueCB(); + OOCStream.QueueCallback right = rightInput.dequeueCB(); boolean leftEos = left == null || left.isEos(); boolean rightEos = right == null || right.isEos(); if(leftEos || rightEos) { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/MappingOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/MappingOOCPrimitive.java index c4a96bc01dc..318c6c84cc4 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/MappingOOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/MappingOOCPrimitive.java @@ -19,7 +19,6 @@ package org.apache.sysds.runtime.ooc.primitives; -import java.util.List; import java.util.function.Function; import org.apache.sysds.runtime.instructions.ooc.OOCStream; @@ -31,27 +30,20 @@ import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; public class MappingOOCPrimitive extends OOCPrimitive { - private final OOCStream _input; private final OOCStreamable _output; private final Function _operation; public MappingOOCPrimitive(OOCStreamable input, OOCStreamable output, Function operation, StreamContext context) { - this(input.getReadStream(), output, operation, context); - } - - private MappingOOCPrimitive(OOCStream input, OOCStreamable output, - Function operation, StreamContext context) { - super(context, input.getPrimitive() == null ? List.of() : List.of(input.getPrimitive())); - _input = input; + super(context, input); _output = output; _operation = operation; } @Override protected void inferPatternsInternal() { - OOCAccessPattern inputPattern = getChildren().isEmpty() ? OOCAccessPattern.ANY : getChildren().stream() - .findFirst().get().getAccessPattern(); + OOCPrimitive dependency = getInputDependency(0); + OOCAccessPattern inputPattern = dependency == null ? OOCAccessPattern.ANY : dependency.getAccessPattern(); _pattern = _pattern.preferred(inputPattern); inferParentPatterns(); } @@ -59,15 +51,17 @@ protected void inferPatternsInternal() { @Override protected void requestPatternInternal(OOCAccessPattern accessPattern) { _pattern = _pattern.preferred(accessPattern); - if(!getChildren().isEmpty()) - getChildren().forEach(c -> c.requestPattern(accessPattern)); + OOCPrimitive dependency = getInputDependency(0); + if(dependency != null) + dependency.requestPattern(accessPattern); } @Override protected void startExecution() { + OOCStream input = getInputReadStream(0); OOCStream output = _output.getWriteStream(); OOCInstructionUtils - .submitAdmittedOOCTasks(_input, output, + .submitAdmittedOOCTasks(input, output, value -> new IndexedMatrixValue(value.getIndexes(), _operation.apply(value)), _allowance, getContext()) .thenRun(this::onComplete); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/MaterializeOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/MaterializeOOCPrimitive.java index e1002e98005..92bc28f8537 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/MaterializeOOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/MaterializeOOCPrimitive.java @@ -19,7 +19,6 @@ package org.apache.sysds.runtime.ooc.primitives; -import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.ToIntFunction; @@ -55,7 +54,7 @@ public MaterializeOOCPrimitive(OOCStreamable source, OOCStor private MaterializeOOCPrimitive(OOCStreamable source, OOCStoreLayout layout, StreamContext context, boolean reusable) { - super(context, source.getPrimitive() == null ? List.of() : List.of(source.getPrimitive())); + super(context, source); _source = source; _layout = layout; _store = new OOCFuture<>(); @@ -100,7 +99,7 @@ protected void requestPatternInternal(OOCAccessPattern accessPattern) { @Override protected void startExecution() { try { - OOCStream source = _source.getReservedReadStream(); + OOCStream source = getInputReadStream(0); MaterializedStore store = _reusable ? new MaterializedStore<>( OOCCacheManager.getGlobalCache(), CachingStream._streamSeq.getNextID()) : new MaterializedStore<>(OOCCacheManager.getGlobalCache(), diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java index 0bc20d54bd4..12d2d3a6d1f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java @@ -64,7 +64,7 @@ protected OOCPrimitive(StreamContext context, OOCStreamable... inputs) { rebuildInputChildren(); } - private OOCPrimitive(StreamContext context) { + protected OOCPrimitive(StreamContext context) { _context = context; _children = new HashSet<>(); _parents = new HashSet<>(); @@ -108,19 +108,19 @@ public final OOCStreamable getInput(int index) { return _inputs.get(index)._source; } - public final OOCPrimitive getChildPrimitiveAt(int index) { - return _inputs.get(index)._primitive; + public final OOCPrimitive getInputDependency(int index) { + return _inputs.get(index)._dependency; } public final void installMaterializedInput(int index, MaterializeOOCPrimitive boundary) { if(hasStartedExecution()) throw new IllegalStateException("Cannot replace an input after primitive execution started."); InputSlot input = _inputs.get(index); - input._primitive = boundary; + input._dependency = boundary; rebuildInputChildren(); } - public final synchronized void transferInputHandle(int index) { + private synchronized void consumeInputHandle(int index) { InputSlot input = _inputs.get(index); if(!input._handleReserved) throw new IllegalStateException("Input " + index + " no longer owns a lazy handle."); @@ -141,13 +141,13 @@ public final void discardInputHandle(int index) { @SuppressWarnings("unchecked") protected final OOCStream getInputReadStream(int index) { - transferInputHandle(index); + consumeInputHandle(index); return (OOCStream) _inputs.get(index)._source.getReservedReadStream(); } protected final OOCFuture> getMaterializedInput(int index) { OOCFuture> materialized = ((MaterializeOOCPrimitive) _inputs - .get(index)._primitive).store(); + .get(index)._dependency).store(); if(materialized == null) throw new IllegalStateException("Input " + index + " was not materialized by the planner."); return materialized; @@ -184,8 +184,8 @@ public final void requestPattern(OOCAccessPattern accessPattern) { private void rebuildInputChildren() { List next = new ArrayList<>(); for(InputSlot input : _inputs) - if(input._primitive != null) - next.add(input._primitive); + if(input._dependency != null) + next.add(input._dependency); for(OOCPrimitive child : _children) if(!next.contains(child)) child._parents.remove(this); @@ -203,12 +203,12 @@ private void rebuildInputChildren() { private static final class InputSlot { private final OOCStreamable _source; - private OOCPrimitive _primitive; + private OOCPrimitive _dependency; private boolean _handleReserved; private InputSlot(OOCStreamable source) { _source = source; - _primitive = source.getPrimitive(); + _dependency = source.getPrimitive(); _handleReserved = true; source.reserveLazyHandle(); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/PlannableDataGenOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/PlannableDataGenOOCPrimitive.java index 83ffcd2acf4..b604e66061a 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/PlannableDataGenOOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/PlannableDataGenOOCPrimitive.java @@ -19,7 +19,6 @@ package org.apache.sysds.runtime.ooc.primitives; -import java.util.List; import java.util.function.Function; import org.apache.sysds.runtime.DMLRuntimeException; @@ -42,7 +41,7 @@ public class PlannableDataGenOOCPrimitive extends OOCPrimitive { public PlannableDataGenOOCPrimitive(OOCStreamable output, Function operation, StreamContext context) { - super(context, List.of()); + super(context); _output = output; _operation = operation; } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/TransposeOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/TransposeOOCPrimitive.java index 18b127bfd09..c2aad428f4d 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/TransposeOOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/TransposeOOCPrimitive.java @@ -19,7 +19,6 @@ package org.apache.sysds.runtime.ooc.primitives; -import java.util.List; import java.util.function.Function; import org.apache.sysds.runtime.instructions.ooc.OOCStream; @@ -32,41 +31,36 @@ import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; public class TransposeOOCPrimitive extends OOCPrimitive { - private final OOCStream _input; private final OOCStreamable _output; private final Function _operation; public TransposeOOCPrimitive(OOCStreamable input, OOCStreamable output, Function operation, StreamContext context) { - this(input.getReadStream(), output, operation, context); - } - - private TransposeOOCPrimitive(OOCStream input, OOCStreamable output, - Function operation, StreamContext context) { - super(context, input.getPrimitive() == null ? List.of() : List.of(input.getPrimitive())); - _input = input; + super(context, input); _output = output; _operation = operation; } @Override protected void inferPatternsInternal() { - _pattern = (getChildren().isEmpty() ? OOCAccessPattern.ANY : getChildren().iterator().next().getAccessPattern()) - .transposed(); + OOCPrimitive dependency = getInputDependency(0); + _pattern = (dependency == null ? OOCAccessPattern.ANY : dependency.getAccessPattern()).transposed(); inferParentPatterns(); } @Override protected void requestPatternInternal(OOCAccessPattern accessPattern) { _pattern = accessPattern; - for(OOCPrimitive child : getChildren()) - child.requestPattern(accessPattern.transposed()); + OOCPrimitive dependency = getInputDependency(0); + if(dependency != null) + dependency.requestPattern(accessPattern.transposed()); } @Override protected void startExecution() { + OOCStream input = getInputReadStream(0); OOCStream output = _output.getWriteStream(); - OOCInstructionUtils.submitAdmittedOOCTasks(_input, output, value -> { + OOCInstructionUtils.submitAdmittedOOCTasks(input, output, value -> { MatrixIndexes indexes = value.getIndexes(); return new IndexedMatrixValue(new MatrixIndexes(indexes.getColumnIndex(), indexes.getRowIndex()), _operation.apply((MatrixBlock) value.getValue())); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedStoreStreamable.java b/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedStoreStreamable.java index 5d3879de72e..42f584ee2f8 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedStoreStreamable.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/MaterializedStoreStreamable.java @@ -208,8 +208,8 @@ private void markMaterializationDone() { @Override public synchronized void reserveLazyHandle() { - if(_deleteScheduled) - throw new DMLRuntimeException("Cannot reserve a reader on a materialized stream scheduled for deletion."); + if(_closed || (_deleteScheduled && _reservedReaders == 0)) + throw new DMLRuntimeException("Cannot reserve a reader on a closed materialized stream."); _reservedReaders++; } diff --git a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java index 0fe0f2e01c3..ab83b474233 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java @@ -110,7 +110,8 @@ public void testPlannerDoubleMaterialize() { sink.start(); Assert.assertEquals(1, sink.getChildren().size()); - Assert.assertTrue(sink.getChildPrimitiveAt(0) instanceof MaterializeOOCPrimitive); + Assert.assertTrue(sink.getInputDependency(0) instanceof MaterializeOOCPrimitive); + Assert.assertSame(sink.getInputDependency(0), sink.getInputDependency(1)); Assert.assertEquals(1, sink._executions); } finally { From 3d419e3482308ee241d552eb83accc3d213224f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:17:01 +0200 Subject: [PATCH 103/132] Bump actions/setup-python from 6 to 7 (#2563) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 6 to 7. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/documentation.yml | 2 +- .github/workflows/javaCodestyle.yml | 2 +- .github/workflows/python.yml | 2 +- .github/workflows/pythonFormatting.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index e2735eb1e2e..8ab254306f5 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -71,7 +71,7 @@ jobs: uses: actions/checkout@v7 - name: Setup Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: 3.7 architecture: 'x64' diff --git a/.github/workflows/javaCodestyle.yml b/.github/workflows/javaCodestyle.yml index 39593e64563..12c70f19fe4 100644 --- a/.github/workflows/javaCodestyle.yml +++ b/.github/workflows/javaCodestyle.yml @@ -93,7 +93,7 @@ jobs: cache: 'maven' - name: Setup Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: '3.11' diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index e00cd6dfe1e..d55f9adc6c0 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -97,7 +97,7 @@ jobs: run: mvn -ntp clean package -P distribution -B -Ddoc.skip=true - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} architecture: 'x64' diff --git a/.github/workflows/pythonFormatting.yml b/.github/workflows/pythonFormatting.yml index a8db5671116..5336bedf037 100644 --- a/.github/workflows/pythonFormatting.yml +++ b/.github/workflows/pythonFormatting.yml @@ -45,7 +45,7 @@ jobs: uses: actions/checkout@v7 - name: Setup Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: '3.x' From 38e34f3417249e01b5d6d8be258a76d6fcce096f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:17:29 +0200 Subject: [PATCH 104/132] Bump docker/login-action from 4 to 4.5.2 (#2571) Bumps [docker/login-action](https://github.com/docker/login-action) from 4 to 4.5.2. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v4...v4.5.2) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.5.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker-cd.yml | 2 +- .github/workflows/docker-release.yml | 2 +- .github/workflows/docker-testImage.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-cd.yml b/.github/workflows/docker-cd.yml index 7819c8daa75..74f7ef8c8f5 100644 --- a/.github/workflows/docker-cd.yml +++ b/.github/workflows/docker-cd.yml @@ -57,7 +57,7 @@ jobs: # https://github.com/docker/login-action - name: Login to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@v4 + uses: docker/login-action@v4.5.2 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index fb7ceea1cbe..9bdef4a8639 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -64,7 +64,7 @@ jobs: # https://github.com/docker/login-action - name: Login to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@v4 + uses: docker/login-action@v4.5.2 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/docker-testImage.yml b/.github/workflows/docker-testImage.yml index 44adf31415c..30c91a6a5fc 100644 --- a/.github/workflows/docker-testImage.yml +++ b/.github/workflows/docker-testImage.yml @@ -55,7 +55,7 @@ jobs: # https://github.com/docker/login-action - name: Login to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@v4 + uses: docker/login-action@v4.5.2 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} From 20146c12a163afd1519e9f5b07dc417bc2402895 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:17:56 +0200 Subject: [PATCH 105/132] Bump actions/setup-node from 6 to 7 (#2546) Bumps [actions/setup-node](https://github.com/actions/setup-node) from 6 to 7. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/setup-node dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/monitoringUITests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/monitoringUITests.yml b/.github/workflows/monitoringUITests.yml index 559cc8be3ab..cda0bf83543 100644 --- a/.github/workflows/monitoringUITests.yml +++ b/.github/workflows/monitoringUITests.yml @@ -58,7 +58,7 @@ jobs: steps: - uses: actions/checkout@v7 - name: Build the application, with Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: # Set always-auth in npmrc always-auth: false # optional, default is false From 08249af5d93ee2a4f55ccbab0bf9d515b9721bc5 Mon Sep 17 00:00:00 2001 From: Jakob-al28 <149481651+Jakob-al28@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:32:46 +0200 Subject: [PATCH 106/132] [SYSTEMDS-3929] Speed up Parquet frame reader/writer (#2528) Rewrites the Parquet frame reader to read columns via parquet's column API. Co-authored-by: Jakob-al28 --- .../apache/sysds/parser/DMLTranslator.java | 1 + .../sysds/runtime/io/FrameReaderFactory.java | 2 + .../sysds/runtime/io/FrameReaderParquet.java | 379 ++++++++++++++---- .../io/FrameReaderParquetParallel.java | 98 ++--- .../sysds/runtime/io/FrameWriterFactory.java | 2 + .../sysds/runtime/io/FrameWriterParquet.java | 237 +++++++---- .../io/FrameWriterParquetParallel.java | 22 +- .../frame/ParquetReaderBenchmark.java | 193 +++++++++ .../frame/ParquetWriterBenchmark.java | 206 ++++++++++ .../io/parquet/FrameParquetSchemaTest.java | 173 ++------ .../parquet/FrameReaderWriterParquetTest.java | 152 +++++++ .../io/parquet/ParquetTestUtils.java | 275 +++++++++++++ .../functions/io/parquet/ReadParquetTest.java | 188 +++++++++ .../io/parquet/WriteParquetTest.java | 180 +++++++++ 14 files changed, 1736 insertions(+), 372 deletions(-) create mode 100644 src/test/java/org/apache/sysds/performance/frame/ParquetReaderBenchmark.java create mode 100644 src/test/java/org/apache/sysds/performance/frame/ParquetWriterBenchmark.java create mode 100644 src/test/java/org/apache/sysds/test/functions/io/parquet/FrameReaderWriterParquetTest.java create mode 100644 src/test/java/org/apache/sysds/test/functions/io/parquet/ParquetTestUtils.java create mode 100644 src/test/java/org/apache/sysds/test/functions/io/parquet/ReadParquetTest.java create mode 100644 src/test/java/org/apache/sysds/test/functions/io/parquet/WriteParquetTest.java diff --git a/src/main/java/org/apache/sysds/parser/DMLTranslator.java b/src/main/java/org/apache/sysds/parser/DMLTranslator.java index a8e1667d049..0db739cc901 100644 --- a/src/main/java/org/apache/sysds/parser/DMLTranslator.java +++ b/src/main/java/org/apache/sysds/parser/DMLTranslator.java @@ -1058,6 +1058,7 @@ public void constructHops(StatementBlock sb) { case LIBSVM: case HDF5: case DELTA: + case PARQUET: // columnar/text formats: no block layout (blocksize -1) ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), -1); break; diff --git a/src/main/java/org/apache/sysds/runtime/io/FrameReaderFactory.java b/src/main/java/org/apache/sysds/runtime/io/FrameReaderFactory.java index 5efbf80b83e..da903543f52 100644 --- a/src/main/java/org/apache/sysds/runtime/io/FrameReaderFactory.java +++ b/src/main/java/org/apache/sysds/runtime/io/FrameReaderFactory.java @@ -51,6 +51,8 @@ public static FrameReader createFrameReader(FileFormat fmt, FileFormatProperties case PROTO: // TODO performance improvement: add parallel reader return new FrameReaderProto(); + case PARQUET: + return binaryParallel ? new FrameReaderParquetParallel() : new FrameReaderParquet(); case DELTA: return textParallel ? new FrameReaderDeltaParallel() : new FrameReaderDelta(); default: diff --git a/src/main/java/org/apache/sysds/runtime/io/FrameReaderParquet.java b/src/main/java/org/apache/sysds/runtime/io/FrameReaderParquet.java index ff23e9ea316..79ca2cff38e 100644 --- a/src/main/java/org/apache/sysds/runtime/io/FrameReaderParquet.java +++ b/src/main/java/org/apache/sysds/runtime/io/FrameReaderParquet.java @@ -20,26 +20,41 @@ import java.io.IOException; import java.io.InputStream; +import java.util.Arrays; +import java.util.Comparator; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.parquet.example.data.Group; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.ColumnReader; +import org.apache.parquet.column.impl.ColumnReadStoreImpl; +import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.hadoop.ParquetFileReader; -import org.apache.parquet.hadoop.ParquetReader; -import org.apache.parquet.hadoop.example.GroupReadSupport; -import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.hadoop.metadata.FileMetaData; import org.apache.parquet.hadoop.util.HadoopInputFile; +import org.apache.parquet.io.api.Converter; +import org.apache.parquet.io.api.GroupConverter; +import org.apache.parquet.io.api.PrimitiveConverter; import org.apache.parquet.schema.MessageType; -import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.apache.parquet.schema.Type; +import org.apache.parquet.schema.Type.Repetition; import org.apache.sysds.common.Types.ValueType; import org.apache.sysds.conf.ConfigurationManager; import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.frame.data.columns.Array; +import org.apache.sysds.runtime.frame.data.columns.ArrayFactory; import org.apache.sysds.runtime.util.HDFSTool; +import org.apache.sysds.runtime.util.UtilFunctions; /** * Single-threaded frame parquet reader. - * + * + * Decodes through parquet-mr's column API ({@link ColumnReadStoreImpl}/{@link ColumnReader}) directly into + * pre-allocated typed column arrays. The output frame is constructed from the filled arrays without copying. Columns + * whose parquet physical type does not match the requested frame value type are converted per cell instead. */ public class FrameReaderParquet extends FrameReader { @@ -54,104 +69,312 @@ public class FrameReaderParquet extends FrameReader { * @return A FrameBlock containing the data read from the Parquet file. */ @Override - public FrameBlock readFrameFromHDFS(String fname, ValueType[] schema, String[] names, long rlen, long clen) throws IOException, DMLRuntimeException { - // Prepare file access + public FrameBlock readFrameFromHDFS(String fname, ValueType[] schema, String[] names, long rlen, long clen) + throws IOException, DMLRuntimeException { Configuration conf = ConfigurationManager.getCachedJobConf(); Path path = new Path(fname); - - // Check existence and non-empty file - if (!HDFSTool.existsFileOnHDFS(path.toString())) { + if(!HDFSTool.existsFileOnHDFS(path.toString())) throw new IOException("File does not exist on HDFS: " + fname); - } - // Allocate output frame block ValueType[] lschema = createOutputSchema(schema, clen); String[] lnames = createOutputNames(names, clen); - FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen); - // Read Parquet file - readParquetFrameFromHDFS(path, conf, ret, lschema, rlen, clen); + Object[] dest = new Object[(int) clen]; + for(int c = 0; c < clen; c++) + dest[c] = ArrayFactory.allocateBacking(lschema[c], (int) rlen); + + readParquetFrameFromHDFS(path, conf, dest, lschema, lnames, rlen); - return ret; + // zero-row output stays a schema-only frame (no zero-length column arrays) + if(rlen == 0) + return new FrameBlock(lschema, lnames, 0); + + Array[] columns = new Array[(int) clen]; + for(int c = 0; c < clen; c++) + columns[c] = ArrayFactory.create(lschema[c], dest[c]); + return new FrameBlock(columns, lnames); } /** - * Reads data from a Parquet file on HDFS and fills the provided FrameBlock. - * The method retrieves the Parquet schema from the file footer, maps the required column names - * to their corresponding indices, and then uses a ParquetReader to iterate over each row. - * Data is extracted based on the column type and set into the output FrameBlock. + * Reads an entire Parquet file (or directory of part files) into the pre-allocated column backing arrays. Part + * files, if any, are read sequentially in name order. * - * @param path The HDFS path to the Parquet file. + * @param path The HDFS path to the Parquet file or directory. * @param conf The Hadoop configuration. - * @param dest The FrameBlock to populate with data. - * @param schema The expected value types for the output columns. + * @param dest The per-column backing arrays to populate. + * @param schema The value types of the output columns. + * @param names The names of the output columns. * @param rlen The expected number of rows. - * @param clen The expected number of columns. */ - protected void readParquetFrameFromHDFS(Path path, Configuration conf, FrameBlock dest, ValueType[] schema, long rlen, long clen) throws IOException { - // Retrieve schema from Parquet footer - ParquetMetadata metadata = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)).getFooter(); - MessageType parquetSchema = metadata.getFileMetaData().getSchema(); - - // Map column names to Parquet schema indices - String[] columnNames = dest.getColumnNames(); - int[] columnIndices = new int[columnNames.length]; - for (int i = 0; i < columnNames.length; i++) { - columnIndices[i] = parquetSchema.getFieldIndex(columnNames[i]); - } + protected void readParquetFrameFromHDFS(Path path, Configuration conf, Object[] dest, ValueType[] schema, + String[] names, long rlen) throws IOException { + FileSystem fs = IOUtilFunctions.getFileSystem(path); + Path[] files = IOUtilFunctions.getSequenceFilePaths(fs, path); + Arrays.sort(files, Comparator.comparing(Path::getName)); - // Read data usind ParquetReader - try (ParquetReader rowReader = ParquetReader.builder(new GroupReadSupport(), path) - .withConf(conf) - .build()) { - - Group group; - int row = 0; - while ((group = rowReader.read()) != null) { - for (int col = 0; col < clen; col++) { - int colIndex = columnIndices[col]; - if (group.getFieldRepetitionCount(colIndex) > 0) { - PrimitiveType.PrimitiveTypeName type = parquetSchema.getType(columnNames[col]).asPrimitiveType().getPrimitiveTypeName(); - switch (type) { - case INT32: - dest.set(row, col, group.getInteger(colIndex, 0)); - break; - case INT64: - dest.set(row, col, group.getLong(colIndex, 0)); - break; - case FLOAT: - dest.set(row, col, group.getFloat(colIndex, 0)); - break; - case DOUBLE: - dest.set(row, col, group.getDouble(colIndex, 0)); - break; - case BOOLEAN: - dest.set(row, col, group.getBoolean(colIndex, 0)); - break; - case BINARY: - dest.set(row, col, group.getBinary(colIndex, 0).toStringUsingUTF8()); - break; - default: - throw new IOException("Unsupported data type: " + type); - } - } else { - dest.set(row, col, null); - } + long off = 0; + for(Path file : files) + off += readSingleParquetFile(file, conf, dest, schema, names, rlen, (int) off); + if(off != rlen) + throw new IOException("Mismatch in row count: expected " + rlen + ", but got " + off); + } + + /** + * Decodes a single Parquet file into the column backing arrays at the given row offset, one row group at a time and + * column-at-a-time within each row group. Thread-safe for distinct row ranges, so the parallel reader assigns each + * file its own offset and shares the output arrays. + * + * @param path The HDFS path to the Parquet file. + * @param conf The Hadoop configuration. + * @param dest The per-column backing arrays to populate. + * @param schema The value types of the output columns. + * @param names The names of the output columns. + * @param rlen The total number of output rows (exclusive upper bound of this file's rows). + * @param rowOffset The row offset of this file's first row. + * @return The number of rows read. + */ + protected int readSingleParquetFile(Path path, Configuration conf, Object[] dest, ValueType[] schema, + String[] names, long rlen, int rowOffset) throws IOException { + final int ncol = schema.length; + try(ParquetFileReader reader = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf))) { + FileMetaData meta = reader.getFooter().getFileMetaData(); + MessageType parquetSchema = meta.getSchema(); + String createdBy = meta.getCreatedBy(); + + // map each requested frame column (by name) to its parquet column descriptor + final ColumnDescriptor[] descs = new ColumnDescriptor[ncol]; + for(int c = 0; c < ncol; c++) + descs[c] = validateDecodable(parquetSchema, names[c]); + + GroupConverter root = dummyConverter(parquetSchema.getFieldCount()); + int off = rowOffset; + PageReadStore pages; + while((pages = reader.readNextRowGroup()) != null) { + int nrow = (int) pages.getRowCount(); + if(off + nrow > rlen) + throw new IOException( + "Mismatch in row count: expected " + rlen + ", but got at least " + (off + nrow)); + ColumnReadStoreImpl store = new ColumnReadStoreImpl(pages, root, parquetSchema, createdBy); + for(int c = 0; c < ncol; c++) { + ColumnReader creader = store.getColumnReader(descs[c]); + int maxDef = descs[c].getMaxDefinitionLevel(); + PrimitiveTypeName ptype = descs[c].getPrimitiveType().getPrimitiveTypeName(); + if(directDecodable(ptype, schema[c])) + decodeColumnInto(creader, maxDef, nrow, ptype, dest[c], off); + else + decodeColumnConvert(creader, maxDef, nrow, ptype, schema[c], dest[c], off); } - row++; + off += nrow; } + return off - rowOffset; + } + } + + /** + * Resolve the descriptor of one parquet column and verify it is decodable: a non-nested primitive of a physical + * type the reader supports (INT96 timestamps and nested/repeated groups are not). + */ + private static ColumnDescriptor validateDecodable(MessageType parquetSchema, String name) throws IOException { + if(!parquetSchema.containsField(name)) + throw new IOException("Column not found in Parquet schema: " + name); + Type t = parquetSchema.getType(name); + if(!t.isPrimitive() || t.isRepetition(Repetition.REPEATED)) + throw new IOException("Nested Parquet columns are not supported: " + name); + PrimitiveTypeName ptype = t.asPrimitiveType().getPrimitiveTypeName(); + switch(ptype) { + case INT32: + case INT64: + case FLOAT: + case DOUBLE: + case BOOLEAN: + case BINARY: + return parquetSchema.getColumnDescription(new String[] {name}); + default: + throw new IOException("Unsupported Parquet type " + ptype + " for column: " + name + + (ptype == PrimitiveTypeName.INT96 ? " (deprecated INT96 timestamps; re-encode as INT64)" : "")); + } + } - // Check frame dimensions - if (row != rlen) { - throw new IOException("Mismatch in row count: expected " + rlen + ", but got " + row); + /** + * Whether the parquet physical type is the one the frame value type's backing array stores, i.e. the column can be + * decoded through the typed direct path without per-value conversion. + */ + private static boolean directDecodable(PrimitiveTypeName ptype, ValueType vt) { + switch(ptype) { + case INT32: + return vt == ValueType.INT32; + case INT64: + return vt == ValueType.INT64; + case FLOAT: + return vt == ValueType.FP32; + case DOUBLE: + return vt == ValueType.FP64; + case BOOLEAN: + return vt == ValueType.BOOLEAN; + default: + return vt == ValueType.STRING; // BINARY + } + } + + /** + * Decode one parquet column of the current row group into a pre-allocated typed array at the given offset. Null + * cells (definition level below max) keep the array default (0 for numerics, null for strings). + */ + private static void decodeColumnInto(ColumnReader creader, int maxDef, int nrow, PrimitiveTypeName ptype, + Object dest, int off) { + final int end = off + nrow; + switch(ptype) { + case INT32: { + int[] a = (int[]) dest; + for(int r = off; r < end; r++) { + if(creader.getCurrentDefinitionLevel() == maxDef) + a[r] = creader.getInteger(); + creader.consume(); + } + break; + } + case INT64: { + long[] a = (long[]) dest; + for(int r = off; r < end; r++) { + if(creader.getCurrentDefinitionLevel() == maxDef) + a[r] = creader.getLong(); + creader.consume(); + } + break; + } + case FLOAT: { + float[] a = (float[]) dest; + for(int r = off; r < end; r++) { + if(creader.getCurrentDefinitionLevel() == maxDef) + a[r] = creader.getFloat(); + creader.consume(); + } + break; + } + case DOUBLE: { + double[] a = (double[]) dest; + for(int r = off; r < end; r++) { + if(creader.getCurrentDefinitionLevel() == maxDef) + a[r] = creader.getDouble(); + creader.consume(); + } + break; + } + case BOOLEAN: { + boolean[] a = (boolean[]) dest; + for(int r = off; r < end; r++) { + if(creader.getCurrentDefinitionLevel() == maxDef) + a[r] = creader.getBoolean(); + creader.consume(); + } + break; + } + default: { // BINARY + String[] a = (String[]) dest; + for(int r = off; r < end; r++) { + if(creader.getCurrentDefinitionLevel() == maxDef) + a[r] = creader.getBinary().toStringUsingUTF8(); + creader.consume(); + } + break; } } } + /** + * Decode one parquet column whose physical type does not match the frame value type, converting each value per cell + * (same semantics as {@link FrameBlock#set(int, int, Object)}, e.g. a DOUBLE file column read into a STRING frame + * column). + */ + private static void decodeColumnConvert(ColumnReader creader, int maxDef, int nrow, PrimitiveTypeName ptype, + ValueType vt, Object dest, int off) { + final int end = off + nrow; + for(int r = off; r < end; r++) { + if(creader.getCurrentDefinitionLevel() == maxDef) + setConverted(dest, vt, r, readValue(creader, ptype)); + creader.consume(); + } + } + + private static Object readValue(ColumnReader creader, PrimitiveTypeName ptype) { + switch(ptype) { + case INT32: + return creader.getInteger(); + case INT64: + return creader.getLong(); + case FLOAT: + return creader.getFloat(); + case DOUBLE: + return creader.getDouble(); + case BOOLEAN: + return creader.getBoolean(); + default: + return creader.getBinary().toStringUsingUTF8(); // BINARY + } + } + + /** + * Store one converted value into the backing array of the given value type. + */ + private static void setConverted(Object dest, ValueType vt, int r, Object val) { + Object converted = UtilFunctions.objectToObject(vt, val); + if(converted == null) + return; + switch(vt) { + case FP64: + ((double[]) dest)[r] = (Double) converted; + break; + case FP32: + ((float[]) dest)[r] = (Float) converted; + break; + case INT64: + case HASH64: + ((long[]) dest)[r] = (Long) converted; + break; + case UINT4: + case UINT8: + case INT32: + case HASH32: + ((int[]) dest)[r] = (Integer) converted; + break; + case BOOLEAN: + ((boolean[]) dest)[r] = (Boolean) converted; + break; + case CHARACTER: + ((char[]) dest)[r] = (Character) converted; + break; + default: + ((String[]) dest)[r] = (String) converted; + break; + } + } + + /** No-op converter tree; the column API requires one, but values are pulled via the typed getters. */ + private static GroupConverter dummyConverter(int nFields) { + final PrimitiveConverter[] leaves = new PrimitiveConverter[nFields]; + for(int i = 0; i < nFields; i++) + leaves[i] = new PrimitiveConverter() { + }; + return new GroupConverter() { + @Override + public Converter getConverter(int fieldIndex) { + return leaves[fieldIndex]; + } + + @Override + public void start() { + } + + @Override + public void end() { + } + }; + } + //not implemented @Override public FrameBlock readFrameFromInputStream(InputStream is, ValueType[] schema, String[] names, long rlen, long clen) throws IOException, DMLRuntimeException { throw new UnsupportedOperationException("Unimplemented method 'readFrameFromInputStream'"); } -} \ No newline at end of file +} diff --git a/src/main/java/org/apache/sysds/runtime/io/FrameReaderParquetParallel.java b/src/main/java/org/apache/sysds/runtime/io/FrameReaderParquetParallel.java index 3d40f53c626..a28a366a317 100644 --- a/src/main/java/org/apache/sysds/runtime/io/FrameReaderParquetParallel.java +++ b/src/main/java/org/apache/sysds/runtime/io/FrameReaderParquetParallel.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; @@ -27,91 +29,79 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.parquet.example.data.Group; -import org.apache.parquet.hadoop.ParquetReader; -import org.apache.parquet.hadoop.example.GroupReadSupport; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.metadata.BlockMetaData; +import org.apache.parquet.hadoop.util.HadoopInputFile; import org.apache.sysds.common.Types.ValueType; import org.apache.sysds.hops.OptimizerUtils; -import org.apache.sysds.runtime.DMLRuntimeException; -import org.apache.sysds.runtime.frame.data.FrameBlock; import org.apache.sysds.runtime.util.CommonThreadPool; /** - * Multi-threaded frame parquet reader. - * + * Multi-threaded frame parquet reader: reads one file per task, each decoding into its own row range of the shared + * column backing arrays. Per-file row offsets are derived from the row counts in the file footers. */ public class FrameReaderParquetParallel extends FrameReaderParquet { - - /** - * Reads a Parquet frame in parallel and populates the provided FrameBlock with the data. - * The method retrieves all file paths from the sequence files at that location, it then determines - * the number of threads to use based on the available files and a configured parallelism setting. - * A thread pool is created to run a reading task for each file concurrently. - * - * @param path The HDFS path to the Parquet file or the directory containing sequence files. - * @param conf The Hadoop configuration. - * @param dest The FrameBlock to be updated with the data read from the files. - * @param schema The expected value types for the frame columns. - * @param rlen The expected number of rows. - * @param clen The expected number of columns. - */ + @Override - protected void readParquetFrameFromHDFS(Path path, Configuration conf, FrameBlock dest, ValueType[] schema, long rlen, long clen) throws IOException, DMLRuntimeException { + protected void readParquetFrameFromHDFS(Path path, Configuration conf, Object[] dest, ValueType[] schema, + String[] names, long rlen) throws IOException { FileSystem fs = IOUtilFunctions.getFileSystem(path); Path[] files = IOUtilFunctions.getSequenceFilePaths(fs, path); + Arrays.sort(files, Comparator.comparing(Path::getName)); int numThreads = Math.min(OptimizerUtils.getParallelBinaryReadParallelism(), files.length); - - // Create and execute read tasks + + long[] offsets = new long[files.length]; + long cumulative = 0; + for(int i = 0; i < files.length; i++) { + offsets[i] = cumulative; + try(ParquetFileReader reader = ParquetFileReader.open(HadoopInputFile.fromPath(files[i], conf))) { + for(BlockMetaData block : reader.getFooter().getBlocks()) + cumulative += block.getRowCount(); + } + } + if(cumulative != rlen) + throw new IOException("Mismatch in row count: expected " + rlen + ", but got " + cumulative); + ExecutorService pool = CommonThreadPool.get(numThreads); try { List tasks = new ArrayList<>(); - for (Path file : files) { - tasks.add(new ReadFileTask(file, conf, dest, schema, clen)); - } + for(int i = 0; i < files.length; i++) + tasks.add(new ReadFileTask(files[i], conf, dest, schema, names, rlen, (int) offsets[i])); - for (Future task : pool.invokeAll(tasks)) { + for(Future task : pool.invokeAll(tasks)) task.get(); - } - } catch (Exception e) { + } + catch(Exception e) { throw new IOException("Failed parallel read of Parquet frame.", e); - } finally { + } + finally { pool.shutdown(); } } private class ReadFileTask implements Callable { - private Path path; - private Configuration conf; - private FrameBlock dest; - @SuppressWarnings("unused") - private ValueType[] schema; - private long clen; + private final Path path; + private final Configuration conf; + private final Object[] dest; + private final ValueType[] schema; + private final String[] names; + private final long rlen; + private final int rowOffset; - public ReadFileTask(Path path, Configuration conf, FrameBlock dest, ValueType[] schema, long clen) { + public ReadFileTask(Path path, Configuration conf, Object[] dest, ValueType[] schema, String[] names, long rlen, + int rowOffset) { this.path = path; this.conf = conf; this.dest = dest; this.schema = schema; - this.clen = clen; + this.names = names; + this.rlen = rlen; + this.rowOffset = rowOffset; } - // When executed, a ParquetReader for the assigned file opens and iterates over each row processing every column. @Override public Object call() throws Exception { - try (ParquetReader reader = ParquetReader.builder(new GroupReadSupport(), path).withConf(conf).build()) { - Group group; - int row = 0; - while ((group = reader.read()) != null) { - for (int col = 0; col < clen; col++) { - if (group.getFieldRepetitionCount(col) > 0) { - dest.set(row, col, group.getValueToString(col, 0)); - } else { - dest.set(row, col, null); - } - } - row++; - } - } + readSingleParquetFile(path, conf, dest, schema, names, rlen, rowOffset); return null; } } diff --git a/src/main/java/org/apache/sysds/runtime/io/FrameWriterFactory.java b/src/main/java/org/apache/sysds/runtime/io/FrameWriterFactory.java index ff38eb395dd..0891ae1397a 100644 --- a/src/main/java/org/apache/sysds/runtime/io/FrameWriterFactory.java +++ b/src/main/java/org/apache/sysds/runtime/io/FrameWriterFactory.java @@ -50,6 +50,8 @@ public static FrameWriter createFrameWriter(FileFormat fmt, FileFormatProperties return binaryParallel ? new FrameWriterBinaryBlockParallel() : new FrameWriterBinaryBlock(); case PROTO: return new FrameWriterProto(); + case PARQUET: + return binaryParallel ? new FrameWriterParquetParallel() : new FrameWriterParquet(); case DELTA: return new FrameWriterDelta(); default: diff --git a/src/main/java/org/apache/sysds/runtime/io/FrameWriterParquet.java b/src/main/java/org/apache/sysds/runtime/io/FrameWriterParquet.java index ccaeeb56d51..775bfde0ce7 100644 --- a/src/main/java/org/apache/sysds/runtime/io/FrameWriterParquet.java +++ b/src/main/java/org/apache/sysds/runtime/io/FrameWriterParquet.java @@ -19,19 +19,23 @@ package org.apache.sysds.runtime.io; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; +import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; -import org.apache.parquet.example.data.Group; -import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.hadoop.ParquetOutputFormat; import org.apache.parquet.hadoop.ParquetWriter; -import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.api.WriteSupport; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.io.api.RecordConsumer; +import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; -import org.apache.parquet.schema.MessageTypeParser; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.apache.parquet.schema.Types; import org.apache.sysds.conf.ConfigurationManager; import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.frame.data.FrameBlock; @@ -44,6 +48,28 @@ */ public class FrameWriterParquet extends FrameWriter { + public enum DictEncoding { + ALL_ON, ALL_OFF, STRING_ONLY + } + + private final CompressionCodecName codec; + private final DictEncoding dictEncoding; + private final long rowGroupSize; + + public FrameWriterParquet() { + this(CompressionCodecName.ZSTD, DictEncoding.STRING_ONLY, ParquetWriter.DEFAULT_BLOCK_SIZE); + } + + public FrameWriterParquet(CompressionCodecName codec, DictEncoding dictEncoding) { + this(codec, dictEncoding, ParquetWriter.DEFAULT_BLOCK_SIZE); + } + + public FrameWriterParquet(CompressionCodecName codec, DictEncoding dictEncoding, long rowGroupSize) { + this.codec = codec; + this.dictEncoding = dictEncoding; + this.rowGroupSize = rowGroupSize; + } + /** * Writes a FrameBlock to a Parquet file on HDFS. * @@ -71,9 +97,9 @@ public final void writeFrameToHDFS(FrameBlock src, String fname, long rlen, long } /** - * Writes the FrameBlock data to a Parquet file using a ParquetWriter. - * The method generates a Parquet schema based on the metadata of the FrameBlock, initializes a ParquetWriter with specified configurations, - * iterates over each row and column, adding values (in batches for improved performance) using type-specific conversions. + * Writes the FrameBlock data to a Parquet file using a ParquetWriter. The method generates a Parquet schema based + * on the metadata of the FrameBlock, initializes a ParquetWriter with specified configurations, iterates over each + * row and column, writing directly to the RecordConsumer, using type-specific conversions. * * @param path The HDFS path where the Parquet file will be written. * @param conf The Hadoop configuration. @@ -87,70 +113,27 @@ protected void writeParquetFrameToHDFS(Path path, Configuration conf, FrameBlock // Create schema based on frame block metadata MessageType schema = createParquetSchema(src); - // TODO:Experiment with different batch sizes? - int batchSize = 1000; - int rowCount = 0; - - // Write data using ParquetWriter //FIXME replace example writer? - try (ParquetWriter writer = ExampleParquetWriter.builder(path) - .withConf(conf) - .withType(schema) - .withCompressionCodec(ParquetWriter.DEFAULT_COMPRESSION_CODEC_NAME) - .withRowGroupSize((long) ParquetWriter.DEFAULT_BLOCK_SIZE) - .withPageSize(ParquetWriter.DEFAULT_PAGE_SIZE) - .withDictionaryEncoding(true) - .build()) - { - - SimpleGroupFactory groupFactory = new SimpleGroupFactory(schema); - - List rowBuffer = new ArrayList<>(batchSize); - - for (int i = 0; i < src.getNumRows(); i++) { - Group group = groupFactory.newGroup(); - for (int j = 0; j < src.getNumColumns(); j++) { - Object value = src.get(i, j); - if (value != null) { - ValueType type = src.getSchema()[j]; - switch (type) { - case STRING: - group.add(src.getColumnNames()[j], value.toString()); - break; - case INT32: - group.add(src.getColumnNames()[j], (int) value); - break; - case INT64: - group.add(src.getColumnNames()[j], (long) value); - break; - case FP32: - group.add(src.getColumnNames()[j], (float) value); - break; - case FP64: - group.add(src.getColumnNames()[j], (double) value); - break; - case BOOLEAN: - group.add(src.getColumnNames()[j], (boolean) value); - break; - default: - throw new IOException("Unsupported value type: " + type); - } - } - } - rowBuffer.add(group); - rowCount++; + String[] columnNames = src.getColumnNames(); + ValueType[] columnTypes = src.getSchema(); - if (rowCount >= batchSize) { - for (Group g : rowBuffer) { - writer.write(g); - } - rowBuffer.clear(); - rowCount = 0; - } - } - - for (Group g : rowBuffer) { - writer.write(g); - } + FrameParquetWriterBuilder writerBuilder = new FrameParquetWriterBuilder(path, schema, src).withConf(conf) + .withCompressionCodec( + CompressionCodecName.fromConf(conf.get(ParquetOutputFormat.COMPRESSION, codec.name()))) + .withRowGroupSize(conf.getLong(ParquetOutputFormat.BLOCK_SIZE, rowGroupSize)) + .withPageSize(conf.getInt(ParquetOutputFormat.PAGE_SIZE, ParquetWriter.DEFAULT_PAGE_SIZE)) + .withDictionaryPageSize( + conf.getInt(ParquetOutputFormat.DICTIONARY_PAGE_SIZE, ParquetWriter.DEFAULT_PAGE_SIZE)) + .withDictionaryEncoding( + conf.getBoolean(ParquetOutputFormat.ENABLE_DICTIONARY, dictEncoding == DictEncoding.ALL_ON)); + + if(dictEncoding == DictEncoding.STRING_ONLY) + for(int j = 0; j < src.getNumColumns(); j++) + if(columnTypes[j] == ValueType.STRING) + writerBuilder = writerBuilder.withDictionaryEncoding(columnNames[j], true); + + try(ParquetWriter writer = writerBuilder.build()) { + for(int i = 0; i < src.getNumRows(); i++) + writer.write(i); } // Delete CRC files created by Hadoop if necessary @@ -164,36 +147,126 @@ protected void writeParquetFrameToHDFS(Path path, Configuration conf, FrameBlock * @return The generated Parquet MessageType schema. */ protected MessageType createParquetSchema(FrameBlock src) { - StringBuilder schemaBuilder = new StringBuilder("message FrameSchema {"); String[] columnNames = src.getColumnNames(); ValueType[] columnTypes = src.getSchema(); + Types.MessageTypeBuilder builder = Types.buildMessage(); for (int i = 0; i < src.getNumColumns(); i++) { - schemaBuilder.append("optional "); switch (columnTypes[i]) { case STRING: - schemaBuilder.append("binary ").append(columnNames[i]).append(" (UTF8);"); + builder.optional(PrimitiveTypeName.BINARY).as(LogicalTypeAnnotation.stringType()) + .named(columnNames[i]); break; case INT32: - schemaBuilder.append("int32 ").append(columnNames[i]).append(";"); + builder.optional(PrimitiveTypeName.INT32).named(columnNames[i]); break; case INT64: - schemaBuilder.append("int64 ").append(columnNames[i]).append(";"); + builder.optional(PrimitiveTypeName.INT64).named(columnNames[i]); break; case FP32: - schemaBuilder.append("float ").append(columnNames[i]).append(";"); + builder.optional(PrimitiveTypeName.FLOAT).named(columnNames[i]); break; case FP64: - schemaBuilder.append("double ").append(columnNames[i]).append(";"); + builder.optional(PrimitiveTypeName.DOUBLE).named(columnNames[i]); break; case BOOLEAN: - schemaBuilder.append("boolean ").append(columnNames[i]).append(";"); + builder.optional(PrimitiveTypeName.BOOLEAN).named(columnNames[i]); break; default: throw new IllegalArgumentException("Unsupported data type: " + columnTypes[i]); } } - schemaBuilder.append("}"); - return MessageTypeParser.parseMessageType(schemaBuilder.toString()); + return builder.named("FrameSchema"); + } + + /** + * WriteSupport implementation that writes rows from a FrameBlock directly to the Parquet RecordConsumer. + */ + private static class FrameWriteSupport extends WriteSupport { + private final MessageType schema; + private final FrameBlock src; + private RecordConsumer recordConsumer; + // constant across all rows + private String[] colNames; + private ValueType[] colTypes; + private int numCols; + + FrameWriteSupport(MessageType schema, FrameBlock src) { + this.schema = schema; + this.src = src; + } + + @Override + public WriteContext init(Configuration configuration) { + Map metadata = new HashMap<>(); + return new WriteContext(schema, metadata); + } + + @Override + public void prepareForWrite(RecordConsumer consumer) { + this.recordConsumer = consumer; + this.colNames = src.getColumnNames(); + this.colTypes = src.getSchema(); + this.numCols = src.getNumColumns(); + } + + @Override + public void write(Integer rowIndex) { + recordConsumer.startMessage(); + for(int j = 0; j < numCols; j++) { + Object value = src.get(rowIndex, j); + if(value != null) { + recordConsumer.startField(colNames[j], j); + switch(colTypes[j]) { + case STRING: + recordConsumer.addBinary(Binary.fromString(value.toString())); + break; + case INT32: + recordConsumer.addInteger((int) value); + break; + case INT64: + recordConsumer.addLong((long) value); + break; + case FP32: + recordConsumer.addFloat((float) value); + break; + case FP64: + recordConsumer.addDouble((double) value); + break; + case BOOLEAN: + recordConsumer.addBoolean((boolean) value); + break; + default: + throw new IllegalArgumentException("Unsupported value type: " + colTypes[j]); + } + recordConsumer.endField(colNames[j], j); + } + } + recordConsumer.endMessage(); + } + } + + /** + * ParquetWriter builder wired to FrameWriteSupport. + */ + private static class FrameParquetWriterBuilder extends ParquetWriter.Builder { + private final MessageType schema; + private final FrameBlock src; + + FrameParquetWriterBuilder(Path path, MessageType schema, FrameBlock src) { + super(path); + this.schema = schema; + this.src = src; + } + + @Override + protected FrameParquetWriterBuilder self() { + return this; + } + + @Override + protected WriteSupport getWriteSupport(Configuration conf) { + return new FrameWriteSupport(schema, src); + } } } diff --git a/src/main/java/org/apache/sysds/runtime/io/FrameWriterParquetParallel.java b/src/main/java/org/apache/sysds/runtime/io/FrameWriterParquetParallel.java index 0ef4431ef47..3efbbbefef0 100644 --- a/src/main/java/org/apache/sysds/runtime/io/FrameWriterParquetParallel.java +++ b/src/main/java/org/apache/sysds/runtime/io/FrameWriterParquetParallel.java @@ -35,17 +35,16 @@ import org.apache.sysds.utils.stats.InfrastructureAnalyzer; /** - * Multi-threaded frame parquet reader. + * Multi-threaded frame parquet writer. * */ public class FrameWriterParquetParallel extends FrameWriterParquet { /** - * Writes the FrameBlock data to HDFS in parallel. - * The method estimates the number of output partitions by comparing the total number of cells in the FrameBlock with the - * HDFS block size. It then determines the number of threads to use based on the parallelism configuration and the - * number of partitions. In case of parallelism, it divides the FrameBlock into chunks and a thread pool is created to - * execute a write task for each partition concurrently. + * Writes the FrameBlock data to HDFS in parallel. The method estimates the number of output partitions by comparing + * the estimated output size of the FrameBlock with the HDFS block size. It then determines the number of threads to + * use based on the parallelism configuration and the number of partitions. In case of parallelism, it divides the + * FrameBlock into chunks and a thread pool is created to execute a write task for each partition concurrently. * * @param path The HDFS path where the Parquet files will be written. * @param conf The Hadoop configuration. @@ -55,14 +54,15 @@ public class FrameWriterParquetParallel extends FrameWriterParquet { protected void writeParquetFrameToHDFS(Path path, Configuration conf, FrameBlock src) throws IOException, DMLRuntimeException { - // Estimate number of output partitions - int numPartFiles = Math.max((int) (src.getNumRows() * src.getNumColumns() / InfrastructureAnalyzer.getHDFSBlockSize()), 1); - + // Estimate output partitions from output size in bytes + int numPartFiles = Math + .max((int) (OptimizerUtils.estimateSizeExactFrame(src.getNumRows(), src.getNumColumns()) / + InfrastructureAnalyzer.getHDFSBlockSize()), 1); + // Determine parallelism int numThreads = Math.min(OptimizerUtils.getParallelBinaryWriteParallelism(), numPartFiles); - // Fall back to sequential write if numThreads <= 1 - if (numThreads <= 1) { + if(!_forcedParallel && numThreads <= 1) { super.writeParquetFrameToHDFS(path, conf, src); return; } diff --git a/src/test/java/org/apache/sysds/performance/frame/ParquetReaderBenchmark.java b/src/test/java/org/apache/sysds/performance/frame/ParquetReaderBenchmark.java new file mode 100644 index 00000000000..b2f4c77bcbd --- /dev/null +++ b/src/test/java/org/apache/sysds/performance/frame/ParquetReaderBenchmark.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sysds.performance.frame; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.io.FrameReaderParquet; +import org.apache.sysds.runtime.io.FrameReaderParquetParallel; +import org.apache.sysds.runtime.io.FrameWriterParquet; +import org.junit.After; +import org.junit.Assume; + +/** + * Parquet reader benchmark comparing the sequential and parallel column-API readers on the TPC-H lineitem dataset. + * + * Results report median and min/max across runs and are appended to temp/benchmark_results.csv for plotting. + * + * The benchmark methods are disabled by default; uncomment the Test annotations to run manually. If the dataset is not + * present the benchmark is skipped with instructions. + */ +public class ParquetReaderBenchmark { + + private static final String TPCH_FILE = "temp/lineitem.tbl"; + private static final String RESULTS_CSV = "temp/benchmark_results.csv"; + private static final String TEMP_FILE = System.getProperty("java.io.tmpdir") + "/systemds_read_bench.parquet"; + private static final int RUNS = 7; + // override on larger machine with -DmaxRows=... (e.g. -DmaxRows=20000000) to test larger row groups. + private static final int MAX_ROWS = Integer.getInteger("maxRows", 2_000_000); + + // TPC-H lineitem schema + private static final ValueType[] LINEITEM_SCHEMA = {ValueType.INT64, ValueType.INT64, ValueType.INT64, + ValueType.INT32, ValueType.FP64, ValueType.FP64, ValueType.FP64, ValueType.FP64, ValueType.STRING, + ValueType.STRING, ValueType.STRING, ValueType.STRING, ValueType.STRING, ValueType.STRING, ValueType.STRING, + ValueType.STRING}; + private static final String[] LINEITEM_NAMES = {"orderkey", "partkey", "suppkey", "linenumber", "quantity", + "extendedprice", "discount", "tax", "returnflag", "linestatus", "shipdate", "commitdate", "receiptdate", + "shipinstruct", "shipmode", "comment"}; + + private PrintWriter csv; + + @After + public void cleanup() { + new File(TEMP_FILE).delete(); + if(csv != null) + csv.close(); + } + + // @Test + public void benchmarkReadTpch() throws Exception { + ReadSpec spec = writeTempParquet(loadLineitemOrSkip()); + runReadBenchmark("read_tpch", spec); + } + + private static final class ReadSpec { + final ValueType[] schema; + final String[] names; + final int rows; + final int cols; + + ReadSpec(ValueType[] schema, String[] names, int rows, int cols) { + this.schema = schema; + this.names = names; + this.rows = rows; + this.cols = cols; + } + } + + private ReadSpec writeTempParquet(FrameBlock data) throws Exception { + int rows = data.getNumRows(), cols = data.getNumColumns(); + ReadSpec spec = new ReadSpec(data.getSchema(), data.getColumnNames(), rows, cols); + new File(TEMP_FILE).delete(); + new FrameWriterParquet().writeFrameToHDFS(data, TEMP_FILE, rows, cols); + return spec; + } + + /** + * Writes a frame to Parquet once, then times reading it back with both readers. + */ + private void runReadBenchmark(String category, ReadSpec spec) throws Exception { + final ValueType[] schema = spec.schema; + final String[] names = spec.names; + final int rows = spec.rows, cols = spec.cols; + + FrameBlock probe = new FrameReaderParquet().readFrameFromHDFS(TEMP_FILE, schema, names, rows, cols); + org.junit.Assert.assertEquals("Row count mismatch", rows, probe.getNumRows()); + org.junit.Assert.assertEquals("Column count mismatch", cols, probe.getNumColumns()); + probe = null; + + openCsv(); + System.out.println( + "\n=== Parquet Read Benchmark [" + category + "] (" + rows + " rows, median of " + RUNS + " runs) ===\n"); + System.out.printf("%-24s %12s %15s %s%n", "Reader", "Median (ms)", "Rows/sec", "[runs]"); + System.out.println("-".repeat(72)); + + timeRead(category, "Sequential", + () -> new FrameReaderParquet().readFrameFromHDFS(TEMP_FILE, schema, names, rows, cols), rows); + timeRead(category, "Parallel", + () -> new FrameReaderParquetParallel().readFrameFromHDFS(TEMP_FILE, schema, names, rows, cols), rows); + System.out.println(); + } + + private interface ReadAction { + FrameBlock run() throws Exception; + } + + private void timeRead(String category, String label, ReadAction action, int rows) throws Exception { + action.run(); // warmup + + long[] times = new long[RUNS]; + for(int run = 0; run < RUNS; run++) { + long start = System.currentTimeMillis(); + action.run(); + times[run] = System.currentTimeMillis() - start; + } + long med = median(times); + long min = Arrays.stream(times).min().orElse(med); + long max = Arrays.stream(times).max().orElse(med); + System.out.printf("%-24s %12d %15.0f %14s %s%n", label, med, rows * 1000.0 / med, meanStd(times), + Arrays.toString(times)); + // columns: benchmark,label,time_ms(median),rows_per_sec,min_ms,max_ms + csv.printf("%s,%s,%d,%.0f,%d,%d%n", category, label, med, rows * 1000.0 / med, min, max); + } + + private static long median(long[] times) { + long[] sorted = times.clone(); + Arrays.sort(sorted); + return sorted[sorted.length / 2]; + } + + private static String meanStd(long[] times) { + double mean = Arrays.stream(times).average().orElse(0); + double var = Arrays.stream(times).mapToDouble(t -> (t - mean) * (t - mean)).average().orElse(0); + return String.format("%.0f+-%.0f ms", mean, Math.sqrt(var)); + } + + private void openCsv() throws Exception { + new File("temp").mkdirs(); + boolean exists = new File(RESULTS_CSV).exists(); + csv = new PrintWriter(new FileWriter(RESULTS_CSV, true)); + if(!exists) + csv.println("benchmark,label,time_ms,rows_per_sec,size_mb,compression_ratio"); + csv.flush(); + } + + private FrameBlock loadLineitemOrSkip() throws Exception { + File f = new File(TPCH_FILE); + if(!f.exists()) { + System.out.println("=== TPC-H read benchmark skipped, dataset not found at " + TPCH_FILE + " ==="); + Assume.assumeTrue("TPC-H dataset not found at " + TPCH_FILE, false); + } + System.out.print("Loading " + f.getPath() + " ... "); + List rows = new ArrayList<>(); + try(BufferedReader br = new BufferedReader(new FileReader(f))) { + String line; + while((line = br.readLine()) != null && rows.size() < MAX_ROWS) { + if(line.isEmpty()) + continue; + if(line.endsWith("|")) + line = line.substring(0, line.length() - 1); + rows.add(line.split("\\|", -1)); + } + } + String[][] arr = rows.toArray(new String[0][]); + System.out.println(arr.length + " rows loaded."); + return new FrameBlock(LINEITEM_SCHEMA, LINEITEM_NAMES, arr); + } + +} diff --git a/src/test/java/org/apache/sysds/performance/frame/ParquetWriterBenchmark.java b/src/test/java/org/apache/sysds/performance/frame/ParquetWriterBenchmark.java new file mode 100644 index 00000000000..aec1f05f70d --- /dev/null +++ b/src/test/java/org/apache/sysds/performance/frame/ParquetWriterBenchmark.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sysds.performance.frame; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.io.FrameWriterParquet; +import org.apache.sysds.runtime.io.FrameWriterParquet.DictEncoding; +import org.junit.After; +import org.junit.Assume; + +/** + * Parquet writer benchmark using the TPC-H lineitem dataset. Writes results to temp/benchmark_results.csv for plotting. + * + * The benchmark methods are disabled by default; uncomment to run manually. If the dataset is not present inside the + * temp dir the corresponding benchmark is skipped with instructions. + * + * The default maxRows=2_000_000, to benchmark row-group size properly, run on a machine with enough RAM for a larger + * frame: # 1. Generate a larger TPC-H lineitem into temp/lineitem.tbl # 2. Raise the maxRows cap, then run the + * benchmark: mvn test -Dtest=ParquetWriterBenchmark#benchmarkRowGroupSizes \ -DmaxRows=60000000 -DargLine="-Xms24g + * -Xmx24g" -DfailIfNoTests=false + */ +public class ParquetWriterBenchmark { + + private static final String TPCH_FILE = "temp/lineitem.tbl"; + private static final String RESULTS_CSV = "temp/benchmark_results.csv"; + private static final String TEMP_FILE = System.getProperty("java.io.tmpdir") + "/systemds_tpch_bench.parquet"; + private static final int RUNS = 3; + private static final int MAX_ROWS = Integer.getInteger("maxRows", 2_000_000); + private static final long[] ROW_GROUP_SIZES = {1024 * 1024, // 1 MB + 8L * 1024 * 1024, // 8 MB + 16L * 1024 * 1024, // 16 MB + 32L * 1024 * 1024, // 32 MB + 64L * 1024 * 1024, // 64 MB + 128L * 1024 * 1024, // 128 MB (Parquet default) + 256L * 1024 * 1024, // 256 MB + 512L * 1024 * 1024 // 512 MB + }; + + // TPC-H lineitem schema + private static final ValueType[] LINEITEM_SCHEMA = {ValueType.INT64, // orderkey + ValueType.INT64, // partkey + ValueType.INT64, // suppkey + ValueType.INT32, // linenumber + ValueType.FP64, // quantity + ValueType.FP64, // extendedprice + ValueType.FP64, // discount + ValueType.FP64, // tax + ValueType.STRING, // returnflag (3 unique values) + ValueType.STRING, // linestatus (2 unique values) + ValueType.STRING, // shipdate + ValueType.STRING, // commitdate + ValueType.STRING, // receiptdate + ValueType.STRING, // shipinstruct (4 unique values) + ValueType.STRING, // shipmode (7 unique values) + ValueType.STRING // comment + }; + + private static final String[] LINEITEM_NAMES = {"orderkey", "partkey", "suppkey", "linenumber", "quantity", + "extendedprice", "discount", "tax", "returnflag", "linestatus", "shipdate", "commitdate", "receiptdate", + "shipinstruct", "shipmode", "comment"}; + + private PrintWriter csv; + + @After + public void cleanup() { + new File(TEMP_FILE).delete(); + if(csv != null) + csv.close(); + } + + // @Test + public void benchmarkDictionaryEncoding() throws Exception { + FrameBlock data = loadOrSkip(); + int rows = data.getNumRows(); + + openCsv(); + System.out.println( + "\n=== TPC-H Dictionary Encoding Benchmark (" + rows + " rows, median of " + RUNS + " runs) ===\n"); + System.out.printf("%-20s %12s %15s%n", "Strategy", "Time (ms)", "Rows/sec"); + System.out.println("-".repeat(52)); + + time("encoding", "ALL_ON", new FrameWriterParquet(CompressionCodecName.UNCOMPRESSED, DictEncoding.ALL_ON), data, + rows); + time("encoding", "ALL_OFF", new FrameWriterParquet(CompressionCodecName.UNCOMPRESSED, DictEncoding.ALL_OFF), + data, rows); + time("encoding", "STRING_ONLY", + new FrameWriterParquet(CompressionCodecName.UNCOMPRESSED, DictEncoding.STRING_ONLY), data, rows); + System.out.println(); + } + + // @Test + public void benchmarkRowGroupSizes() throws Exception { + FrameBlock data = loadOrSkip(); + int rows = data.getNumRows(); + + openCsv(); + System.out + .println("\n=== TPC-H Row Group Size Benchmark (" + rows + " rows, median of " + RUNS + " runs) ===\n"); + System.out.printf("%-20s %12s %15s%n", "Row Group Size", "Time (ms)", "Rows/sec"); + System.out.println("-".repeat(52)); + + for(long rowGroupSize : ROW_GROUP_SIZES) { + String label = (rowGroupSize / (1024 * 1024)) + "MB"; + time("row_group_sizes", label, + new FrameWriterParquet(CompressionCodecName.ZSTD, DictEncoding.ALL_ON, rowGroupSize), data, rows); + } + System.out.println(); + } + + private FrameBlock loadOrSkip() throws Exception { + File f = new File(TPCH_FILE); + if(!f.exists()) { + System.out.println(); + System.out.println("==================================================="); + System.out.println("TPC-H benchmark skipped, dataset not found"); + System.out.println("To reproduce:"); + System.out.println(" 1. Install DuckDB: https://duckdb.org"); + System.out.println(" 2. Open shell: duckdb"); + System.out.println(" 3. Run in DuckDB: INSTALL tpch;"); + System.out.println(" LOAD tpch;"); + System.out.println(" CALL dbgen(sf=1);"); + System.out.println(" COPY lineitem TO '/temp/lineitem.tbl'"); + System.out.println(" (DELIMITER '|', HEADER false);"); + System.out.println("==================================================="); + Assume.assumeTrue("TPC-H dataset not found at " + TPCH_FILE, false); + } + return loadLineitem(f); + } + + private void openCsv() throws Exception { + new File("temp").mkdirs(); + boolean exists = new File(RESULTS_CSV).exists(); + csv = new PrintWriter(new FileWriter(RESULTS_CSV, true)); + if(!exists) + csv.println("benchmark,label,time_ms,rows_per_sec,size_mb,compression_ratio"); + csv.flush(); + } + + private void time(String category, String label, FrameWriterParquet writer, FrameBlock data, int rows) + throws Exception { + new File(TEMP_FILE).delete(); + writer.writeFrameToHDFS(data, TEMP_FILE, rows, data.getNumColumns()); // warmup + + long[] times = new long[RUNS]; + for(int run = 0; run < RUNS; run++) { + new File(TEMP_FILE).delete(); + long start = System.currentTimeMillis(); + writer.writeFrameToHDFS(data, TEMP_FILE, rows, data.getNumColumns()); + times[run] = System.currentTimeMillis() - start; + } + long med = median(times); + System.out.printf("%-20s %12d %15.0f%n", label, med, rows * 1000.0 / med); + csv.printf("%s,%s,%d,%.0f,,%n", category, label, med, rows * 1000.0 / med); + } + + private static long median(long[] times) { + long[] sorted = times.clone(); + Arrays.sort(sorted); + return sorted[sorted.length / 2]; + } + + private static FrameBlock loadLineitem(File f) throws Exception { + System.out.print("Loading " + f.getPath() + " ... "); + List rows = new ArrayList<>(); + try(BufferedReader br = new BufferedReader(new FileReader(f))) { + String line; + while((line = br.readLine()) != null && rows.size() < MAX_ROWS) { + if(line.isEmpty()) + continue; + if(line.endsWith("|")) + line = line.substring(0, line.length() - 1); + rows.add(line.split("\\|", -1)); + } + } + String[][] data = rows.toArray(new String[0][]); + System.out.println(data.length + " rows loaded."); + return new FrameBlock(LINEITEM_SCHEMA, LINEITEM_NAMES, data); + } +} diff --git a/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java index 1e4334891ed..cc1412b1606 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java @@ -19,9 +19,9 @@ package org.apache.sysds.test.functions.io.parquet; -import org.junit.Assert; import org.junit.Test; +import org.apache.sysds.test.TestUtils; import org.apache.sysds.common.Types.ValueType; import org.apache.sysds.runtime.frame.data.FrameBlock; import org.apache.sysds.runtime.io.FrameReader; @@ -55,169 +55,48 @@ public void setUp() { /** * Test for sequential writer and reader - * + * */ @Test - public void testParquetWriteReadAllSchemaTypes() { - String fname = output("Rout"); - - // Define a schema with one column per type - ValueType[] schema = new ValueType[] { - ValueType.FP64, - ValueType.FP32, - ValueType.INT32, - ValueType.INT64, - ValueType.BOOLEAN, - ValueType.STRING - }; - - // Create an empty frame block with the above schema - FrameBlock fb = new FrameBlock(schema); - - // Populate frame block - Object[][] rows = new Object[][] { - { 1.0, 1.1f, 10, 100L, true, "A" }, - { 2.0, 2.1f, 20, 200L, false, "B" }, - { 3.0, 3.1f, 30, 300L, true, "C" }, - { 4.0, 4.1f, 40, 400L, false, "D" }, - { 5.0, 5.1f, 50, 500L, true, "E" } - }; - - for (Object[] row : rows) { - fb.appendRow(row); - } - - System.out.println(fb); - - int numRows = fb.getNumRows(); - int numCols = fb.getNumColumns(); - - // Write the FrameBlock to a Parquet file using the sequential writer - try { - FrameWriter writer = new FrameWriterParquet(); - writer.writeFrameToHDFS(fb, fname, numRows, numCols); - } - catch (IOException e) { - e.printStackTrace(); - Assert.fail("Failed to write frame block to Parquet: " + e.getMessage()); - } - - // Read the Parquet file back into a new FrameBlock - FrameBlock fbRead = null; - try { - FrameReader reader = new FrameReaderParquet(); - String[] colNames = fb.getColumnNames(); - fbRead = reader.readFrameFromHDFS(fname, schema, colNames, numRows, numCols); - } - catch (IOException e) { - e.printStackTrace(); - Assert.fail("Failed to read frame block from Parquet: " + e.getMessage()); - } - - // Compare the original and the read frame blocks - compareFrameBlocks(fb, fbRead, 1e-6); + public void testParquetWriteReadAllSchemaTypes() throws IOException { + runWriteReadAllSchemaTypes(new FrameWriterParquet(), new FrameReaderParquet(), output("Rout")); } /** * Test for multithreaded writer and reader - * + * */ @Test - public void testParquetWriteReadAllSchemaTypesParallel() { - String fname = output("Rout_parallel"); - - ValueType[] schema = new ValueType[] { - ValueType.FP64, - ValueType.FP32, - ValueType.INT32, - ValueType.INT64, - ValueType.BOOLEAN, - ValueType.STRING - }; + public void testParquetWriteReadAllSchemaTypesParallel() throws IOException { + runWriteReadAllSchemaTypes(new FrameWriterParquetParallel(), new FrameReaderParquetParallel(), + output("Rout_parallel")); + } + + private static void runWriteReadAllSchemaTypes(FrameWriter writer, FrameReader reader, String fname) + throws IOException { + // Define a schema with one column per type + ValueType[] schema = new ValueType[] {ValueType.FP64, ValueType.FP32, ValueType.INT32, ValueType.INT64, + ValueType.BOOLEAN, ValueType.STRING}; + // Create an empty frame block with the above schema FrameBlock fb = new FrameBlock(schema); - Object[][] rows = new Object[][] { - { 1.0, 1.1f, 10, 100L, true, "A" }, - { 2.0, 2.1f, 20, 200L, false, "B" }, - { 3.0, 3.1f, 30, 300L, true, "C" }, - { 4.0, 4.1f, 40, 400L, false, "D" }, - { 5.0, 5.1f, 50, 500L, true, "E" } - }; + // Populate frame block + Object[][] rows = new Object[][] {{1.0, 1.1f, 10, 100L, true, "A"}, {2.0, 2.1f, 20, 200L, false, "B"}, + {3.0, 3.1f, 30, 300L, true, "C"}, {4.0, 4.1f, 40, 400L, false, "D"}, {5.0, 5.1f, 50, 500L, true, "E"}}; - for (Object[] row : rows) { + for(Object[] row : rows) fb.appendRow(row); - } int numRows = fb.getNumRows(); int numCols = fb.getNumColumns(); - try { - FrameWriter writer = new FrameWriterParquetParallel(); - writer.writeFrameToHDFS(fb, fname, numRows, numCols); - } - catch (IOException e) { - e.printStackTrace(); - Assert.fail("Failed to write frame block to Parquet (parallel): " + e.getMessage()); - } - - FrameBlock fbRead = null; - try { - FrameReader reader = new FrameReaderParquetParallel(); - String[] colNames = fb.getColumnNames(); - fbRead = reader.readFrameFromHDFS(fname, schema, colNames, numRows, numCols); - } - catch (IOException e) { - e.printStackTrace(); - Assert.fail("Failed to read frame block from Parquet (parallel): " + e.getMessage()); - } - - compareFrameBlocks(fb, fbRead, 1e-6); - } + writer.writeFrameToHDFS(fb, fname, numRows, numCols); - private void compareFrameBlocks(FrameBlock expected, FrameBlock actual, double eps) { - Assert.assertEquals("Number of rows mismatch", expected.getNumRows(), actual.getNumRows()); - Assert.assertEquals("Number of columns mismatch", expected.getNumColumns(), actual.getNumColumns()); - - int rows = expected.getNumRows(); - int cols = expected.getNumColumns(); - - for (int i = 0; i < rows; i++) { - for (int j = 0; j < cols; j++) { - Object expVal = expected.get(i, j); - Object actVal = actual.get(i, j); - ValueType vt = expected.getSchema()[j]; - - // Handle nulls first - if(expVal == null || actVal == null) { - Assert.assertEquals("Mismatch at (" + i + "," + j + ")", expVal, actVal); - } else { - switch(vt) { - case FP64: - case FP32: - double dExp = ((Number) expVal).doubleValue(); - double dAct = ((Number) actVal).doubleValue(); - Assert.assertEquals("Mismatch at (" + i + "," + j + ")", dExp, dAct, eps); - break; - case INT32: - case INT64: - long lExp = ((Number) expVal).longValue(); - long lAct = ((Number) actVal).longValue(); - Assert.assertEquals("Mismatch at (" + i + "," + j + ")", lExp, lAct); - break; - case BOOLEAN: - boolean bExp = (Boolean) expVal; - boolean bAct = (Boolean) actVal; - Assert.assertEquals("Mismatch at (" + i + "," + j + ")", bExp, bAct); - break; - case STRING: - Assert.assertEquals("Mismatch at (" + i + "," + j + ")", expVal.toString(), actVal.toString()); - break; - default: - Assert.fail("Unsupported type in comparison: " + vt); - } - } - } - } + String[] colNames = fb.getColumnNames(); + FrameBlock fbRead = reader.readFrameFromHDFS(fname, schema, colNames, numRows, numCols); + + // Compare the original and the read frame blocks + TestUtils.compareFrames(fb, fbRead, false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameReaderWriterParquetTest.java b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameReaderWriterParquetTest.java new file mode 100644 index 00000000000..0a8f9156122 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameReaderWriterParquetTest.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.io.parquet; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.IOException; + +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.io.FrameReaderParquet; +import org.apache.sysds.runtime.io.FrameWriterParquet; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; + +/** + * Random-frame write/read round-trip test for the parquet frame reader/writer. + */ +public class FrameReaderWriterParquetTest { + + private static final String FILENAME = "target/testTemp/functions/io/parquet/FrameReaderWriterParquetTest/frame.parquet"; + + // Parquet-supported value types + private static final ValueType[] SCHEMA = {ValueType.FP64, ValueType.FP32, ValueType.INT32, ValueType.INT64, + ValueType.BOOLEAN, ValueType.STRING}; + + @Test + public void testSingleRowSingleCol() throws IOException { + runWriteReadRoundTrip(1, 1, 4669201); + } + + @Test + public void testSingleRowMultiCol() throws IOException { + runWriteReadRoundTrip(1, 6, 4669201); + } + + @Test + public void testMultiRowSingleCol() throws IOException { + runWriteReadRoundTrip(21, 1, 4669201); + } + + @Test + public void testMultiRowMultiCol() throws IOException { + runWriteReadRoundTrip(42, 5, 4669201); + } + + @Test + public void testLargerFrame() throws IOException { + runWriteReadRoundTrip(694, 6, 4669201); + } + + @Test + public void testValueTypeEdgeCases() throws IOException { + // type min/max, empty string, special chars (comma/quote/newline/unicode), column name with space + ValueType[] schema = {ValueType.FP32, ValueType.FP64, ValueType.INT32, ValueType.INT64, ValueType.BOOLEAN, + ValueType.STRING}; + String[] names = {"f 32", "f64", "i32", "i64", "b", "s"}; + String[][] data = { + {String.valueOf(Float.MAX_VALUE), String.valueOf(Double.MAX_VALUE), String.valueOf(Integer.MAX_VALUE), + String.valueOf(Long.MAX_VALUE), "true", ""}, + {String.valueOf(-Float.MAX_VALUE), String.valueOf(-Double.MAX_VALUE), String.valueOf(Integer.MIN_VALUE), + String.valueOf(Long.MIN_VALUE), "false", "a,b\"c\nd"}, + {"0.0", "0.0", "0", "0", "true", "unicode_é中"}}; + FrameBlock in = new FrameBlock(schema, names, data); + + new FrameWriterParquet().writeFrameToHDFS(in, FILENAME, 3, 6); + FrameBlock out = new FrameReaderParquet().readFrameFromHDFS(FILENAME, schema, names, 3, 6); + + TestUtils.compareFrames(in, out, false); + } + + @Test + public void testNullsInStringColumn() throws IOException { + // Numeric columns from String[][] convert null to 0. Only test string nulls here. + // Numeric nulls are covered by the Spark-written userdata1/all files in ReadParquetTest. + ValueType[] schema = {ValueType.STRING, ValueType.STRING}; + String[] names = {"a", "b"}; + String[][] data = {{"x", "y"}, {null, null}, {"p", null}}; + FrameBlock in = new FrameBlock(schema, names, data); + + new FrameWriterParquet().writeFrameToHDFS(in, FILENAME, 3, 2); + FrameBlock out = new FrameReaderParquet().readFrameFromHDFS(FILENAME, schema, names, 3, 2); + + org.junit.Assert.assertNull(out.get(1, 0)); + org.junit.Assert.assertNull(out.get(1, 1)); + org.junit.Assert.assertNull(out.get(2, 1)); + org.junit.Assert.assertNotNull(out.get(0, 0)); + org.junit.Assert.assertNotNull(out.get(2, 0)); + TestUtils.compareFrames(in, out, false); + } + + @Test + public void testMismatchedSchemaRead() throws IOException { + // requested frame types differ from the parquet physical types -> per-cell conversion path + ValueType[] writeSchema = {ValueType.INT32, ValueType.FP32, ValueType.BOOLEAN, ValueType.FP64, + ValueType.STRING}; + String[] names = {"i32", "f32", "b", "f64", "s"}; + String[][] data = {{"1", "1.5", "true", "2.5", "7"}, {"2", "2.5", "false", "3.5", null}}; + FrameBlock in = new FrameBlock(writeSchema, names, data); + + new FrameWriterParquet().writeFrameToHDFS(in, FILENAME, 2, 5); + + ValueType[] readSchema = {ValueType.INT64, ValueType.FP64, ValueType.STRING, ValueType.STRING, ValueType.FP64}; + FrameBlock out = new FrameReaderParquet().readFrameFromHDFS(FILENAME, readSchema, names, 2, 5); + + assertEquals(1L, out.get(0, 0)); + assertEquals(1.5, ((Number) out.get(0, 1)).doubleValue(), 0.0); + assertEquals("true", out.get(0, 2)); + assertEquals("2.5", out.get(0, 3)); + assertEquals(7.0, ((Number) out.get(0, 4)).doubleValue(), 0.0); + // numeric nulls keep the array default + assertEquals(0.0, ((Number) out.get(1, 4)).doubleValue(), 0.0); + } + + private void runWriteReadRoundTrip(int rows, int cols, long seed) throws IOException { + try { + ValueType[] schema = new ValueType[cols]; + for(int i = 0; i < cols; i++) + schema[i] = SCHEMA[i % SCHEMA.length]; + + FrameBlock writeBlock = TestUtils.generateRandomFrameBlock(rows, schema, seed); + + new FrameWriterParquet().writeFrameToHDFS(writeBlock, FILENAME, rows, cols); + FrameBlock readBlock = new FrameReaderParquet().readFrameFromHDFS(FILENAME, schema, + writeBlock.getColumnNames(), rows, cols); + + TestUtils.compareFrames(writeBlock, readBlock, false); + } + catch(Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/src/test/java/org/apache/sysds/test/functions/io/parquet/ParquetTestUtils.java b/src/test/java/org/apache/sysds/test/functions/io/parquet/ParquetTestUtils.java new file mode 100644 index 00000000000..0ae2145839c --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/io/parquet/ParquetTestUtils.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.io.parquet; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.sql.Timestamp; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.metadata.BlockMetaData; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.hadoop.util.HadoopInputFile; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.spark.sql.DataFrameWriter; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.test.AutomatedTestBase; + +class ParquetTestUtils { + + static class ParquetMetadataInfo { + String[] names; + ValueType[] schema; + long rlen; + long clen; + } + + static ParquetMetadataInfo inferMetadata(String fname) throws IOException { + Configuration conf = ConfigurationManager.getCachedJobConf(); + Path path = new Path(fname); + + ParquetMetadata metadata; + try(ParquetFileReader r = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf))) { + metadata = r.getFooter(); + } + MessageType parquetSchema = metadata.getFileMetaData().getSchema(); + + int fieldCount = parquetSchema.getFieldCount(); + String[] names = new String[fieldCount]; + ValueType[] schema = new ValueType[fieldCount]; + + for(int i = 0; i < fieldCount; i++) { + names[i] = parquetSchema.getFieldName(i); + PrimitiveType.PrimitiveTypeName type = parquetSchema.getType(i).asPrimitiveType().getPrimitiveTypeName(); + switch(type) { + case INT32: + schema[i] = ValueType.INT32; + break; + case INT64: + schema[i] = ValueType.INT64; + break; + case FLOAT: + schema[i] = ValueType.FP32; + break; + case DOUBLE: + schema[i] = ValueType.FP64; + break; + case BOOLEAN: + schema[i] = ValueType.BOOLEAN; + break; + case BINARY: + schema[i] = ValueType.STRING; + break; + default: + throw new IOException("Unsupported parquet type: " + type + " in column " + names[i]); + } + } + + long rlen = 0; + for(BlockMetaData block : metadata.getBlocks()) + rlen += block.getRowCount(); + + ParquetMetadataInfo info = new ParquetMetadataInfo(); + info.names = names; + info.schema = schema; + info.rlen = rlen; + info.clen = fieldCount; + return info; + } + + static SparkSession sparkSession() { + return AutomatedTestBase.createSystemDSSparkSession("parquet-frame-tests", "local[1]"); + } + + /** + * Reads a parquet file (or directory of part files) through Spark, into a FrameBlock with the given column layout, + * projecting the columns in the requested order. + * + * @param fname parquet file or directory + * @param schema value types of the requested columns + * @param names names of the requested columns + * @return the frame as read by Spark + */ + static FrameBlock sparkReadAsFrame(String fname, ValueType[] schema, String[] names) { + Dataset df = sparkSession().read().parquet(fname).select(names[0], + Arrays.copyOfRange(names, 1, names.length)); + return toFrameBlock(df.collectAsList(), schema, names); + } + + /** Converts collected Spark rows into a FrameBlock */ + static FrameBlock toFrameBlock(List rows, ValueType[] schema, String[] names) { + FrameBlock fb = new FrameBlock(schema, names); + fb.ensureAllocatedColumns(rows.size()); + for(int r = 0; r < rows.size(); r++) + for(int c = 0; c < schema.length; c++) + fb.set(r, c, rows.get(r).get(c)); + return fb; + } + + /** + * Generates the public test files (userdata1, alltypes_plain, all) with Spark's DataFrameWriter, covering all + * supported physical types and null values. One file is gzip-compressed to also cover a non-default codec on read; + * the others use Spark's default (snappy). + * + * @param outDir directory the generated files are written into + * @return map from file name (e.g. "userdata1") to its generated file path + */ + static Map generatePublicTestFiles(File outDir) throws Exception { + SparkSession spark = sparkSession(); + + Map files = new LinkedHashMap<>(); + files.put("userdata1", writeTestFile(spark, outDir, "userdata1", userdata1Rows(), userdata1Schema(), null)); + files.put("alltypes_plain", + writeTestFile(spark, outDir, "alltypes_plain", alltypesPlainRows(), alltypesPlainSchema(), "gzip")); + files.put("all", writeTestFile(spark, outDir, "all", allRows(), allSchema(), null)); + return files; + } + + /** + * Writes a file containing a deprecated INT96 timestamp column (id, ts, name), forced via + * spark.sql.parquet.outputTimestampType so the encoding does not depend on the Spark version. + * + * @param outDir directory the file is written into + * @return the generated file path + */ + static String writeInt96TimestampFile(File outDir) throws IOException { + SparkSession spark = sparkSession(); + StructType schema = new StructType( + new StructField[] {new StructField("id", DataTypes.IntegerType, true, Metadata.empty()), + new StructField("ts", DataTypes.TimestampType, true, Metadata.empty()), + new StructField("name", DataTypes.StringType, true, Metadata.empty()),}); + List rows = Arrays.asList(RowFactory.create(1, Timestamp.valueOf("2016-02-03 07:55:29"), "Amanda"), + RowFactory.create(2, Timestamp.valueOf("2016-02-03 17:04:03"), "Albert"), + RowFactory.create(3, null, "Evelyn")); + spark.conf().set("spark.sql.parquet.outputTimestampType", "INT96"); + try { + return writeTestFile(spark, outDir, "int96", rows, schema, null); + } + finally { + spark.conf().unset("spark.sql.parquet.outputTimestampType"); + } + } + + private static StructType userdata1Schema() { + return new StructType(new StructField[] {new StructField("id", DataTypes.IntegerType, true, Metadata.empty()), + new StructField("first_name", DataTypes.StringType, true, Metadata.empty()), + new StructField("salary", DataTypes.DoubleType, true, Metadata.empty()),}); + } + + private static List userdata1Rows() { + return Arrays.asList(RowFactory.create(1, "Amanda", 49756.53), RowFactory.create(2, "Albert", 150280.17), + RowFactory.create(3, "Evelyn", 144972.51), RowFactory.create(4, "Denise", null), + RowFactory.create(5, "Carlos", 75500.34)); + } + + private static StructType alltypesPlainSchema() { + return new StructType(new StructField[] {new StructField("id", DataTypes.IntegerType, true, Metadata.empty()), + new StructField("bool_col", DataTypes.BooleanType, true, Metadata.empty()), + new StructField("tinyint_col", DataTypes.IntegerType, true, Metadata.empty()), + new StructField("smallint_col", DataTypes.IntegerType, true, Metadata.empty()), + new StructField("bigint_col", DataTypes.LongType, true, Metadata.empty()), + new StructField("float_col", DataTypes.FloatType, true, Metadata.empty()), + new StructField("double_col", DataTypes.DoubleType, true, Metadata.empty()), + new StructField("date_string_col", DataTypes.StringType, true, Metadata.empty()), + new StructField("string_col", DataTypes.StringType, true, Metadata.empty()),}); + } + + private static List alltypesPlainRows() { + return Arrays.asList(RowFactory.create(1, true, 1, 10, 100L, 1.5f, 2.25, "03/01/09", "row-1"), + RowFactory.create(2, false, 2, 20, 200L, 2.5f, 4.5, "03/02/09", "row-2"), + RowFactory.create(3, true, 3, 30, 300L, 3.5f, 6.75, "03/03/09", "row-3"), + RowFactory.create(4, false, 4, 40, 400L, 4.5f, 9.0, "03/04/09", "row-4"), + RowFactory.create(5, true, 5, 50, 500L, 5.5f, 11.25, "03/05/09", "row-5"), + RowFactory.create(6, false, 6, 60, 600L, 6.5f, 13.5, "03/06/09", "row-6"), + RowFactory.create(7, true, 7, 70, 700L, 7.5f, 15.75, "03/07/09", "row-7"), + RowFactory.create(8, false, 8, 80, 800L, 8.5f, 18.0, "03/08/09", "row-8")); + } + + private static StructType allSchema() { + return new StructType( + new StructField[] {new StructField("PassengerId", DataTypes.IntegerType, true, Metadata.empty()), + new StructField("Survived", DataTypes.IntegerType, true, Metadata.empty()), + new StructField("Pclass", DataTypes.IntegerType, true, Metadata.empty()), + new StructField("Name", DataTypes.StringType, true, Metadata.empty()), + new StructField("Sex", DataTypes.StringType, true, Metadata.empty()), + new StructField("Age", DataTypes.DoubleType, true, Metadata.empty()), + new StructField("Fare", DataTypes.DoubleType, true, Metadata.empty()), + new StructField("Embarked", DataTypes.StringType, true, Metadata.empty()),}); + } + + private static List allRows() { + return Arrays.asList(RowFactory.create(1, 0, 3, "Braund, Mr. Owen Harris", "male", 22.0, 7.25, "S"), + RowFactory.create(2, 1, 1, "Cumings, Mrs. John Bradley", "female", 38.0, 71.2833, "C"), + RowFactory.create(3, 1, 3, "Heikkinen, Miss. Laina", "female", 26.0, 7.925, "S"), + RowFactory.create(4, 1, 1, "Futrelle, Mrs. Jacques Heath", "female", 35.0, 53.1, "S"), + RowFactory.create(5, 0, 3, "Allen, Mr. William Henry", "male", null, 8.05, "S"), + RowFactory.create(6, 0, 3, "Moran, Mr. James", "male", null, 8.4583, "Q"), + RowFactory.create(7, 0, 1, "McCarthy, Mr. Timothy J", "male", 54.0, 51.8625, "S"), + RowFactory.create(8, 0, 3, "Palsson, Master. Gosta Leonard", "male", 2.0, 21.075, "S")); + } + + // Spark writes a directory of part files, so we force one partition and rename it to a single file. + private static String writeTestFile(SparkSession spark, File outDir, String name, List rows, StructType schema, + String codec) throws IOException { + Dataset df = spark.createDataFrame(rows, schema); + File tmpDir = new File(outDir, "_tmp_" + name); + DataFrameWriter writer = df.coalesce(1).write().mode(SaveMode.Overwrite); + if(codec != null) + writer = writer.option("compression", codec); + writer.parquet(tmpDir.getPath()); + + File[] parts = tmpDir.listFiles((d, n) -> n.startsWith("part-") && n.endsWith(".parquet")); + if(parts == null || parts.length != 1) + throw new IOException("expected exactly 1 part file in " + tmpDir); + + File dest = new File(outDir, name + ".parquet"); + Files.copy(parts[0].toPath(), dest.toPath(), StandardCopyOption.REPLACE_EXISTING); + deleteRecursive(tmpDir); + return dest.getPath(); + } + + private static void deleteRecursive(File f) { + File[] children = f.listFiles(); + if(children != null) + for(File c : children) + deleteRecursive(c); + f.delete(); + } +} diff --git a/src/test/java/org/apache/sysds/test/functions/io/parquet/ReadParquetTest.java b/src/test/java/org/apache/sysds/test/functions/io/parquet/ReadParquetTest.java new file mode 100644 index 00000000000..3e5fc17c907 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/io/parquet/ReadParquetTest.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.io.parquet; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Map; + +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.io.FrameReaderParquet; +import org.apache.sysds.runtime.io.FrameReaderParquetParallel; +import org.apache.sysds.runtime.io.FrameWriterParquet; +import org.apache.sysds.test.functions.io.parquet.ParquetTestUtils.ParquetMetadataInfo; +import org.apache.sysds.test.TestUtils; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Verifies the parquet frame readers against Spark's parquet reader on Spark-written files. + */ +public class ReadParquetTest { + + // Generated once per test class with Spark's DataFrameWriter + private static File testFileDir; + private static String[] FILENAMES; + + @BeforeClass + public static void generateTestFiles() throws Exception { + testFileDir = Files.createTempDirectory("systemds_parquet_public_test_files").toFile(); + Map files = ParquetTestUtils.generatePublicTestFiles(testFileDir); + FILENAMES = new String[] {files.get("userdata1"), files.get("alltypes_plain"), files.get("all")}; + } + + @AfterClass + public static void cleanupTestFiles() { + deleteRecursive(testFileDir); + } + + private static void deleteRecursive(File f) { + File[] children = f.listFiles(); + if(children != null) + for(File c : children) + deleteRecursive(c); + f.delete(); + } + + @Test + public void testReadMatchesSpark() throws Exception { + for(String filename : FILENAMES) { + ParquetMetadataInfo info = ParquetTestUtils.inferMetadata(filename); + + FrameBlock expected = ParquetTestUtils.sparkReadAsFrame(filename, info.schema, info.names); + FrameBlock actual = new FrameReaderParquet().readFrameFromHDFS(filename, info.schema, info.names, info.rlen, + info.clen); + + TestUtils.compareFrames(expected, actual, false); + } + } + + @Test + public void testColumnSubsetProjection() throws Exception { + // request a reordered subset of columns (d, a, c; skip b) + File temp = Files.createTempFile("systemds_subset_parquet", ".parquet").toFile(); + try { + ValueType[] fullSchema = {ValueType.INT64, ValueType.STRING, ValueType.FP64, ValueType.BOOLEAN}; + String[] fullNames = {"a", "b", "c", "d"}; + FrameBlock original = new FrameBlock(fullSchema, fullNames, + new String[][] {{"10", "x", "1.5", "true"}, {"20", "y", "2.5", "false"}, {"30", "z", "3.5", "true"}}); + new FrameWriterParquet().writeFrameToHDFS(original, temp.getPath(), 3, 4); + + ValueType[] subSchema = {ValueType.BOOLEAN, ValueType.INT64, ValueType.FP64}; + String[] subNames = {"d", "a", "c"}; + + FrameBlock expected = ParquetTestUtils.sparkReadAsFrame(temp.getPath(), subSchema, subNames); + FrameBlock actual = new FrameReaderParquet().readFrameFromHDFS(temp.getPath(), subSchema, subNames, 3, 3); + + TestUtils.compareFrames(expected, actual, false); + } + finally { + temp.delete(); + } + } + + @Test + public void testInt96ColumnRejectedOthersReadable() throws Exception { + String file = ParquetTestUtils.writeInt96TimestampFile(testFileDir); + + ValueType[] fullSchema = {ValueType.INT32, ValueType.INT64, ValueType.STRING}; + String[] fullNames = {"id", "ts", "name"}; + try { + new FrameReaderParquet().readFrameFromHDFS(file, fullSchema, fullNames, 3, 3); + Assert.fail("Expected rejection of the INT96 column"); + } + catch(IOException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains("INT96")); + } + + ValueType[] subSchema = {ValueType.INT32, ValueType.STRING}; + String[] subNames = {"id", "name"}; + FrameBlock expected = ParquetTestUtils.sparkReadAsFrame(file, subSchema, subNames); + FrameBlock actual = new FrameReaderParquet().readFrameFromHDFS(file, subSchema, subNames, 3, 2); + TestUtils.compareFrames(expected, actual, false); + } + + @Test + public void testEmptyFile() throws Exception { + File temp = Files.createTempFile("systemds_empty_parquet", ".parquet").toFile(); + try { + ValueType[] schema = {ValueType.INT64, ValueType.STRING}; + String[] names = {"a", "b"}; + FrameBlock empty = new FrameBlock(schema, names, new String[0][]); + new FrameWriterParquet().writeFrameToHDFS(empty, temp.getPath(), 0, 2); + + FrameBlock result = new FrameReaderParquet().readFrameFromHDFS(temp.getPath(), schema, names, 0, 2); + + Assert.assertEquals("Empty file should yield 0 rows", 0, result.getNumRows()); + Assert.assertEquals(2, result.getNumColumns()); + } + finally { + temp.delete(); + } + } + + @Test + public void testReadSparkOutputDirectory() throws Exception { + // raw Spark output layout: a directory with _SUCCESS marker, .crc files and uuid part names; + // repartition shuffles rows across the part files, so values are verified keyed by id + File dir = new File(testFileDir, "spark_dir_out"); + ValueType[] schema = {ValueType.INT64, ValueType.FP64, ValueType.STRING}; + String[] names = {"id", "val", "name"}; + SparkSession spark = ParquetTestUtils.sparkSession(); + spark.range(100).selectExpr("id", "id * 0.5d as val", "concat('r', id) as name").repartition(2).write() + .mode(SaveMode.Overwrite).parquet(dir.getPath()); + Assert.assertTrue(new File(dir, "_SUCCESS").exists()); + + FrameBlock serial = new FrameReaderParquet().readFrameFromHDFS(dir.getPath(), schema, names, 100, 3); + Assert.assertEquals(100, serial.getNumRows()); + boolean[] seen = new boolean[100]; + for(int r = 0; r < 100; r++) { + int id = ((Long) serial.get(r, 0)).intValue(); + Assert.assertFalse("duplicate id " + id, seen[id]); + seen[id] = true; + Assert.assertEquals(id * 0.5, (Double) serial.get(r, 1), 0); + Assert.assertEquals("r" + id, serial.get(r, 2)); + } + + FrameBlock parallel = new FrameReaderParquetParallel().readFrameFromHDFS(dir.getPath(), schema, names, 100, 3); + TestUtils.compareFrames(serial, parallel, false); + } + + @Test + public void testParallelReaderMatchesSequential() throws Exception { + for(String filename : FILENAMES) { + ParquetMetadataInfo info = ParquetTestUtils.inferMetadata(filename); + + FrameReaderParquet sequential = new FrameReaderParquet(); + FrameBlock expected = sequential.readFrameFromHDFS(filename, info.schema, info.names, info.rlen, info.clen); + + FrameReaderParquetParallel parallel = new FrameReaderParquetParallel(); + FrameBlock actual = parallel.readFrameFromHDFS(filename, info.schema, info.names, info.rlen, info.clen); + + TestUtils.compareFrames(expected, actual, false); + } + } +} diff --git a/src/test/java/org/apache/sysds/test/functions/io/parquet/WriteParquetTest.java b/src/test/java/org/apache/sysds/test/functions/io/parquet/WriteParquetTest.java new file mode 100644 index 00000000000..46434da26a2 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/io/parquet/WriteParquetTest.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.io.parquet; + +import java.io.File; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Map; + +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.io.FrameReaderParquet; +import org.apache.sysds.runtime.io.FrameReaderParquetParallel; +import org.apache.sysds.runtime.io.FrameWriterParquet; +import org.apache.sysds.runtime.io.FrameWriterParquetParallel; +import org.apache.sysds.test.TestUtils; +import org.apache.sysds.test.functions.io.parquet.ParquetTestUtils.ParquetMetadataInfo; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Verifies the parquet frame writers against Spark's parquet reader. + * + * Column 0 carries the row index so Spark reads of multi-file outputs can be compared order-independently (sorted by + * id), since engines do not guarantee row order across files. + */ +public class WriteParquetTest { + + private static final String TEMP_FILE = System.getProperty("java.io.tmpdir") + + "/systemds_write_parquet_test.parquet"; + private static final String TEMP_PAR_PATH = System.getProperty("java.io.tmpdir") + + "/systemds_write_parquet_test_par"; + + private static final ValueType[] SCHEMA = {ValueType.INT64, ValueType.STRING, ValueType.FP64, ValueType.BOOLEAN, + ValueType.INT32, ValueType.FP32}; + private static final long SEED = 4669201; + + // See ParquetTestUtils.generatePublicTestFiles(): these are generated with Spark's DataFrameWriter + private static File testFileDir; + private static String[] PUBLIC_FILES; + + @BeforeClass + public static void generateTestFiles() throws Exception { + testFileDir = Files.createTempDirectory("systemds_parquet_public_test_files").toFile(); + Map files = ParquetTestUtils.generatePublicTestFiles(testFileDir); + PUBLIC_FILES = new String[] {files.get("userdata1"), files.get("alltypes_plain"), files.get("all")}; + } + + @AfterClass + public static void cleanupTestFiles() { + File[] children = testFileDir.listFiles(); + if(children != null) + for(File f : children) + f.delete(); + testFileDir.delete(); + } + + @After + public void cleanup() { + new File(TEMP_FILE).delete(); + deleteRecursive(new File(TEMP_PAR_PATH)); + } + + private static void deleteRecursive(File f) { + if(f.isDirectory()) + for(File c : f.listFiles()) + deleteRecursive(c); + f.delete(); + } + + @Test + public void testSparkReadsMultiPartFiles() throws Exception { + FrameBlock original = indexedRandomFrame(60); + writeTwoParts(original); + + FrameBlock result = sparkReadSorted(TEMP_PAR_PATH, original.getColumnNames()); + TestUtils.compareFrames(original, result, false); + } + + @Test + public void testMultiPartFileRoundtrip() throws Exception { + FrameBlock original = indexedRandomFrame(60); + writeTwoParts(original); + + FrameBlock result = new FrameReaderParquetParallel().readFrameFromHDFS(TEMP_PAR_PATH, SCHEMA, + original.getColumnNames(), 60, SCHEMA.length); + + TestUtils.compareFrames(original, result, false); + } + + @Test + public void testSequentialReaderMultiPartFileRoundtrip() throws Exception { + // the serial reader must also expand a directory into its part files, not just the parallel one + FrameBlock original = indexedRandomFrame(60); + writeTwoParts(original); + + FrameBlock result = new FrameReaderParquet().readFrameFromHDFS(TEMP_PAR_PATH, SCHEMA, original.getColumnNames(), + 60, SCHEMA.length); + + TestUtils.compareFrames(original, result, false); + } + + @Test + public void testForcedParallelWriterRoundTrip() throws Exception { + FrameBlock original = indexedRandomFrame(20); + + FrameWriterParquetParallel writer = new FrameWriterParquetParallel(); + writer.setForcedParallel(true); + writer.writeFrameToHDFS(original, TEMP_PAR_PATH, 20, SCHEMA.length); + + FrameBlock result = new FrameReaderParquetParallel().readFrameFromHDFS(TEMP_PAR_PATH, SCHEMA, + original.getColumnNames(), 20, SCHEMA.length); + + TestUtils.compareFrames(original, result, false); + } + + @Test + public void testRoundtripPublicFiles() throws Exception { + for(String filename : PUBLIC_FILES) { + ParquetMetadataInfo info = ParquetTestUtils.inferMetadata(filename); + + FrameReaderParquet reader = new FrameReaderParquet(); + FrameBlock original = reader.readFrameFromHDFS(filename, info.schema, info.names, info.rlen, info.clen); + + FrameWriterParquet writer = new FrameWriterParquet(); + writer.writeFrameToHDFS(original, TEMP_FILE, original.getNumRows(), original.getNumColumns()); + + FrameBlock result = reader.readFrameFromHDFS(TEMP_FILE, info.schema, info.names, info.rlen, info.clen); + + TestUtils.compareFrames(original, result, false); + + FrameBlock sparkResult = ParquetTestUtils.sparkReadAsFrame(TEMP_FILE, info.schema, info.names); + TestUtils.compareFrames(original, sparkResult, false); + } + } + + /** Random frame over all parquet-supported value types, with the row index in column 0. */ + private static FrameBlock indexedRandomFrame(int rows) { + FrameBlock fb = TestUtils.generateRandomFrameBlock(rows, SCHEMA, SEED); + for(int r = 0; r < rows; r++) + fb.set(r, 0, (long) r); + return fb; + } + + private static void writeTwoParts(FrameBlock fb) throws Exception { + int rows = fb.getNumRows(); + new File(TEMP_PAR_PATH).mkdir(); + FrameWriterParquet writer = new FrameWriterParquet(); + writer.writeFrameToHDFS(fb.slice(0, rows / 2 - 1), TEMP_PAR_PATH + "/part-0.parquet", rows / 2, SCHEMA.length); + writer.writeFrameToHDFS(fb.slice(rows / 2, rows - 1), TEMP_PAR_PATH + "/part-1.parquet", rows - rows / 2, + SCHEMA.length); + } + + private static FrameBlock sparkReadSorted(String path, String[] names) { + Dataset df = ParquetTestUtils.sparkSession().read().parquet(path) + .select(names[0], Arrays.copyOfRange(names, 1, names.length)).sort(names[0]); + return ParquetTestUtils.toFrameBlock(df.collectAsList(), SCHEMA, names); + } +} From 4db79bd3ee14c7e15128120ed4b9e0b4844720eb Mon Sep 17 00:00:00 2001 From: Jakob-al28 <149481651+Jakob-al28@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:52:49 +0200 Subject: [PATCH 107/132] [SYSTEMDS-3949] Extend Delta Spark round trip tests (#2580) Add three scenarios only an external engine can set up: a log that has rolled into a parquet checkpoint with a delete after it, DATE/TIMESTAMP/DECIMAL columns that have no SystemDS value type and must be rejected on the Delta type and a table SystemDS creates that Spark then appends to. Also stop countParquet from counting checkpoint files as data files. --- .../io/DeltaFrameSparkContractTest.java | 103 +++++++++++++++--- .../io/DeltaFrameSparkInteropTest.java | 44 ++++++-- .../component/io/DeltaFrameTestUtils.java | 11 +- 3 files changed, 135 insertions(+), 23 deletions(-) diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaFrameSparkContractTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameSparkContractTest.java index 5687461c851..7ace4b10d0a 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaFrameSparkContractTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameSparkContractTest.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.File; import java.nio.file.Files; @@ -33,7 +34,9 @@ import org.apache.commons.io.FileUtils; import org.apache.sysds.common.Types.ValueType; +import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.frame.data.FrameBlock; +import org.apache.sysds.runtime.io.DeltaKernelUtils; import org.apache.sysds.runtime.io.FrameReaderDelta; import org.apache.sysds.runtime.io.FrameReaderDeltaParallel; import org.apache.spark.sql.Dataset; @@ -50,7 +53,7 @@ /** * Regression tests pinning the Delta table layouts that the direct column-API decode path (and its kernel-engine * fallback for deletion vectors and partitioned tables) must honor beyond plain flat reads. Each case is a table layout - * the SystemDS writer never produces itself, so it must be created by the reference engine (Spark/Delta). + * or schema the SystemDS writer never produces itself, so it must be created by the reference engine (Spark/Delta). * * dvFeatureEnabledNoDeleteRead covers a table whose protocol carries the {@code deletionVectors} reader feature but has * no deleted rows - a distinct case from {@link DeltaFrameSparkInteropTest#sparkDeletionVectorsSystemdsRead}, which @@ -62,6 +65,11 @@ * handler must surface it as nulls rather than fail. partitionedTableRead covers partition values, which are not stored * in the data files and must be spliced back in. idColumnMappingRead covers a table using * {@code delta.columnMapping.mode = id}, where columns must be resolvable by parquet field id rather than logical name. + * + * checkpointBackedHistoryRead covers a table whose log has rolled into a parquet checkpoint, with a remove action after + * it, so the snapshot comes from a checkpoint plus trailing commits rather than plain per-commit JSON. + * unsupportedColumnTypeRead covers Delta types with no SystemDS value type. Parquet stores them as types the direct + * decode reads fine, so the rejection has to come from the Delta type before any decoding starts. */ @net.jcip.annotations.NotThreadSafe public class DeltaFrameSparkContractTest { @@ -106,7 +114,7 @@ public void dvFeatureEnabledNoDeleteRead() throws Exception { String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { spark.conf().set(DV_DEFAULT, "true"); - indexedDataFrame(rows).write().format("delta").save(tablePath); + indexedDataFrame(0, rows).write().format("delta").save(tablePath); assertFrameMatchesIds(new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), rows, "serial-dvfeat"); @@ -129,7 +137,7 @@ public void schemaEvolutionAddedColumnRead() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_frame_evo_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - indexedDataFrame(oldRows).write().format("delta").save(tablePath); + indexedDataFrame(0, oldRows).write().format("delta").save(tablePath); evolvedDataFrame(oldRows, allRows).write().format("delta").mode("append").option("mergeSchema", "true") .save(tablePath); @@ -168,7 +176,7 @@ public void partitionedTableRead() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_frame_part_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - indexedDataFrame(rows).write().format("delta").partitionBy("c3").save(tablePath); + indexedDataFrame(0, rows).write().format("delta").partitionBy("c3").save(tablePath); assertFrameMatchesIds(new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), rows, "serial-part"); @@ -192,7 +200,7 @@ public void idColumnMappingRead() throws Exception { try { spark.sql("CREATE TABLE delta.`" + tablePath + "` (c0 BIGINT, c1 DOUBLE, c2 STRING, c3 BOOLEAN) " + "USING delta TBLPROPERTIES ('delta.columnMapping.mode'='id')"); - indexedDataFrame(rows).write().format("delta").mode("append").save(tablePath); + indexedDataFrame(0, rows).write().format("delta").mode("append").save(tablePath); assertFrameMatchesIds(new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), rows, "serial-idmap"); @@ -206,6 +214,70 @@ public void idColumnMappingRead() throws Exception { } } + @Test + public void checkpointBackedHistoryRead() throws Exception { + // two appends past delta.checkpointInterval=2 put a parquet checkpoint in + // _delta_log; the delete after it adds a remove action the replay must apply on + // top of the checkpoint + int perCommit = 50, deleteBelow = 30, rows = 2 * perCommit; + Path dir = Files.createTempDirectory("sysds_delta_frame_ckpt_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + spark.sql("CREATE TABLE delta.`" + tablePath + "` (c0 BIGINT, c1 DOUBLE, c2 STRING, c3 BOOLEAN) " + + "USING delta TBLPROPERTIES ('delta.checkpointInterval' = '2')"); + indexedDataFrame(0, perCommit).write().format("delta").mode("append").save(tablePath); + indexedDataFrame(perCommit, rows).write().format("delta").mode("append").save(tablePath); + spark.sql("DELETE FROM delta.`" + tablePath + "` WHERE c0 < " + deleteBelow); + assertTrue("expected a checkpoint in _delta_log", DeltaFrameTestUtils.countCheckpoints(tablePath) > 0); + + // add actions replayed out of a checkpoint must still carry the numRecords + // statistic, otherwise the read silently drops to the buffered path + DeltaKernelUtils.ScanHandle handle = DeltaKernelUtils.openScan(DeltaKernelUtils.createEngine(), + DeltaKernelUtils.qualify(tablePath)); + assertTrue("checkpointed log should still expose per-file row counts", handle.hasExactRowCounts()); + + assertFrameMatchesIds(new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), + deleteBelow, rows, "serial-ckpt"); + assertFrameMatchesIds( + new FrameReaderDeltaParallel().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), deleteBelow, + rows, "parallel-ckpt"); + } + finally { + spark.sql("DROP TABLE IF EXISTS delta.`" + tablePath + "`"); + FileUtils.deleteQuietly(dir.toFile()); + } + } + + @Test + public void unsupportedColumnTypeRead() throws Exception { + // parquet stores these as int32/int64/fixed_len_byte_array, which the direct + // decode would read, so the rejection has to come from the Delta type. + assertUnsupportedColumnRejected("c1 DATE"); + assertUnsupportedColumnRejected("c1 TIMESTAMP"); + assertUnsupportedColumnRejected("c1 DECIMAL(10,2)"); + } + + private void assertUnsupportedColumnRejected(String columnDdl) throws Exception { + Path dir = Files.createTempDirectory("sysds_delta_frame_unsupported_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + // the schema alone drives the rejection, so no data has to be written + spark.sql("CREATE TABLE delta.`" + tablePath + "` (c0 BIGINT, " + columnDdl + ") USING delta"); + try { + new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1); + fail("expected read of unsupported column type to fail: " + columnDdl); + } + catch(DMLRuntimeException e) { + assertTrue("message should name the offending column: " + e.getMessage(), + e.getMessage().contains("c1")); + } + } + finally { + spark.sql("DROP TABLE IF EXISTS delta.`" + tablePath + "`"); + FileUtils.deleteQuietly(dir.toFile()); + } + } + // deterministic, exactly-representable cell values keyed by the row id in column 0 private static double dval(int id) { return id * 0.5 - 1.0; @@ -223,15 +295,15 @@ private static String vval(int id) { return "v" + id; } - /** Spark DataFrame with columns c0..c3 (long/double/string/boolean) keyed by the row id in c0. */ - private Dataset indexedDataFrame(int rows) { + /** Spark DataFrame with columns c0..c3 (long/double/string/boolean) keyed by the row id in c0, ids [from,to). */ + private Dataset indexedDataFrame(int from, int to) { StructType schema = DataTypes .createStructType(new StructField[] {DataTypes.createStructField("c0", DataTypes.LongType, false), DataTypes.createStructField("c1", DataTypes.DoubleType, false), DataTypes.createStructField("c2", DataTypes.StringType, false), DataTypes.createStructField("c3", DataTypes.BooleanType, false)}); - List data = new ArrayList<>(rows); - for(int r = 0; r < rows; r++) + List data = new ArrayList<>(to - from); + for(int r = from; r < to; r++) data.add(RowFactory.create((long) r, dval(r), sval(r), bval(r))); return spark.createDataFrame(data, schema); } @@ -252,16 +324,21 @@ private Dataset evolvedDataFrame(int from, int to) { /** Asserts {@code out} holds exactly ids [0,rows) with the exact per-id values in c1..c3. */ private static void assertFrameMatchesIds(FrameBlock out, int rows, String tag) { - assertEquals(tag + " rows", rows, out.getNumRows()); + assertFrameMatchesIds(out, 0, rows, tag); + } + + /** Asserts {@code out} holds exactly ids [fromId,toId) with the exact per-id values in c1..c3. */ + private static void assertFrameMatchesIds(FrameBlock out, int fromId, int toId, String tag) { + assertEquals(tag + " rows", toId - fromId, out.getNumRows()); assertEquals(tag + " cols", 4, out.getNumColumns()); assertEquals(tag + " c0 type", ValueType.INT64, out.getSchema()[0]); assertEquals(tag + " c1 type", ValueType.FP64, out.getSchema()[1]); assertEquals(tag + " c2 type", ValueType.STRING, out.getSchema()[2]); assertEquals(tag + " c3 type", ValueType.BOOLEAN, out.getSchema()[3]); - boolean[] seen = new boolean[rows]; - for(int r = 0; r < rows; r++) { + boolean[] seen = new boolean[toId]; + for(int r = 0; r < out.getNumRows(); r++) { int id = ((Number) out.get(r, 0)).intValue(); - assertTrue(tag + ": unexpected/duplicate id " + id, id >= 0 && id < rows && !seen[id]); + assertTrue(tag + ": unexpected/duplicate id " + id, id >= fromId && id < toId && !seen[id]); seen[id] = true; assertEquals(tag + " id" + id + " c1", dval(id), ((Number) out.get(r, 1)).doubleValue(), 1e-9); assertEquals(tag + " id" + id + " c2", sval(id), out.get(r, 2).toString()); diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaFrameSparkInteropTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameSparkInteropTest.java index d17d9ae4005..9bebf0e0bb6 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaFrameSparkInteropTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameSparkInteropTest.java @@ -59,9 +59,11 @@ * cannot catch a table that SystemDS writes in a way other Delta engines reject (or vice versa). These tests close that * gap by routing a mixed-type (long/double/string/boolean) frame through two independent engines: *
    - *
  • SystemDS writes -> Spark/Delta reads (our output is spec-compliant), and
    • + *
  • SystemDS writes -> Spark/Delta reads (our output is spec-compliant),
    • *
  • Spark/Delta writes -> SystemDS reads, including a multi-file layout and a table with deletion vectors / a - * second commit that the SystemDS writer never produces itself.
    • + * second commit that the SystemDS writer never produces itself, and + *
  • SystemDS creates the table and Spark/Delta appends to it, so a table with a mixed-engine commit history must + * still read back as one table.
    • *
* *

@@ -144,7 +146,7 @@ public void sparkWriteSystemdsReadMultiFile() throws Exception { // the reference Delta engine writes a multi-file mixed-type table; both the // serial and parallel SystemDS frame readers must reconstruct it cell-for-cell. int rows = 600; - Dataset df = indexedDataFrame(rows).repartition(3); // -> multiple data files + Dataset df = indexedDataFrame(0, rows).repartition(3); // -> multiple data files Path dir = Files.createTempDirectory("sysds_delta_frame_p2s_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { @@ -176,7 +178,7 @@ public void sparkDeletionVectorsSystemdsRead() throws Exception { // enable deletion vectors for tables created in this block, then delete a // row range so Delta records a DV rather than rewriting the data files. spark.conf().set(DV_DEFAULT, "true"); - indexedDataFrame(rows).write().format("delta").save(tablePath); + indexedDataFrame(0, rows).write().format("delta").save(tablePath); spark.sql("DELETE FROM delta.`" + tablePath + "` WHERE c0 < " + deleteBelow); Set expected = idRange(deleteBelow, rows); @@ -197,6 +199,30 @@ public void sparkDeletionVectorsSystemdsRead() throws Exception { } } + @Test + public void systemdsCreateSparkAppendRead() throws Exception { + // the SystemDS writer always creates from scratch, so Spark has to do the + // appending. both commits have to come back in a single read. + int firstRows = 300, appendRows = 250; + Path dir = Files.createTempDirectory("sysds_delta_frame_s2p_"); + String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); + try { + FrameBlock in = indexedFrame(firstRows); + new FrameWriterDelta().writeFrameToHDFS(in, tablePath, firstRows, in.getNumColumns()); + indexedDataFrame(firstRows, firstRows + appendRows).write().format("delta").mode("append").save(tablePath); + + Set expected = idRange(0, firstRows + appendRows); + assertFrameMatchesIds(new FrameReaderDelta().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), + expected, "serial-mixed"); + assertFrameMatchesIds( + new FrameReaderDeltaParallel().readFrameFromHDFS(tablePath, NO_SCHEMA, NO_NAMES, -1, -1), expected, + "parallel-mixed"); + } + finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } + private static final String DV_DEFAULT = "spark.databricks.delta.properties.defaults.enableDeletionVectors"; // deterministic, exactly-representable cell values keyed by the row id in column 0 @@ -227,16 +253,18 @@ private static FrameBlock indexedFrame(int rows) { return fb; } - /** Spark DataFrame mirroring {@link #indexedFrame} with columns c0..c3 (long/double/string/boolean). */ - private Dataset indexedDataFrame(int rows) { + /** + * Spark DataFrame mirroring {@link #indexedFrame} with columns c0..c3 (long/double/string/boolean), ids [from,to). + */ + private Dataset indexedDataFrame(int from, int to) { StructType schema = DataTypes .createStructType(new StructField[] {DataTypes.createStructField("c0", DataTypes.LongType, false), DataTypes.createStructField("c1", DataTypes.DoubleType, false), DataTypes.createStructField("c2", DataTypes.StringType, false), DataTypes.createStructField("c3", DataTypes.BooleanType, false)}); - List data = new ArrayList<>(rows); - for(int r = 0; r < rows; r++) + List data = new ArrayList<>(to - from); + for(int r = from; r < to; r++) data.add(RowFactory.create((long) r, dval(r), sval(r), bval(r))); return spark.createDataFrame(data, schema); } diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaFrameTestUtils.java b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameTestUtils.java index fe025c40eae..122df1e372c 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaFrameTestUtils.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaFrameTestUtils.java @@ -31,10 +31,17 @@ private DeltaFrameTestUtils() { // utility class } - /** Count the parquet data files under a Delta table directory. */ + /** Count the parquet data files under a Delta table directory, ignoring checkpoints in the log. */ public static long countParquet(String tablePath) throws Exception { try(Stream s = Files.walk(new File(tablePath).toPath())) { - return s.filter(p -> p.toString().endsWith(".parquet")).count(); + return s.filter(p -> p.toString().endsWith(".parquet") && !p.toString().contains("_delta_log")).count(); + } + } + + /** Count the checkpoint files under a Delta table's {@code _delta_log}. */ + public static long countCheckpoints(String tablePath) throws Exception { + try(Stream s = Files.walk(new File(tablePath, "_delta_log").toPath())) { + return s.filter(p -> p.toString().endsWith(".checkpoint.parquet")).count(); } } } From 5327c3b93e8dba42ca8ca9ef7cd3562379afb70e Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Fri, 7 Aug 2026 08:35:28 +0200 Subject: [PATCH 108/132] [SYSTEMDS-3947] Introduce robustness check in hyperparameter tuning This patch adds a min-max variance measure to Scuro's hyperparameter tuning functionality in order to avoid overfitting to the validation score. Additionally, it adds the missing parameter setters to multiple representations. --- .../scuro/drsearch/hyperparameter_tuner.py | 100 +++++++++++++++--- .../scuro/representations/color_histogram.py | 5 + .../systemds/scuro/representations/lstm.py | 14 ++- .../scuro/representations/mlp_averaging.py | 3 + .../systemds/scuro/representations/resnet.py | 1 + .../representations/swin_video_transformer.py | 2 + .../text_context_with_indices.py | 5 + .../systemds/scuro/representations/tfidf.py | 2 + .../timeseries_representations.py | 8 ++ .../systemds/scuro/representations/vgg.py | 2 + .../scuro/representations/word2vec.py | 3 + .../systemds/scuro/representations/x3d.py | 5 +- 12 files changed, 128 insertions(+), 22 deletions(-) diff --git a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py index 4f04bffcbe5..9407bdf1250 100644 --- a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py +++ b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py @@ -40,6 +40,10 @@ from systemds.scuro.utils.checkpointing import CheckpointManager +def _wandb_safe_tag(s: str, max_len: int = 64) -> str: + return s if len(s) <= max_len else s[: max_len - 3] + "..." + + def _get_params_for_node(node_id, params): return { k.split("-")[-1]: v for k, v in params.items() if k.startswith(node_id + "-") @@ -53,6 +57,7 @@ def _param_values_to_spec( return {"name": full_name, "type": "categorical", "domain": list(param_values)} if isinstance(param_values, tuple) and len(param_values) == 2: lo, hi = param_values + lo, hi = min(lo, hi), max(lo, hi) if isinstance(lo, int) and isinstance(hi, int): return {"name": full_name, "type": "integer", "domain": (lo, hi)} return {"name": full_name, "type": "real", "domain": (float(lo), float(hi))} @@ -456,16 +461,25 @@ def visit_node(node_id): modalities_override = ( self._get_cached_modalities_for_task(task, modality_ids) if mm_opt else None ) + + baseline_params, baseline_raw_scores = self.evaluate_dag_config( + dag, + {}, + node_order, + modality_ids, + task, + modalities_override=modalities_override, + ) + baseline = ( + baseline_params, + [ + self._score_value(baseline_raw_scores[0]), + self._score_value_list(baseline_raw_scores[1]), + self._score_value(baseline_raw_scores[2]), + ], + ) + if not hyperparams: - # TODO: extract the information from the unimodal optimization results - baseline = self.evaluate_dag_config( - dag, - {}, - node_order, - modality_ids, - task, - modalities_override=modalities_override, - ) all_results = [baseline] else: param_specs = self._build_param_specs(hyperparams) @@ -482,6 +496,7 @@ def visit_node(node_id): initial_config=None, rep_name=rep_name, ) + all_results.append(baseline) if not all_results: return None @@ -491,14 +506,37 @@ def get_score(result): if isinstance(score, PerformanceMeasure): return score.average_scores[self.scoring_metric] elif isinstance(score, list): - return score[1] + score = score[1] + + if isinstance(score, list): + score = np.mean(score) return score - if self.maximize_metric: - best_params, best_score = max(all_results, key=get_score) - else: - best_params, best_score = min(all_results, key=get_score) + best_params, best_score = all_results[0][0], get_score(all_results[0]) + for params, score in all_results[1:]: + candidate_score = get_score((params, score)) + if self._is_better(candidate_score, best_score): + best_params, best_score = params, candidate_score + + if hyperparams and best_params != baseline_params: + baseline_folds = self._score_value_list(baseline_raw_scores[1]) + candidate_folds = next(s for p, s in all_results if p == best_params)[1] + if baseline_folds is not None and candidate_folds is not None: + accept, candidate_range, baseline_range = self._should_accept_optimized( + baseline_folds, candidate_folds + ) + if not accept: + self.logger.info( + f"{rep_name}: optimized config too variable across folds " + f"(range={candidate_range:.4f} vs baseline={baseline_range:.4f}) " + "— keeping baseline" + ) + best_params, best_score = baseline_params, get_score(baseline) + else: + self.logger.warning( + f"{rep_name}: fold-level scores unavailable, skipping variance gate" + ) tuning_time = time.time() - start_time best_result = HyperparamResult( @@ -572,6 +610,23 @@ def _score_value(self, score: Any) -> float: return score.average_scores.get(self.scoring_metric, np.nan) return score + def _score_value_list(self, score: Any) -> List[float]: + if isinstance(score, PerformanceMeasure): + return score.scores.get(self.scoring_metric, []) + return [score] + + def _should_accept_optimized( + self, + baseline_folds: List[float], + candidate_folds: List[float], + range_threshold: float = 0.03, + ) -> Tuple[bool, float, float]: + baseline_range = max(baseline_folds) - min(baseline_folds) + candidate_range = max(candidate_folds) - min(candidate_folds) + if candidate_range > baseline_range * (1 + range_threshold): + return False, candidate_range, baseline_range + return True, candidate_range, baseline_range + def _is_better(self, candidate_score: float, best_score: float) -> bool: if np.isnan(candidate_score): return False @@ -789,7 +844,10 @@ def _search_best_configs( "project": self.wandb_project, "entity": self.wandb_entity, "group": self.wandb_group or task.model.name, - "tags": self.wandb_tags + [rep_name, task.model.name], + "tags": [ + _wandb_safe_tag(t) + for t in (self.wandb_tags + [rep_name, task.model.name]) + ], "name": f"{task.model.name}-{rep_name}-{int(time.time())}", "config": { "task": task.model.name, @@ -835,10 +893,18 @@ def objective(trial: optuna.Trial) -> float: seen[self._config_key(params)] = ( params, - [train_score, val_score, test_score], + [train_score, self._score_value_list(scores[1]), test_score], + ) + + trial_results.append( + (params, [train_score, self._score_value_list(scores[1]), test_score]) ) + val_folds = self._score_value_list(scores[1]) + if val_folds is not None and len(val_folds) > 1: + lam = 0.5 + robust_val = np.mean(val_folds) - lam * np.std(val_folds, ddof=1) + return robust_val - trial_results.append((params, [train_score, val_score, test_score])) return val_score callbacks = [c for c in [wandb_cb] if c is not None] diff --git a/src/main/python/systemds/scuro/representations/color_histogram.py b/src/main/python/systemds/scuro/representations/color_histogram.py index d1c7175b166..993b179fb8b 100644 --- a/src/main/python/systemds/scuro/representations/color_histogram.py +++ b/src/main/python/systemds/scuro/representations/color_histogram.py @@ -49,6 +49,11 @@ def __init__( super().__init__( "ColorHistogram", ModalityType.EMBEDDING, self._get_parameters() ) + if params is not None: + color_space = params.get("color_space", color_space) + bins = params.get("bins", bins) + normalize = params.get("normalize", normalize) + aggregation = params.get("aggregation", aggregation) self.color_space = color_space self.bins = bins self.normalize = normalize diff --git a/src/main/python/systemds/scuro/representations/lstm.py b/src/main/python/systemds/scuro/representations/lstm.py index c15776284ce..08d5bc8af78 100644 --- a/src/main/python/systemds/scuro/representations/lstm.py +++ b/src/main/python/systemds/scuro/representations/lstm.py @@ -51,19 +51,27 @@ def __init__( "depth": [1, 2, 3], "dropout_rate": [0.1, 0.2, 0.3, 0.4, 0.5], "learning_rate": [0.001, 0.0001, 0.01, 0.1], - "epochs": [10, 2050, 100, 200], + "epochs": [10, 20, 50, 100, 200], "batch_size": [8, 16, 32, 64, 128], } super().__init__("LSTM", parameters) + if params is not None: + width = params.get("width", width) + depth = params.get("depth", depth) + dropout_rate = params.get("dropout_rate", dropout_rate) + learning_rate = params.get("learning_rate", learning_rate) + epochs = params.get("epochs", epochs) + batch_size = params.get("batch_size", batch_size) + self.width = int(width) self.depth = int(depth) self.dropout_rate = float(dropout_rate) self.learning_rate = float(learning_rate) self.epochs = int(epochs) self.batch_size = int(batch_size) - + self.device = get_device() self.needs_training = True self.needs_alignment = True self.model = None @@ -180,7 +188,6 @@ def execute(self, modalities: List[Modality], labels: np.ndarray = None): self.input_dim = X.shape[2] self.model = self._build_model(self.input_dim, self.num_classes) - self.device = get_device_for_model(self.model, memory_factor=1.5) self.model = self.model.to(self.device) if self.is_multilabel: @@ -245,7 +252,6 @@ def apply_representation(self, modalities: List[Modality]) -> np.ndarray: X = self._prepare_data(modalities) - self.device = get_device_for_model(self.model, memory_factor=1.5) self.model = self.model.to(self.device) X_tensor = torch.FloatTensor(X) diff --git a/src/main/python/systemds/scuro/representations/mlp_averaging.py b/src/main/python/systemds/scuro/representations/mlp_averaging.py index 8c8d67a06ec..fb71424d738 100644 --- a/src/main/python/systemds/scuro/representations/mlp_averaging.py +++ b/src/main/python/systemds/scuro/representations/mlp_averaging.py @@ -54,6 +54,9 @@ def __init__(self, output_dim=512, batch_size=32, params=None): "batch_size": [8, 16, 32, 64, 128], } super().__init__("MLPAveraging", parameters) + if params is not None: + output_dim = params.get("output_dim", output_dim) + batch_size = params.get("batch_size", batch_size) self.output_dim = output_dim self.batch_size = batch_size self.device = None diff --git a/src/main/python/systemds/scuro/representations/resnet.py b/src/main/python/systemds/scuro/representations/resnet.py index 1202748aa63..26a3350e9c8 100644 --- a/src/main/python/systemds/scuro/representations/resnet.py +++ b/src/main/python/systemds/scuro/representations/resnet.py @@ -56,6 +56,7 @@ def __init__( if params is not None: self.batch_size = int(params.get("batch_size", batch_size)) self.layer_name = params.get("layer_name", layer_name) + model_name = params.get("model_name", model_name) else: self.batch_size = batch_size self.layer_name = layer_name diff --git a/src/main/python/systemds/scuro/representations/swin_video_transformer.py b/src/main/python/systemds/scuro/representations/swin_video_transformer.py index 39191f2f252..7bb40e278f3 100644 --- a/src/main/python/systemds/scuro/representations/swin_video_transformer.py +++ b/src/main/python/systemds/scuro/representations/swin_video_transformer.py @@ -59,6 +59,8 @@ def __init__(self, layer_name="avgpool", params=None): } self.data_type = torch.float32 super().__init__("SwinVideoTransformer", ModalityType.EMBEDDING, parameters) + if params is not None: + layer_name = params.get("layer_name", layer_name) self.layer_name = layer_name self.model = swin3d_t(weights=models.video.Swin3D_T_Weights.KINETICS400_V1) self.device = get_device_for_model(self.model, memory_factor=1.5) diff --git a/src/main/python/systemds/scuro/representations/text_context_with_indices.py b/src/main/python/systemds/scuro/representations/text_context_with_indices.py index 1a341af1e3a..4de53698d7a 100644 --- a/src/main/python/systemds/scuro/representations/text_context_with_indices.py +++ b/src/main/python/systemds/scuro/representations/text_context_with_indices.py @@ -296,6 +296,11 @@ def __init__(self, max_words=55, overlap=0.5, stride=None, params=None): "stride": [10, 15, 20, 30], } super().__init__("OverlappingSplit", parameters) + if params is not None: + max_words = params.get("max_words", max_words) + overlap = params.get("overlap", overlap) + overlap_words = int(max_words * overlap) + stride = params.get("stride", max_words - overlap_words) self.max_words = max_words self.overlap = overlap self.stride = stride diff --git a/src/main/python/systemds/scuro/representations/tfidf.py b/src/main/python/systemds/scuro/representations/tfidf.py index 0b603f247e3..3c3d894c173 100644 --- a/src/main/python/systemds/scuro/representations/tfidf.py +++ b/src/main/python/systemds/scuro/representations/tfidf.py @@ -36,6 +36,8 @@ class TfIdf(UnimodalRepresentation): def __init__(self, min_df=2, output_file=None, params=None): parameters = {"min_df": [min_df, 4, 8]} super().__init__("TF-IDF", ModalityType.EMBEDDING, parameters) + if params is not None: + min_df = params.get("min_df", min_df) self.min_df = int(min_df) self.output_file = output_file self.data_type = np.float32 diff --git a/src/main/python/systemds/scuro/representations/timeseries_representations.py b/src/main/python/systemds/scuro/representations/timeseries_representations.py index 14fcacf724f..6bf7f38d132 100644 --- a/src/main/python/systemds/scuro/representations/timeseries_representations.py +++ b/src/main/python/systemds/scuro/representations/timeseries_representations.py @@ -199,6 +199,8 @@ def compute_feature(self, signal, axis=-1): class ACF(TimeSeriesRepresentation): def __init__(self, k=1, params=None): super().__init__("ACF", {"k": [1, 2, 5, 10, 20, 25, 50, 100, 200, 500]}) + if params is not None: + k = params.get("k", k) self.k = k def compute_feature(self, signal, axis=-1): @@ -246,6 +248,8 @@ def compute_feature(self, signal, axis=-1): class SpectralCentroid(TimeSeriesRepresentation): def __init__(self, fs=1.0, params=None): super().__init__("SpectralCentroid", parameters={"fs": [0.5, 1.0, 2.0]}) + if params is not None: + fs = params.get("fs", fs) self.fs = fs def compute_feature(self, signal, axis=-1): @@ -271,6 +275,10 @@ def __init__(self, fs=1.0, f1=0.0, f2=0.5, params=None): "BandpowerFFT", parameters={"fs": [0.5, 1.0], "f1": [0.0, 1.0], "f2": [0.5, 1.0]}, ) + if params is not None: + fs = params.get("fs", fs) + f1 = params.get("f1", f1) + f2 = params.get("f2", f2) self.fs = fs self.f1 = f1 self.f2 = f2 diff --git a/src/main/python/systemds/scuro/representations/vgg.py b/src/main/python/systemds/scuro/representations/vgg.py index 35bc07d8a29..c2b56e8d6bd 100644 --- a/src/main/python/systemds/scuro/representations/vgg.py +++ b/src/main/python/systemds/scuro/representations/vgg.py @@ -56,6 +56,8 @@ def __init__( self.model = self.model.to(self.device) parameters = self._get_parameters() super().__init__("VGG19", ModalityType.EMBEDDING, parameters) + if params is not None: + layer = params.get("layer_name", layer) self.output_file = output_file self.layer_name = layer self.model.eval() diff --git a/src/main/python/systemds/scuro/representations/word2vec.py b/src/main/python/systemds/scuro/representations/word2vec.py index a744bc8db37..bc1c8791f20 100644 --- a/src/main/python/systemds/scuro/representations/word2vec.py +++ b/src/main/python/systemds/scuro/representations/word2vec.py @@ -49,6 +49,9 @@ def __init__(self, vector_size=150, min_count=1, output_file=None, params=None): "min_count": [1, 2, 4, 8], } super().__init__("Word2Vec", ModalityType.EMBEDDING, parameters) + if params is not None: + vector_size = params.get("vector_size", vector_size) + min_count = params.get("min_count", min_count) self.vector_size = vector_size self.min_count = min_count self.output_file = output_file diff --git a/src/main/python/systemds/scuro/representations/x3d.py b/src/main/python/systemds/scuro/representations/x3d.py index ace4cf4b8ca..bba22434fc4 100644 --- a/src/main/python/systemds/scuro/representations/x3d.py +++ b/src/main/python/systemds/scuro/representations/x3d.py @@ -50,6 +50,9 @@ def __init__( self, layer="classifier.1", model_name="s3d", output_file=None, params=None ): self.data_type = torch.float32 + if params is not None: + model_name = params.get("model_name", model_name) + layer = params.get("layer_name", layer) self.model_name = model_name parameters = self._get_parameters() super().__init__("X3D", ModalityType.EMBEDDING, parameters) @@ -127,7 +130,7 @@ def model_name(self, model_name): def _get_parameters(self, high_level=True): parameters = {"model_name": [], "layer_name": []} - for m in ["c3d", "s3d"]: + for m in ["r3d", "s3d"]: parameters["model_name"].append(m) # TODO: add embedding dimensions for each layer From 9e9902a31f662408241470c1e381af3d4b25d2eb Mon Sep 17 00:00:00 2001 From: ywcb00 Date: Fri, 7 Aug 2026 14:12:58 +0200 Subject: [PATCH 109/132] [SYSTEMDS-3958] Add XML options for sparsity rewrites. Closes #2579. --- .../java/org/apache/sysds/conf/DMLConfig.java | 6 +++ .../org/apache/sysds/hops/OptimizerUtils.java | 5 +- .../sysds/hops/estim/EstimationUtils.java | 49 +++++++++++++++++++ .../sysds/hops/rewrite/ProgramRewriter.java | 7 ++- ...riteMatrixMultChainOptimizationSparse.java | 14 ++++-- .../rewrite/RewriteMatrixChainDPTest.java | 46 ++++++++++++++--- .../RewriteMatrixMultChainOptSparseTest.java | 40 +++++++++++++-- ...ewriteMatrixMultChainOptTransposeTest.java | 6 +-- 8 files changed, 149 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/apache/sysds/conf/DMLConfig.java b/src/main/java/org/apache/sysds/conf/DMLConfig.java index 3a0829922a5..b08c2864597 100644 --- a/src/main/java/org/apache/sysds/conf/DMLConfig.java +++ b/src/main/java/org/apache/sysds/conf/DMLConfig.java @@ -43,6 +43,7 @@ import org.apache.sysds.hops.codegen.SpoofCompiler.CompilerType; import org.apache.sysds.hops.codegen.SpoofCompiler.GeneratorAPI; import org.apache.sysds.hops.codegen.SpoofCompiler.PlanSelector; +import org.apache.sysds.hops.estim.EstimationUtils.EstimatorType; import org.apache.sysds.hops.fedplanner.FTypes.FederatedPlanner; import org.apache.sysds.lops.Compression; import org.apache.sysds.lops.compile.linearization.IDagLinearizerFactory.DagLinearizer; @@ -97,6 +98,8 @@ public class DMLConfig public static final String NATIVE_BLAS = "sysds.native.blas"; public static final String NATIVE_BLAS_DIR = "sysds.native.blas.directory"; public static final String DAG_LINEARIZATION = "sysds.compile.linearization"; + public static final String SPARSITY_REWRITES = "sysds.rewrites.sparsity.enabled"; // boolean + public static final String SPARSITY_ESTIMATOR = "sysds.rewrites.sparsity.estimator"; // see EstiamtionUtils.EstimatorType public static final String CODEGEN = "sysds.codegen.enabled"; //boolean public static final String CODEGEN_API = "sysds.codegen.api"; // see SpoofCompiler.API public static final String CODEGEN_COMPILER = "sysds.codegen.compiler"; //see SpoofCompiler.CompilerType @@ -188,6 +191,8 @@ public class DMLConfig _defaultVals.put(COMPRESSED_TRANSPOSE, "auto"); _defaultVals.put(COMPRESSED_TRANSFORMENCODE, "false"); _defaultVals.put(DAG_LINEARIZATION, DagLinearizer.DEPTH_FIRST.name()); + _defaultVals.put(SPARSITY_REWRITES, "false"); + _defaultVals.put(SPARSITY_ESTIMATOR, EstimatorType.BASIC_AVG.name()); _defaultVals.put(CODEGEN, "false" ); _defaultVals.put(CODEGEN_API, GeneratorAPI.JAVA.name() ); _defaultVals.put(CODEGEN_COMPILER, CompilerType.AUTO.name() ); @@ -476,6 +481,7 @@ public String getConfigInfo() { COMPRESSED_LINALG, COMPRESSED_LOSSY, COMPRESSED_VALID_COMPRESSIONS, COMPRESSED_OVERLAPPING, COMPRESSED_SAMPLING_RATIO, COMPRESSED_SOFT_REFERENCE_COUNT, COMPRESSED_COCODE, COMPRESSED_TRANSPOSE, COMPRESSED_TRANSFORMENCODE, DAG_LINEARIZATION, + SPARSITY_REWRITES, SPARSITY_ESTIMATOR, CODEGEN, CODEGEN_API, CODEGEN_COMPILER, CODEGEN_OPTIMIZER, CODEGEN_PLANCACHE, CODEGEN_LITERALS, STATS_MAX_WRAP_LEN, LINEAGECACHESPILL, COMPILERASSISTED_RW, BUFFERPOOL_LIMIT, MEMORY_MANAGER, PRINT_GPU_MEMORY_INFO, AVAILABLE_GPUS, SYNCHRONIZE_GPU, EAGER_CUDA_FREE, GPU_RULE_BASED_PLACEMENT, diff --git a/src/main/java/org/apache/sysds/hops/OptimizerUtils.java b/src/main/java/org/apache/sysds/hops/OptimizerUtils.java index 04850cf8637..4c5edbdcc28 100644 --- a/src/main/java/org/apache/sysds/hops/OptimizerUtils.java +++ b/src/main/java/org/apache/sysds/hops/OptimizerUtils.java @@ -200,10 +200,9 @@ public enum MemoryManager { public static boolean ALLOW_SUM_PRODUCT_REWRITES2 = true; /** - * Enables additional mmchain optimizations. In the future, this might be merged with - * ALLOW_SUM_PRODUCT_REWRITES. + * Enables transpose mmchain optimizations. In the future, this might be merged with ALLOW_SUM_PRODUCT_REWRITES. */ - public static boolean ALLOW_ADVANCED_MMCHAIN_REWRITES = false; + public static boolean ALLOW_TRANSPOSE_MMCHAIN_REWRITES = false; /** * Enables a DPSize inspired algorithm rewrite for MMChain with transposes diff --git a/src/main/java/org/apache/sysds/hops/estim/EstimationUtils.java b/src/main/java/org/apache/sysds/hops/estim/EstimationUtils.java index eeca0f115fc..b0552343152 100644 --- a/src/main/java/org/apache/sysds/hops/estim/EstimationUtils.java +++ b/src/main/java/org/apache/sysds/hops/estim/EstimationUtils.java @@ -30,6 +30,55 @@ public abstract class EstimationUtils { + /** + * Enumeration for the sparsity estimators supported + */ + public enum EstimatorType { + BASIC_AVG, + BASIC_WORST, + BITSET_MM, + DM, + LG, + MNC, + MNC_LIM, + MNC_EXT, + RS, + SAMPLE, + SAMPLE_RA; + + /** + * @return a sparsity estimator object corresponding to this estimator type + */ + public SparsityEstimator getEstimator() { + switch(this) { + case BASIC_AVG: + return new EstimatorBasicAvg(); + case BASIC_WORST: + return new EstimatorBasicWorst(); + case BITSET_MM: + return new EstimatorBitsetMM(); + case DM: + return new EstimatorDensityMap(); + case LG: + return new EstimatorLayeredGraph(); + case MNC: + return new EstimatorMatrixHistogram(); + case MNC_LIM: + return new EstimatorMatrixHistogram(false); + case MNC_EXT: + return new EstimatorMatrixHistogram(true); + case RS: + return new EstimatorRowWise(); + case SAMPLE: + return new EstimatorSample(); + case SAMPLE_RA: + return new EstimatorSampleRa(); + default: + throw new DMLRuntimeException("Unknown sparsity estimator " + this.toString()); + } + } + } + /** * This utility function computes the exact output nnz * of a self matrix product without need to materialize diff --git a/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriter.java b/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriter.java index efc3de5a655..73add6c7af0 100644 --- a/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriter.java +++ b/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriter.java @@ -24,6 +24,7 @@ import org.apache.sysds.api.DMLScript; import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; import org.apache.sysds.conf.CompilerConfig.ConfigType; import org.apache.sysds.hops.Hop; import org.apache.sysds.hops.OptimizerUtils; @@ -139,9 +140,11 @@ public ProgramRewriter(boolean staticRewrites, boolean dynamicRewrites) if( OptimizerUtils.ALLOW_NEW_MMCHAIN_REWRITE ) { _dagRuleSet.add( new RewriteMatrixMultChainWithTransOptimization() ); } - if(OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES){ + if(OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES){ _dagRuleSet.add( new RewriteMatrixMultChainOptimizationTranspose() ); //dependency: cse - _dagRuleSet.add( new RewriteMatrixMultChainOptimizationSparse() ); //dependency: cse + } + if(ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.SPARSITY_REWRITES)) { + _dagRuleSet.add( new RewriteMatrixMultChainOptimizationSparse() ); } if( OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION ) { _dagRuleSet.add( new RewriteAlgebraicSimplificationDynamic() ); //dependencies: cse diff --git a/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java b/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java index 80b71a1c902..5ab9d57e44c 100644 --- a/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java +++ b/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java @@ -23,10 +23,13 @@ import java.util.List; import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; import org.apache.sysds.hops.Hop; import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.hops.estim.MMNode; -import org.apache.sysds.hops.estim.EstimatorBasicAvg; +import org.apache.sysds.hops.estim.SparsityEstimator; +import org.apache.sysds.hops.estim.EstimationUtils.EstimatorType; import org.apache.sysds.hops.estim.SparsityEstimator.OpCode; /** @@ -35,7 +38,7 @@ * * Solution: Classic Dynamic Programming * Approach: Currently, the approach based only on matrix dimensions - * and sparsity estimates using the MNC sketch + * and sparsity estimates using the basic average estimator * Goal: To reduce the number of computations in the run-time * (map-reduce) layer */ @@ -85,9 +88,10 @@ private static int[][] mmChainDPSparse(double[] dimArray, MMNode[] sketchArray, } //compute cost-optimal chains for increasing chain sizes - EstimatorBasicAvg estim = new EstimatorBasicAvg(); - for( int l = 2; l <= size; l++ ) { // chain length - for( int i = 0; i < size - l + 1; i++ ) { + SparsityEstimator estim = EstimatorType.valueOf(ConfigurationManager.getDMLConfig() + .getTextValue(DMLConfig.SPARSITY_ESTIMATOR)).getEstimator(); + for(int l = 2; l <= size; l++) { // chain length + for(int i = 0; i < size - l + 1; i++) { int j = i + l - 1; // find cost of (i,j) dpMatrix[i][j] = Double.MAX_VALUE; diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java index 64af7415f88..60b491b8141 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java @@ -21,7 +21,15 @@ import org.junit.Assert; import org.junit.Test; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; + import org.apache.sysds.common.Types.ExecMode; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.hops.recompile.Recompiler; import org.apache.sysds.test.AutomatedTestBase; @@ -123,19 +131,33 @@ public void setUp() { private void runTestMatrixChainDP(String testName) { ExecMode platformOld = rtplatform; - boolean rewritesOld = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; - boolean newMMchain1 = OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES; - boolean newMMchain2 = OptimizerUtils.ALLOW_NEW_MMCHAIN_REWRITE; + boolean oldFlag1 = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; + boolean oldFlag2 = OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES; + boolean oldFlag3 = OptimizerUtils.ALLOW_NEW_MMCHAIN_REWRITE; + DMLConfig oldDMLConfig = ConfigurationManager.getDMLConfig(); try { rtplatform = ExecMode.SINGLE_NODE; OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = true; - OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES = true; + OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES = true; OptimizerUtils.ALLOW_NEW_MMCHAIN_REWRITE = true; TestConfiguration config = getTestConfiguration(testName); loadTestConfiguration(config); + try { + DMLConfig dmlConfig = new DMLConfig(getCurConfigFile().getPath()); + dmlConfig.setTextValue(DMLConfig.SPARSITY_REWRITES, "true"); + overwriteCurrentConfig(dmlConfig); + } + catch(FileNotFoundException fnfe) { + Assert.fail("Could not find DML config file: " + + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); + } + catch(IOException ioe) { + Assert.fail("Could not overwrite the DML configuration file. " + ioe.getMessage()); + } + String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testName + ".dml"; @@ -300,11 +322,21 @@ private void runTestMatrixChainDP(String testName) { } } } finally { - OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewritesOld; - OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES = newMMchain1; - OptimizerUtils.ALLOW_NEW_MMCHAIN_REWRITE = newMMchain2; + OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag1; + OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES = oldFlag2; + OptimizerUtils.ALLOW_NEW_MMCHAIN_REWRITE = oldFlag3; + try { + overwriteCurrentConfig(oldDMLConfig); + } + catch(IOException ioe) { + Assert.fail("Unable to restore the previous DML configuration. " + ioe.getMessage()); + } rtplatform = platformOld; Recompiler.reinitRecompiler(); } } + + private void overwriteCurrentConfig(DMLConfig config) throws IOException { + Files.write(getCurConfigFile().toPath(), config.serializeDMLConfig().getBytes(StandardCharsets.UTF_8)); + } } diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java index d4d676dc0e2..bf9acd9e52a 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java @@ -23,6 +23,8 @@ import org.apache.log4j.Logger; import org.apache.log4j.spi.LoggingEvent; import org.apache.sysds.common.Opcodes; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.hops.recompile.Recompiler; import org.apache.sysds.runtime.matrix.data.MatrixValue; @@ -37,6 +39,10 @@ import org.junit.Test; import org.junit.runner.RunWith; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -103,21 +109,37 @@ public void testMatrixMultChainOptSparseRewrites() { } private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { - boolean oldFlag1 = OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES; + boolean oldFlag1 = OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES; boolean oldFlag2 = OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES; + DMLConfig oldDMLConfig = ConfigurationManager.getDMLConfig(); try { TestConfiguration config = getTestConfiguration(TEST_NAME); loadTestConfiguration(config); + try { + DMLConfig dmlConfig = new DMLConfig(getCurConfigFile().getPath()); + dmlConfig.setTextValue(DMLConfig.SPARSITY_REWRITES, String.valueOf(rewrites)); + overwriteCurrentConfig(dmlConfig); + } + catch(FileNotFoundException fnfe) { + Assert.fail("Could not find DML config file: " + + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); + } + catch(IOException ioe) { + Assert.fail("Could not overwrite the DML configuration file. " + ioe.getMessage()); + } + String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + TEST_NAME + ".dml"; - programArgs = new String[] {"-explain", "hops", "-stats", "-args", input("X"), input("Y"), output("R")}; + programArgs = new String[] {"-explain", "hops", "-stats", + "-args", input("X"), input("Y"), output("R")}; fullRScriptName = HOME + TEST_NAME + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); - OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES = rewrites; + OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES = rewrites; OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES = rewrites; + double[][] X = getRandomMatrix(rows, cols, -1, 1, sparsities[0], 7); double[][] Y = getRandomMatrix(cols, 1, -1, 1, sparsities[1], 3); long X_nnz = Stream.of(X).mapToLong(row -> DoubleStream.of(row).filter(val -> val != 0).count()).sum(); @@ -164,9 +186,19 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { } } finally { - OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES = oldFlag1; + OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES = oldFlag1; OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES = oldFlag2; + try { + overwriteCurrentConfig(oldDMLConfig); + } + catch(IOException ioe) { + Assert.fail("Unable to restore the previous DML configuration. " + ioe.getMessage()); + } Recompiler.reinitRecompiler(); } } + + private void overwriteCurrentConfig(DMLConfig config) throws IOException { + Files.write(getCurConfigFile().toPath(), config.serializeDMLConfig().getBytes(StandardCharsets.UTF_8)); + } } diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptTransposeTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptTransposeTest.java index 72ae5384298..e062f6f0420 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptTransposeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptTransposeTest.java @@ -93,7 +93,7 @@ public void testMMChainFourNoRewrite() { private void testMMChainWithTransposeOperator(String testname, int numOptTranspositions, int numOriginalTranspositions, boolean rewrites) { - boolean oldFlag = OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES; + boolean oldFlag = OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES; try { TestConfiguration config = getTestConfiguration(testname); loadTestConfiguration(config); @@ -104,7 +104,7 @@ private void testMMChainWithTransposeOperator(String testname, int numOptTranspo fullRScriptName = HOME + testname + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); - OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES = rewrites; + OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES = rewrites; //execute tests runTest(true, false, null, -1); @@ -124,7 +124,7 @@ private void testMMChainWithTransposeOperator(String testname, int numOptTranspo } finally { - OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES = oldFlag; + OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES = oldFlag; Recompiler.reinitRecompiler(); } } From b33375518b6aecf282d7ff223037dcf68446d69e Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Fri, 7 Aug 2026 15:44:33 +0200 Subject: [PATCH 110/132] [SYSTEMDS-3891] Wire OOC Instructions with New Primitives --- .../ooc/BinaryOOCInstruction.java | 63 ++++++++----------- .../ooc/DataGenOOCInstruction.java | 42 +++++-------- .../ParameterizedBuiltinOOCInstruction.java | 5 +- .../ooc/TernaryOOCInstruction.java | 59 ++++++++++------- .../sysds/test/functions/ooc/SeqTest.java | 5 ++ 5 files changed, 82 insertions(+), 92 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/BinaryOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/BinaryOOCInstruction.java index c252cd3d1ec..89b6d911000 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/BinaryOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/BinaryOOCInstruction.java @@ -30,6 +30,7 @@ import org.apache.sysds.runtime.matrix.operators.BinaryOperator; import org.apache.sysds.runtime.matrix.operators.Operator; import org.apache.sysds.runtime.matrix.operators.ScalarOperator; +import org.apache.sysds.runtime.ooc.store.CountingLiveness; import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; public class BinaryOOCInstruction extends ComputationOOCInstruction { @@ -67,8 +68,8 @@ protected void processMatrixMatrixInstruction(ExecutionContext ec) { OOCStream qOut = new SubscribableTaskQueue<>(); ec.getMatrixObject(output).setStreamHandle(qOut); - final boolean known1 = (m1.getNumRows() >= 0 && m1.getNumColumns() >= 0); - final boolean known2 = (m2.getNumRows() >= 0 && m2.getNumColumns() >= 0); + final boolean known1 = m1.getNumRows() >= 0 && m1.getNumColumns() >= 0 && m1.getBlocksize() > 0; + final boolean known2 = m2.getNumRows() >= 0 && m2.getNumColumns() >= 0 && m2.getBlocksize() > 0; // If dimensions are unknown, we cannot safely detect broadcasting. // Fall back to strict key-based join and let downstream operators validate as needed. @@ -94,36 +95,28 @@ protected void processMatrixMatrixInstruction(ExecutionContext ec) { boolean isRowBroadcast = m1.getNumRows() > 1 && m2.getNumRows() == 1; if (isColBroadcast && !isRowBroadcast) { - OOCStream qIn1 = m1.getStreamHandle(); - OOCStream qIn2 = m2.getStreamHandle(); - final long maxProcessesPerBroadcast = (m1.getNumColumns() + m1.getBlocksize() - 1) / m1.getBlocksize(); - - broadcastJoinOOC(qIn1, qIn2, qOut, (tmp1, b) -> { - IndexedMatrixValue tmpOut = new IndexedMatrixValue(); - tmpOut.set(tmp1.getIndexes(), - tmp1.getValue().binaryOperations((BinaryOperator)_optr, b.getValue().getValue(), tmpOut.getValue())); - - if (b.incrProcessCtrAndGet() >= maxProcessesPerBroadcast) - b.release(); - - return tmpOut; - }, tmp -> tmp.getIndexes().getRowIndex()); + int broadcastBlocks = Math.toIntExact(m2.getDataCharacteristics().getNumRowBlocks()); + int usesPerBlock = Math.toIntExact(m1.getDataCharacteristics().getNumColBlocks()); + OOCInstructionUtils.indexedBroadcastMap(m1.getStreamable(), m2.getStreamable(), qOut, + tmp -> Math.toIntExact(tmp.getIndexes().getRowIndex() - 1), + () -> new CountingLiveness(broadcastBlocks, usesPerBlock), (tmp, broadcast) -> { + IndexedMatrixValue tmpOut = new IndexedMatrixValue(); + tmpOut.set(tmp.getIndexes(), tmp.getValue().binaryOperations((BinaryOperator) _optr, + broadcast.getValue(), tmpOut.getValue())); + return tmpOut; + }, getContext()); } else if (isRowBroadcast && !isColBroadcast) { - OOCStream qIn1 = m1.getStreamHandle(); - OOCStream qIn2 = m2.getStreamHandle(); - final long maxProcessesPerBroadcast = (m1.getNumRows() + m1.getBlocksize() - 1) / m1.getBlocksize(); - - broadcastJoinOOC(qIn1, qIn2, qOut, (tmp1, b) -> { - IndexedMatrixValue tmpOut = new IndexedMatrixValue(); - tmpOut.set(tmp1.getIndexes(), - tmp1.getValue().binaryOperations((BinaryOperator)_optr, b.getValue().getValue(), tmpOut.getValue())); - - if (b.incrProcessCtrAndGet() >= maxProcessesPerBroadcast) - b.release(); - - return tmpOut; - }, tmp -> tmp.getIndexes().getColumnIndex()); + int broadcastBlocks = Math.toIntExact(m2.getDataCharacteristics().getNumColBlocks()); + int usesPerBlock = Math.toIntExact(m1.getDataCharacteristics().getNumRowBlocks()); + OOCInstructionUtils.indexedBroadcastMap(m1.getStreamable(), m2.getStreamable(), qOut, + tmp -> Math.toIntExact(tmp.getIndexes().getColumnIndex() - 1), + () -> new CountingLiveness(broadcastBlocks, usesPerBlock), (tmp, broadcast) -> { + IndexedMatrixValue tmpOut = new IndexedMatrixValue(); + tmpOut.set(tmp.getIndexes(), tmp.getValue().binaryOperations((BinaryOperator) _optr, + broadcast.getValue(), tmpOut.getValue())); + return tmpOut; + }, getContext()); } else { if (m1.getNumColumns() != m2.getNumColumns() || m1.getNumRows() != m2.getNumRows()) @@ -144,15 +137,9 @@ protected void processScalarMatrixInstruction(ExecutionContext ec) { //create thread and process binary operation MatrixObject min = ec.getMatrixObject(input1.isMatrix() ? input1 : input2); - OOCStream qIn = min.getStreamHandle(); OOCStream qOut = createWritableStream(); ec.getMatrixObject(output).setStreamHandle(qOut); - - mapOOC(qIn, qOut, tmp -> { - IndexedMatrixValue tmpOut = new IndexedMatrixValue(); - tmpOut.set(tmp.getIndexes(), - tmp.getValue().scalarOperations(sc_op, new MatrixBlock())); - return tmpOut; - }); + OOCInstructionUtils.equiMapBlock(min.getStreamable(), qOut, + block -> block.scalarOperations(sc_op, new MatrixBlock()), getContext()); } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/DataGenOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/DataGenOOCInstruction.java index c5c2e299cbb..51a0e414a7d 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/DataGenOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/DataGenOOCInstruction.java @@ -34,10 +34,8 @@ import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.matrix.data.LibMatrixDatagen; import org.apache.sysds.runtime.matrix.data.MatrixBlock; -import org.apache.sysds.runtime.matrix.data.MatrixIndexes; import org.apache.sysds.runtime.matrix.data.RandomMatrixGenerator; import org.apache.sysds.runtime.matrix.operators.UnaryOperator; -import org.apache.sysds.runtime.ooc.stream.StreamContext; import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; import org.apache.sysds.runtime.util.UtilFunctions; @@ -251,34 +249,22 @@ else if(method == Types.OpOpDG.SEQ) { final int maxK = (int) UtilFunctions.getSeqLength(lfrom, lto, lincr); final double finalLincr = lincr; + ec.getDataCharacteristics(output.getName()).set(maxK, 1, blen, -1); - OOCInstructionUtils.submitOOCTask(() -> { - int k = 0; - double curFrom = lfrom; - double curTo; - MatrixBlock mb; - - while (k < maxK) { - long desiredLen = Math.min(blen, maxK - k); - curTo = curFrom + (desiredLen - 1) * finalLincr; - long actualLen = UtilFunctions.getSeqLength(curFrom, curTo, finalLincr); - - if (actualLen != desiredLen) { - // Then we add / subtract a small correction term - curTo += (actualLen < desiredLen) ? finalLincr / 2 : -finalLincr / 2; - - if (UtilFunctions.getSeqLength(curFrom, curTo, finalLincr) != desiredLen) - throw new DMLRuntimeException("OOC seq could not construct the right number of elements."); - } - - mb = MatrixBlock.seqOperations(curFrom, curTo, finalLincr); - qOut.enqueue(new IndexedMatrixValue(new MatrixIndexes(1 + k / blen, 1), mb)); - curFrom = mb.get(mb.getNumRows() - 1, 0) + finalLincr; - k += blen; + OOCInstructionUtils.dataGen(qOut, idx -> { + long offset = (idx.getRowIndex() - 1) * blen; + long desiredLen = Math.min(blen, maxK - offset); + double curFrom = lfrom + offset * finalLincr; + double curTo = curFrom + (desiredLen - 1) * finalLincr; + long actualLen = UtilFunctions.getSeqLength(curFrom, curTo, finalLincr); + + if(actualLen != desiredLen) { + curTo += actualLen < desiredLen ? finalLincr / 2 : -finalLincr / 2; + if(UtilFunctions.getSeqLength(curFrom, curTo, finalLincr) != desiredLen) + throw new DMLRuntimeException("OOC seq could not construct the right number of elements."); } - - qOut.closeInput(); - }, new StreamContext(_callerId, getExtendedOpcode()).addOutStream(qOut)); + return MatrixBlock.seqOperations(curFrom, curTo, finalLincr); + }, getContext()); } else throw new NotImplementedException(); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ParameterizedBuiltinOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ParameterizedBuiltinOOCInstruction.java index 2f71d0e4538..d46c606768a 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ParameterizedBuiltinOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ParameterizedBuiltinOOCInstruction.java @@ -40,6 +40,7 @@ import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.runtime.matrix.operators.Operator; import org.apache.sysds.runtime.matrix.operators.SimpleOperator; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; import org.apache.sysds.runtime.util.UtilFunctions; import java.util.ArrayList; @@ -92,13 +93,13 @@ public void processInstruction(ExecutionContext ec) { throw new NotImplementedException(); } else{ MatrixObject targetObj = ec.getMatrixObject(params.get("target")); - OOCStream qIn = targetObj.getStreamHandle(); OOCStream qOut = createWritableStream(); double pattern = Double.parseDouble(params.get("pattern")); double replacement = Double.parseDouble(params.get("replacement")); - mapOOC(qIn, qOut, tmp -> new IndexedMatrixValue(tmp.getIndexes(), tmp.getValue().replaceOperations(new MatrixBlock(), pattern, replacement))); + OOCInstructionUtils.equiMapBlock(targetObj.getStreamable(), qOut, + block -> block.replaceOperations(new MatrixBlock(), pattern, replacement), getContext()); ec.getMatrixObject(output).setStreamHandle(qOut); } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/TernaryOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/TernaryOOCInstruction.java index 6036647cc7f..7b91b16d237 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/TernaryOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/TernaryOOCInstruction.java @@ -35,6 +35,7 @@ import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.runtime.matrix.operators.Operator; import org.apache.sysds.runtime.matrix.operators.TernaryOperator; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; public class TernaryOOCInstruction extends ComputationOOCInstruction { @@ -106,27 +107,20 @@ private void processSingleMatrixInstruction(ExecutionContext ec, int matrixPos) MatrixBlock s2 = input2.isMatrix() ? null : getScalarInputBlock(ec, input2); MatrixBlock s3 = input3.isMatrix() ? null : getScalarInputBlock(ec, input3); - OOCStream qIn = mo.getStreamHandle(); OOCStream qOut = createWritableStream(); ec.getMatrixObject(output).setStreamHandle(qOut); - mapOOC(qIn, qOut, tmp -> { - IndexedMatrixValue outVal = new IndexedMatrixValue(); - MatrixBlock op1 = resolveOperandBlock(1, tmp, null, matrixPos, -1, s1, s2, s3); - MatrixBlock op2 = resolveOperandBlock(2, tmp, null, matrixPos, -1, s1, s2, s3); - MatrixBlock op3 = resolveOperandBlock(3, tmp, null, matrixPos, -1, s1, s2, s3); - outVal.set(tmp.getIndexes(), - op1.ternaryOperations((TernaryOperator)_optr, op2, op3, new MatrixBlock())); - return outVal; - }); + OOCInstructionUtils.equiMapBlock(mo.getStreamable(), qOut, block -> { + MatrixBlock op1 = resolveOperandBlock(1, block, null, matrixPos, -1, s1, s2, s3); + MatrixBlock op2 = resolveOperandBlock(2, block, null, matrixPos, -1, s1, s2, s3); + MatrixBlock op3 = resolveOperandBlock(3, block, null, matrixPos, -1, s1, s2, s3); + return op1.ternaryOperations((TernaryOperator) _optr, op2, op3, new MatrixBlock()); + }, getContext()); } private void processTwoMatrixInstruction(ExecutionContext ec, int leftPos, int rightPos) { MatrixObject left = getMatrixObject(ec, leftPos); MatrixObject right = getMatrixObject(ec, rightPos); - OOCStream leftStream = left.getStreamHandle(); - OOCStream rightStream = right.getStreamHandle(); - MatrixBlock s1 = input1.isMatrix() ? null : getScalarInputBlock(ec, input1); MatrixBlock s2 = input2.isMatrix() ? null : getScalarInputBlock(ec, input2); MatrixBlock s3 = input3.isMatrix() ? null : getScalarInputBlock(ec, input3); @@ -134,15 +128,26 @@ private void processTwoMatrixInstruction(ExecutionContext ec, int leftPos, int r OOCStream qOut = createWritableStream(); ec.getMatrixObject(output).setStreamHandle(qOut); - joinOOC(leftStream, rightStream, qOut, (l, r) -> { - IndexedMatrixValue outVal = new IndexedMatrixValue(); - MatrixBlock op1 = resolveOperandBlock(1, l, r, leftPos, rightPos, s1, s2, s3); - MatrixBlock op2 = resolveOperandBlock(2, l, r, leftPos, rightPos, s1, s2, s3); - MatrixBlock op3 = resolveOperandBlock(3, l, r, leftPos, rightPos, s1, s2, s3); - outVal.set(l.getIndexes(), - op1.ternaryOperations((TernaryOperator)_optr, op2, op3, new MatrixBlock())); - return outVal; - }, IndexedMatrixValue::getIndexes); + if(left.getDataCharacteristics().dimsKnown() && right.getDataCharacteristics().dimsKnown()) { + OOCInstructionUtils.equiJoin(left.getStreamable(), right.getStreamable(), qOut, (l, r) -> { + MatrixBlock op1 = resolveOperandBlock(1, l, r, leftPos, rightPos, s1, s2, s3); + MatrixBlock op2 = resolveOperandBlock(2, l, r, leftPos, rightPos, s1, s2, s3); + MatrixBlock op3 = resolveOperandBlock(3, l, r, leftPos, rightPos, s1, s2, s3); + return op1.ternaryOperations((TernaryOperator) _optr, op2, op3, new MatrixBlock()); + }, getContext()); + } + else { + OOCStream leftStream = left.getStreamHandle(); + OOCStream rightStream = right.getStreamHandle(); + joinOOC(leftStream, rightStream, qOut, (l, r) -> { + IndexedMatrixValue outVal = new IndexedMatrixValue(); + MatrixBlock op1 = resolveOperandBlock(1, l, r, leftPos, rightPos, s1, s2, s3); + MatrixBlock op2 = resolveOperandBlock(2, l, r, leftPos, rightPos, s1, s2, s3); + MatrixBlock op3 = resolveOperandBlock(3, l, r, leftPos, rightPos, s1, s2, s3); + outVal.set(l.getIndexes(), op1.ternaryOperations((TernaryOperator) _optr, op2, op3, new MatrixBlock())); + return outVal; + }, IndexedMatrixValue::getIndexes); + } } private void processThreeMatrixInstruction(ExecutionContext ec) { @@ -185,10 +190,16 @@ private MatrixBlock getScalarInputBlock(ExecutionContext ec, CPOperand operand) private MatrixBlock resolveOperandBlock(int operandPos, IndexedMatrixValue left, IndexedMatrixValue right, int leftPos, int rightPos, MatrixBlock s1, MatrixBlock s2, MatrixBlock s3) { + return resolveOperandBlock(operandPos, left == null ? null : (MatrixBlock) left.getValue(), + right == null ? null : (MatrixBlock) right.getValue(), leftPos, rightPos, s1, s2, s3); + } + + private MatrixBlock resolveOperandBlock(int operandPos, MatrixBlock left, MatrixBlock right, int leftPos, + int rightPos, MatrixBlock s1, MatrixBlock s2, MatrixBlock s3) { if(operandPos == leftPos && left != null) - return (MatrixBlock) left.getValue(); + return left; if(operandPos == rightPos && right != null) - return (MatrixBlock) right.getValue(); + return right; if(operandPos == 1) return s1; diff --git a/src/test/java/org/apache/sysds/test/functions/ooc/SeqTest.java b/src/test/java/org/apache/sysds/test/functions/ooc/SeqTest.java index f7855b93e2d..47761e85c30 100644 --- a/src/test/java/org/apache/sysds/test/functions/ooc/SeqTest.java +++ b/src/test/java/org/apache/sysds/test/functions/ooc/SeqTest.java @@ -53,6 +53,11 @@ public void testSeq2() { runSeqTest(0, 15.9, 0.01); } + @Test + public void testDescendingSeq() { + runSeqTest(10, 0, -0.1); + } + private void runSeqTest(double from, double to, double incr) { Types.ExecMode platformOld = setExecMode(Types.ExecMode.SINGLE_NODE); From 6077976e69a15e95bcec8f46a6a781daac312142 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Sat, 8 Aug 2026 12:39:46 +0200 Subject: [PATCH 111/132] [SYSTEMDS-3891] OOC Non-blocking Join and Primitive Simplification --- .../ooc/primitives/BroadcastOOCPrimitive.java | 13 +- .../primitives/GroupedReduceOOCPrimitive.java | 15 +-- .../ooc/primitives/JoinOOCPrimitive.java | 117 ++++++++++++------ .../primitives/MaterializeOOCPrimitive.java | 6 - .../runtime/ooc/primitives/OOCPrimitive.java | 15 +++ .../PlannableDataGenOOCPrimitive.java | 3 +- .../runtime/ooc/util/OOCInstructionUtils.java | 19 +++ .../ooc/OOCInstructionUtilsTest.java | 32 +++++ 8 files changed, 147 insertions(+), 73 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/BroadcastOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/BroadcastOOCPrimitive.java index e2a9be63c25..49980c28959 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/BroadcastOOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/BroadcastOOCPrimitive.java @@ -51,7 +51,6 @@ public final class BroadcastOOCPrimitive extends OOCPrimitive { private final Supplier _liveness; private final BiFunction _operation; private final AtomicBoolean _cleaned; - private final AtomicBoolean _failed; private final AtomicBoolean _sourceComplete; private final AtomicInteger _active; private MaterializedStore _store; @@ -70,7 +69,6 @@ public BroadcastOOCPrimitive(OOCStreamable streamed, _liveness = liveness; _operation = operation; _cleaned = new AtomicBoolean(); - _failed = new AtomicBoolean(); _sourceComplete = new AtomicBoolean(); _active = new AtomicInteger(1); } @@ -100,7 +98,7 @@ protected void startExecution() { _outputStream = _output.getWriteStream(); _ready = new SubscribableTaskQueue<>(); getContext().addOutStream(_outputStream, _ready); - OOCInstructionUtils.submitOOCTasks(_ready, callback -> process(callback.get()), getContext()) + OOCInstructionUtils.submitCloseableOOCTasks(_ready, this::process, getContext()) .whenComplete((ignored, error) -> { try { if(error != null) @@ -232,7 +230,6 @@ private void process(BroadcastWork work) { fail(failure); } finally { - work.close(); if(budget != null) budget.close(); completeOne(); @@ -255,14 +252,6 @@ private void completeOne() { } } - private void fail(Throwable error) { - if(!_failed.compareAndSet(false, true)) - return; - DMLRuntimeException failure = DMLRuntimeException.of(error); - _outputStream.propagateFailure(failure); - getContext().failAll(failure); - } - private void cleanup() { if(!_cleaned.compareAndSet(false, true)) return; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/GroupedReduceOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/GroupedReduceOOCPrimitive.java index aa67497206e..b380848aa7d 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/GroupedReduceOOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/GroupedReduceOOCPrimitive.java @@ -49,7 +49,6 @@ public final class GroupedReduceOOCPrimitive extends OOCPrimitive { private final OOCStreamable _output; private final BiFunction _merge; private final AtomicBoolean _cleaned; - private final AtomicBoolean _failed; private final AtomicBoolean _sourceComplete; private final AtomicInteger _active; private final AtomicInteger _finalizedGroups; @@ -66,7 +65,6 @@ public GroupedReduceOOCPrimitive(OOCStreamable input, OOCStr _output = output; _merge = merge; _cleaned = new AtomicBoolean(); - _failed = new AtomicBoolean(); _sourceComplete = new AtomicBoolean(); _active = new AtomicInteger(1); _finalizedGroups = new AtomicInteger(); @@ -100,7 +98,7 @@ protected void startExecution() { getContext().addInStream(input).addOutStream(_outputStream, _ready); _table = new StateTable<>(OOCCacheManager.getGlobalCache(), CachingStream._streamSeq.getNextID()); - OOCInstructionUtils.submitOOCTasks(_ready, callback -> process(callback.get()), getContext()) + OOCInstructionUtils.submitCloseableOOCTasks(_ready, this::process, getContext()) .whenComplete((ignored, error) -> { try { _outputStream.closeInput(); @@ -222,7 +220,6 @@ private void process(MergeWork work) { catch(Throwable failure) { if(merged != null) merged.release(); - work.close(); budget.close(); fail(failure); completeOne(); @@ -276,7 +273,7 @@ private void completeOne() { int remaining = _active.decrementAndGet(); if(remaining != 0) return; - if(!_failed.get() && _finalizedGroups.get() != _numGroups) + if(!hasFailed() && _finalizedGroups.get() != _numGroups) fail(new DMLRuntimeException( "Grouped reduction completed " + _finalizedGroups.get() + " of " + _numGroups + " row groups.")); try { @@ -287,14 +284,6 @@ private void completeOne() { } } - private void fail(Throwable error) { - if(!_failed.compareAndSet(false, true)) - return; - DMLRuntimeException failure = DMLRuntimeException.of(error); - _outputStream.propagateFailure(failure); - getContext().failAll(failure); - } - private void cleanup() { if(!_cleaned.compareAndSet(false, true)) return; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/JoinOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/JoinOOCPrimitive.java index a4ef2c30ced..0ec802b181c 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/JoinOOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/JoinOOCPrimitive.java @@ -19,7 +19,8 @@ package org.apache.sysds.runtime.ooc.primitives; -import java.util.concurrent.ExecutionException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; import org.apache.sysds.runtime.DMLRuntimeException; @@ -44,6 +45,9 @@ public class JoinOOCPrimitive extends OOCPrimitive { private final OOCStreamable _right; private final OOCStreamable _output; private final BiFunction _operation; + private final AtomicInteger _pending = new AtomicInteger(1); + private final AtomicInteger _unmatched = new AtomicInteger(); + private final CompletableFuture _pendingCompletion = new CompletableFuture<>(); private StateTable _table; public JoinOOCPrimitive(OOCStreamable left, OOCStreamable right, @@ -87,16 +91,13 @@ protected void startExecution() { long taskBytes = outputBytes + 2 * inputBytes; getContext().addOutStream(output); - OOCInstructionUtils.submitOOCTasks(matches, callback -> { - try(JoinWork work = callback.get()) { - IndexedMatrixValue mleft = work._left.get(); - IndexedMatrixValue mright = work._right.get(); - OOCUtils.enqueueExact(output, - new IndexedMatrixValue(mleft.getIndexes(), - _operation.apply((MatrixBlock) mleft.getValue(), (MatrixBlock) mright.getValue())), - work._budget); - } - }, callback -> true, (index, callback) -> callback.get().close(), getContext()).thenRun(() -> { + CompletableFuture processing = OOCInstructionUtils.submitCloseableOOCTasks(matches, (JoinWork work) -> { + IndexedMatrixValue mleft = work._left.get(); + IndexedMatrixValue mright = work._right.get(); + OOCUtils.enqueueExact(output, new IndexedMatrixValue(mleft.getIndexes(), + _operation.apply((MatrixBlock) mleft.getValue(), (MatrixBlock) mright.getValue())), work._budget); + }, getContext()); + CompletableFuture.allOf(processing, _pendingCompletion).thenRun(() -> { try { _table.close(); onComplete(); @@ -113,7 +114,6 @@ protected void startExecution() { private void drive(OOCStream leftInput, OOCStream rightInput, OOCStream matches, long taskBytes) { long cols = _right.getDataCharacteristics().getNumColBlocks(); - int unmatched = 0; try { while(true) { OOCStream.QueueCallback left = leftInput.dequeueCB(); @@ -129,23 +129,26 @@ private void drive(OOCStream leftInput, OOCStream callback, boolean left, long cols, long taskBytes, + private void accept(OOCStream.QueueCallback callback, boolean left, long cols, long taskBytes, OOCStream matches) { if(callback == null) - return 0; + return; OOCStream.QueueCallback owned = null; ReservationBudget budget = null; + boolean pending = false; try { owned = callback.keepOpen(); callback.close(); @@ -155,25 +158,18 @@ private int accept(OOCStream.QueueCallback callback, boolean long row = value.getIndexes().getRowIndex() - 1; long col = value.getIndexes().getColumnIndex() - 1; int slot = Math.toIntExact(row * cols + col); + _pending.incrementAndGet(); + pending = true; OOCFuture future = StateTableUtils.putOrTake(_table, slot, owned, budget); owned = null; - StateTableUtils.Match match = await(future); - if(match == null) - return 1; - JoinWork work = left ? new JoinWork(match.left(), match.right(), budget) : new JoinWork(match.right(), - match.left(), budget); + ReservationBudget pendingBudget = budget; budget = null; - try { - matches.enqueue(work); - work = null; - } - finally { - if(work != null) - work.close(); - } - return -1; + future.whenComplete((match, error) -> matchReady(match, left, pendingBudget, error, matches)); + pending = false; } finally { + if(pending) + completePending(matches); if(callback != null) callback.close(); if(owned != null) @@ -183,16 +179,57 @@ private int accept(OOCStream.QueueCallback callback, boolean } } - private static StateTableUtils.Match await(OOCFuture future) { + private void matchReady(StateTableUtils.Match match, boolean left, ReservationBudget budget, Throwable error, + OOCStream matches) { + JoinWork work = null; try { - return future.get(); + if(error != null) + throw DMLRuntimeException.of(error); + if(match == null) { + _unmatched.incrementAndGet(); + return; + } + _unmatched.decrementAndGet(); + work = left ? new JoinWork(match.left(), match.right(), budget) : new JoinWork(match.right(), match.left(), + budget); + match = null; + budget = null; + matches.enqueue(work); + work = null; + } + catch(Throwable failure) { + fail(failure); + } + finally { + if(work != null) + work.close(); + if(match != null) { + match.left().close(); + match.right().close(); + } + if(budget != null) + budget.close(); + completePending(matches); } - catch(InterruptedException error) { - Thread.currentThread().interrupt(); - throw new DMLRuntimeException(error); + } + + private void completePending(OOCStream matches) { + if(_pending.decrementAndGet() != 0) + return; + try { + int unmatched = _unmatched.get(); + if(unmatched != 0) + fail(new DMLRuntimeException("Join inputs contain " + unmatched + " unmatched blocks")); + else { + try { + matches.closeInput(); + } + catch(Exception ignored) { + } + } } - catch(ExecutionException error) { - throw DMLRuntimeException.of(error.getCause()); + finally { + _pendingCompletion.complete(null); } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/MaterializeOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/MaterializeOOCPrimitive.java index 92bc28f8537..4c5842fb607 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/MaterializeOOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/MaterializeOOCPrimitive.java @@ -23,7 +23,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.ToIntFunction; -import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.instructions.ooc.CachingStream; import org.apache.sysds.runtime.instructions.ooc.OOCStream; import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; @@ -130,11 +129,6 @@ protected void startExecution() { } } - private void fail(Throwable error) { - if(getContext() != null) - getContext().failAll(DMLRuntimeException.of(error)); - } - private void finish() { if(_finished.compareAndSet(false, true)) onComplete(); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java index 12d2d3a6d1f..45b7a3ff031 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/OOCPrimitive.java @@ -26,6 +26,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.instructions.ooc.OOCStream; import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; @@ -46,6 +47,7 @@ public abstract class OOCPrimitive { private final List _inputs; private final AtomicBoolean _started; private final AtomicBoolean _executionStarted; + private final AtomicBoolean _failed; protected OOCAccessPattern _pattern; protected MemoryAllowance _allowance; @@ -71,6 +73,7 @@ protected OOCPrimitive(StreamContext context) { _inputs = new ArrayList<>(); _started = new AtomicBoolean(); _executionStarted = new AtomicBoolean(); + _failed = new AtomicBoolean(); _pattern = OOCAccessPattern.UNSET; } @@ -165,6 +168,18 @@ public final void tryStartExecution() { } } + protected final boolean fail(Throwable error) { + if(!_failed.compareAndSet(false, true)) + return false; + if(_context != null) + _context.failAll(DMLRuntimeException.of(error)); + return true; + } + + protected final boolean hasFailed() { + return _failed.get(); + } + public final void onComplete() { for(int i = 0; i < _inputs.size(); i++) discardInputHandle(i); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/PlannableDataGenOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/PlannableDataGenOOCPrimitive.java index b604e66061a..55d2a318083 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/PlannableDataGenOOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/PlannableDataGenOOCPrimitive.java @@ -21,7 +21,6 @@ import java.util.function.Function; -import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.instructions.ooc.OOCStream; import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; @@ -77,7 +76,7 @@ protected void startExecution() { budget.close(); } }, getContext()).thenRun(output::closeInput).exceptionally(error -> { - output.propagateFailure(DMLRuntimeException.of(error)); + fail(error); return null; }).thenRun(this::onComplete); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java index f86c2360608..e93876dfc03 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java @@ -119,6 +119,25 @@ public static CompletableFuture submitOOCTasks(OOCStream queue, return submitOOCTasks(List.of(queue), (i, callback) -> consumer.accept(callback), null, null, context); } + public static CompletableFuture submitCloseableOOCTasks(OOCStream queue, + Consumer consumer, StreamContext context) { + return submitOOCTasks(List.of(queue), (index, callback) -> { + try(T value = callback.get()) { + consumer.accept(value); + } + catch(Exception error) { + throw DMLRuntimeException.of(error); + } + }, null, (index, callback) -> { + try { + callback.get().close(); + } + catch(Exception error) { + throw DMLRuntimeException.of(error); + } + }, context); + } + public static CompletableFuture submitAdmittedOOCTasks(OOCStream in, OOCStream out, Function operation, MemoryAllowance allowance, StreamContext context) { diff --git a/src/test/java/org/apache/sysds/test/component/ooc/OOCInstructionUtilsTest.java b/src/test/java/org/apache/sysds/test/component/ooc/OOCInstructionUtilsTest.java index a889cc73e83..7737f45e9fd 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/OOCInstructionUtilsTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/OOCInstructionUtilsTest.java @@ -64,6 +64,38 @@ public void testSubmitTasksClosesCallbacksAfterCompletion() throws Exception { Assert.assertEquals(1, released.get()); } + @Test + public void testSubmitCloseableOOCTasks() throws Exception { + SubscribableTaskQueue source = new SubscribableTaskQueue<>(); + AtomicInteger processed = new AtomicInteger(); + AtomicInteger closed = new AtomicInteger(); + CompletableFuture completion = OOCInstructionUtils.submitCloseableOOCTasks(source, + (OwnedTask work) -> processed.addAndGet(work._value), new StreamContext().addOutStream()); + + source.enqueue(new OwnedTask(1, closed)); + source.enqueue(new OwnedTask(2, closed)); + source.closeInput(); + completion.get(10, TimeUnit.SECONDS); + + Assert.assertEquals(3, processed.get()); + Assert.assertEquals(2, closed.get()); + } + + private static final class OwnedTask implements AutoCloseable { + private final int _value; + private final AtomicInteger _closed; + + private OwnedTask(int value, AtomicInteger closed) { + _value = value; + _closed = closed; + } + + @Override + public void close() { + _closed.incrementAndGet(); + } + } + @Test public void testSubmitTasksWaitsForAllStreams() throws Exception { SubscribableTaskQueue first = new SubscribableTaskQueue<>(); From e672d4fbcef8be7fb159599a16d9d4c6d961e700 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Tue, 11 Aug 2026 18:39:47 +0200 Subject: [PATCH 112/132] [SYSTEMDS-3956] Add AI Policy Assisted-by: AI --- AGENTS.md | 68 +++++++++++++++++++++++++++++++++++++++++++++++++ CONTRIBUTING.md | 23 +++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000000..027edc20841 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,68 @@ + + +# Instructions for Apache SystemDS + +> [!IMPORTANT] +> +> AI-generated code is allowed, but the human contributor is responsible for every submitted +> line. Read and follow [CONTRIBUTING.md](CONTRIBUTING.md) before making changes. + +## Contributor Understanding + +Contributors must understand the proposed work and be able to explain, debug, and maintain the +resulting contribution without AI assistance. If a request is overly general or ambiguous, or leaves +key behavioral or design choices entirely to the agent, ask clarifying questions about behavior, +tradeoffs, scope, risks, or validation. + +## Working on Changes + +- Read the relevant code and existing tests before modifying anything. +- Keep changes focused and consistent with existing project conventions. +- Run relevant tests and clearly report anything that was not tested. +- Treat generated code and text as drafts requiring human review. +- Do not add overly verbose comments or comments that restate the code. +- Prefer simple solutions. Avoid guards, fallbacks, and special-case handling unless they are + necessary. + +## Project Interactions + +Agents may perform local analysis, including creating private review notes. Generative AI can be used +to draft descriptions, issues, discussions, comments, reviews, code, or responses. Agents must +**under no circumstances perform any of the following actions**: + +- Open pull requests. +- Open issues on GitHub or JIRA. +- Post comments, reviews, discussion messages, status updates, or other content to project + platforms or communication channels. +- Send project-related emails or chat messages. +- Push commits, branches, tags, or other changes. + +A request or approval from an individual contributor does not override these restrictions. + +## Disclosure + +AI use must be disclosed in the pull request and commit message if it meaningfully contributed +to the submitted work: + +```text +Assisted-by: AI +``` + +Remind the contributor of this requirement before they commit or submit the work. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 91ff5a42bef..d86b30ac227 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,6 +42,29 @@ let's make sure the changes are consistent with the guidelines and coding style. transferred to the SystemDS team. The benefit of the contribution is to be compared against the cost of maintaining the feature. +## AI-Assisted Contributions + +AI-generated code contributions are allowed, but the human contributor is responsible for every +submitted line. Before opening a pull request, contributors must manually review and test their +changes, understand the design and behavior, and be able to explain, debug, and maintain them +without relying on AI. AI use must be disclosed in the pull request and commit message if it +meaningfully contributed to the submitted work: + +```text +Assisted-by: AI +``` + +Minor assistance such as inline autocomplete, spelling corrections, or grammar corrections does +not need to be disclosed. When in doubt, disclose the use of AI. + +Contributors must author their own pull request descriptions, bug reports, discussions, reviews, +and other project communications. Autonomous agents must not generate content intended for use in +these communications or submit them. + +Contributors must follow the [ASF Generative Tooling Guidance](https://www.apache.org/legal/generative-tooling.html). +Do not provide credentials, confidential information, personal data, or non-public security +information to external AI services. + ## Code Style We suggest applying a code formatter to the written code. Generally, this is done automatically. From 2b6212f8d28659649dfd4d0b6f87c4ba5a40c989 Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Fri, 14 Aug 2026 20:00:07 +0200 Subject: [PATCH 113/132] [SYSTEMDS-3940] Improve Scuro Node Executor In this patch the node executor for Representation DAGs is improved and made mode efficient. Additionally, the code was cleaned up. Assisted-by: AI --- .../scuro/drsearch/hyperparameter_tuner.py | 20 +- .../scuro/drsearch/modality_result_cache.py | 136 ++ .../scuro/drsearch/modality_shared_memory.py | 28 +- .../systemds/scuro/drsearch/node_executor.py | 1260 +++++++++-------- .../systemds/scuro/drsearch/node_scheduler.py | 224 ++- .../scuro/drsearch/unimodal_optimizer.py | 97 +- .../systemds/scuro/drsearch/worker_pool.py | 277 ++++ .../scuro/representations/representation.py | 121 ++ .../systemds/scuro/utils/memory_utility.py | 115 +- src/main/python/tests/scuro/data_generator.py | 4 +- 10 files changed, 1581 insertions(+), 701 deletions(-) create mode 100644 src/main/python/systemds/scuro/drsearch/modality_result_cache.py create mode 100644 src/main/python/systemds/scuro/drsearch/worker_pool.py diff --git a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py index 9407bdf1250..a62b990fa99 100644 --- a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py +++ b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py @@ -297,6 +297,7 @@ def __init__( wandb_entity: Optional[str] = None, wandb_group: Optional[str] = None, wandb_tags: Optional[List[str]] = None, + enable_checkpointing: bool = False, ): self.tasks = tasks self.unimodal_optimization_results = optimization_results @@ -333,6 +334,7 @@ def __init__( self.wandb_group = wandb_group self.wandb_tags = wandb_tags or [] self._wandb_run = None + self.enable_checkpointing = enable_checkpointing def get_modalities_by_id(self, modality_ids: List[int]) -> Modality: modalities = [] @@ -415,14 +417,18 @@ def tune_unimodal_representations(self, max_eval_per_rep: Optional[int] = None): ) ) self.optimization_results.add_result(results) - self._checkpoint_manager.increment(task.model.name, len(results)) - self._checkpoint_manager.checkpoint_if_due( - self.optimization_results.results, - ) + if self.enable_checkpointing: + self._checkpoint_manager.increment( + task.model.name, len(results) + ) + self._checkpoint_manager.checkpoint_if_due( + self.optimization_results.results, + ) except Exception: - self._checkpoint_manager.save_checkpoint( - self.optimization_results.results, {} - ) + if self.enable_checkpointing: + self._checkpoint_manager.save_checkpoint( + self.optimization_results.results, {} + ) raise if self.save_results: diff --git a/src/main/python/systemds/scuro/drsearch/modality_result_cache.py b/src/main/python/systemds/scuro/drsearch/modality_result_cache.py new file mode 100644 index 00000000000..0848702cbe1 --- /dev/null +++ b/src/main/python/systemds/scuro/drsearch/modality_result_cache.py @@ -0,0 +1,136 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- +from typing import Any, Dict, List, Optional + +from systemds.scuro.drsearch.modality_shared_memory import unlink_shm +from systemds.scuro.utils.static_variables import DEBUG + + +class RefCountResultCache: + def __init__(self): + self.cache: Dict[str, Any] = {} + self.ref_count: Dict[str, int] = {} + self.memory_usage_per_node: Dict[str, int] = {} + self.shared_memory_names: Dict[str, List[str]] = {} + self._shm_retain_count: Dict[str, int] = {} + + def get(self, node_id: str) -> Any: + return self.cache[node_id] + + def add_result( + self, + node_id: str, + result: Any, + shm_name: Optional[str] = None, + resident_bytes: Optional[int] = None, + shm_bytes: int = 0, + ): + if shm_name is not None: + self.shared_memory_names[node_id] = [shm_name] + self.cache[node_id] = result + self.memory_usage_per_node[node_id] = int(resident_bytes or 0) + int( + shm_bytes or 0 + ) + if DEBUG: + print( + f"Node {node_id} has a CPU memory usage of " + f"{self.memory_usage_per_node[node_id]/1024**3:.5f} GB" + + ( + f" ({int(shm_bytes or 0)/1024**3:.5f} GB of it shared memory)" + if shm_name is not None + else "" + ) + ) + + def inc_ref(self, node_id: str): + self.ref_count[node_id] = self.ref_count.get(node_id, 0) + 1 + + def dec_ref(self, node_id: str): + if node_id not in self.ref_count: + return + self.ref_count[node_id] -= 1 + if self.ref_count[node_id] <= 0: + self.ref_count[node_id] = 0 + self._try_cleanup_node(node_id) + + def clear(self, node_id: str): + self.ref_count[node_id] = 0 + self._try_cleanup_node(node_id) + + def retain_shm_names(self, shm_names: List[str]) -> List[str]: + retained: List[str] = [] + for shm_name in shm_names: + if not shm_name: + continue + self._shm_retain_count[shm_name] = ( + self._shm_retain_count.get(shm_name, 0) + 1 + ) + retained.append(shm_name) + return retained + + def release_shm_names(self, shm_names: List[str]) -> None: + nodes_to_recheck: List[str] = [] + for shm_name in shm_names: + if not shm_name: + continue + count = self._shm_retain_count.get(shm_name, 0) - 1 + if count <= 0: + self._shm_retain_count.pop(shm_name, None) + else: + self._shm_retain_count[shm_name] = count + for node_id, node_names in self.shared_memory_names.items(): + if shm_name in node_names and node_id not in nodes_to_recheck: + nodes_to_recheck.append(node_id) + for node_id in nodes_to_recheck: + self._try_cleanup_node(node_id) + + def __len__(self): + return len(self.cache) + + def get_memory_total_memory_usage(self): + return sum(self.memory_usage_per_node.values()) + + def _shm_names_in_use(self, shm_names: List[str]) -> bool: + return any(self._shm_retain_count.get(name, 0) > 0 for name in shm_names) + + def _try_cleanup_node(self, node_id: str) -> None: + if self.ref_count.get(node_id, 0) > 0: + return + shm_names = self.shared_memory_names.get(node_id, []) + if shm_names and self._shm_names_in_use(shm_names): + return + self.cache.pop(node_id, None) + self.ref_count.pop(node_id, None) + self.memory_usage_per_node.pop(node_id, None) + self._cleanup_shared_memory(node_id) + + def _cleanup_shared_memory(self, node_id: str): + names = self.shared_memory_names.pop(node_id, []) + for shm_name in names: + unlink_shm(shm_name) + + def cleanup_all(self): + self._shm_retain_count.clear() + for node_id in list(self.shared_memory_names.keys()): + self.ref_count.pop(node_id, None) + self.cache.pop(node_id, None) + self.memory_usage_per_node.pop(node_id, None) + self._cleanup_shared_memory(node_id) diff --git a/src/main/python/systemds/scuro/drsearch/modality_shared_memory.py b/src/main/python/systemds/scuro/drsearch/modality_shared_memory.py index d4092b90cfc..e68d7195a2d 100644 --- a/src/main/python/systemds/scuro/drsearch/modality_shared_memory.py +++ b/src/main/python/systemds/scuro/drsearch/modality_shared_memory.py @@ -20,11 +20,29 @@ # ------------------------------------------------------------- from typing import Any, List, Tuple import numpy as np -from multiprocessing import shared_memory +from multiprocessing import shared_memory, resource_tracker SHARED_MEMORY_MIN_BYTES = 1 * 1024 * 1024 +def _untrack(shm: shared_memory.SharedMemory) -> None: + try: + resource_tracker.unregister(shm._name, "shared_memory") + except Exception: + pass + + +def unlink_shm(name: str) -> None: + try: + shm = shared_memory.SharedMemory(name=name) + shm.close() + shm.unlink() + except FileNotFoundError: + pass + except Exception: + pass + + class SharedStringList: def __init__( self, shm_name: str, offsets: List[Tuple[int, int]], payload_nbytes: int @@ -37,6 +55,7 @@ def __init__( def _ensure_attached(self): if self._shm is None: self._shm = shared_memory.SharedMemory(name=self.shm_name) + _untrack(self._shm) def __len__(self): return len(self.offsets) @@ -91,6 +110,7 @@ def __init__( def _ensure_attached(self): if self._shm is None: self._shm = shared_memory.SharedMemory(name=self.shm_name) + _untrack(self._shm) self._buffer = np.ndarray( (self.total_elems,), dtype=self._dtype, buffer=self._shm.buf ) @@ -146,6 +166,7 @@ def __init__(self, shm_name: str, dtype_str: str, shape: tuple): def _ensure_attached(self): if self._shm is None: self._shm = shared_memory.SharedMemory(name=self.shm_name) + _untrack(self._shm) self._arr = np.ndarray(self.shape, dtype=self._dtype, buffer=self._shm.buf) self._arr.setflags(write=False) @@ -215,6 +236,7 @@ def __init__( def _ensure_attached(self): if self._shm is None: self._shm = shared_memory.SharedMemory(name=self.shm_name) + _untrack(self._shm) self._buffer = np.ndarray( (self.total_elems,), dtype=self._dtype, buffer=self._shm.buf ) @@ -330,6 +352,7 @@ def add_shared_memory_candidate(data: Any, resident_bytes: int = 0) -> bool: resident_bytes, max(2 * 1024 * 1024, len(offsets) * 64) ) shm.close() + _untrack(shm) return data, shm.name, data_nbytes, resident_bytes elif _is_shared_ndarray_candidate(data): arr = data @@ -344,6 +367,7 @@ def add_shared_memory_candidate(data: Any, resident_bytes: int = 0) -> bool: resident_bytes = min(resident_bytes, 2 * 1024 * 1024) shm.close() + _untrack(shm) return data, shm.name, data_nbytes, resident_bytes elif _is_nested_shared_memory_candidate(data): leaves: List[np.ndarray] = [] @@ -379,6 +403,7 @@ def add_shared_memory_candidate(data: Any, resident_bytes: int = 0) -> bool: resident_bytes, max(2 * 1024 * 1024, len(offsets) * 64) ) shm.close() + _untrack(shm) return data, shm.name, data_nbytes, resident_bytes elif _is_string_list_shared_memory_candidate(data): encoded = [s.encode("utf-8") for s in data] @@ -398,6 +423,7 @@ def add_shared_memory_candidate(data: Any, resident_bytes: int = 0) -> bool: resident_bytes, max(2 * 1024 * 1024, len(str_offsets) * 32) ) shm.close() + _untrack(shm) return data, shm.name, data_nbytes, resident_bytes return None, None, 0, resident_bytes diff --git a/src/main/python/systemds/scuro/drsearch/node_executor.py b/src/main/python/systemds/scuro/drsearch/node_executor.py index a6a7ffe2ca4..ec5d9f40c36 100644 --- a/src/main/python/systemds/scuro/drsearch/node_executor.py +++ b/src/main/python/systemds/scuro/drsearch/node_executor.py @@ -18,229 +18,131 @@ # under the License. # # ------------------------------------------------------------- -from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, wait -from dataclasses import dataclass +import multiprocessing as mp import os -from multiprocessing import shared_memory +import time +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Union + +import torch + from systemds.scuro import Modality +from systemds.scuro.drsearch.modality_result_cache import RefCountResultCache from systemds.scuro.drsearch.modality_shared_memory import ( add_shared_memory_candidate, collect_shm_names_from_payload, + unlink_shm, ) from systemds.scuro.drsearch.node_scheduler import MemoryAwareNodeScheduler from systemds.scuro.drsearch.representation_dag import ( RepresentationDag, RepresentationNode, ) - -import threading -import numpy as np -from typing import Any, Dict, List, Optional -import multiprocessing as mp -import psutil -import time -import torch from systemds.scuro.drsearch.task import PerformanceMeasure +from systemds.scuro.drsearch.worker_pool import PersistentWorkerPool, create_mp_context +from systemds.scuro.representations.aggregated_representation import ( + AggregatedRepresentation, +) from systemds.scuro.representations.context import Context from systemds.scuro.representations.dimensionality_reduction import ( DimensionalityReduction, ) -from systemds.scuro.representations.aggregated_representation import ( - AggregatedRepresentation, +from systemds.scuro.representations.representation import ( + RepresentationStats, + infer_stats_from_data, ) -from systemds.scuro.representations.representation import RepresentationStats from systemds.scuro.representations.unimodal import UnimodalRepresentation from systemds.scuro.utils.checkpointing import CheckpointManager -import threading -import time -import psutil -import os +from systemds.scuro.utils.memory_utility import ( + MemoryMeasurement, + cleanup_gpu, + cpu_memory_budget_bytes, + estimate_modality_bytes, + is_cuda_oom, + measure_memory_during, +) from systemds.scuro.utils.static_variables import DEBUG +_MAX_NODE_RETRIES = int(os.environ.get("SCURO_MAX_NODE_RETRIES", "3")) -def measure_peak_rss_during(fn, *args, sample_s=0.01, **kwargs): - proc = psutil.Process(os.getpid()) - baseline = proc.memory_info().rss - peak = baseline - stop = threading.Event() - - def sampler(): - nonlocal peak - while not stop.is_set(): - rss = proc.memory_info().rss - if rss > peak: - peak = rss - time.sleep(sample_s) - t = threading.Thread(target=sampler, daemon=True) - t.start() +def _run_gpu_op(fn, gpu_id: Optional[int]): + if gpu_id is None or not torch.cuda.is_available(): + return fn() try: - out = fn(*args, **kwargs) + try: + return fn() + except Exception as e: + if is_cuda_oom(e): + cleanup_gpu(gpu_id) + _WORKER_OP_CACHE.clear() + return fn() + raise finally: - stop.set() - t.join() + cleanup_gpu(gpu_id) - return out, (peak - baseline), peak +_WORKER_OP_CACHE: Dict[str, Any] = {} -class RefCountResultCache: - def __init__(self): - self.cache = {} - self.ref_count = {} - self.memory_usage_per_node = {} - self.shared_memory_names = {} - self._shm_retain_count: Dict[str, int] = {} - def get(self, node_id: str) -> Any: - return self.cache[node_id] +def _instantiate_operation(node): + cache_key = None + if getattr(node.operation, "cache_in_worker", False): + try: + params_repr = repr(sorted(node.parameters.items(), key=lambda kv: kv[0])) + cache_key = ( + f"{node.operation.__module__}.{node.operation.__qualname__}" + f"|{params_repr}" + ) + except Exception: + cache_key = None + if cache_key is not None and cache_key in _WORKER_OP_CACHE: + return _WORKER_OP_CACHE[cache_key] + operation = node.operation(params=node.parameters) + if cache_key is not None: + _WORKER_OP_CACHE[cache_key] = operation + return operation + + +def _infer_actual_output_stats( + transformed_modality: Any, +) -> Optional[RepresentationStats]: + if transformed_modality is None or not hasattr(transformed_modality, "data"): + return None + return infer_stats_from_data(transformed_modality.data) - def add_result(self, node_id: str, result: Any): - resident_bytes = result.calculate_memory_usage() - shared_backing_bytes = 0 - if hasattr(result, "data"): - try: - data, shm_name, data_nbytes, resident_bytes = ( - add_shared_memory_candidate(result.data, resident_bytes) - ) - if data is not None: - result.data = data - self.shared_memory_names[node_id] = [shm_name] - shared_backing_bytes = data_nbytes - except Exception as e: - print( - f"Failed to move cache entry {node_id} to shared memory, falling back to RAM: {e}" - ) +def _offload_to_shared_memory(result: Any): + if result is None or not hasattr(result, "data"): + return None, None, 0, None - self.cache[node_id] = result - self.memory_usage_per_node[node_id] = int(resident_bytes) - if DEBUG: - print( - f"Node {node_id} has a CPU memory usage of {self.memory_usage_per_node[node_id]/1024**3:.5f} GB" - + ( - f" (shared-memory backing: {shared_backing_bytes/1024**3:.5f} GB)" - if shared_backing_bytes > 0 - else "" - ) - ) + actual_stats = _infer_actual_output_stats(result) + shm_name = None + resident_bytes = None + shm_bytes = 0 + try: + resident_bytes = result.calculate_memory_usage() + data, shm_name, shm_bytes, resident_bytes = add_shared_memory_candidate( + result.data, resident_bytes + ) + if data is not None: + result._data = data + except Exception as e: + shm_name = None + shm_bytes = 0 + print(f"Failed to move worker result to shared memory: {e}") - def inc_ref(self, node_id: str): - if node_id not in self.ref_count: - self.ref_count[node_id] = 0 - self.ref_count[node_id] += 1 - - def dec_ref(self, node_id: str): - if node_id not in self.ref_count: - return - self.ref_count[node_id] -= 1 - if self.ref_count[node_id] <= 0: - self.ref_count[node_id] = 0 - self._try_cleanup_node(node_id) - - def clear(self, node_id: str): - if node_id in self.ref_count: - self.ref_count[node_id] = 0 - self._try_cleanup_node(node_id) - - def retain_shm_names(self, shm_names: List[str]) -> List[str]: - retained: List[str] = [] - for shm_name in shm_names: - if not shm_name: - continue - self._shm_retain_count[shm_name] = ( - self._shm_retain_count.get(shm_name, 0) + 1 - ) - retained.append(shm_name) - return retained - - def release_shm_names(self, shm_names: List[str]) -> None: - nodes_to_recheck: List[str] = [] - for shm_name in shm_names: - if not shm_name: - continue - count = self._shm_retain_count.get(shm_name, 0) - 1 - if count <= 0: - self._shm_retain_count.pop(shm_name, None) - else: - self._shm_retain_count[shm_name] = count - for node_id, node_names in self.shared_memory_names.items(): - if shm_name in node_names and node_id not in nodes_to_recheck: - nodes_to_recheck.append(node_id) - for node_id in nodes_to_recheck: - self._try_cleanup_node(node_id) - - def __len__(self): - return len(self.cache) - - def get_memory_total_memory_usage(self): - return sum(self.memory_usage_per_node.values()) - - def _shm_names_in_use(self, shm_names: List[str]) -> bool: - return any(self._shm_retain_count.get(name, 0) > 0 for name in shm_names) - - def _try_cleanup_node(self, node_id: str) -> None: - if self.ref_count.get(node_id, 0) > 0: - return - shm_names = self.shared_memory_names.get(node_id, []) - if shm_names and self._shm_names_in_use(shm_names): - return - self.cache.pop(node_id, None) - self.ref_count.pop(node_id, None) - self.memory_usage_per_node.pop(node_id, None) - self._cleanup_shared_memory(node_id) - - def _cleanup_shared_memory(self, node_id: str): - names = self.shared_memory_names.pop(node_id, []) - for shm_name in names: - try: - shm = shared_memory.SharedMemory(name=shm_name) - shm.close() - shm.unlink() - except FileNotFoundError: - pass - except Exception: - pass - - def cleanup_all(self): - self._shm_retain_count.clear() - for node_id in list(self.shared_memory_names.keys()): - self.ref_count.pop(node_id, None) - self.cache.pop(node_id, None) - self.memory_usage_per_node.pop(node_id, None) - self._cleanup_shared_memory(node_id) - - -def _execute_multiple_reps_for_leaf_dependencies( - nodes: List[RepresentationNode], - modalities: List[Modality], - gpu_id: Optional[int], -): - representations = [] - node_id_by_representation = {} - for node in nodes: - operation = node.operation(params=node.parameters) - if hasattr(operation, "gpu_id"): - operation.gpu_id = gpu_id - representations.append(operation) - node_id_by_representation[operation.name] = node.node_id - - modality_results = modalities[0].apply_representations( - representations, parallel=True - ) - return { - "results": modality_results, - "node_id_by_representation": node_id_by_representation, - } + return shm_name, resident_bytes, int(shm_bytes or 0), actual_stats -def _execute_node_worker(node, input_mods, task, rep_cache, gpu_id): +def _execute_node_worker(node, input_mods: List[Any], gpu_id: Optional[int]): start_time = time.perf_counter() if gpu_id is not None: device = torch.device(f"cuda:{gpu_id}") torch.cuda.set_device(device) torch.cuda.reset_peak_memory_stats(device) - node_operation = node.operation(params=node.parameters) + node_operation = _instantiate_operation(node) operation_name = node_operation.name if DEBUG: print(f"Executing node {node.node_id} {operation_name} on GPU {gpu_id}") @@ -258,47 +160,55 @@ def _run_node_op(): return node_operation.transform(input_mods[0]) elif isinstance(node_operation, UnimodalRepresentation): pushdown_config = node.parameters.get("_pushdown_aggregation", None) - agg = None - if pushdown_config is not None: - agg = AggregatedRepresentation(params=pushdown_config) - if rep_cache is not None and node_operation.name in rep_cache: - return rep_cache[node_operation.name] + agg = ( + AggregatedRepresentation(params=pushdown_config) + if pushdown_config is not None + else None + ) return input_mods[0].apply_representation( node_operation, aggregation=agg ) return input_mods[0].apply_representation(node_operation) else: fusion_op = node_operation - if hasattr(fusion_op, "needs_training") and fusion_op.needs_training: + if getattr(fusion_op, "needs_training", False): return input_mods[0].combine_with_training( - input_mods[1:], fusion_op, task + input_mods[1:], fusion_op, None ) return input_mods[0].combine(input_mods[1:], fusion_op) gpu_peak_bytes = -1 - peak_delta_bytes = -1 - peak_abs_rss = -1 + measurement = None if DEBUG: - result, peak_delta_bytes, peak_abs_rss = measure_peak_rss_during( - _run_node_op, + input_resident = sum(estimate_modality_bytes(m) for m in input_mods) + result, measurement = measure_memory_during( + lambda: _run_gpu_op(_run_node_op, gpu_id), + input_resident_bytes=input_resident, sample_s=0.01, ) gpu_peak_bytes = ( torch.cuda.max_memory_allocated(device) if gpu_id is not None else 0 ) else: - result = _run_node_op() + result = _run_gpu_op(_run_node_op, gpu_id) + + shm_name, resident_bytes, shm_bytes, actual_stats = _offload_to_shared_memory( + result + ) end_time = time.perf_counter() - pid = os.getpid() return { "result": result, - "peak_bytes": peak_delta_bytes, - "peak_abs_rss_bytes": peak_abs_rss, + "result_shm_name": shm_name, + "result_resident_bytes": resident_bytes, + "result_shm_bytes": shm_bytes, + "actual_stats": actual_stats, + "memory": measurement, + "peak_bytes": measurement.increment_bytes if measurement else -1, "gpu_peak_bytes": gpu_peak_bytes, "operation_name": operation_name, "start_time": start_time, "end_time": end_time, - "pid": pid, + "pid": os.getpid(), } @@ -307,7 +217,7 @@ def _execute_task_worker( task: Any, modality: Any, gpu_id: Optional[int], - aggregation: AggregatedRepresentation = None, + aggregation=None, ) -> Dict[str, Any]: start_time = time.perf_counter() if DEBUG: @@ -335,55 +245,150 @@ def _run_task(): return scores, end - start gpu_peak_bytes = -1 - peak_delta_bytes = -1 + measurement = None if DEBUG: - gpu_peak_bytes = ( - torch.cuda.max_memory_allocated(device) if gpu_id is not None else 0 - ) - result, peak_delta_bytes, peak_abs_rss = measure_peak_rss_during( - _run_task, + result, measurement = measure_memory_during( + lambda: _run_gpu_op(_run_task, gpu_id), + input_resident_bytes=estimate_modality_bytes(modality), sample_s=0.01, ) - - print( - f"Task {task_node_id} has a CPU peak memory usage of {peak_delta_bytes/1024**3:.2f} GB, and a GPU peak memory usage of {gpu_peak_bytes/1024**3:.2f} GB" + gpu_peak_bytes = ( + torch.cuda.max_memory_allocated(device) if gpu_id is not None else 0 ) else: - result = _run_task() + result = _run_gpu_op(_run_task, gpu_id) end_time = time.perf_counter() - pid = os.getpid() return { "scores": result[0], "task_time": result[1], - "peak_bytes": peak_delta_bytes, + "memory": measurement, + "peak_bytes": measurement.increment_bytes if measurement else -1, "gpu_peak_bytes": gpu_peak_bytes, "start_time": start_time, "end_time": end_time, - "pid": pid, + "pid": os.getpid(), + } + + +def _execute_leaf_batch_worker(nodes: List[Any], modality: Any, gpu_id: Optional[int]): + node_id_by_representation = {} + + def _run(): + representations = [] + for node in nodes: + operation = node.operation(params=node.parameters) + if hasattr(operation, "gpu_id"): + operation.gpu_id = gpu_id + representations.append(operation) + node_id_by_representation[operation.name] = node.node_id + return modality.apply_representations(representations, parallel=True) + + modality_results = _run_gpu_op(_run, gpu_id) + shm_info = {} + for representation_name, transformed_modality in modality_results.items(): + shm_name, resident_bytes, shm_bytes, actual_stats = _offload_to_shared_memory( + transformed_modality + ) + shm_info[representation_name] = { + "shm_name": shm_name, + "resident_bytes": resident_bytes, + "shm_bytes": shm_bytes, + "actual_stats": actual_stats, + } + return { + "results": modality_results, + "node_id_by_representation": node_id_by_representation, + "shm_info": shm_info, } +def _load_leaf_worker(modality: Any) -> Dict[str, Any]: + if hasattr(modality, "extract_raw_data") and not modality.has_data(): + modality.extract_raw_data() + + data = modality.data + resident_bytes = 0 + try: + resident_bytes = modality.estimate_memory_bytes() + except Exception: + resident_bytes = 0 + + wrapped, shm_name, _, resident_bytes = add_shared_memory_candidate( + data, resident_bytes + ) + if wrapped is not None: + data = wrapped + + return {"data": data, "metadata": modality.metadata, "shm_name": shm_name} + + +def _dispatch_node(payload, gpu_id): + node, input_mods = payload + return _execute_node_worker(node, input_mods, gpu_id) + + +def _dispatch_task(payload, gpu_id): + task_node_id, task, modality, aggregation = payload + return _execute_task_worker(task_node_id, task, modality, gpu_id, aggregation) + + +def _dispatch_leaf_batch(payload, gpu_id): + nodes, modality = payload + return _execute_leaf_batch_worker(nodes, modality, gpu_id) + + +def _dispatch_load_leaf(payload, _gpu_id): + (modality,) = payload + return _load_leaf_worker(modality) + + +_WORKER_DISPATCH = { + "node": _dispatch_node, + "task": _dispatch_task, + "leaf_batch": _dispatch_leaf_batch, + "load_leaf": _dispatch_load_leaf, +} + + +@dataclass +class _NodeUnit: + node_id: str + + +@dataclass +class _BatchUnit: + node_ids: List[str] + + +@dataclass +class ResultEntry: + val_score: PerformanceMeasure = None + train_score: PerformanceMeasure = None + test_score: PerformanceMeasure = None + representation_time: float = 0.0 + task_time: float = 0.0 + dag: RepresentationDag = None + tradeoff_score: float = 0.0 + + class NodeExecutor: def __init__( self, dags: List[RepresentationDag], modalities: List[Modality], tasks: List[Any], - checkpoint_manager: Optional[CheckpointManager] = None, max_num_workers: int = -1, result_path: Optional[str] = None, - enable_checkpointing: bool = True, + enable_checkpointing: bool = False, + worker_pool: Optional[PersistentWorkerPool] = None, ): self.enable_checkpointing = enable_checkpointing - available_total_cpu = ( - float(psutil.virtual_memory().available) - - float(psutil.virtual_memory().available) * 0.30 - ) + available_total_cpu = cpu_memory_budget_bytes() self.dags = dags self.scheduler = MemoryAwareNodeScheduler( dags, modalities, tasks, available_total_cpu ) - self.checkpoint_manager = CheckpointManager( + self._checkpoint_manager = CheckpointManager( checkpoint_dir=result_path if result_path is not None else os.getcwd(), prefix=f"node_executor_checkpoint_{modalities[0].modality_id}_", checkpoint_every=1, @@ -394,257 +399,281 @@ def __init__( if max_num_workers != -1 else mp.cpu_count() ) - self.modalities = modalities - self.tasks = tasks - self.result_cache = RefCountResultCache() - self.memory_usage_checkpoint = CheckpointManager( + self._modalities = modalities + self._tasks = tasks + self._result_path = result_path + self._result_cache = RefCountResultCache() + self._memory_usage_checkpoint = CheckpointManager( checkpoint_dir=result_path if result_path is not None else os.getcwd(), prefix=f"memory_usage_checkpoint_{modalities[0].modality_id}_", checkpoint_every=1, resume=False, ) - self.statistics = {} - self.statistics["worker_stats"] = {} - self.statistics["node_stats"] = {} + self._memory_usage_data: Dict[str, Any] = {} + self.statistics = {"worker_stats": {}, "node_stats": {}} + + self._node_attempts: Dict[str, int] = {} + + self._job_units: Dict[int, Union[_NodeUnit, _BatchUnit]] = {} + self._job_retained_shm: Dict[int, List[str]] = {} + self._leaf_shm_names: List[str] = [] + self._task_results: Dict[str, ResultEntry] = {} + + self._owns_pool = worker_pool is None + if worker_pool is None: + cpu_count = os.cpu_count() or 1 + threads_per_worker = max(1, cpu_count // max(1, self.max_num_workers)) + worker_pool = PersistentWorkerPool( + self.max_num_workers, + _WORKER_DISPATCH, + ctx=create_mp_context(), + threads_per_worker=threads_per_worker, + ) + self._pool = worker_pool - def _shm_names_for_submit( - self, parent_node_ids: List[str], payload_data: Any - ) -> List[str]: - names: List[str] = [] - for parent_id in parent_node_ids or []: - names.extend(self.result_cache.shared_memory_names.get(parent_id, [])) - if parent_node_ids: - names.extend(collect_shm_names_from_payload(payload_data)) - else: - names.extend(getattr(self, "_leaf_shm_names", [])) - names.extend(collect_shm_names_from_payload(payload_data)) - # preserve order, drop duplicates - return list(dict.fromkeys(names)) - - def _retain_for_submit( - self, parent_node_ids: List[str], payload_data: Any - ) -> List[str]: - return self.result_cache.retain_shm_names( - self._shm_names_for_submit(parent_node_ids, payload_data) + def _requeue_or_give_up(self, node_id: str, reason: str) -> bool: + attempts = self._node_attempts.get(node_id, 0) + 1 + self._node_attempts[node_id] = attempts + if attempts > _MAX_NODE_RETRIES: + print( + f"[node_executor] giving up on node {node_id} after {attempts} " + f"failed attempts ({reason}); marking it permanently failed and " + f"continuing with the rest of the search.", + flush=True, + ) + self.scheduler.add_failed_node(node_id, reason) + return False + print( + f"[node_executor] node {node_id} did not complete (attempt " + f"{attempts}/{_MAX_NODE_RETRIES + 1}): {reason}. Re-queuing it for " + f"another attempt.", + flush=True, ) + self.scheduler.requeue_node(node_id) + return True - def _release_for_future(self, retained_shm_names: List[str]) -> None: - if retained_shm_names: - self.result_cache.release_shm_names(retained_shm_names) - - def run(self) -> None: - task_results = {} - memory_usage_data = {} - - self._materialize_leaf_modalities_in_shared_memory() - - ctx = mp.get_context("spawn") - with ProcessPoolExecutor( - max_workers=self.max_num_workers, mp_context=ctx - ) as executor: - future_to_node_id = {} - future_to_retained_shm: Dict[Any, List[str]] = {} - - def submit_nodes_with_leaf_dependencies(node_ids: List[str]): - nodes = [self.scheduler.mapping[node_id] for node_id in node_ids] - gpu_id = nodes[0].gpu_id - self.scheduler.move_to_running(node_ids) - - retained = self._retain_for_submit([], self.modalities[0].data) - future = executor.submit( - _execute_multiple_reps_for_leaf_dependencies, - nodes, - self.modalities, - gpu_id, - ) - future_to_node_id[future] = node_ids - future_to_retained_shm[future] = retained - - def submit_node(node_id: str): - node = self.scheduler.mapping[node_id] - gpu_id = node.gpu_id - parent_node_ids = self.scheduler.get_valid_parents(node_id) - parent_results = None - if parent_node_ids: - parent_results = [ - self.result_cache.get(parent_node_id) - for parent_node_id in parent_node_ids - ] - - if self._is_task_node(node): - # potentially batch task nodes and then execute them together - # by the the same task type (index, gpu vs cpu) - # either enough nodes to batch or enough time to batch whatever happens first - - task_result = ResultEntry( - dag=self._get_dag_from_node_ids(node_id), - representation_time=parent_results[0].transform_time, - ) - task_results[node_id] = task_result - task_idx = int(node.parameters.get("_task_idx", 0)) - payload_data = ( - self.modalities[0] - if parent_results is None - else parent_results[0] - ) - retained = self._retain_for_submit(parent_node_ids, payload_data) - aggregation = node.aggregation - - future = executor.submit( - _execute_task_worker, - node_id, - self.tasks[task_idx], - payload_data, - gpu_id, - aggregation, - ) - else: - payload_data = ( - self.modalities if parent_results is None else parent_results - ) - retained = self._retain_for_submit(parent_node_ids, payload_data) - future = executor.submit( - _execute_node_worker, - node, - payload_data, - None, - None, - gpu_id, + def _release_parents(self, node_id: str) -> None: + for parent_id in self.scheduler.get_valid_parents(node_id): + self._result_cache.dec_ref(parent_id) + + def _retain_for_submit(self, parent_ids: List[str], payload: Any) -> List[str]: + names: List[str] = [] + for parent_id in parent_ids or []: + names.extend(self._result_cache.shared_memory_names.get(parent_id, [])) + if not parent_ids: + names.extend(self._leaf_shm_names) + names.extend(collect_shm_names_from_payload(payload)) + names = list(dict.fromkeys(names)) + return self._result_cache.retain_shm_names(names) + + def _load_leaf_modalities(self) -> None: + for modality in self._modalities: + if getattr(modality, "has_data", None) and modality.has_data(): + continue + attempts = 0 + while True: + self._pool.submit("load_leaf", (modality,), gpu_id=None) + jr = self._pool.wait() + if jr.ok: + break + attempts += 1 + if attempts > _MAX_NODE_RETRIES: + raise RuntimeError( + f"Failed to load leaf modality {modality.modality_id}: " + f"{jr.error}" ) - self.scheduler.move_to_running(node_id) - future_to_node_id[future] = node_id - future_to_retained_shm[future] = retained - - def submit_new_ready_nodes(): - ready_nodes = self.scheduler.get_runnable().copy() - for node_id in ready_nodes: - if isinstance(node_id, list): - submit_nodes_with_leaf_dependencies(node_id) - continue - submit_node(node_id) - - submit_new_ready_nodes() - - while future_to_node_id or not self.scheduler.is_finished(): - if not future_to_node_id: - submit_new_ready_nodes() + modality._data = jr.value["data"] + modality.metadata = jr.value["metadata"] + shm_name = jr.value.get("shm_name") + if shm_name is not None: + self._leaf_shm_names.append(shm_name) + + def _cleanup_leaf_shared_memory(self) -> None: + for shm_name in self._leaf_shm_names: + unlink_shm(shm_name) + self._leaf_shm_names = [] + + def _submit_node(self, node_id: str) -> None: + node = self.scheduler.mapping[node_id] + gpu_id = node.gpu_id + parent_ids = self.scheduler.get_valid_parents(node_id) + parent_results = ( + [self._result_cache.get(pid) for pid in parent_ids] if parent_ids else None + ) + + if self._is_task_node(node): + task_idx = int(node.parameters.get("_task_idx", 0)) + payload = ( + self._modalities[0] if parent_results is None else parent_results[0] + ) + self._task_results[node_id] = ResultEntry( + dag=self._get_dag_from_node_ids(node_id), + representation_time=payload.transform_time, + ) + retained = self._retain_for_submit(parent_ids, payload) + self.scheduler.begin_execution(node_id) + self.scheduler.move_to_running(node_id) + job_id = self._pool.submit( + "task", + (node_id, self._tasks[task_idx], payload, node.aggregation), + gpu_id=gpu_id, + ) + else: + payload = self._modalities if parent_results is None else parent_results + retained = self._retain_for_submit(parent_ids, payload) + self.scheduler.begin_execution(node_id) + self.scheduler.move_to_running(node_id) + job_id = self._pool.submit("node", (node, payload), gpu_id=gpu_id) + + self._job_units[job_id] = _NodeUnit(node_id) + self._job_retained_shm[job_id] = retained + + def _submit_leaf_batch(self, node_ids: List[str]) -> None: + nodes = [self.scheduler.mapping[nid] for nid in node_ids] + gpu_id = nodes[0].gpu_id + retained = self._retain_for_submit([], self._modalities[0].data) + for nid in node_ids: + self.scheduler.begin_execution(nid) + self.scheduler.move_to_running(node_ids) + job_id = self._pool.submit( + "leaf_batch", (nodes, self._modalities[0]), gpu_id=gpu_id + ) + self._job_units[job_id] = _BatchUnit(node_ids) + self._job_retained_shm[job_id] = retained + + def _fill_pipeline(self) -> None: + ready = self.scheduler.get_runnable().copy() + for entry in ready: + if not self._pool.has_idle_worker: + break + if isinstance(entry, list): + self._submit_leaf_batch(entry) + else: + if not self.scheduler.can_start_now(entry): continue + self._submit_node(entry) + + def _record_stats(self, node_id: str, pid: int, start_time: float, end_time: float): + node_stats = self.statistics["node_stats"] + worker_stats = self.statistics["worker_stats"] + node_stats[node_id] = {"start_time": start_time, "end_time": end_time} + entry = worker_stats.get(pid) + if entry is None: + worker_stats[pid] = { + "start_time": start_time, + "end_time": end_time, + "busy_time": end_time - start_time, + "num_jobs": 1, + } + else: + entry["start_time"] = min(entry["start_time"], start_time) + entry["end_time"] = max(entry["end_time"], end_time) + entry["busy_time"] += end_time - start_time + entry["num_jobs"] += 1 + + def _process_result(self, jr) -> None: + unit = self._job_units.pop(jr.job_id, None) + retained = self._job_retained_shm.pop(jr.job_id, []) + try: + if unit is None: + return + if not jr.ok: + self._handle_job_failure(unit, jr) + return + if isinstance(unit, _BatchUnit): + self._handle_batch_success(jr.value) + else: + self._handle_node_success(unit.node_id, jr.value) + finally: + if retained: + self._result_cache.release_shm_names(retained) + + def _handle_job_failure(self, unit: Union[_NodeUnit, _BatchUnit], jr): + reason = jr.error or "unknown worker failure" + if jr.cuda_oom: + reason += " (CUDA out of memory)" + node_ids = unit.node_ids if isinstance(unit, _BatchUnit) else [unit.node_id] + for node_id in node_ids: + requeued = self._requeue_or_give_up(node_id, reason) + if not requeued: + self._release_parents(node_id) + + def _handle_node_success(self, node_id: str, value: Dict[str, Any]) -> None: + node = self.scheduler.mapping[node_id] + if "pid" in value: + self._record_stats( + node_id, value["pid"], value["start_time"], value["end_time"] + ) - done, _ = wait( - set(future_to_node_id.keys()), return_when=FIRST_COMPLETED + if self._is_task_node(node): + entry = self._task_results[node_id] + entry.task_time = value["task_time"] + entry.train_score = value["scores"][0].average_scores + entry.val_score = value["scores"][1].average_scores + entry.test_score = value["scores"][2].average_scores + if self.enable_checkpointing: + self._checkpoint_manager.increment(node_id) + self._checkpoint_manager.checkpoint_if_due( + self._task_results, self._discard_report() + ) + self._checkpoint_memory_usage( + node_id, + value["peak_bytes"], + value["gpu_peak_bytes"], + "task", + None, + measurement=value.get("memory"), ) + self._release_parents(node_id) + self.scheduler.complete_node(node_id) + else: + self._handle_modality_result( + value["result"], + node_id, + value["peak_bytes"], + value["gpu_peak_bytes"], + value["operation_name"], + actual_stats=value.get("actual_stats"), + shm_name=value.get("result_shm_name"), + resident_bytes=value.get("result_resident_bytes"), + shm_bytes=value.get("result_shm_bytes", 0), + measurement=value.get("memory"), + ) - for future in done: - node_id = future_to_node_id.pop(future) - retained_shm = future_to_retained_shm.pop(future, []) - try: - result = future.result() - - if isinstance(node_id, list): - results = result["results"] - node_id_by_representation = result[ - "node_id_by_representation" - ] - for ( - representation, - transformed_modality, - ) in results.items(): - batch_node_id = node_id_by_representation[ - representation - ] - self._handle_modality_result( - transformed_modality, - batch_node_id, - None, - None, - memory_usage_data, - representation, - ) - submit_new_ready_nodes() - continue - - peak_bytes = result["peak_bytes"] - gpu_peak_bytes = result["gpu_peak_bytes"] - node = self.scheduler.mapping[node_id] - self.statistics["worker_stats"][result["pid"]] = { - "start_time": result["start_time"], - "end_time": result["end_time"], - } - self.statistics["node_stats"][node_id] = { - "start_time": result["start_time"], - "end_time": result["end_time"], - } - if self._is_task_node(node): - task_results[node_id].task_time = result["task_time"] - task_results[node_id].train_score = result["scores"][ - 0 - ].average_scores - task_results[node_id].val_score = result["scores"][ - 1 - ].average_scores - task_results[node_id].test_score = result["scores"][ - 2 - ].average_scores - if self.enable_checkpointing: - self.checkpoint_manager.increment(node_id) - self.checkpoint_manager.checkpoint_if_due(task_results) - self._checkpoint_memory_usage( - node_id, - peak_bytes, - gpu_peak_bytes, - "task", - memory_usage_data, - None, - ) - - parent_node_ids = self.scheduler.get_valid_parents(node_id) - for parent_node_id in parent_node_ids: - self.result_cache.dec_ref(parent_node_id) - self.scheduler.complete_node(node_id) - else: - transformed_modality = result["result"] - self._handle_modality_result( - transformed_modality, - node_id, - peak_bytes, - gpu_peak_bytes, - memory_usage_data, - result["operation_name"], - ) - - submit_new_ready_nodes() - except Exception: - parent_node_ids = [] - if not isinstance(node_id, list): - parent_node_ids = self.scheduler.get_valid_parents(node_id) - for parent_node_id in parent_node_ids: - self.result_cache.dec_ref(parent_node_id) - if not isinstance(node_id, list): - self.scheduler.add_failed_node(node_id) - raise - finally: - self._release_for_future(retained_shm) - - assert not self.result_cache.ref_count - assert not self.result_cache._shm_retain_count - - self.result_cache.cleanup_all() - self._cleanup_leaf_shared_memory() - return { - "task_results": list(task_results.values()), - "statistics": self.statistics, - } + def _handle_batch_success(self, value: Dict[str, Any]) -> None: + results = value["results"] + node_id_by_representation = value["node_id_by_representation"] + shm_info = value.get("shm_info", {}) + for representation, transformed_modality in results.items(): + node_id = node_id_by_representation[representation] + info = shm_info.get(representation, {}) + self._handle_modality_result( + transformed_modality, + node_id, + None, + None, + representation, + actual_stats=info.get("actual_stats"), + shm_name=info.get("shm_name"), + resident_bytes=info.get("resident_bytes"), + shm_bytes=info.get("shm_bytes", 0), + ) def _handle_modality_result( self, transformed_modality: Any, node_id: str, - peak_bytes: int, - gpu_peak_bytes: int, - memory_usage_data, + peak_bytes: Optional[int], + gpu_peak_bytes: Optional[int], operation_name: str, + actual_stats: Optional[RepresentationStats] = None, + shm_name: Optional[str] = None, + resident_bytes: Optional[int] = None, + shm_bytes: int = 0, + measurement: Optional[MemoryMeasurement] = None, ): - actual_stats = self._infer_actual_output_stats(transformed_modality) + if actual_stats is None: + actual_stats = _infer_actual_output_stats(transformed_modality) estimated_stats = self.scheduler.node_stats.get(node_id) if actual_stats is not None and ( @@ -660,169 +689,222 @@ def _handle_modality_result( peak_bytes, gpu_peak_bytes, operation_name, - memory_usage_data, - transformed_modality.data, + actual_stats, + measurement=measurement, ) - before_bytes = self.result_cache.get_memory_total_memory_usage() - self._manage_result_cache(node_id, transformed_modality) - after_bytes = self.result_cache.get_memory_total_memory_usage() + before_bytes = self._result_cache.get_memory_total_memory_usage() + self._manage_result_cache( + node_id, + transformed_modality, + shm_name=shm_name, + resident_bytes=resident_bytes, + shm_bytes=shm_bytes, + ) + after_bytes = self._result_cache.get_memory_total_memory_usage() self.scheduler.update_cpu_memory_in_use(after_bytes - before_bytes) self.scheduler.complete_node(node_id) - def _materialize_leaf_modalities_in_shared_memory(self): - self._leaf_shm_names = [] - for modality in self.modalities: - if hasattr(modality, "extract_raw_data") and not modality.has_data(): - modality.extract_raw_data() - data, shm_name, _, _ = add_shared_memory_candidate(modality.data) - if shm_name is not None: - modality.data = data - self._leaf_shm_names.append(shm_name) + def _manage_result_cache( + self, + node_id: str, + result: Any, + shm_name: Optional[str] = None, + resident_bytes: Optional[int] = None, + shm_bytes: int = 0, + ): + self._release_parents(node_id) - def _cleanup_leaf_shared_memory(self): - for shm_name in getattr(self, "_leaf_shm_names", []): - try: - shm = shared_memory.SharedMemory(name=shm_name) - shm.close() - shm.unlink() - except FileNotFoundError: - pass - except Exception: - pass - self._leaf_shm_names = [] + children = self.scheduler.get_children(node_id) + if children: + for _ in children: + self._result_cache.inc_ref(node_id) + self._result_cache.add_result( + node_id, + result, + shm_name=shm_name, + resident_bytes=resident_bytes, + shm_bytes=shm_bytes, + ) + elif shm_name is not None: + unlink_shm(shm_name) def _checkpoint_memory_usage( self, node_id: str, - peak_bytes: int, - gpu_peak_bytes: int, + peak_bytes: Optional[int], + gpu_peak_bytes: Optional[int], operation_name: str, - data, - result, + actual_stats: Optional[RepresentationStats], + measurement: Optional[MemoryMeasurement] = None, ): - self.memory_usage_checkpoint.increment(node_id) + if self.enable_checkpointing: + self._memory_usage_checkpoint.increment(node_id) + + if measurement is not None: + peak_bytes = measurement.footprint_bytes shape = None if DEBUG: - shape = self._print_node_stats(node_id, result, operation_name) - if peak_bytes > self.scheduler.node_resources[node_id][0]: - print( - f"UNDERESTIMATED PEAK MEMORY: Peak bytes: {peak_bytes/1024**3:.2f} GB, Estimated CPU bytes: {self.scheduler.node_resources[node_id][0]/1024**3:.2f} GB for node {node_id}: {operation_name}" - ) - if gpu_peak_bytes > self.scheduler.node_resources[node_id][1]: - print( - f"UNDERESTIMATED GPU PEAK MEMORY: GPU peak bytes: {gpu_peak_bytes/1024**3:.2f} GB, Estimated GPU bytes: {self.scheduler.node_resources[node_id][1]/1024**3:.2f} GB for node {node_id}: {operation_name}" - ) - if self.scheduler.node_resources[node_id][0] >= peak_bytes * 2: - print( - f"Peak bytes: {peak_bytes/1024**3:.2f} GB, Estimated CPU bytes: {self.scheduler.node_resources[node_id][0]/1024**3:.2f} GB, 200% of estimated for node {node_id}: {operation_name}" - ) - if self.scheduler.node_resources[node_id][1] > gpu_peak_bytes * 2: - print( - f"GPU peak bytes: {gpu_peak_bytes/1024**3:.2f} GB, Estimated GPU bytes: {self.scheduler.node_resources[node_id][1]/1024**3:.2f} GB, 200% of estimated for node {node_id}: {operation_name}" - ) - data[node_id] = { - "cpu_peak_bytes": peak_bytes, - "gpu_peak_bytes": gpu_peak_bytes, - "operation_name": operation_name, - "estimated_cpu_bytes": self.scheduler.node_resources[node_id][0], - "estimated_gpu_bytes": self.scheduler.node_resources[node_id][1], - "shape": shape, - } - self.memory_usage_checkpoint.checkpoint_if_due(data) - - def _print_node_stats(self, node_id: str, result: Any, operation_name: str): - if ( - result is not None - and operation_name != "BoW" - and not operation_name.endswith("Split") - ): - node_stats = self.scheduler.node_stats[node_id] - shape = None - if isinstance(result[0], list): - if isinstance(result[0][0], np.ndarray): - shape = (len(result[0]), *result[0][0].shape) - elif isinstance(result[0][0], list): - shape = (len(result[0]), *result[0][0][0].shape) - else: - shape = (len(result[0]), *result[0][0].shape) - else: - shape = result[0].shape - print( - f"Node {node_id} {operation_name} should have shape of {node_stats.num_instances, node_stats.output_shape}, actual shape: {len(result), shape} output shape is known: {node_stats.output_shape_is_known}" - ) - if node_stats.output_shape_is_known: - assert ( - len(result) == node_stats.num_instances - ), f"Node {node_id} {operation_name} should have {node_stats.num_instances} instances, actual: {len(result)}" - # assert ( - # shape == node_stats.output_shape - # ), f"Node {node_id} {operation_name} should have shape of {node_stats.output_shape}, actual shape: {shape}" - return shape - - def _infer_actual_output_stats( - self, transformed_modality: Any - ) -> Optional[RepresentationStats]: - if transformed_modality is None or not hasattr(transformed_modality, "data"): + shape = self._print_node_stats(node_id, actual_stats, operation_name) + est_cpu, est_gpu = self.scheduler.node_resources[node_id] + if peak_bytes is not None and peak_bytes >= 0: + if peak_bytes > est_cpu: + print( + f"UNDERESTIMATED PEAK MEMORY: Peak bytes: {peak_bytes/1024**3:.2f} GB, " + f"Estimated CPU bytes: {est_cpu/1024**3:.2f} GB for node {node_id}: {operation_name}" + ) + if est_cpu >= peak_bytes * 2: + print( + f"Peak bytes: {peak_bytes/1024**3:.2f} GB, Estimated CPU bytes: " + f"{est_cpu/1024**3:.2f} GB, >200% of estimated for node {node_id}: {operation_name}" + ) + if gpu_peak_bytes is not None and gpu_peak_bytes >= 0: + if gpu_peak_bytes > est_gpu: + print( + f"UNDERESTIMATED GPU PEAK MEMORY: GPU peak bytes: {gpu_peak_bytes/1024**3:.2f} GB, " + f"Estimated GPU bytes: {est_gpu/1024**3:.2f} GB for node {node_id}: {operation_name}" + ) + if est_gpu > gpu_peak_bytes * 2: + print( + f"GPU peak bytes: {gpu_peak_bytes/1024**3:.2f} GB, Estimated GPU bytes: " + f"{est_gpu/1024**3:.2f} GB, >200% of estimated for node {node_id}: {operation_name}" + ) + if self.enable_checkpointing: + self._memory_usage_data[node_id] = { + "cpu_peak_bytes": peak_bytes if peak_bytes is not None else -1, + "gpu_peak_bytes": gpu_peak_bytes if gpu_peak_bytes is not None else -1, + "operation_name": operation_name, + "estimated_cpu_bytes": self.scheduler.node_resources[node_id][0], + "estimated_gpu_bytes": self.scheduler.node_resources[node_id][1], + "shape": shape, + "cpu_increment_bytes": ( + measurement.increment_bytes if measurement else -1 + ), + "cpu_footprint_bytes": ( + measurement.footprint_bytes if measurement else -1 + ), + "input_resident_bytes": ( + measurement.input_resident_bytes if measurement else -1 + ), + "traced_peak_bytes": ( + measurement.traced_peak_bytes if measurement else -1 + ), + "rss_delta_bytes": measurement.rss_delta_bytes if measurement else -1, + "num_instances": getattr(actual_stats, "num_instances", None), + "output_shape": getattr(actual_stats, "output_shape", None), + "dtype": str(getattr(actual_stats, "dtype", None)), + "container": str(getattr(actual_stats, "container", None)), + } + self._memory_usage_checkpoint.checkpoint_if_due(self._memory_usage_data) + + def _print_node_stats( + self, + node_id: str, + actual_stats: Optional[RepresentationStats], + operation_name: str, + ): + if actual_stats is None: return None - - data = transformed_modality.data - - if isinstance(data, np.ndarray): - if data.ndim == 0: - return RepresentationStats(1, (1,), output_shape_is_known=True) - num_instances = int(data.shape[0]) - output_shape = ( - tuple(int(d) for d in data.shape[1:]) if data.ndim > 1 else (1,) - ) - return RepresentationStats( - num_instances, output_shape, output_shape_is_known=True - ) - - if isinstance(data, list) and len(data) > 0 and isinstance(data[0], np.ndarray): - num_instances = len(data) - first_shape = tuple(int(d) for d in data[0].shape) - same_shape = all( - isinstance(x, np.ndarray) and x.shape == data[0].shape for x in data - ) - return RepresentationStats( - num_instances, - first_shape, - output_shape_is_known=bool(same_shape), + node_stats = self.scheduler.node_stats.get(node_id) + shape = actual_stats.output_shape + if node_stats is not None: + print( + f"Node {node_id} {operation_name} should have shape of " + f"{node_stats.num_instances, node_stats.output_shape}, actual shape: " + f"{actual_stats.num_instances, shape} output shape is known: " + f"{node_stats.output_shape_is_known}" ) + return shape - return None - - def _manage_result_cache(self, node_id: str, result: Any): - parent_node_ids = self.scheduler.get_valid_parents(node_id) - for parent_node_id in parent_node_ids: - self.result_cache.dec_ref(parent_node_id) - - if self.scheduler.get_children(node_id): - for _ in self.scheduler.get_children(node_id): - self.result_cache.inc_ref(node_id) - self.result_cache.add_result(node_id, result) - - def _get_nodes_by_ids(self, nodes_ids: List[str]) -> List[RepresentationNode]: - return [self.scheduler.mapping[node_id] for node_id in nodes_ids] - - def _get_dag_from_node_ids(self, node_id: str) -> RepresentationDag: + def _get_dag_from_node_ids(self, node_id: str) -> Optional[RepresentationDag]: for dag in self.dags: if dag.root_node_id == node_id: return dag return None + def _describe_node(self, node_id: str) -> str: + names = [] + seen = set() + current = node_id + while current is not None and current not in seen: + seen.add(current) + node = self.scheduler.mapping.get(current) + if node is None: + break + if node.operation is not None: + try: + names.append(node.operation().name) + except Exception: + names.append( + getattr(node.operation, "__name__", str(node.operation)) + ) + parent_ids = [pid for pid in self.scheduler.get_valid_parents(current)] + current = parent_ids[0] if len(parent_ids) == 1 else None + if len(parent_ids) > 1: + names.append( + "[" + + ", ".join(self._describe_node(pid) for pid in parent_ids) + + "]" + ) + names.reverse() + return " -> ".join(names) if names else node_id + + def _discard_report(self) -> Dict[str, Any]: + return { + "failed_nodes": { + node_id: { + "representation": self._describe_node(node_id), + "reason": reason, + } + for node_id, reason in self.scheduler.failed_node_reasons.items() + }, + "blocked_memory_nodes": { + node_id: { + "representation": self._describe_node(node_id), + "reason": reason, + } + for node_id, reason in self.scheduler.blocked_memory_reasons.items() + }, + "cpu_fallback_nodes": { + node_id: { + "representation": self._describe_node(node_id), + "reason": reason, + } + for node_id, reason in self.scheduler.cpu_fallback_reasons.items() + }, + "deadlock": self.scheduler.deadlock, + "deadlock_reason": self.scheduler.deadlock_reason, + } + @staticmethod def _is_task_node(node: RepresentationNode) -> bool: return bool(getattr(node, "parameters", {}).get("_node_kind") == "task") + def run(self) -> Dict[str, Any]: + self._task_results = {} + self._load_leaf_modalities() -@dataclass -class ResultEntry: - val_score: PerformanceMeasure = None - train_score: PerformanceMeasure = None - test_score: PerformanceMeasure = None - representation_time: float = 0.0 - task_time: float = 0.0 - dag: RepresentationDag = None - tradeoff_score: float = 0.0 + try: + self._fill_pipeline() + while self._job_units or not self.scheduler.is_finished(): + self._fill_pipeline() + if not self._job_units: + continue + jr = self._pool.wait() + self._process_result(jr) + self._fill_pipeline() + finally: + if self._owns_pool: + self._pool.shutdown() + self._result_cache.cleanup_all() + self._cleanup_leaf_shared_memory() + + if self.enable_checkpointing: + self._checkpoint_manager.save_checkpoint( + self._task_results, self._discard_report() + ) + + return { + "task_results": list(self._task_results.values()), + "statistics": self.statistics, + } diff --git a/src/main/python/systemds/scuro/drsearch/node_scheduler.py b/src/main/python/systemds/scuro/drsearch/node_scheduler.py index 209f4503860..1ca681e88ad 100644 --- a/src/main/python/systemds/scuro/drsearch/node_scheduler.py +++ b/src/main/python/systemds/scuro/drsearch/node_scheduler.py @@ -19,6 +19,7 @@ # # ------------------------------------------------------------- from __future__ import annotations +import os import re from typing import List, Dict, Optional, Any from collections import defaultdict, deque @@ -29,9 +30,15 @@ RepresentationNode, ) from systemds.scuro.modality.modality import Modality +from systemds.scuro.representations.representation import ( + stats_dtype, + stats_itemsize, +) from systemds.scuro.utils.memory_utility import gpu_memory_info from systemds.scuro.utils.static_variables import DEBUG +_MAX_GPU_SCHEDULE_ATTEMPTS = int(os.environ.get("SCURO_MAX_GPU_SCHEDULE_ATTEMPTS", "3")) + class MemoryAwareNodeScheduler: @@ -66,37 +73,82 @@ def __init__( self.success = False self.deadlock = False self.ready_nodes = [] - self.running_nodes = [] + self._ready_set = set() + self.running_nodes = set() self.completed_nodes = [] + self._completed_set = set() self.failed_nodes = [] + self.failed_node_reasons: Dict[str, str] = {} self.blocked_memory_nodes_perm = [] + self.blocked_memory_reasons: Dict[str, str] = {} self.cancelled_nodes = [] + self.deadlock_reason: Optional[str] = None + self.gpu_wait_attempts: Dict[str, int] = {} + self.cpu_fallback_nodes: List[str] = [] + self.cpu_fallback_reasons: Dict[str, str] = {} + self._candidates = { + node_id + for node_id in self.topo_order + if node_id not in self.leaves and self.unresolved_parents[node_id] == 0 + } self.n_gpu = ( torch.cuda.device_count() if torch and torch.cuda.is_available() else 0 ) + leaf_cached = sum(self.node_resources[node][0] for node in self.leaves) self.memory_stats = { - "cpu_in_use": sum([self.node_resources[node][0] for node in self.leaves]), + "cpu_cached": leaf_cached, + "cpu_in_flight": 0, "gpu_in_use": { info["index"]: int(info["total_b"] - info["free_b"]) for info in self.gpu_memory_info }, } + self._cpu_reserved_nodes: Dict[str, int] = {} self._initialized = False + def _total_cpu_in_use(self) -> float: + return self.memory_stats["cpu_cached"] + self.memory_stats["cpu_in_flight"] + + def _pending_admitted_cpu_bytes(self) -> int: + total = 0 + for node_id in self._ready_set: + if node_id in self._cpu_reserved_nodes: + continue + resources = self.node_resources.get(node_id) + if resources: + total += resources[0] + return int(total) + + def can_start_now(self, node_id: str) -> bool: + resources = self.node_resources.get(node_id) + if not resources: + return True + if not self._cpu_reserved_nodes: + return True + return resources[0] <= self.memory_budget["cpu"] - self._total_cpu_in_use() + def update_cpu_memory_in_use(self, delta_bytes: int): - self.memory_stats["cpu_in_use"] += delta_bytes + self.memory_stats["cpu_cached"] += delta_bytes def get_runnable(self) -> List[RepresentationNode]: runnable_nodes = self._get_runnable_nodes() + admitted_bytes = self._pending_admitted_cpu_bytes() + for node in runnable_nodes: - ok, gpu_id = self._check_memory_constraints(node) + if node in self._ready_set: + continue + ok, gpu_id = self._check_memory_constraints(node, admitted_bytes) if ok: + admitted_bytes += self.node_resources[node][0] self.mapping[node].gpu_id = gpu_id - self._reserve_memory(node, gpu_id) + self._candidates.discard(node) self.ready_nodes.append(node) + self._ready_set.add(node) contains_leaf = [] for node in self.ready_nodes: + if isinstance(node, list): + continue if any(re.fullmatch(r"leaf_\d+", i) for i in self.mapping[node].inputs): for mod in self.modalities: if ( @@ -115,16 +167,7 @@ def get_runnable(self) -> List[RepresentationNode]: return self.ready_nodes def _get_runnable_nodes(self) -> List[str]: - runnable_nodes = [] - for node in self.topo_order: - if ( - node not in self.leaves - and self.unresolved_parents[node] == 0 - and node not in self.running_nodes - and node not in self.completed_nodes - and node not in self.ready_nodes - ): - runnable_nodes.append(node) + runnable_nodes = list(self._candidates) def _score(node_id: str): release_bytes = 0 @@ -134,32 +177,54 @@ def _score(node_id: str): and self.remaining_children.get(parent_id, 0) == 1 ): release_bytes += self.node_resources[parent_id][0] - return (-release_bytes, node_id not in self.roots) + return (-release_bytes, node_id not in self.roots, node_id) runnable_nodes.sort(key=_score) return runnable_nodes - def add_failed_node(self, node_id: str): + def add_failed_node(self, node_id: str, reason: str = "unknown failure"): self.failed_nodes.append(node_id) - self.running_nodes.remove(node_id) + self.failed_node_reasons[node_id] = reason + self.running_nodes.discard(node_id) + self._release_execution_memory(node_id, self.mapping[node_id].gpu_id) - self._release_memory(node_id, self.mapping[node_id].gpu_id) + def requeue_node(self, node_id: str) -> None: + self.running_nodes.discard(node_id) + self._release_execution_memory(node_id, self.mapping[node_id].gpu_id) + self._candidates.add(node_id) + + def begin_execution(self, node_id: str) -> None: + gpu_id = self.mapping[node_id].gpu_id + cpu_mem, gpu_mem = self.node_resources[node_id] + if gpu_id is not None and gpu_mem > 0: + self.memory_stats["gpu_in_use"][gpu_id] += gpu_mem + if cpu_mem > 0 and node_id not in self._cpu_reserved_nodes: + self._cpu_reserved_nodes[node_id] = int(cpu_mem) + self.memory_stats["cpu_in_flight"] += int(cpu_mem) def move_to_running(self, node_id: str | list): self.ready_nodes.remove(node_id) if isinstance(node_id, list): - self.running_nodes.extend(node_id) + self._ready_set.difference_update(node_id) + self.running_nodes.update(node_id) else: - self.running_nodes.append(node_id) + self._ready_set.discard(node_id) + self.running_nodes.add(node_id) def complete_node(self, node_id: str): - self.running_nodes.remove(node_id) + self.running_nodes.discard(node_id) self.completed_nodes.append(node_id) - self._release_memory(node_id, self.mapping[node_id].gpu_id) - self.topo_order.remove(node_id) + self._completed_set.add(node_id) + self._release_execution_memory(node_id, self.mapping[node_id].gpu_id) for child_id in self.children[node_id]: self.parent_refcounts[child_id] -= 1 self.unresolved_parents[child_id] -= 1 + if ( + self.unresolved_parents[child_id] == 0 + and child_id not in self.leaves + and child_id not in self._completed_set + ): + self._candidates.add(child_id) for parent_id in self.parents.get(node_id, set()): if self.remaining_children.get(parent_id, 0) > 0: @@ -209,9 +274,9 @@ def update_node_stats_and_reestimate_descendants( for desc_id in descendants: if ( desc_id in self.leaves - or desc_id in self.ready_nodes + or desc_id in self._ready_set or desc_id in self.running_nodes - or desc_id in self.completed_nodes + or desc_id in self._completed_set ): continue @@ -219,9 +284,10 @@ def update_node_stats_and_reestimate_descendants( if not parent_ids: continue - input_stats = self.node_stats.get(parent_ids[0]) - if input_stats is None: + parent_stats = [self.node_stats.get(pid) for pid in parent_ids] + if any(s is None for s in parent_stats): continue + input_stats = parent_stats[0] if len(parent_stats) == 1 else parent_stats if desc_id not in self.roots: operation = self.mapping[desc_id].operation( @@ -232,7 +298,9 @@ def update_node_stats_and_reestimate_descendants( peak_memory["cpu_peak_bytes"] += ( 64 * 1024 + 512 * input_stats_for_overhead.num_instances ) - output_stats = operation.get_output_stats(input_stats) + output_stats = self._resolve_output_stats( + operation.get_output_stats(input_stats), input_stats + ) self.node_resources[desc_id] = ( int(peak_memory["cpu_peak_bytes"]), @@ -288,29 +356,65 @@ def _is_deadlock(self) -> bool: return blocked def not_enough_memory(self) -> bool: - for node_id in self._get_pending_nodes(): + if self.running_nodes or self.ready_nodes: + return False + for node_id in self._candidates: cpu_mem, gpu_mem = self.node_resources[node_id] - if cpu_mem > self.memory_budget["cpu"] - self.memory_stats["cpu_in_use"]: + if cpu_mem > self.memory_budget["cpu"] - self._total_cpu_in_use(): + self.deadlock_reason = ( + f"node {node_id} needs {cpu_mem / 1024**3:.2f} GB CPU but only " + f"{(self.memory_budget['cpu'] - self._total_cpu_in_use()) / 1024**3:.2f} " + f"GB is free and nothing is running to release more" + ) return True if gpu_mem > 0.0 and self.n_gpu > 0: gpu_id = self._gpu_with_most_free_memory(gpu_mem) if gpu_id is None: + if ( + self.gpu_wait_attempts.get(node_id, 0) + <= _MAX_GPU_SCHEDULE_ATTEMPTS + ): + continue + self.deadlock_reason = ( + f"node {node_id} needs {gpu_mem / 1024**3:.2f} GB GPU memory " + f"but no GPU has enough free and nothing is running to release more" + ) return True - return self.memory_stats["cpu_in_use"] > self.memory_budget["cpu"] + return False - def _check_memory_constraints(self, node_id: str) -> bool: + def _check_memory_constraints(self, node_id: str, pending_bytes: int = 0) -> bool: cpu_mem, gpu_mem = self.node_resources[node_id] gpu_id = None - if cpu_mem > self.memory_budget["cpu"] - self.memory_stats["cpu_in_use"]: + if ( + cpu_mem + > self.memory_budget["cpu"] - self._total_cpu_in_use() - pending_bytes + ): if cpu_mem > self.memory_budget["cpu"]: self.blocked_memory_nodes_perm.append(node_id) - self.topo_order.remove(node_id) + self.blocked_memory_reasons[node_id] = ( + f"estimated CPU peak {cpu_mem / 1024**3:.2f} GB exceeds the " + f"total CPU memory budget of " + f"{self.memory_budget['cpu'] / 1024**3:.2f} GB" + ) + self._candidates.discard(node_id) return False, None if gpu_mem > 0.0 and self.n_gpu > 0: gpu_id = self._gpu_with_most_free_memory(gpu_mem) if gpu_id is None: + attempts = self.gpu_wait_attempts.get(node_id, 0) + 1 + self.gpu_wait_attempts[node_id] = attempts + if attempts > _MAX_GPU_SCHEDULE_ATTEMPTS: + reason = ( + f"no GPU had {gpu_mem / 1024**3:.2f} GB free after " + f"{attempts} scheduling attempts; running on CPU instead" + ) + if DEBUG: + print(f"Node {node_id}: {reason}") + self.cpu_fallback_nodes.append(node_id) + self.cpu_fallback_reasons[node_id] = reason + return True, None if DEBUG: print(f"Node {node_id} has no available GPU") return False, None @@ -330,23 +434,17 @@ def _gpu_with_most_free_memory(self, memory_needed): return free_memory.index(max(free_memory)) def _get_pending_nodes(self) -> List[str]: - return [ - node_id - for node_id in self.topo_order - if node_id not in self.leaves and self.unresolved_parents[node_id] == 0 - ] - - def _reserve_memory(self, node_id: str, gpu_id: int) -> bool: - cpu_mem, gpu_mem = self.node_resources[node_id] - self.memory_stats["cpu_in_use"] += cpu_mem - if gpu_id is not None: - self.memory_stats["gpu_in_use"][gpu_id] += gpu_mem + return list(self._candidates) - def _release_memory(self, node_id: str, gpu_id: int) -> bool: - cpu_mem, gpu_mem = self.node_resources[node_id] - self.memory_stats["cpu_in_use"] -= cpu_mem - if gpu_id is not None: + def _release_execution_memory(self, node_id: str, gpu_id: int) -> None: + _, gpu_mem = self.node_resources[node_id] + if gpu_id is not None and gpu_mem > 0: self.memory_stats["gpu_in_use"][gpu_id] -= gpu_mem + reserved = self._cpu_reserved_nodes.pop(node_id, 0) + if reserved: + self.memory_stats["cpu_in_flight"] = max( + 0, self.memory_stats["cpu_in_flight"] - reserved + ) def _get_nodes_from_dags( self, dags: List[RepresentationDag] @@ -430,7 +528,9 @@ def _estimate_node_resources(self): 64 * 1024 + 512 * input_stats_for_overhead.num_instances ) # Placeholder for transformed modality creation overhead peak_memory["cpu_peak_bytes"] *= 1 - output_stats = operation.get_output_stats(input_stats) + output_stats = self._resolve_output_stats( + operation.get_output_stats(input_stats), input_stats + ) node_resources[node] = ( int(peak_memory["cpu_peak_bytes"]), int(peak_memory["gpu_peak_bytes"]), @@ -461,9 +561,11 @@ def _stats_for_overhead(input_stats: Any) -> Any: return input_stats @staticmethod - def _stats_to_bytes(stats: Optional[Any], dtype_size: int = 4) -> int: + def _stats_to_bytes(stats: Optional[Any], dtype_size: Optional[int] = None) -> int: if stats is None: return 0 + if dtype_size is None: + dtype_size = stats_itemsize(stats) num_instances = int(getattr(stats, "num_instances", 0)) output_shape = tuple(getattr(stats, "output_shape", ())) numel = 1 @@ -473,3 +575,21 @@ def _stats_to_bytes(stats: Optional[Any], dtype_size: int = 4) -> int: except Exception: numel *= 1 return max(0, int(num_instances * numel * dtype_size)) + + @staticmethod + def _resolve_output_stats(output_stats: Any, input_stats: Any) -> Any: + if output_stats is None or getattr(output_stats, "dtype", None) is not None: + return output_stats + + sources = input_stats if isinstance(input_stats, list) else [input_stats] + sources = [s for s in sources if s is not None] + sources = [s for s in sources if stats_itemsize(s) > 0] + if not sources: + return output_stats + + widest = max(sources, key=lambda s: stats_itemsize(s)) + try: + output_stats.dtype = stats_dtype(widest) + except AttributeError: + pass + return output_stats diff --git a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py index a67cbe12029..a632c97e973 100644 --- a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py +++ b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py @@ -202,45 +202,41 @@ def optimize_parallel(self, n_workers=None): if n_workers is None: n_workers = min(len(self.modalities), mp.cpu_count()) - with mp.Manager() as manager: - - ctx = mp.get_context("spawn") - with ProcessPoolExecutor(max_workers=n_workers, mp_context=ctx) as executor: - future_to_modality = { - executor.submit( - self._process_modality, - modality, - self._checkpoint_manager.skip_remaining_by_key.get( - modality.modality_id, 0 - ) - / len(self.tasks), - scheduler=None, - ): modality - for modality in self.modalities - } - - for future in as_completed(future_to_modality): - modality = future_to_modality[future] - try: - results = future.result() - self._merge_results(results) - new_count = self._count_results(results.results) - self._checkpoint_manager.increment( - modality.modality_id, new_count - ) - self._checkpoint_manager.checkpoint_if_due( - self.operator_performance.results, - ) - except Exception as e: - print(f"Error processing modality {modality.modality_id}: {e}") - import traceback - - traceback.print_exc() - self._checkpoint_manager.save_checkpoint( - self.operator_performance.results, - {}, - ) - continue + ctx = mp.get_context("spawn") + with ProcessPoolExecutor(max_workers=n_workers, mp_context=ctx) as executor: + future_to_modality = { + executor.submit( + self._process_modality, + modality, + self._checkpoint_manager.skip_remaining_by_key.get( + modality.modality_id, 0 + ) + / len(self.tasks), + scheduler=None, + ): modality + for modality in self.modalities + } + + for future in as_completed(future_to_modality): + modality = future_to_modality[future] + try: + results = future.result() + self._merge_results(results) + new_count = self._count_results(results.results) + self._checkpoint_manager.increment(modality.modality_id, new_count) + self._checkpoint_manager.checkpoint_if_due( + self.operator_performance.results, + ) + except Exception as e: + print(f"Error processing modality {modality.modality_id}: {e}") + import traceback + + traceback.print_exc() + self._checkpoint_manager.save_checkpoint( + self.operator_performance.results, + {}, + ) + continue def optimize(self): if self.resume: @@ -265,10 +261,11 @@ def optimize(self): ) self._merge_results(local_result) new_count = self._count_results(local_result.results) - self._checkpoint_manager.increment(modality.modality_id, new_count) - self._checkpoint_manager.checkpoint_if_due( - self.operator_performance.results - ) + if self.enable_checkpointing: + self._checkpoint_manager.increment(modality.modality_id, new_count) + self._checkpoint_manager.checkpoint_if_due( + self.operator_performance.results + ) if self.save_all_results: self.store_results(f"{modality.modality_id}_unimodal_results.pkl") except Exception as e: @@ -276,9 +273,10 @@ def optimize(self): import traceback traceback.print_exc() - self._checkpoint_manager.save_checkpoint( - self.operator_performance.results, {} - ) + if self.enable_checkpointing: + self._checkpoint_manager.save_checkpoint( + self.operator_performance.results, {} + ) raise return execution_time @@ -338,9 +336,8 @@ def _process_modality(self, modality, skip_remaining: int = 0, scheduler=None): expanded_dags_with_task_roots, [modality], self.tasks, - self._checkpoint_manager, - self.max_num_workers, - self.result_path, + max_num_workers=self.max_num_workers, + result_path=self.result_path, enable_checkpointing=self.enable_checkpointing, ) start_time = time.perf_counter() @@ -617,7 +614,7 @@ def temporal_context_operators(self, modality, builder, leaf_id): for context_operator in context_operators: for window_size, num_window in zip(window_lengths, num_windows): context_operator_instance = context_operator(agg()) - if hasattr(context_operator, "num_windows"): + if hasattr(context_operator_instance, "num_windows"): context_operator_instance.num_windows = num_window elif hasattr(context_operator_instance, "window_size"): context_operator_instance.window_size = window_size diff --git a/src/main/python/systemds/scuro/drsearch/worker_pool.py b/src/main/python/systemds/scuro/drsearch/worker_pool.py new file mode 100644 index 00000000000..7e78862a104 --- /dev/null +++ b/src/main/python/systemds/scuro/drsearch/worker_pool.py @@ -0,0 +1,277 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- +import itertools +import multiprocessing as mp +import multiprocessing.connection as mp_connection +import os +import signal +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional + +import torch + +from systemds.scuro.utils.memory_utility import is_cuda_oom + +_THREAD_ENV_VARS = ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "NUMEXPR_NUM_THREADS", + "VECLIB_MAXIMUM_THREADS", + "BLIS_NUM_THREADS", +) + + +def _resolve_thread_count(num_threads: int) -> int: + explicit = os.environ.get("OMP_NUM_THREADS") + if explicit: + try: + num_threads = int(explicit) + except ValueError: + pass + return max(1, int(num_threads)) + + +def set_thread_env_before_spawn(num_threads: int) -> None: + num_threads = _resolve_thread_count(num_threads) + for var in _THREAD_ENV_VARS: + os.environ[var] = str(num_threads) + + +def _worker_initializer(num_threads: int) -> None: + num_threads = _resolve_thread_count(num_threads) + for var in _THREAD_ENV_VARS: + os.environ[var] = str(num_threads) + try: + torch.set_num_threads(num_threads) + except Exception: + pass + + +@dataclass +class _Job: + job_id: int + kind: str + payload: tuple + gpu_id: Optional[int] = None + + +@dataclass +class _JobResult: + job_id: int + ok: bool + pid: Optional[int] + value: Any = None + error: Optional[str] = None + cuda_oom: bool = False + worker_died: bool = False + + +def _worker_main( + job_q, result_q, dispatch: Dict[str, Callable], num_threads: int +) -> None: + _worker_initializer(num_threads) + while True: + job = job_q.get() + if job is None: + return + try: + value = dispatch[job.kind](job.payload, job.gpu_id) + result_q.put(_JobResult(job.job_id, True, os.getpid(), value=value)) + except Exception as e: + result_q.put( + _JobResult( + job.job_id, + False, + os.getpid(), + error=f"{type(e).__name__}: {e}", + cuda_oom=is_cuda_oom(e), + ) + ) + + +def _describe_worker_death(exitcode: Optional[int]) -> str: + if exitcode is None: + return "exit code unknown" + if exitcode < 0: + try: + sig = signal.Signals(-exitcode) + except ValueError: + return f"killed by signal {-exitcode}" + hint = { + signal.SIGKILL: " (often the OOM killer or an explicit kill -9)", + signal.SIGSEGV: " (segmentation fault, often a native library crash, e.g. CUDA/BLAS)", + signal.SIGABRT: " (abort, often a C-level assertion or CUDA error)", + signal.SIGBUS: " (bus error, often a full /dev/shm or a shared-memory issue)", + }.get(sig, "") + return f"killed by signal {sig.name} ({-exitcode}){hint}" + return f"exited with status {exitcode}" + + +def create_mp_context(): + ctx_name = os.environ.get("SCURO_MP_CONTEXT", "spawn") + try: + return mp.get_context(ctx_name) + except ValueError: + return mp.get_context("spawn") + + +class PersistentWorkerPool: + def __init__( + self, + n_workers: int, + dispatch: Dict[str, Callable], + ctx=None, + threads_per_worker: int = 1, + ): + self._ctx = ctx or create_mp_context() + self._dispatch = dispatch + self._threads_per_worker = max(1, int(threads_per_worker)) + self._result_q = self._ctx.Queue() + self._job_counter = itertools.count() + self._workers: Dict[int, Dict[str, Any]] = {} + self._idle_pids: List[int] = [] + self._running: Dict[int, tuple] = {} + for _ in range(max(1, n_workers)): + self._spawn_worker() + + def _spawn_worker(self) -> None: + set_thread_env_before_spawn(self._threads_per_worker) + job_q = self._ctx.Queue() + p = self._ctx.Process( + target=_worker_main, + args=(job_q, self._result_q, self._dispatch, self._threads_per_worker), + daemon=True, + ) + p.start() + self._workers[p.pid] = {"process": p, "job_q": job_q} + self._idle_pids.append(p.pid) + + @property + def has_idle_worker(self) -> bool: + return len(self._idle_pids) > 0 + + @property + def num_in_flight(self) -> int: + return len(self._running) + + def submit(self, kind: str, payload: tuple, gpu_id: Optional[int] = None) -> int: + if not self._idle_pids: + raise RuntimeError("submit() called with no idle worker available") + job_id = next(self._job_counter) + job = _Job(job_id, kind, payload, gpu_id) + pid = self._idle_pids.pop() + self._running[job_id] = (pid, job) + self._workers[pid]["job_q"].put(job) + return job_id + + def wait(self) -> _JobResult: + while True: + sentinel_to_pid = { + w["process"].sentinel: pid for pid, w in self._workers.items() + } + ready = mp_connection.wait( + [self._result_q._reader, *sentinel_to_pid.keys()] + ) + if self._result_q._reader in ready: + jr = self._result_q.get() + entry = self._running.pop(jr.job_id, None) + if entry is not None: + pid, _job = entry + if pid in self._workers: + self._idle_pids.append(pid) + return jr + for r in ready: + dead_pid = sentinel_to_pid.get(r) + if dead_pid is None: + continue + result = self._replace_dead_worker(dead_pid) + if result is not None: + return result + + def _replace_dead_worker(self, pid: int) -> Optional[_JobResult]: + w = self._workers.pop(pid, None) + if w is None: + return None + try: + if pid in self._idle_pids: + self._idle_pids.remove(pid) + except ValueError: + pass + try: + w["process"].join(timeout=1) + except Exception: + pass + exitcode = w["process"].exitcode + try: + w["job_q"].close() + w["job_q"].join_thread() + except Exception: + pass + + failed_job_id = None + for job_id, (running_pid, _job) in self._running.items(): + if running_pid == pid: + failed_job_id = job_id + break + if failed_job_id is not None: + self._running.pop(failed_job_id, None) + + self._spawn_worker() + + if failed_job_id is None: + return None + return _JobResult( + failed_job_id, + False, + pid, + error=f"worker process died ({_describe_worker_death(exitcode)})", + worker_died=True, + ) + + def shutdown(self) -> None: + for w in self._workers.values(): + try: + w["job_q"].put(None) + except Exception: + pass + for w in self._workers.values(): + try: + w["process"].join(timeout=5) + if w["process"].is_alive(): + w["process"].kill() + w["process"].join(timeout=2) + except Exception: + pass + for w in self._workers.values(): + try: + w["job_q"].close() + w["job_q"].join_thread() + except Exception: + pass + try: + self._result_q.close() + self._result_q.join_thread() + except Exception: + pass + self._workers.clear() + self._idle_pids.clear() + self._running.clear() diff --git a/src/main/python/systemds/scuro/representations/representation.py b/src/main/python/systemds/scuro/representations/representation.py index d83553ec6e1..c7b6d69d730 100644 --- a/src/main/python/systemds/scuro/representations/representation.py +++ b/src/main/python/systemds/scuro/representations/representation.py @@ -20,8 +20,18 @@ # ------------------------------------------------------------- import abc from dataclasses import dataclass +from typing import Any, Optional + +import numpy as np + from systemds.scuro.utils.identifier import Identifier +CONTAINER_ARRAY = "ndarray" +CONTAINER_LIST = "list_of_ndarray" +CONTAINER_RAGGED = "ragged" +NDARRAY_OBJECT_OVERHEAD_BYTES = 112 +DEFAULT_DTYPE = np.dtype(np.float32) + @dataclass class RepresentationStats: @@ -29,6 +39,117 @@ class RepresentationStats: output_shape: tuple output_shape_is_known: bool = True aggregate_dim: tuple = (0,) + dtype: Optional[Any] = None + container: str = CONTAINER_ARRAY + shape_variance: float = 0.0 + + +def stats_dtype(stats) -> np.dtype: + dtype = getattr(stats, "dtype", None) + if dtype is None: + return DEFAULT_DTYPE + + if type(dtype).__module__ == "torch": + try: + resolved = np.dtype(str(dtype).rsplit(".", 1)[-1]) + return resolved if resolved.itemsize else DEFAULT_DTYPE + except TypeError: + return DEFAULT_DTYPE + try: + resolved = np.dtype(dtype) + except TypeError: + return DEFAULT_DTYPE + + if resolved.itemsize == 0: + return DEFAULT_DTYPE + return resolved + + +def stats_itemsize(stats) -> int: + return int(stats_dtype(stats).itemsize) + + +def stats_num_elements(stats) -> int: + n = 1 + for dim in getattr(stats, "output_shape", ()) or (): + n *= int(dim) + return int(n) + + +def stats_bytes(stats, quantile: float = 0.0) -> int: + num_instances = int(getattr(stats, "num_instances", 0) or 0) + per_instance = stats_num_elements(stats) * stats_itemsize(stats) + total = num_instances * per_instance + + container = getattr(stats, "container", CONTAINER_ARRAY) + if container in (CONTAINER_LIST, CONTAINER_RAGGED): + total += num_instances * NDARRAY_OBJECT_OVERHEAD_BYTES + + if quantile > 0.0: + variance = float(getattr(stats, "shape_variance", 0.0) or 0.0) + if variance > 0.0: + z = {0.9: 1.282, 0.95: 1.645, 0.99: 2.326}.get(quantile, 1.645) + total = int(total * (1.0 + z * variance)) + + return int(total) + + +def infer_stats_from_data(data) -> Optional[RepresentationStats]: + if data is None: + return None + + if isinstance(data, np.ndarray): + if data.ndim == 0: + return RepresentationStats( + 1, (1,), output_shape_is_known=True, dtype=data.dtype + ) + num_instances = int(data.shape[0]) + output_shape = tuple(int(d) for d in data.shape[1:]) if data.ndim > 1 else (1,) + return RepresentationStats( + num_instances, + output_shape, + output_shape_is_known=True, + dtype=data.dtype, + container=CONTAINER_ARRAY, + ) + + if isinstance(data, list) and len(data) > 0 and isinstance(data[0], np.ndarray): + num_instances = len(data) + first_shape = tuple(int(d) for d in data[0].shape) + sizes = [x.size for x in data if isinstance(x, np.ndarray)] + same_shape = len(sizes) == num_instances and all( + x.shape == data[0].shape for x in data if isinstance(x, np.ndarray) + ) + variance = 0.0 + if not same_shape and sizes: + mean = sum(sizes) / len(sizes) + if mean > 0: + spread = (sum((s - mean) ** 2 for s in sizes) / len(sizes)) ** 0.5 + variance = spread / mean + return RepresentationStats( + num_instances, + first_shape, + output_shape_is_known=bool(same_shape), + dtype=data[0].dtype, + container=CONTAINER_LIST if same_shape else CONTAINER_RAGGED, + shape_variance=variance, + ) + + return None + + +def derive_stats(stats: RepresentationStats, **overrides) -> RepresentationStats: + fields = dict( + num_instances=stats.num_instances, + output_shape=stats.output_shape, + output_shape_is_known=getattr(stats, "output_shape_is_known", True), + aggregate_dim=getattr(stats, "aggregate_dim", (0,)), + dtype=getattr(stats, "dtype", None), + container=getattr(stats, "container", CONTAINER_ARRAY), + shape_variance=getattr(stats, "shape_variance", 0.0), + ) + fields.update(overrides) + return RepresentationStats(**fields) class Representation: diff --git a/src/main/python/systemds/scuro/utils/memory_utility.py b/src/main/python/systemds/scuro/utils/memory_utility.py index 0d3cd9d786c..88698fa53cc 100644 --- a/src/main/python/systemds/scuro/utils/memory_utility.py +++ b/src/main/python/systemds/scuro/utils/memory_utility.py @@ -18,13 +18,31 @@ # under the License. # # ------------------------------------------------------------- +import os import resource import sys +import threading +import time +import tracemalloc +from dataclasses import dataclass import numpy as np from sympy import Dict import torch -from typing import List, Tuple +from typing import List, Optional, Tuple import psutil +import gc + +_CPU_MEMORY_BUDGET_FRACTION = float( + os.environ.get("SCURO_CPU_MEMORY_BUDGET_FRACTION", "0.5") +) +_CPU_MEMORY_BUDGET_GB = os.environ.get("SCURO_CPU_MEMORY_BUDGET_GB") + + +def cpu_memory_budget_bytes() -> float: + budget = float(psutil.virtual_memory().available) * _CPU_MEMORY_BUDGET_FRACTION + if _CPU_MEMORY_BUDGET_GB: + budget = min(budget, float(_CPU_MEMORY_BUDGET_GB) * 1024**3) + return budget def get_model_size_mb(model: torch.nn.Module) -> float: @@ -192,3 +210,98 @@ def estimate_modality_bytes(modality) -> int: metadata = getattr(modality, "metadata", None) metadata_bytes = estimate_numpy_like_bytes(metadata) return int(data_bytes + metadata_bytes) + + +def is_cuda_oom(exc: BaseException) -> bool: + if isinstance(exc, torch.cuda.OutOfMemoryError): + return True + msg = str(exc).lower() + return ( + "cuda out of memory" in msg + or "cudamalloc" in msg + or "cuda error: out of memory" in msg + ) + + +def cleanup_gpu(gpu_id: Optional[int]) -> None: + if gpu_id is None or not torch.cuda.is_available(): + return + device = torch.device(f"cuda:{gpu_id}") + torch.cuda.set_device(device) + torch.cuda.synchronize(device) + gc.collect() + torch.cuda.empty_cache() + + +@dataclass +class MemoryMeasurement: + increment_bytes: int + footprint_bytes: int + input_resident_bytes: int + traced_peak_bytes: int + rss_delta_bytes: int + peak_abs_rss_bytes: int + + +def merge_memory_measurements( + measurements: List[Optional[MemoryMeasurement]], +) -> Optional[MemoryMeasurement]: + present = [m for m in measurements if m is not None] + if not present: + return None + return MemoryMeasurement( + increment_bytes=max(m.increment_bytes for m in present), + footprint_bytes=max(m.footprint_bytes for m in present), + input_resident_bytes=max(m.input_resident_bytes for m in present), + traced_peak_bytes=max(m.traced_peak_bytes for m in present), + rss_delta_bytes=max(m.rss_delta_bytes for m in present), + peak_abs_rss_bytes=max(m.peak_abs_rss_bytes for m in present), + ) + + +def measure_memory_during( + fn, *args, input_resident_bytes: int = 0, sample_s: float = 0.01, **kwargs +): + proc = psutil.Process(os.getpid()) + baseline_rss = proc.memory_info().rss + peak_rss = baseline_rss + stop = threading.Event() + + def sampler(): + nonlocal peak_rss + while not stop.is_set(): + rss = proc.memory_info().rss + if rss > peak_rss: + peak_rss = rss + time.sleep(sample_s) + + owns_tracing = not tracemalloc.is_tracing() + if owns_tracing: + tracemalloc.start() + else: + tracemalloc.reset_peak() + traced_baseline, _ = tracemalloc.get_traced_memory() + + t = threading.Thread(target=sampler, daemon=True) + t.start() + try: + out = fn(*args, **kwargs) + finally: + stop.set() + t.join() + _, traced_peak = tracemalloc.get_traced_memory() + if owns_tracing: + tracemalloc.stop() + + traced_increment = max(int(traced_peak) - int(traced_baseline), 0) + rss_delta = max(int(peak_rss) - int(baseline_rss), 0) + increment = max(traced_increment, rss_delta) + + return out, MemoryMeasurement( + increment_bytes=increment, + footprint_bytes=increment + int(input_resident_bytes), + input_resident_bytes=int(input_resident_bytes), + traced_peak_bytes=traced_increment, + rss_delta_bytes=rss_delta, + peak_abs_rss_bytes=int(peak_rss), + ) diff --git a/src/main/python/tests/scuro/data_generator.py b/src/main/python/tests/scuro/data_generator.py index 937fd622d85..b30946fb7df 100644 --- a/src/main/python/tests/scuro/data_generator.py +++ b/src/main/python/tests/scuro/data_generator.py @@ -222,7 +222,9 @@ def create_audio_data(self, num_instances, max_audio_length): data = [ [ random.random() - for _ in range(random.randint(max_audio_length * 0.9, max_audio_length)) + for _ in range( + random.randint(int(max_audio_length * 0.9), max_audio_length) + ) ] for _ in range(num_instances) ] From 6401a624021dabe2d3ea5b5cc90e5cfbcc1c048a Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Tue, 18 Aug 2026 12:06:41 +0200 Subject: [PATCH 114/132] [SYSTEMDS-3956] Extend AI Policy for IP Provenance --- CONTRIBUTING.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d86b30ac227..f060557ecc4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,9 +61,12 @@ Contributors must author their own pull request descriptions, bug reports, discu and other project communications. Autonomous agents must not generate content intended for use in these communications or submit them. -Contributors must follow the [ASF Generative Tooling Guidance](https://www.apache.org/legal/generative-tooling.html). -Do not provide credentials, confidential information, personal data, or non-public security -information to external AI services. +Contributors must ensure that AI-generated contributions do not introduce incompatible licenses or +contain undisclosed third-party material, and that the terms of service of all tools used do not +conflict with the [Open Source Definition](https://opensource.org/osd/). Review the +[ASF Generative Tooling Guidance](https://www.apache.org/legal/generative-tooling.html) for further +information. Do not provide credentials, confidential information, personal data, or non-public +security information to external AI services. ## Code Style From e2e1f5de2ba115de2107b8a88b3eca96afa1d8e0 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:51:12 +0200 Subject: [PATCH 115/132] [SYSTEMDS-3891] Wire OOC Aggregate Unary and Bugfix Race Condition Assisted-by: AI --- .../ooc/AggregateUnaryOOCInstruction.java | 177 ++++++++++-------- .../sysds/runtime/ooc/cache/OOCCacheImpl.java | 33 +++- .../primitives/GroupedReduceOOCPrimitive.java | 113 +++++++++-- .../runtime/ooc/util/OOCInstructionUtils.java | 10 +- .../test/component/ooc/OOCPrimitiveTest.java | 38 ++++ 5 files changed, 267 insertions(+), 104 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/AggregateUnaryOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/AggregateUnaryOOCInstruction.java index ac4e9bac919..0de35832f32 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/AggregateUnaryOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/AggregateUnaryOOCInstruction.java @@ -35,6 +35,9 @@ import org.apache.sysds.runtime.matrix.operators.AggregateUnaryOperator; import org.apache.sysds.runtime.matrix.operators.Operator; import org.apache.sysds.runtime.meta.DataCharacteristics; +import org.apache.sysds.runtime.ooc.primitives.GroupedReduceOOCPrimitive; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; +import org.apache.sysds.runtime.ooc.util.OOCUtils; import java.util.HashMap; @@ -72,88 +75,108 @@ public static AggregateUnaryOOCInstruction parseInstruction(String str) { public void processInstruction( ExecutionContext ec ) { //TODO support all types of aggregations, currently only full aggregation, row aggregation and column aggregation - //setup operators and input queue AggregateUnaryOperator aggun = (AggregateUnaryOperator) getOperator(); MatrixObject min = ec.getMatrixObject(input1); + DataCharacteristics chars = ec.getDataCharacteristics(input1.getName()); + int blen = chars != null && chars.getBlocksize() > 0 ? chars.getBlocksize() : ConfigurationManager + .getBlocksize(); + + if(!aggun.isRowAggregate() && !aggun.isColAggregate()) { + processScalarAggregate(ec, min, aggun, blen); + return; + } + if(OOCUtils.getNumBlocks(chars) > 0) { + processPlannerMatrixAggregate(ec, min, aggun, blen); + return; + } + OOCStream qIn = min.getStreamHandle(); - int blen = ConfigurationManager.getBlocksize(); - - if (aggun.isRowAggregate() || aggun.isColAggregate()) { - DataCharacteristics chars = ec.getDataCharacteristics(input1.getName()); - // number of blocks to process per aggregation idx (row or column dim) - long emitThreshold = aggun.isRowAggregate()? chars.getNumColBlocks() : chars.getNumRowBlocks(); - OOCMatrixBlockTracker aggTracker = new OOCMatrixBlockTracker(emitThreshold); - HashMap corrs = new HashMap<>(); // correction blocks - - OOCStream qOut = createWritableStream(); - OOCStream qLocal = createWritableStream(); - - ec.getMatrixObject(output).setStreamHandle(qOut); - - // per-block aggregation (parallel map) - mapOOC(qIn, qLocal, tmp -> { - MatrixIndexes midx = aggun.isRowAggregate() ? - new MatrixIndexes(tmp.getIndexes().getRowIndex(), 1) : - new MatrixIndexes(1, tmp.getIndexes().getColumnIndex()); - - MatrixBlock ltmp = (MatrixBlock) ((MatrixBlock) tmp.getValue()) - .aggregateUnaryOperations(aggun, new MatrixBlock(), blen, tmp.getIndexes()); - return new IndexedMatrixValue(midx, ltmp); - }); - - // global reduce - addOutStream(qOut); - submitOOCTasks(qLocal, callback -> { - IndexedMatrixValue partial = callback.get(); - synchronized(aggTracker) { - long idx = aggun.isRowAggregate() ? partial.getIndexes().getRowIndex() : partial.getIndexes() - .getColumnIndex(); - - MatrixBlock ret = aggTracker.get(idx); - boolean ready; - if(ret != null) { - MatrixBlock corr = corrs.get(idx); - OperationsOnMatrixValues.incrementalAggregation(ret, - _aop.existsCorrection() ? corr : null, (MatrixBlock) partial.getValue(), _aop, - true); - ready = aggTracker.incrementCount(idx); - } - else { - ret = (MatrixBlock) partial.getValue(); - MatrixBlock corr = _aop.existsCorrection() ? new MatrixBlock(ret.getNumRows(), - ret.getNumColumns(), false) : null; - ready = aggTracker.putAndIncrementCount(idx, ret); - if(!ready && _aop.existsCorrection()) - corrs.put(idx, corr); - } - - if(ready) { - ret.dropLastRowsOrColumns(_aop.correction); - qOut.enqueue(new IndexedMatrixValue(partial.getIndexes(), ret)); - aggTracker.remove(idx); - corrs.remove(idx); - } + long emitThreshold = aggun.isRowAggregate() ? chars.getNumColBlocks() : chars.getNumRowBlocks(); + OOCMatrixBlockTracker aggTracker = new OOCMatrixBlockTracker(emitThreshold); + HashMap corrs = new HashMap<>(); + OOCStream qOut = createWritableStream(); + OOCStream qLocal = createWritableStream(); + ec.getMatrixObject(output).setStreamHandle(qOut); + + mapOOC(qIn, qLocal, tmp -> { + MatrixIndexes midx = aggun.isRowAggregate() ? new MatrixIndexes(tmp.getIndexes().getRowIndex(), + 1) : new MatrixIndexes(1, tmp.getIndexes().getColumnIndex()); + MatrixBlock ltmp = (MatrixBlock) ((MatrixBlock) tmp.getValue()).aggregateUnaryOperations(aggun, + new MatrixBlock(), blen, tmp.getIndexes()); + return new IndexedMatrixValue(midx, ltmp); + }); + + addOutStream(qOut); + submitOOCTasks(qLocal, callback -> { + IndexedMatrixValue partial = callback.get(); + synchronized(aggTracker) { + long idx = aggun.isRowAggregate() ? partial.getIndexes().getRowIndex() : partial.getIndexes() + .getColumnIndex(); + MatrixBlock ret = aggTracker.get(idx); + boolean ready; + if(ret != null) { + MatrixBlock corr = corrs.get(idx); + OperationsOnMatrixValues.incrementalAggregation(ret, _aop.existsCorrection() ? corr : null, + (MatrixBlock) partial.getValue(), _aop, true); + ready = aggTracker.incrementCount(idx); + } + else { + ret = (MatrixBlock) partial.getValue(); + MatrixBlock corr = _aop.existsCorrection() ? new MatrixBlock(ret.getNumRows(), ret.getNumColumns(), + false) : null; + ready = aggTracker.putAndIncrementCount(idx, ret); + if(!ready && _aop.existsCorrection()) + corrs.put(idx, corr); + } + if(ready) { + ret.dropLastRowsOrColumns(_aop.correction); + qOut.enqueue(new IndexedMatrixValue(partial.getIndexes(), ret)); + aggTracker.remove(idx); + corrs.remove(idx); } - }).thenRun(qOut::closeInput); - } - // full aggregation - else { - OOCStream qLocal = createWritableStream(); - - mapOOC(qIn, qLocal, tmp -> (MatrixBlock) tmp.getValue() - .aggregateUnaryOperations(aggun, new MatrixBlock(), blen, tmp.getIndexes())); - - MatrixBlock ltmp; - int extra = _aop.correction.getNumRemovedRowsColumns(); - MatrixBlock ret = new MatrixBlock(1, 1 + extra, _aop.initialValue); - MatrixBlock corr = new MatrixBlock(1,1+extra,false); - while((ltmp = qLocal.dequeue()) != LocalTaskQueue.NO_MORE_TASKS) { - OperationsOnMatrixValues.incrementalAggregation( - ret, _aop.existsCorrection() ? corr : null, ltmp, _aop, true); } + }).thenRun(qOut::closeInput); + } - //create scalar output - ec.setScalarOutput(output.getName(), new DoubleObject(ret.get(0, 0))); - } + private void processPlannerMatrixAggregate(ExecutionContext ec, MatrixObject input, AggregateUnaryOperator operator, + int blocksize) { + OOCStream outputStream = createWritableStream(); + ec.getMatrixObject(output).setStreamHandle(outputStream); + GroupedReduceOOCPrimitive.Grouping grouping = operator + .isRowAggregate() ? GroupedReduceOOCPrimitive.Grouping.ROW_BLOCKS : GroupedReduceOOCPrimitive.Grouping.COL_BLOCKS; + OOCInstructionUtils.groupedReduceIndexed(input.getStreamable(), outputStream, grouping, + value -> aggregatePartial(value, operator, blocksize), this::mergeAggregate, this::finalizeAggregate, + getContext()); + } + + private void processScalarAggregate(ExecutionContext ec, MatrixObject input, AggregateUnaryOperator operator, + int blocksize) { + OOCStream partials = createWritableStream(); + mapOOC(input.getStreamHandle(), partials, value -> aggregatePartial(value, operator, blocksize)); + + int extra = _aop.correction.getNumRemovedRowsColumns(); + MatrixBlock result = new MatrixBlock(1, 1 + extra, _aop.initialValue); + MatrixBlock correction = new MatrixBlock(1, 1 + extra, false); + MatrixBlock partial; + while((partial = partials.dequeue()) != LocalTaskQueue.NO_MORE_TASKS) + OperationsOnMatrixValues.incrementalAggregation(result, _aop.existsCorrection() ? correction : null, + partial, _aop, true); + ec.setScalarOutput(output.getName(), new DoubleObject(result.get(0, 0))); + } + + private static MatrixBlock aggregatePartial(IndexedMatrixValue value, AggregateUnaryOperator operator, + int blocksize) { + return (MatrixBlock) value.getValue().aggregateUnaryOperations(operator, new MatrixBlock(), blocksize, + value.getIndexes()); + } + + private MatrixBlock mergeAggregate(MatrixBlock left, MatrixBlock right) { + OperationsOnMatrixValues.incrementalAggregation(left, null, right, _aop, true); + return left; + } + + private MatrixBlock finalizeAggregate(MatrixBlock block) { + block.dropLastRowsOrColumns(_aop.correction); + return block; } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheImpl.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheImpl.java index 9b0008e84ab..90f81d43bed 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheImpl.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheImpl.java @@ -341,7 +341,7 @@ else if(meta.entry.getDataUnsafe() != null) { readFuture = null; } else if(meta.readFuture == null) { - meta.entry.setState(BlockState.READING); + awaitRead(meta); OOCFuture scheduled = _ioHandler.scheduleRead(meta.entry); meta.readFuture = scheduled; readFuture = scheduled; @@ -354,8 +354,10 @@ else if(meta.readFuture == null) { } }); } - else + else { + awaitRead(meta); readFuture = meta.readFuture; + } } if(releaseReserved) { allowance.release(reservedBytes); @@ -372,19 +374,24 @@ else if(meta.readFuture == null) { try { if(ex != null) { release = true; + synchronized(OOCCacheImpl.this) { + finishRead(meta); + } allowance.release(reservedBytes); result.completeExceptionally(ex); return; } BlockEntry pinned; synchronized(OOCCacheImpl.this) { - if(getMeta(meta.entry) != meta || meta.entry.getDataUnsafe() == null) { + if(getMeta(meta.entry) != meta) { release = true; - if(meta.entry.getState() == BlockState.READING) - meta.entry.setState(BlockState.COLD); pinned = null; } else { + finishRead(meta); + if(meta.entry.getDataUnsafe() == null) + throw new IllegalStateException( + "Backing read left no data for entry: " + meta.entry.getKey()); completion = pinResident(meta); Statistics.incrementOOCEvictionGet(); pinned = meta.entry; @@ -404,6 +411,18 @@ else if(meta.readFuture == null) { return result; } + private void awaitRead(EntryMeta meta) { + meta.readWaiters++; + clearLive(meta.entry); + meta.entry.setState(BlockState.READING); + } + + private void finishRead(EntryMeta meta) { + meta.readWaiters = Math.max(0, meta.readWaiters - 1); + if(meta.readWaiters == 0 && meta.entry.getState() == BlockState.READING) + meta.entry.setState(BlockState.COLD); + } + private DeferredCompletion pinResident(EntryMeta meta) { BlockEntry entry = meta.entry; if(isCacheOwned(entry)) { @@ -624,7 +643,8 @@ private EvictController getOrCreateEvictController(long streamId) { } private void removeIfUnused(EntryMeta meta) { - if(meta.entry.getReferenceCount() > 0 || meta.entry.getPinCount() > 0 || meta.deferredUnpin != null) + if(meta.entry.getReferenceCount() > 0 || meta.entry.getPinCount() > 0 || meta.deferredUnpin != null || + meta.readWaiters > 0) return; BlockEntry entry = meta.entry; if(isCacheOwned(entry)) @@ -705,6 +725,7 @@ private static class EntryMeta { private final BlockEntry entry; private boolean backed; private OOCFuture readFuture; + private int readWaiters; private CacheUnpinHandle deferredUnpin; private EntryMeta(BlockEntry entry) { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/GroupedReduceOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/GroupedReduceOOCPrimitive.java index b380848aa7d..c94c2108dcf 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/GroupedReduceOOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/GroupedReduceOOCPrimitive.java @@ -22,6 +22,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; +import java.util.function.Function; import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.instructions.ooc.CachingStream; @@ -45,9 +46,16 @@ import org.apache.sysds.runtime.ooc.util.OOCUtils; public final class GroupedReduceOOCPrimitive extends OOCPrimitive { + public enum Grouping { + ROW_BLOCKS, COL_BLOCKS + } + private final OOCStreamable _input; private final OOCStreamable _output; + private final Grouping _grouping; + private final Function _partial; private final BiFunction _merge; + private final Function _finish; private final AtomicBoolean _cleaned; private final AtomicBoolean _sourceComplete; private final AtomicInteger _active; @@ -60,10 +68,21 @@ public final class GroupedReduceOOCPrimitive extends OOCPrimitive { public GroupedReduceOOCPrimitive(OOCStreamable input, OOCStreamable output, BiFunction merge, StreamContext context) { + this(input, output, Grouping.ROW_BLOCKS, value -> (MatrixBlock) value.getValue(), merge, Function.identity(), + context); + } + + public GroupedReduceOOCPrimitive(OOCStreamable input, OOCStreamable output, + Grouping grouping, Function partial, + BiFunction merge, Function finish, + StreamContext context) { super(context, input); _input = input; _output = output; + _grouping = grouping; + _partial = partial; _merge = merge; + _finish = finish; _cleaned = new AtomicBoolean(); _sourceComplete = new AtomicBoolean(); _active = new AtomicInteger(1); @@ -72,17 +91,21 @@ public GroupedReduceOOCPrimitive(OOCStreamable input, OOCStr @Override protected void inferPatternsInternal() { - _pattern = OOCAccessPattern.ROW_MAJOR; + _pattern = groupingPattern(); for(OOCPrimitive child : getChildren()) - child.requestPattern(OOCAccessPattern.ROW_MAJOR); + child.requestPattern(_pattern); inferParentPatterns(); } @Override protected void requestPatternInternal(OOCAccessPattern accessPattern) { - _pattern = OOCAccessPattern.ROW_MAJOR; + _pattern = groupingPattern(); for(OOCPrimitive child : getChildren()) - child.requestPattern(OOCAccessPattern.ROW_MAJOR); + child.requestPattern(_pattern); + } + + private OOCAccessPattern groupingPattern() { + return _grouping == Grouping.COL_BLOCKS ? OOCAccessPattern.COL_MAJOR : OOCAccessPattern.ROW_MAJOR; } @Override @@ -91,8 +114,7 @@ protected void startExecution() { if(inputDc == null || !inputDc.dimsKnown() || inputDc.getBlocksize() <= 0) throw new DMLRuntimeException("Grouped OOC reduction requires known input dimensions and block size."); OOCStream input = getInputReadStream(0); - _numGroups = Math.toIntExact(inputDc.getNumRowBlocks()); - _groupSize = Math.toIntExact(inputDc.getNumColBlocks()); + configureGroups(inputDc); _outputStream = _output.getWriteStream(); _ready = new SubscribableTaskQueue<>(); getContext().addInStream(input).addOutStream(_outputStream, _ready); @@ -121,6 +143,25 @@ protected void startExecution() { admitted.setSubscriber(this::accept); } + private void configureGroups(DataCharacteristics inputDc) { + long rowBlocks = inputDc.getNumRowBlocks(); + long colBlocks = inputDc.getNumColBlocks(); + switch(_grouping) { + case ROW_BLOCKS: + _numGroups = Math.toIntExact(rowBlocks); + _groupSize = Math.toIntExact(colBlocks); + break; + case COL_BLOCKS: + _numGroups = Math.toIntExact(colBlocks); + _groupSize = Math.toIntExact(rowBlocks); + break; + default: + throw new IllegalStateException("Unsupported grouped-reduce grouping: " + _grouping); + } + if(_numGroups <= 0 || _groupSize <= 0) + throw new DMLRuntimeException("Grouped OOC reduction requires non-empty input block geometry."); + } + private void accept(OOCStream.QueueCallback callback) { if(callback.isEos() || callback.isFailure()) { try(callback) { @@ -140,11 +181,11 @@ private void accept(OOCStream.QueueCallback callback) { try(callback) { budget = AllocatedOOCStream.detachBudget(callback).enableReuse(); IndexedMatrixValue input = callback.get(); - int group = Math.toIntExact(input.getIndexes().getRowIndex() - 1); - if(group < 0 || group >= _numGroups) - throw new DMLRuntimeException("Invalid grouped-reduce row block: " + (group + 1)); - IndexedMatrixValue value = new IndexedMatrixValue(new MatrixIndexes(group + 1L, 1), input.getValue()); - payload = payload(value, budget); + int group = group(input.getIndexes()); + MatrixBlock partial = _partial.apply(input); + if(partial == null) + throw new DMLRuntimeException("Grouped OOC reduction produced a null partial block."); + payload = payload(partialValue(group, 1, partial), budget); reduce(group, payload, budget); payload = null; budget = null; @@ -211,9 +252,11 @@ private void process(MergeWork work) { IndexedMatrixValue right = work._incoming.value(); int count = Math.addExact(multiplicity(left), multiplicity(right)); if(count > _groupSize) - throw new DMLRuntimeException("Too many partial tiles for grouped-reduce row " + (work._group + 1)); + throw new DMLRuntimeException("Too many partial tiles for grouped-reduce group " + (work._group + 1)); MatrixBlock value = _merge.apply((MatrixBlock) left.getValue(), (MatrixBlock) right.getValue()); - merged = payload(new IndexedMatrixValue(new MatrixIndexes(work._group + 1L, count), value), budget); + if(value == null) + throw new DMLRuntimeException("Grouped OOC reduction produced a null merged block."); + merged = payload(partialValue(work._group, count, value), budget); work.releaseIncoming(); released = work.closeExistingAsync(); } @@ -240,14 +283,18 @@ private void process(MergeWork work) { } private void finalizeGroup(int group, ManagedPayload payload, ReservationBudget budget) { - IndexedMatrixValue accumulated = payload.value(); - IndexedMatrixValue output = new IndexedMatrixValue(new MatrixIndexes(group + 1L, 1), accumulated.getValue()); - payload.release(); try { - OOCUtils.enqueueExact(_outputStream, output, budget); + MatrixBlock outputBlock = _finish.apply((MatrixBlock) payload.value().getValue()); + if(outputBlock == null) + throw new DMLRuntimeException("Grouped OOC reduction produced a null final block."); + payload.release(); + payload = null; + OOCUtils.enqueueExact(_outputStream, new IndexedMatrixValue(outputIndexes(group), outputBlock), budget); _finalizedGroups.incrementAndGet(); } catch(Throwable failure) { + if(payload != null) + payload.release(); budget.close(); fail(failure); } @@ -260,8 +307,34 @@ private static ManagedPayload payload(IndexedMatrixValue val return new ManagedPayload<>(value, bytes, budget); } - private static int multiplicity(IndexedMatrixValue value) { - return Math.toIntExact(value.getIndexes().getColumnIndex()); + private int group(MatrixIndexes indexes) { + long group = switch(_grouping) { + case ROW_BLOCKS -> indexes.getRowIndex() - 1; + case COL_BLOCKS -> indexes.getColumnIndex() - 1; + }; + if(group < 0 || group >= _numGroups) + throw new DMLRuntimeException("Invalid grouped-reduce group index: " + group); + return Math.toIntExact(group); + } + + private IndexedMatrixValue partialValue(int group, int count, MatrixBlock value) { + MatrixIndexes indexes = switch(_grouping) { + case ROW_BLOCKS -> new MatrixIndexes(group + 1L, count); + case COL_BLOCKS -> new MatrixIndexes(count, group + 1L); + }; + return new IndexedMatrixValue(indexes, value); + } + + private int multiplicity(IndexedMatrixValue value) { + return Math.toIntExact( + _grouping == Grouping.COL_BLOCKS ? value.getIndexes().getRowIndex() : value.getIndexes().getColumnIndex()); + } + + private MatrixIndexes outputIndexes(int group) { + return switch(_grouping) { + case ROW_BLOCKS -> new MatrixIndexes(group + 1L, 1); + case COL_BLOCKS -> new MatrixIndexes(1, group + 1L); + }; } private void finishSource() { @@ -275,7 +348,7 @@ private void completeOne() { return; if(!hasFailed() && _finalizedGroups.get() != _numGroups) fail(new DMLRuntimeException( - "Grouped reduction completed " + _finalizedGroups.get() + " of " + _numGroups + " row groups.")); + "Grouped reduction completed " + _finalizedGroups.get() + " of " + _numGroups + " groups.")); try { _ready.closeInput(); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java index e93876dfc03..4f2bcc34853 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java @@ -103,7 +103,15 @@ public static void indexedBroadcastMap(OOCStreamable streame public static void rowGroupedReduce(OOCStreamable input, OOCStream output, BiFunction merge, StreamContext context) { - output.assignPrimitive(new GroupedReduceOOCPrimitive(input, output, merge, context)); + groupedReduceIndexed(input, output, GroupedReduceOOCPrimitive.Grouping.ROW_BLOCKS, + value -> (MatrixBlock) value.getValue(), merge, Function.identity(), context); + } + + public static void groupedReduceIndexed(OOCStreamable input, + OOCStream output, GroupedReduceOOCPrimitive.Grouping grouping, + Function partial, BiFunction merge, + Function finish, StreamContext context) { + output.assignPrimitive(new GroupedReduceOOCPrimitive(input, output, grouping, partial, merge, finish, context)); } public static int getComputeInFlight() { diff --git a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java index ab83b474233..c94e0ac5f1d 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java @@ -39,6 +39,7 @@ import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; import org.apache.sysds.runtime.ooc.planning.OOCStoreLayout; +import org.apache.sysds.runtime.ooc.primitives.GroupedReduceOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.MaterializeOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.OOCPrimitive; import org.apache.sysds.runtime.ooc.store.CountingLiveness; @@ -187,6 +188,43 @@ public void testDataGenMapTransposePipeline() { values); } + @Test + public void testGroupedReduceModes() { + Assert.assertEquals(Map.of("1,1", 136d, "2,1", 166d), + runGroupedReduce(GroupedReduceOOCPrimitive.Grouping.ROW_BLOCKS, 2, 1)); + Assert.assertEquals(Map.of("1,1", 132d, "1,2", 134d, "1,3", 136d), + runGroupedReduce(GroupedReduceOOCPrimitive.Grouping.COL_BLOCKS, 1, 3)); + } + + private static Map runGroupedReduce(GroupedReduceOOCPrimitive.Grouping grouping, long outputRows, + long outputCols) { + SubscribableTaskQueue input = new SubscribableTaskQueue<>(); + SubscribableTaskQueue output = new SubscribableTaskQueue<>(); + input.setData(new MatrixObject(ValueType.FP64, "/dev/null", + new MetaDataFormat(new MatrixCharacteristics(2, 3, 1), FileFormat.BINARY))); + output.setData(new MatrixObject(ValueType.FP64, "/dev/null", + new MetaDataFormat(new MatrixCharacteristics(outputRows, outputCols, 1), FileFormat.BINARY))); + for(long[] indexes : List.of(new long[] {2, 3}, new long[] {1, 1}, new long[] {2, 1}, new long[] {1, 3}, + new long[] {1, 2}, new long[] {2, 2})) + input.enqueue(new IndexedMatrixValue(new MatrixIndexes(indexes[0], indexes[1]), + new MatrixBlock(1, 1, indexes[0] * 10d + indexes[1]))); + input.closeInput(); + OOCInstructionUtils.groupedReduceIndexed(input, output, grouping, value -> (MatrixBlock) value.getValue(), + (left, right) -> new MatrixBlock(1, 1, left.get(0, 0) + right.get(0, 0)), + value -> new MatrixBlock(1, 1, value.get(0, 0) + 100), new StreamContext()); + + output.start(); + Map values = new HashMap<>(); + OOCStream.QueueCallback callback; + while((callback = output.dequeueCB()) != null) + try(OOCStream.QueueCallback current = callback) { + IndexedMatrixValue value = current.get(); + values.put(value.getIndexes().getRowIndex() + "," + value.getIndexes().getColumnIndex(), + value.getValue().get(0, 0)); + } + return values; + } + @Test public void testJoinOutOfOrder() { SubscribableTaskQueue left = new SubscribableTaskQueue<>(); From e1376a04804d2b4187358201e1c7d640a8bedcd4 Mon Sep 17 00:00:00 2001 From: Wenliang Cao <59223665+WenliangCao@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:45:44 +0200 Subject: [PATCH 116/132] [SYSTEMDS-3863] Add PowerTransformer built-in functions Closes 2499. Signed-off-by: Grigorii Turchenko --- docs/site/builtins-reference.md | 74 ++++ scripts/builtin/powerTransform.dml | 382 ++++++++++++++++++ scripts/builtin/powerTransformApply.dml | 131 ++++++ .../org/apache/sysds/common/Builtins.java | 2 + .../part2/BuiltinPowerTransformTest.java | 266 ++++++++++++ .../functions/builtin/powerTransform.R | 71 ++++ .../functions/builtin/powerTransform.dml | 48 +++ .../functions/builtin/powerTransformApply.dml | 33 ++ src/test/scripts/installDependencies.R | 1 + 9 files changed, 1008 insertions(+) create mode 100644 scripts/builtin/powerTransform.dml create mode 100644 scripts/builtin/powerTransformApply.dml create mode 100644 src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinPowerTransformTest.java create mode 100644 src/test/scripts/functions/builtin/powerTransform.R create mode 100644 src/test/scripts/functions/builtin/powerTransform.dml create mode 100644 src/test/scripts/functions/builtin/powerTransformApply.dml diff --git a/docs/site/builtins-reference.md b/docs/site/builtins-reference.md index 22b335866cb..8dcc1ef61b9 100644 --- a/docs/site/builtins-reference.md +++ b/docs/site/builtins-reference.md @@ -72,6 +72,8 @@ limitations under the License. * [`outlier`-Function](#outlier-function) * [`outlierByDB`-Function](#outlierByDB-function) * [`pnmf`-Function](#pnmf-function) + * [`powerTransform`-Function](#powerTransform-function) + * [`powerTransformApply`-Function](#powerTransformApply-function) * [`scale`-Function](#scale-function) * [`setdiff`-Function](#setdiff-function) * [`sherlock`-Function](#sherlock-function) @@ -1840,6 +1842,78 @@ X = rand(rows = 50, cols = 10) [W, H] = pnmf(X = X, rnk = 2, eps = 10^-8, maxi = 10, verbose = TRUE) ``` +## `powerTransform`-Function + +The `powerTransform`-function estimates one power parameter per column and transforms the input matrix. It uses +Yeo-Johnson by default and can optionally use Box-Cox for strictly positive data. +NaN entries are preserved, while parameter estimation and standardization use the non-NaN entries in each column. + +### Usage + +```r +powerTransform(X, method="yeo-johnson", standardize=TRUE) +``` + +### Arguments + +| Name | Type | Default | Description | +| :---------- | :------------- | :-------------- | :---------- | +| X | Matrix[Double] | required | Matrix of feature vectors. | +| method | String | `"yeo-johnson"` | Transformation method: `"yeo-johnson"` or `"box-cox"`. | +| standardize | Boolean | TRUE | Whether to center and scale the transformed columns. | + +### Returns + +| Type | Description | +| :------------- | :---------- | +| Matrix[Double] | Transformed matrix. | +| Matrix[Double] | Row vector of estimated power parameters. | +| Matrix[Double] | Row vector of transformed column means, or an empty matrix when standardization is disabled. | +| Matrix[Double] | Row vector of transformed column scales, or an empty matrix when standardization is disabled. | + +### Example + +```r +X = matrix("-2 -1 0 1 2 4", rows=6, cols=1) +[Y, lambdas, means, scales] = powerTransform(X=X) +``` + +## `powerTransformApply`-Function + +The `powerTransformApply`-function transforms a matrix using parameters previously returned by `powerTransform`. +NaN entries are preserved in the transformed matrix. + +### Usage + +```r +powerTransformApply(X, lambdas, means, scales, method="yeo-johnson") +``` + +### Arguments + +| Name | Type | Default | Description | +| :------ | :------------- | :--------------- | :---------- | +| X | Matrix[Double] | required | Matrix of feature vectors. | +| lambdas | Matrix[Double] | required | Row vector of fitted power parameters. | +| means | Matrix[Double] | required | Row vector of fitted means, or an empty matrix to skip standardization. | +| scales | Matrix[Double] | required | Row vector of fitted scales, or an empty matrix to skip standardization. | +| method | String | `"yeo-johnson"` | Transformation method used during fitting. | + +### Returns + +| Type | Description | +| :------------- | :---------- | +| Matrix[Double] | Transformed matrix. | + +### Example + +```r +X = matrix("-2 -1 0 1 2 4", rows=6, cols=1) +[Y, lambdas, means, scales] = powerTransform(X=X) +Xnew = matrix("-3 0 3", rows=3, cols=1) +Ynew = powerTransformApply(X=Xnew, lambdas=lambdas, means=means, scales=scales) +``` + ## `scale`-Function diff --git a/scripts/builtin/powerTransform.dml b/scripts/builtin/powerTransform.dml new file mode 100644 index 00000000000..331f3290679 --- /dev/null +++ b/scripts/builtin/powerTransform.dml @@ -0,0 +1,382 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# Power transformation using the selected method. +# Reduces feature skewness by estimating and applying an optimal transformation parameter for each column. +# +# INPUT: +# ------------------------------------------------------------------------------------- +# X Input feature matrix of shape n-by-m +# method Power transformation method: "yeo-johnson" (default) or "box-cox" +# standardize Whether to normalize transformed columns to zero mean and unit variance +# ------------------------------------------------------------------------------------- +# +# OUTPUT: +# ------------------------------------------------------------------------------------- +# Y Power-transformed matrix of shape n-by-m +# lambdas Estimated lambda parameters of shape 1-by-m, one per column +# means Transformed column means of shape 1-by-m, or an empty matrix when not standardized +# scales Transformed column scales of shape 1-by-m, or an empty matrix when not standardized +# ------------------------------------------------------------------------------------- + +m_powerTransform = function( + Matrix[Double] X, + String method="yeo-johnson", + Boolean standardize=TRUE) + return ( + Matrix[Double] Y, + Matrix[Double] lambdas, + Matrix[Double] means, + Matrix[Double] scales) +{ + if (method != "yeo-johnson" & method != "box-cox") { + stop("powerTransform: unsupported method '" + method + + "'; expected 'yeo-johnson' or 'box-cox'") + } + + validatedX = replace(target=X, pattern=NaN, replacement=1.0) + if (method == "box-cox" & min(validatedX) <= 0.0) { + stop("powerTransform: Box-Cox requires strictly positive input") + } + + m = ncol(X) + lambdas = matrix(1.0, rows=1, cols=m) # Initialize first, then replace each column with the best lambdas + + # Estimate lambda for each column separately + for (j in 1:m){ + x = X[,j] + xObserved = removeEmpty(target=x, margin="rows", select=(is.na(x) == 0)) + observedN = nrow(xObserved) + + # Yeo-Johnson leaves constant columns unchanged; Box-Cox rejects them + if (observedN == 0) { + lambdas[1,j] = 1.0 + } + else if (max(xObserved) == min(xObserved)) { + if (method == "yeo-johnson") { + lambdas[1,j] = 1.0; + } + else { + stop("powerTransform: Box-Cox does not support constant columns") + } + } + else{ + lambdas[1,j] = ptEstimateLambda(xObserved, method); + } + } + + # Apply the fitted transformation before optional standardization + emptyStats = matrix(0.0, rows=0, cols=0) + Y = powerTransformApply(X, lambdas, emptyStats, emptyStats, method); + + means = matrix(0.0, rows=0, cols=0) + scales = matrix(0.0, rows=0, cols=0) + + if (standardize) { + means = matrix(0.0, rows=1, cols=m) + scales = matrix(1.0, rows=1, cols=m) + + for (j in 1:m) { + y = Y[,j] + yObserved = removeEmpty(target=y, margin="rows", select=(is.na(y) == 0)) + observedN = nrow(yObserved) + + if (observedN > 0) { + means[1,j] = mean(yObserved) + scale = sqrt(sum((yObserved - means[1,j])^2) / observedN) + if (!is.na(scale) & !is.infinite(scale) & scale != 0.0) { + scales[1,j] = scale + } + } + + Y[,j] = ifelse(is.na(y), NaN, (y - means[1,j]) / scales[1,j]) + } + } +} +ptEstimateLambda = function(Matrix[Double] x, String method) + return (Double lambda) +{ + lower = -2.0; + upper = 2.0; + + if (method == "box-cox") { + jacTerm = sum(log(x)) + } + else { + jacTerm = sum(sign(x) * log(abs(x) + 1.0)) + } + + lambda = ptBrentSearch(x, lower, upper, method, jacTerm); +} + +# Compute negative log likelihood; lower lambda score is better + +ptNegLogLikelihood = function( + Matrix[Double] x, + Double lambda, + String method, + Double jacTerm) + return (Double negLogLikelihood) +{ + eps = 1e-12 + if (method == "box-cox") { + if (abs(lambda) < eps) { + y = log(x) + } + else { + y = (x^lambda - 1.0) / lambda + } + } + else { + nonnegative = x >= 0 + xPos = ifelse(nonnegative, x, 0.0) + xNeg = ifelse(nonnegative, 0.0, x) + + if (abs(lambda) < eps) { + yPos = log(xPos + 1.0) + } + else { + yPos = ((xPos + 1.0)^lambda - 1.0) / lambda + } + + if (abs(lambda - 2.0) < eps) { + yNeg = -log(1.0 - xNeg) + } + else { + yNeg = -((1.0 - xNeg)^(2.0 - lambda) - 1.0) / (2.0 - lambda) + } + + y = ifelse(nonnegative, yPos, yNeg) + } + + n = nrow(x); + yMean = mean(y); + yVariance = sum((y - yMean)^2) / n; + + if (is.na(yVariance) | is.infinite(yVariance) | yVariance <= 0.0) { + negLogLikelihood = 1e300 + } + else { + logLikelihood = -n / 2.0 * log(yVariance) + (lambda - 1.0) * jacTerm; + negLogLikelihood = -logLikelihood; + if (is.na(negLogLikelihood) | is.infinite(negLogLikelihood)) { + negLogLikelihood = 1e300 + } + } +} + +# Minimize the negative log likelihood with Brent optimization +ptBrentSearch = function( + Matrix[Double] x, + Double lower, + Double upper, + String method, + Double jacTerm) + return (Double lambdaOptimal) +{ + # Expand the initial interval until it brackets a minimum + goldenRatio = 1.618034; + maxBracketIterations = 1000; + lowerScore = ptNegLogLikelihood(x, lower, method, jacTerm); + upperScore = ptNegLogLikelihood(x, upper, method, jacTerm); + + lambdaOptimal = 1.0 + bestScore = ptNegLogLikelihood(x, lambdaOptimal, method, jacTerm) + if (lowerScore < bestScore) { + lambdaOptimal = lower + bestScore = lowerScore + } + if (upperScore < bestScore) { + lambdaOptimal = upper + bestScore = upperScore + } + + if (lowerScore < upperScore) { + xa = upper; + fa = upperScore; + xb = lower; + fb = lowerScore; + } + else { + xa = lower; + fa = lowerScore; + xb = upper; + fb = upperScore; + } + + initialXc = xb + goldenRatio * (xb - xa); + initialFc = ptNegLogLikelihood(x, initialXc, method, jacTerm); + xc = initialXc; + fc = initialFc; + if (fc < bestScore) { + lambdaOptimal = xc + bestScore = fc + } + bracketIteration = 0; + while ((fc < fb) & (bracketIteration < maxBracketIterations)) { + nextXc = xc + goldenRatio * (xc - xb); + nextFc = ptNegLogLikelihood(x, nextXc, method, jacTerm); + xa = xb; + fa = fb; + xb = xc; + fb = fc; + xc = nextXc; + fc = nextFc; + if (fc < bestScore) { + lambdaOptimal = xc + bestScore = fc + } + bracketIteration = bracketIteration + 1; + } + + validBracket = bracketIteration < maxBracketIterations & + (((fb < fa) & (fb <= fc)) | ((fb <= fa) & (fb < fc))) + + if (validBracket) { + a = min(xa, xc); + b = max(xa, xc); + + goldenMean = 0.3819660112501051; + sqrtEpsilon = sqrt(2.2e-16); + tolerance = 1.48e-8; + maxIterations = 500; + + xf = a + goldenMean * (b - a); + nfc = xf; + fulc = xf; + fx = ptNegLogLikelihood(x, xf, method, jacTerm); + fnfc = fx; + ffulc = fx; + if (fx < bestScore) { + lambdaOptimal = xf + bestScore = fx + } + + rat = 0.0; + e = 0.0; + midpoint = 0.5 * (a + b); + tol1 = sqrtEpsilon * abs(xf) + tolerance / 3.0; + tol2 = 2.0 * tol1; + + iteration = 0; + while ((abs(xf - midpoint) > (tol2 - 0.5 * (b - a))) & + (iteration < maxIterations)) { + goldenStep = TRUE; + + if (abs(e) > tol1) { + goldenStep = FALSE; + r = (xf - nfc) * (fx - ffulc); + q = (xf - fulc) * (fx - fnfc); + p = (xf - fulc) * q - (xf - nfc) * r; + q = 2.0 * (q - r); + + if (q > 0.0) { + p = -p; + } + + q = abs(q); + previousE = e; + e = rat; + + if ((q > 0.0) & (abs(p) < abs(0.5 * q * previousE)) & + (p > q * (a - xf)) & (p < q * (b - xf))) { + rat = p / q; + candidate = xf + rat; + + if (((candidate - a) < tol2) | ((b - candidate) < tol2)) { + if (midpoint >= xf) { + rat = tol1; + } + else { + rat = -tol1; + } + } + } + else { + goldenStep = TRUE; + } + } + + if (goldenStep) { + if (xf >= midpoint) { + e = a - xf; + } + else { + e = b - xf; + } + rat = goldenMean * e; + } + + if (rat >= 0.0) { + candidate = xf + max(abs(rat), tol1); + } + else { + candidate = xf - max(abs(rat), tol1); + } + + fCandidate = ptNegLogLikelihood(x, candidate, method, jacTerm); + if (fCandidate < bestScore) { + lambdaOptimal = candidate + bestScore = fCandidate + } + + if (fCandidate <= fx) { + if (candidate >= xf) { + a = xf; + } + else { + b = xf; + } + + fulc = nfc; + ffulc = fnfc; + nfc = xf; + fnfc = fx; + xf = candidate; + fx = fCandidate; + } + else { + if (candidate < xf) { + a = candidate; + } + else { + b = candidate; + } + + if ((fCandidate <= fnfc) | (nfc == xf)) { + fulc = nfc; + ffulc = fnfc; + nfc = candidate; + fnfc = fCandidate; + } + else if ((fCandidate <= ffulc) | (fulc == xf) | (fulc == nfc)) { + fulc = candidate; + ffulc = fCandidate; + } + } + + midpoint = 0.5 * (a + b); + tol1 = sqrtEpsilon * abs(xf) + tolerance / 3.0; + tol2 = 2.0 * tol1; + iteration = iteration + 1; + } + } +} diff --git a/scripts/builtin/powerTransformApply.dml b/scripts/builtin/powerTransformApply.dml new file mode 100644 index 00000000000..4d8e2797068 --- /dev/null +++ b/scripts/builtin/powerTransformApply.dml @@ -0,0 +1,131 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# Applies a fitted power transformation and optional standardization. +# Transforms each feature using its previously estimated lambda and scaling parameters. +# +# INPUT: +# ------------------------------------------------------------------------------------- +# X Input feature matrix of shape n-by-m +# lambdas Precomputed lambda parameters of shape 1-by-m, one per column +# means Transformed column means of shape 1-by-m; empty to skip standardization +# scales Transformed column scales of shape 1-by-m; empty to skip standardization +# method Power transformation method: "yeo-johnson" (default) or "box-cox" +# ------------------------------------------------------------------------------------- +# +# OUTPUT: +# ------------------------------------------------------------------------------------- +# Y Power-transformed matrix of shape n-by-m +# ------------------------------------------------------------------------------------- + + +m_powerTransformApply = function( + Matrix[Double] X, + Matrix[Double] lambdas, + Matrix[Double] means, + Matrix[Double] scales, + String method="yeo-johnson") + return (Matrix[Double] Y) +{ + n = nrow(X) + m = ncol(X) + + if (method != "yeo-johnson" & method != "box-cox") { + stop("powerTransformApply: unsupported method '" + method + + "'; expected 'yeo-johnson' or 'box-cox'") + } + + validatedX = replace(target=X, pattern=NaN, replacement=1.0) + if (method == "box-cox" & min(validatedX) <= 0.0) { + stop("powerTransformApply: Box-Cox requires strictly positive input") + } + + if (nrow(lambdas) != 1 | ncol(lambdas) != m) { + stop("powerTransformApply: lambdas must have shape 1-by-ncol(X)") + } + + hasMeans = nrow(means) > 0 | ncol(means) > 0 + hasScales = nrow(scales) > 0 | ncol(scales) > 0 + + if (hasMeans != hasScales) { + stop("powerTransformApply: means and scales must either both be provided or both be empty") + } + + if (hasMeans & (nrow(means) != 1 | ncol(means) != m | + nrow(scales) != 1 | ncol(scales) != m)) { + stop("powerTransformApply: means and scales must have shape 1-by-ncol(X)") + } + + Y = matrix(0.0, rows=n, cols=m) + + # Handle boundary points (0 and 2) + eps = 1e-12 + + # Loop over columns for transformation + for (j in 1:m){ + x = X[,j] + nanMask = is.na(x) + lambda_j = as.scalar(lambdas[1,j]) + + if (method == "box-cox") { + x = replace(target=x, pattern=NaN, replacement=1.0) + if (abs(lambda_j) < eps) { + y = log(x) + } + else { + y = (x^lambda_j - 1.0) / lambda_j + } + } + else { + x = replace(target=x, pattern=NaN, replacement=0.0) + nonnegative = x >= 0 + + # Use intermediate inputs to avoid invalid domains in scoring or fractional powers + x_pos = ifelse(nonnegative, x, 0.0) + x_neg = ifelse(nonnegative, 0.0, x) + + # Transform nonnegative values (x>=0) + if (abs(lambda_j) < eps) { + y_pos = log(x_pos + 1) + } + else { + y_pos = ((x_pos + 1)^lambda_j - 1) / lambda_j + } + + # Transform negative values (x<0) + if (abs(lambda_j - 2) < eps) { + y_neg = -log(1 - x_neg) + } + else { + y_neg = -((1 - x_neg)^(2 - lambda_j) - 1) / (2 - lambda_j) + } + + # Combine the two branches + y = ifelse(nonnegative, y_pos, y_neg) + } + + Y[,j] = ifelse(nanMask, NaN, y) + } + + if (hasMeans) { + Y = (Y - means) / scales + } +} diff --git a/src/main/java/org/apache/sysds/common/Builtins.java b/src/main/java/org/apache/sysds/common/Builtins.java index c77a2e9d866..d6e1954bc16 100644 --- a/src/main/java/org/apache/sysds/common/Builtins.java +++ b/src/main/java/org/apache/sysds/common/Builtins.java @@ -274,6 +274,8 @@ public enum Builtins { PCAINVERSE("pcaInverse", true), PCATRANSFORM("pcaTransform", true), PNMF("pnmf", true), + POWERTRANSFORM("powerTransform", true), + POWERTRANSFORMAPPLY("powerTransformApply", true), PPCA("ppca", true), PPRED("ppred", false), PROD("prod", false), diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinPowerTransformTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinPowerTransformTest.java new file mode 100644 index 00000000000..8b020fe8d6b --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinPowerTransformTest.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.builtin.part2; + +import java.util.HashMap; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.sysds.common.Types.ExecMode; +import org.apache.sysds.common.Types.ExecType; +import org.apache.sysds.runtime.DMLScriptException; +import org.apache.sysds.runtime.matrix.data.MatrixValue.CellIndex; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; + +public class BuiltinPowerTransformTest extends AutomatedTestBase { + private static final String TRANSFORM_TEST_NAME = "powerTransform"; + private static final String APPLY_TEST_NAME = "powerTransformApply"; + private static final String TEST_DIR = "functions/builtin/"; + private static final String TEST_CLASS_DIR = TEST_DIR + BuiltinPowerTransformTest.class.getSimpleName() + "/"; + + private static final double REFERENCE_EPS = 1e-4; + private static final double APPLY_EPS = 1e-9; + + @Override + public void setUp() { + addTestConfiguration(TRANSFORM_TEST_NAME, + new TestConfiguration(TEST_CLASS_DIR, TRANSFORM_TEST_NAME, new String[] {"Y", "L", "S"})); + addTestConfiguration(APPLY_TEST_NAME, + new TestConfiguration(TEST_CLASS_DIR, APPLY_TEST_NAME, new String[] {"Y"})); + } + + @Test + public void testPowerTransformYeoJohnsonDefaultDenseCP() { + runPowerTransformYeoJohnsonDefaultDenseTest(ExecType.CP); + } + + @Test + public void testPowerTransformYeoJohnsonDefaultDenseSpark() { + runPowerTransformYeoJohnsonDefaultDenseTest(ExecType.SPARK); + } + + private void runPowerTransformYeoJohnsonDefaultDenseTest(ExecType execType) { + double[][] input = {{-2, 1, 5}, {-1, 1, 5}, {0, 2, 5}, {1, 3, 5}, {2, 6, 5}, {4, 12, 5}}; + runPowerTransformTest(execType, "default", true, input, false); + } + + @Test + public void testPowerTransformBoxCoxUnstandardizedDenseCP() { + double[][] input = {{1.0, 1.0}, {2.0, 1.1}, {3.0, 1.2}, {4.0, 1.3}, {5.0, 1.4}, {6.0, 1.5}, {7.0, 2.0}, + {8.0, 8.0}}; + runPowerTransformTest("box-cox", false, input, false); + } + + @Test + public void testPowerTransformYeoJohnsonLambdaAboveInitialInterval() { + double[][] input = {{0.00}, {0.97}, {0.98}, {0.99}, {1.00}}; + runPowerTransformTest("yeo-johnson", false, input, false); + assertLambdaOutsideInitialInterval(true); + } + + @Test + public void testPowerTransformYeoJohnsonPreservesNaNCP() { + double[][] input = {{-2, 1}, {-1, Double.NaN}, {Double.NaN, 2}, {1, 4}, {2, 8}}; + runPowerTransformTest("yeo-johnson", true, input, false, true, false); + assertNaNPositions(input); + } + + @Test + public void testPowerTransformBoxCoxLambdaBelowInitialInterval() { + double[][] input = {{1.00}, {1.01}, {1.02}, {1.03}, {10.0}}; + runPowerTransformTest("box-cox", false, input, false); + assertLambdaOutsideInitialInterval(false); + } + + @Test + public void testPowerTransformBoxCoxFallsBackToFiniteLambdaCP() { + double[][] input = {{1e-100}, {1e-50}, {1.0}, {1e50}, {1e100}}; + runPowerTransformTest("box-cox", false, input, false, false); + double lambda = readDMLMatrixFromOutputDir("L").get(new CellIndex(1, 1)); + Assert.assertTrue(Double.isFinite(lambda)); + assertNaNPositions(input); + } + + @Test + public void testPowerTransformBoxCoxRejectsNonPositiveInput() { + double[][] input = {{0, 1}, {1, 2}}; + runPowerTransformTest("box-cox", false, input, true); + } + + @Test + public void testPowerTransformApplyYeoJohnsonDenseCP() { + runPowerTransformApplyYeoJohnsonDenseTest(ExecType.CP); + } + + @Test + public void testPowerTransformApplyYeoJohnsonDenseSpark() { + runPowerTransformApplyYeoJohnsonDenseTest(ExecType.SPARK); + } + + private void runPowerTransformApplyYeoJohnsonDenseTest(ExecType execType) { + double[][] input = {{-2, -2, -2}, {-1, -1, -1}, {0, 0, 0}, {1, 1, 1}, {2, 2, 2}}; + double[][] expected = {{-3, -1.5, -1.03944491546724}, {-1.33333333333333, -1, -0.877258872223978}, + {-0.333333333333333, -0.5, -0.6}, {0.128764787039964, 0, 0}, {0.399074859112073, 0.5, 1}}; + runPowerTransformApplyTest(execType, "yeo-johnson", true, input, expected, false); + } + + @Test + public void testPowerTransformApplyBoxCoxDenseCP() { + double[][] input = {{0.5, 0.5, 0.5}, {1.0, 1.0, 1.0}, {2.0, 2.0, 2.0}, {4.0, 4.0, 4.0}, {8.0, 8.0, 8.0}}; + double[][] expected = {{-0.693147180559945, -0.5, -0.375}, {0, 0, 0}, {0.693147180559945, 1, 1.5}, + {1.38629436111989, 3, 7.5}, {2.07944154167984, 7, 31.5}}; + runPowerTransformApplyTest(ExecType.CP, "box-cox", false, input, expected, false); + } + + @Test + public void testPowerTransformApplyBoxCoxPreservesNaNCP() { + double[][] input = {{0.5, Double.NaN, 0.5}, {Double.NaN, 1.0, 1.0}, {2.0, 2.0, Double.NaN}}; + double[][] expected = {{-0.693147180559945, Double.NaN, -0.375}, {Double.NaN, 0, 0}, + {0.693147180559945, 1, Double.NaN}}; + runPowerTransformApplyTest(ExecType.CP, "box-cox", false, input, expected, false); + } + + @Test + public void testPowerTransformApplyBoxCoxRejectsNonPositiveInput() { + double[][] input = {{0, 1, 2}, {1, 2, 3}}; + runPowerTransformApplyTest(ExecType.CP, "box-cox", false, input, null, true); + } + + private void runPowerTransformTest(String method, boolean standardize, double[][] input, boolean shouldFail) { + runPowerTransformTest(ExecType.CP, method, standardize, input, shouldFail, true, true); + } + + private void runPowerTransformTest(ExecType execType, String method, boolean standardize, double[][] input, + boolean shouldFail) { + runPowerTransformTest(execType, method, standardize, input, shouldFail, true, true); + } + + private void runPowerTransformTest(String method, boolean standardize, double[][] input, boolean shouldFail, + boolean compareReference) { + runPowerTransformTest(ExecType.CP, method, standardize, input, shouldFail, compareReference, true); + } + + private void runPowerTransformTest(String method, boolean standardize, double[][] input, boolean shouldFail, + boolean compareReference, boolean compareTransformed) { + runPowerTransformTest(ExecType.CP, method, standardize, input, shouldFail, compareReference, + compareTransformed); + } + + private void runPowerTransformTest(ExecType execType, String method, boolean standardize, double[][] input, + boolean shouldFail, boolean compareReference, boolean compareTransformed) { + ExecMode oldExecMode = setExecMode(execType); + + try { + loadTestConfiguration(getTestConfiguration(TRANSFORM_TEST_NAME)); + + String home = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = home + TRANSFORM_TEST_NAME + ".dml"; + fullRScriptName = home + TRANSFORM_TEST_NAME + ".R"; + programArgs = new String[] {"-args", input("X"), output("Y"), output("L"), output("S"), method, + Boolean.toString(standardize)}; + + if(compareReference) { + String referenceMethod = method.equals("default") ? "yeo-johnson" : method; + rCmd = getRCmd(inputDir(), expectedDir(), referenceMethod, Boolean.toString(standardize)); + } + + writeInputMatrixWithMTD("X", input, true); + runTest(true, shouldFail, shouldFail ? DMLScriptException.class : null, -1); + if(shouldFail) + return; + + if(compareReference) { + runRScript(true); + if(compareTransformed) + compareOutput("Y", REFERENCE_EPS); + compareOutput("L", REFERENCE_EPS); + compareOutput("S", REFERENCE_EPS); + } + } + catch(Exception exception) { + throw new RuntimeException(exception); + } + finally { + resetExecMode(oldExecMode); + } + } + + private void runPowerTransformApplyTest(ExecType execType, String method, boolean standardize, double[][] input, + double[][] expected, boolean shouldFail) { + ExecMode oldExecMode = setExecMode(execType); + + try { + loadTestConfiguration(getTestConfiguration(APPLY_TEST_NAME)); + + String home = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = home + APPLY_TEST_NAME + ".dml"; + programArgs = new String[] {"-args", input("X"), input("L"), input("M"), input("S"), output("Y"), method, + Boolean.toString(standardize)}; + + double[][] L = {{0, 1, 2}}; + double[][] means = {{0.5, 1.0, 1.5}}; + double[][] scales = {{1.5, 2.0, 2.5}}; + + writeInputMatrixWithMTD("X", input, true); + writeInputMatrixWithMTD("L", L, true); + writeInputMatrixWithMTD("M", means, true); + writeInputMatrixWithMTD("S", scales, true); + if(!shouldFail) + writeExpectedMatrix("Y", expected); + + runTest(true, shouldFail, shouldFail ? DMLScriptException.class : null, -1); + if(shouldFail) + return; + + compareResults(APPLY_EPS); + } + catch(Exception exception) { + throw new RuntimeException(exception); + } + finally { + resetExecMode(oldExecMode); + } + } + + private void compareOutput(String name, double tolerance) { + HashMap dmlResult = readDMLMatrixFromOutputDir(name); + HashMap rResult = readRMatrixFromExpectedDir(name); + TestUtils.compareMatrices(dmlResult, rResult, tolerance, "DML", "R"); + } + + private void assertLambdaOutsideInitialInterval(boolean above) { + double lambda = readDMLMatrixFromOutputDir("L").get(new CellIndex(1, 1)); + Assert.assertTrue("Expected lambda outside the initial interval, but was " + lambda, + above ? lambda > 2.0 : lambda < -2.0); + } + + private void assertNaNPositions(double[][] input) { + HashMap output = readDMLMatrixFromOutputDir("Y"); + for(int i = 0; i < input.length; i++) { + for(int j = 0; j < input[i].length; j++) { + double value = output.getOrDefault(new CellIndex(i + 1, j + 1), 0.0); + Assert.assertEquals(Double.isNaN(input[i][j]), Double.isNaN(value)); + } + } + } +} diff --git a/src/test/scripts/functions/builtin/powerTransform.R b/src/test/scripts/functions/builtin/powerTransform.R new file mode 100644 index 00000000000..251fb139911 --- /dev/null +++ b/src/test/scripts/functions/builtin/powerTransform.R @@ -0,0 +1,71 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +library("Matrix") +suppressPackageStartupMessages(library("recipes")) + +args <- commandArgs(TRUE) +X <- as.matrix(readMM(paste(args[1], "X.mtx", sep = ""))) +method <- args[3] +standardize <- as.logical(args[4]) + +colnames(X) <- paste0("V", seq_len(ncol(X))) +data <- as.data.frame(X) +transform <- recipe(~ ., data = data) + +if (method == "box-cox") { + transform <- step_BoxCox( + transform, + all_numeric(), + limits = c(-20, 20), + num_unique = 2 + ) +} else { + transform <- step_YeoJohnson( + transform, + all_numeric(), + limits = c(-20, 20), + num_unique = 2 + ) +} + +fitted <- prep(transform, training = data) +estimates <- tidy(fitted, number = 1) +lambdas <- matrix(1.0, nrow = 1, ncol = ncol(X)) +lambdas[1, match(estimates$terms, colnames(X))] <- estimates$value +Y <- as.matrix(bake(fitted, new_data = data)) + +if (standardize) { + observed <- colSums(!is.na(Y)) + means <- colMeans(Y, na.rm = TRUE) + means[is.nan(means)] <- 0 + centered <- sweep(Y, 2, means) + scales <- sqrt(colSums(centered^2, na.rm = TRUE) / observed) + scales[observed == 0 | scales == 0 | is.nan(scales)] <- 1 + Y <- sweep(centered, 2, scales, "/") + state <- rbind(means, scales) +} else { + state <- matrix(0.0, nrow = 2, ncol = ncol(X)) +} + +writeMM(as(Y, "CsparseMatrix"), paste(args[2], "Y", sep = "")) +writeMM(as(lambdas, "CsparseMatrix"), paste(args[2], "L", sep = "")) +writeMM(as(state, "CsparseMatrix"), paste(args[2], "S", sep = "")) diff --git a/src/test/scripts/functions/builtin/powerTransform.dml b/src/test/scripts/functions/builtin/powerTransform.dml new file mode 100644 index 00000000000..d7dfba4759d --- /dev/null +++ b/src/test/scripts/functions/builtin/powerTransform.dml @@ -0,0 +1,48 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = read($1); + +if ($5 == "default") { + [Y, lambdas, means, scales] = powerTransform(X=X); + standardize = TRUE; +} +else { + standardize = as.boolean($6); + [Y, lambdas, means, scales] = powerTransform( + X=X, + method=$5, + standardize=standardize + ); +} + +if (standardize) { + state = rbind(means, scales); +} +else { + assert(nrow(means) == 0 & ncol(means) == 0); + assert(nrow(scales) == 0 & ncol(scales) == 0); + state = matrix(0.0, rows=2, cols=ncol(X)); +} + +write(Y, $2); +write(lambdas, $3); +write(state, $4); diff --git a/src/test/scripts/functions/builtin/powerTransformApply.dml b/src/test/scripts/functions/builtin/powerTransformApply.dml new file mode 100644 index 00000000000..87615b8daef --- /dev/null +++ b/src/test/scripts/functions/builtin/powerTransformApply.dml @@ -0,0 +1,33 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = read($1); +lambdas = read($2); +means = read($3); +scales = read($4); + +if (!as.boolean($7)) { + means = matrix(0.0, rows=0, cols=0); + scales = matrix(0.0, rows=0, cols=0); +} + +Y = powerTransformApply(X, lambdas, means, scales, $6); +write(Y, $5); diff --git a/src/test/scripts/installDependencies.R b/src/test/scripts/installDependencies.R index 60642fa8ed4..0bbe8960682 100644 --- a/src/test/scripts/installDependencies.R +++ b/src/test/scripts/installDependencies.R @@ -52,6 +52,7 @@ custom_install("boot"); custom_install("matrixStats"); custom_install("outliers"); custom_install("caret"); +custom_install("recipes"); custom_install("sigmoid"); custom_install("DescTools"); custom_install("mice"); From b810e54bc930f90fb42f3c9c40a0087199ec3c3d Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:53:13 +0200 Subject: [PATCH 117/132] [SYSTEMDS-3891] Add Generic OOC Reduce Assisted-by: AI --- .../instructions/ooc/TSMMOOCInstruction.java | 27 ++-- .../runtime/ooc/cache/OOCCacheManager.java | 3 +- .../ooc/cache/legacy/OOCCacheScheduler.java | 2 +- .../cache/legacy/OOCLRUCacheScheduler.java | 12 +- .../runtime/ooc/memory/CachedAllowance.java | 22 +-- .../ooc/memory/InMemoryQueueCallback.java | 47 +++--- .../ooc/primitives/ReduceOOCPrimitive.java | 146 ++++++++++++++++++ .../runtime/ooc/util/OOCInstructionUtils.java | 7 + .../sysds/runtime/ooc/util/OOCUtils.java | 2 +- .../test/component/ooc/OOCPrimitiveTest.java | 24 +++ .../component/ooc/StateTableUtilsTest.java | 4 +- .../ooc/memory/OOCMemoryAllowanceTest.java | 27 ++-- 12 files changed, 247 insertions(+), 76 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/primitives/ReduceOOCPrimitive.java diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/TSMMOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/TSMMOOCInstruction.java index 37b2ba93a77..0707601b12f 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/TSMMOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/TSMMOOCInstruction.java @@ -28,7 +28,6 @@ import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; import org.apache.sysds.runtime.controlprogram.context.ExecutionContext; -import org.apache.sysds.runtime.controlprogram.parfor.LocalTaskQueue; import org.apache.sysds.runtime.functionobjects.Multiply; import org.apache.sysds.runtime.functionobjects.Plus; import org.apache.sysds.runtime.instructions.InstructionUtils; @@ -41,6 +40,7 @@ import org.apache.sysds.runtime.matrix.operators.AggregateOperator; import org.apache.sysds.runtime.matrix.operators.BinaryOperator; import org.apache.sysds.runtime.matrix.operators.Operator; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; public class TSMMOOCInstruction extends ComputationOOCInstruction { private final MMTSJType _type; @@ -109,24 +109,15 @@ public void processInstruction(ExecutionContext ec) { } private void processSingleOutputTileInstruction(ExecutionContext ec, MatrixObject min) { - OOCStream qIn = min.getStreamHandle(); + OOCStream out = createWritableStream(); + ec.getMatrixObject(output).setStreamHandle(out); BinaryOperator plus = InstructionUtils.parseBinaryOperator(Opcodes.PLUS.toString()); - MatrixBlock resultBlock = null; - - OOCStream tmpStream = createWritableStream(); - mapOOC(qIn, tmpStream, - tmp -> ((MatrixBlock) tmp.getValue()) - .transposeSelfMatrixMultOperations(new MatrixBlock(), _type)); - - MatrixBlock tmp; - while((tmp = tmpStream.dequeue()) != LocalTaskQueue.NO_MORE_TASKS) { - if(resultBlock == null) - resultBlock = tmp; - else - resultBlock.binaryOperationsInPlace(plus, tmp); - } - - ec.setMatrixOutput(output.getName(), resultBlock); + OOCInstructionUtils.reduce(min.getStreamable(), out, + value -> new IndexedMatrixValue(new MatrixIndexes(1, 1), + ((MatrixBlock) value.getValue()).transposeSelfMatrixMultOperations(new MatrixBlock(), _type)), + (left, right) -> new IndexedMatrixValue(new MatrixIndexes(1, 1), + ((MatrixBlock) left.getValue()).binaryOperationsInPlace(plus, right.getValue())), + value -> ((MatrixBlock) value.getValue()).getExactSerializedSize(), getContext()); } private long getJoinIndex(IndexedMatrixValue value) { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java index 5cec3ae981d..4bde5c9df4d 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCCacheManager.java @@ -281,7 +281,8 @@ public static boolean canClaimMemory() { return getCache().isWithinLimits() && OOCInstruction.getComputeInFlight() <= OOCInstruction.getComputeBackpressureThreshold(); } - public static OOCCacheScheduler.HandoverHandle handover(BlockKey key, InMemoryQueueCallback callback) { + public static OOCCacheScheduler.HandoverHandle handover(BlockKey key, + InMemoryQueueCallback callback) { return getCache().handover(key, callback); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java index ad161e95303..d8ff79dd797 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java @@ -106,7 +106,7 @@ interface HandoverHandle { OOCStream.QueueCallback reclaim(); } - HandoverHandle handover(BlockKey key, InMemoryQueueCallback callback); + HandoverHandle handover(BlockKey key, InMemoryQueueCallback callback); /** * Places a new source-backed block in the cache and registers the location with the IO handler. The entry is diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java index 3f5601adbae..96de368ccc9 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java @@ -293,7 +293,7 @@ public BlockEntry putAndPin(BlockKey key, Object data, long size) { } @Override - public HandoverHandle handover(BlockKey key, InMemoryQueueCallback callback) { + public HandoverHandle handover(BlockKey key, InMemoryQueueCallback callback) { if(!this._running) throw new IllegalStateException("Cache scheduler has been shut down."); PendingHandover handover = new PendingHandover(key, callback); @@ -1085,7 +1085,7 @@ private void registerWaiter(BlockKey key, DeferredReadRequest request, int index } private boolean commitHandover(PendingHandover pending) { - InMemoryQueueCallback callback = pending.takeForCommit(); + InMemoryQueueCallback callback = pending.takeForCommit(); if(callback == null) return false; try { @@ -1135,12 +1135,12 @@ private DeferredReadWaiter(DeferredReadRequest request, int index) { private static class PendingHandover implements HandoverHandle { private final BlockKey _key; private final CompletableFuture _completionFuture; - private InMemoryQueueCallback _callback; + private InMemoryQueueCallback _callback; private boolean _committed; private boolean _cancelled; private boolean _committing; - private PendingHandover(BlockKey key, InMemoryQueueCallback callback) { + private PendingHandover(BlockKey key, InMemoryQueueCallback callback) { _key = key; _completionFuture = new CompletableFuture<>(); _callback = callback; @@ -1180,11 +1180,11 @@ private synchronized boolean isCancelled() { return _cancelled; } - private synchronized InMemoryQueueCallback takeForCommit() { + private synchronized InMemoryQueueCallback takeForCommit() { if(_committed || _cancelled || _committing) return null; _committing = true; - InMemoryQueueCallback callback = _callback; + InMemoryQueueCallback callback = _callback; _callback = null; return callback; } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/memory/CachedAllowance.java b/src/main/java/org/apache/sysds/runtime/ooc/memory/CachedAllowance.java index ffba3910b26..d2375b80c1e 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/memory/CachedAllowance.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/memory/CachedAllowance.java @@ -56,12 +56,12 @@ public CachedAllowance(MemoryBroker broker) { _handoverSchedulingRequested = false; } - public void handover(InMemoryQueueCallback callback, int index) { + public void handover(InMemoryQueueCallback callback, int index) { if(callback == null) throw new IllegalArgumentException("Cannot hand over null callback."); callback.transferOwnershipBlocking(this); - InMemoryQueueCallback root = (InMemoryQueueCallback) callback.keepOpen(); + InMemoryQueueCallback root = callback.keepOpen(); callback.close(); root.getHandle().attachCachedAllowance(this, index); @@ -88,7 +88,7 @@ public OOCStream.QueueCallback tryGet(int index) { while(true) { BlockKey cacheKey = null; OOCCacheScheduler.HandoverHandle handover = null; - InMemoryQueueCallback local = null; + InMemoryQueueCallback local = null; synchronized(entry) { if(entry._local != null && entry._handover == null) @@ -124,7 +124,7 @@ else if(entry._cacheKey != null) if(!future.isDone()) return null; boolean committed = future.join(); - InMemoryQueueCallback localToClose = null; + InMemoryQueueCallback localToClose = null; synchronized(entry) { if(entry._handover != handover) continue; @@ -171,8 +171,8 @@ public CompletableFuture> get(int in throw DMLRuntimeException.of(ex.getCause() == null ? ex : ex.getCause()); return committed == true; }).thenCompose(committed -> { - InMemoryQueueCallback localToClose = null; - InMemoryQueueCallback local = null; + InMemoryQueueCallback localToClose = null; + InMemoryQueueCallback local = null; BlockKey key; synchronized(entry) { @@ -215,7 +215,7 @@ public void clear(int index) { while(true) { OOCCacheScheduler.HandoverHandle handover = null; BlockKey forgetKey = null; - InMemoryQueueCallback localToClose = null; + InMemoryQueueCallback localToClose = null; synchronized(entry) { if(entry._local != null && entry._handover == null) { @@ -410,7 +410,7 @@ private long tryStartCacheHandover(SlotEntry entry) { if(bytes <= 0) return 0; - InMemoryQueueCallback retained = (InMemoryQueueCallback) entry._local.keepOpen(); + InMemoryQueueCallback retained = entry._local.keepOpen(); try { entry._cacheKey = new BlockKey(_streamId, _nextBlockId.getAndIncrement()); entry._handover = OOCCacheManager.handover(entry._cacheKey, retained); @@ -448,7 +448,7 @@ private void finishPendingHandover(SlotEntry entry) { onFinishedHandover(bytes); } - private void closeRoot(InMemoryQueueCallback local) { + private void closeRoot(InMemoryQueueCallback local) { local.getHandle().detachCachedAllowance(); local.close(); } @@ -486,12 +486,12 @@ private void ensureCapacity(int index) { } private static final class SlotEntry { - private InMemoryQueueCallback _local; + private InMemoryQueueCallback _local; private BlockKey _cacheKey; private OOCCacheScheduler.HandoverHandle _handover; private long _pendingBytes; - private SlotEntry(InMemoryQueueCallback local) { + private SlotEntry(InMemoryQueueCallback local) { _local = local; } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/memory/InMemoryQueueCallback.java b/src/main/java/org/apache/sysds/runtime/ooc/memory/InMemoryQueueCallback.java index 7fafc042e41..ae712305cb7 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/memory/InMemoryQueueCallback.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/memory/InMemoryQueueCallback.java @@ -21,36 +21,38 @@ import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.instructions.ooc.OOCStream; -import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; - import java.util.concurrent.atomic.AtomicInteger; -public class InMemoryQueueCallback implements OOCStream.QueueCallback { - private CallbackHandle _handle; +public class InMemoryQueueCallback implements OOCStream.QueueCallback { + private CallbackHandle _handle; private boolean _closed; - public InMemoryQueueCallback(IndexedMatrixValue result, DMLRuntimeException failure, MemoryAllowance allow, - long reservedBytes) { - _handle = new CallbackHandle(result, failure, allow, reservedBytes); + public InMemoryQueueCallback(T result, DMLRuntimeException failure, MemoryAllowance allow, long reservedBytes) { + _handle = new CallbackHandle<>(result, failure, allow, reservedBytes); _closed = false; } - private InMemoryQueueCallback(CallbackHandle handle) { + public InMemoryQueueCallback(ManagedPayload payload) { + this(payload.value(), null, payload.owner(), payload.bytes()); + payload.transfer(); + } + + private InMemoryQueueCallback(CallbackHandle handle) { _handle = handle; _closed = false; } @Override - public IndexedMatrixValue get() { + public T get() { return _handle.get(); } @Override - public synchronized OOCStream.QueueCallback keepOpen() { + public synchronized InMemoryQueueCallback keepOpen() { if(_closed) throw new IllegalStateException("Cannot keep open a closed callback"); _handle._refCtr.incrementAndGet(); - return new InMemoryQueueCallback(_handle); + return new InMemoryQueueCallback<>(_handle); } @Override @@ -126,20 +128,19 @@ public boolean isFailure() { return _handle._failure != null; } - CallbackHandle getHandle() { + CallbackHandle getHandle() { return _handle; } - static final class CallbackHandle { - private volatile IndexedMatrixValue _result; + static final class CallbackHandle { + private volatile T _result; private final AtomicInteger _refCtr; private MemoryAllowance _allow; private long _reservedBytes; private volatile DMLRuntimeException _failure; private int _cacheIdx; - private CallbackHandle(IndexedMatrixValue result, DMLRuntimeException failure, MemoryAllowance allow, - long reservedBytes) { + private CallbackHandle(T result, DMLRuntimeException failure, MemoryAllowance allow, long reservedBytes) { _result = result; _failure = failure; _refCtr = new AtomicInteger(1); @@ -148,7 +149,7 @@ private CallbackHandle(IndexedMatrixValue result, DMLRuntimeException failure, M _cacheIdx = -1; } - private IndexedMatrixValue get() { + private T get() { if(_failure != null) throw _failure; return _result; @@ -174,8 +175,8 @@ boolean isExclusiveToRoot() { return _refCtr.get() == 1; } - private synchronized IndexedMatrixValue takeManagedResultForHandover() { - IndexedMatrixValue result = _result; + private synchronized T takeManagedResultForHandover() { + T result = _result; _result = null; return result; } @@ -188,14 +189,14 @@ private void closeFinal() { } } - public IndexedMatrixValue takeManagedResultForHandover() { + public T takeManagedResultForHandover() { return _handle.takeManagedResultForHandover(); } - public synchronized ManagedPayload extractManagedPayload() { + public synchronized ManagedPayload extractManagedPayload() { if(_closed) throw new IllegalStateException("Cannot extract a managed payload from a closed callback."); - CallbackHandle handle = _handle; + CallbackHandle handle = _handle; synchronized(handle) { if(handle._failure != null) throw handle._failure; @@ -203,7 +204,7 @@ public synchronized ManagedPayload extractManagedPayload() { throw new IllegalStateException("Cannot extract a managed payload while callback aliases exist."); if(handle._cacheIdx >= 0) throw new IllegalStateException("Cannot extract a managed payload from a cached-slot callback."); - IndexedMatrixValue result = handle._result; + T result = handle._result; if(result == null) throw new IllegalStateException("Cannot extract a managed payload from an empty callback."); long bytes = handle._reservedBytes; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/ReduceOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/ReduceOOCPrimitive.java new file mode 100644 index 00000000000..9f5ee85feda --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/ReduceOOCPrimitive.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.primitives; + +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.ToLongFunction; + +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; +import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; +import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; +import org.apache.sysds.runtime.ooc.memory.ManagedPayload; +import org.apache.sysds.runtime.ooc.memory.ReservationBudget; +import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; +import org.apache.sysds.runtime.ooc.stream.AllocatedOOCStream; +import org.apache.sysds.runtime.ooc.stream.StreamContext; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; +import org.apache.sysds.runtime.ooc.util.OOCUtils; + +public final class ReduceOOCPrimitive extends OOCPrimitive { + private final OOCStreamable _input; + private final OOCStreamable _output; + private final Function _partial; + private final BiFunction _merge; + private final ToLongFunction _size; + private ManagedPayload _accumulator; + + public ReduceOOCPrimitive(OOCStreamable input, OOCStreamable output, Function partial, + BiFunction merge, ToLongFunction size, StreamContext context) { + super(context, input); + _input = input; + _output = output; + _partial = partial; + _merge = merge; + _size = size; + } + + @Override + protected void inferPatternsInternal() { + _pattern = OOCAccessPattern.ANY; + inferParentPatterns(); + } + + @Override + protected void requestPatternInternal(OOCAccessPattern accessPattern) { + _pattern = OOCAccessPattern.ANY; + } + + @Override + protected void startExecution() { + OOCStream input = getInputReadStream(0); + OOCStream output = _output.getWriteStream(); + long inputBytes = OOCUtils.estimateOutputTileBytes(_input.getDataCharacteristics()); + long outputBytes = OOCUtils.estimateOutputTileBytes(_output.getDataCharacteristics()); + long taskBytes = OOCCacheManager.getGlobalCache().maxPhysicalPinBytes(inputBytes) + 2 * outputBytes; + AllocatedOOCStream admitted = new AllocatedOOCStream<>(input, _allowance, ignored -> taskBytes); + getContext().addOutStream(output); + OOCInstructionUtils.submitOOCTasks(admitted, callback -> { + ReservationBudget budget = null; + ManagedPayload partial = null; + try { + budget = AllocatedOOCStream.detachBudget(callback).enableReuse(); + O value = _partial.apply(callback.get()); + long bytes = _size.applyAsLong(value); + budget.reserveBlocking(bytes); + partial = new ManagedPayload<>(value, bytes, budget); + synchronized(this) { + if(_accumulator != null) { + value = _merge.apply(_accumulator.value(), partial.value()); + bytes = _size.applyAsLong(value); + budget.reserveBlocking(bytes); + _accumulator.release(); + partial.release(); + partial = new ManagedPayload<>(value, bytes, budget); + } + _accumulator = partial; + partial = null; + budget.close(); + budget = null; + } + } + catch(Throwable error) { + fail(error); + throw DMLRuntimeException.of(error); + } + finally { + if(partial != null) + partial.release(); + if(budget != null) + budget.close(); + } + }, getContext()).thenRun(() -> { + ManagedPayload result; + synchronized(this) { + result = _accumulator; + _accumulator = null; + } + try { + if(hasFailed()) { + if(result != null) + result.release(); + return; + } + if(result == null) + throw new DMLRuntimeException("Cannot reduce an empty OOC stream"); + OOCStream.QueueCallback callback = new InMemoryQueueCallback<>(result); + try { + output.enqueue(callback); + callback = null; + } + finally { + if(callback != null) + callback.close(); + } + output.closeInput(); + } + catch(Throwable error) { + if(result != null) + result.release(); + fail(error); + } + finally { + onComplete(); + } + }); + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java index 4f2bcc34853..3eedb32b8e6 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java @@ -31,6 +31,7 @@ import java.util.function.Function; import java.util.function.Supplier; import java.util.function.ToIntFunction; +import java.util.function.ToLongFunction; import org.apache.sysds.api.DMLScript; import org.apache.sysds.runtime.DMLRuntimeException; @@ -47,6 +48,7 @@ import org.apache.sysds.runtime.ooc.primitives.JoinOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.MappingOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.PlannableDataGenOOCPrimitive; +import org.apache.sysds.runtime.ooc.primitives.ReduceOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.TransposeOOCPrimitive; import org.apache.sysds.runtime.ooc.stats.OOCEventLog; import org.apache.sysds.runtime.ooc.store.MaterializedStore; @@ -114,6 +116,11 @@ public static void groupedReduceIndexed(OOCStreamable input, output.assignPrimitive(new GroupedReduceOOCPrimitive(input, output, grouping, partial, merge, finish, context)); } + public static void reduce(OOCStreamable input, OOCStream output, Function partial, + BiFunction merge, ToLongFunction size, StreamContext context) { + output.assignPrimitive(new ReduceOOCPrimitive<>(input, output, partial, merge, size, context)); + } + public static int getComputeInFlight() { return COMPUTE_IN_FLIGHT.get(); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java index 7981736753f..c1a0e6439c1 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java @@ -163,7 +163,7 @@ public static void enqueueExact(OOCStream out, IndexedMatrix OOCStream.QueueCallback callback = null; try { budget.reserveBlocking(bytes); - callback = new InMemoryQueueCallback(value, null, budget, bytes); + callback = new InMemoryQueueCallback<>(value, null, budget, bytes); budget.close(); out.enqueue(callback); callback = null; diff --git a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java index c94e0ac5f1d..336a7aecfd0 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java @@ -188,6 +188,30 @@ public void testDataGenMapTransposePipeline() { values); } + @Test + public void testReduce() { + SubscribableTaskQueue input = new SubscribableTaskQueue<>(); + SubscribableTaskQueue output = new SubscribableTaskQueue<>(); + input.setData(new MatrixObject(ValueType.FP64, "/dev/null", + new MetaDataFormat(new MatrixCharacteristics(2, 3, 1), FileFormat.BINARY))); + output.setData(new MatrixObject(ValueType.FP64, "/dev/null", + new MetaDataFormat(new MatrixCharacteristics(1, 1, 1), FileFormat.BINARY))); + for(long[] indexes : List.of(new long[] {2, 3}, new long[] {1, 1}, new long[] {2, 1}, new long[] {1, 3}, + new long[] {1, 2}, new long[] {2, 2})) + input.enqueue(new IndexedMatrixValue(new MatrixIndexes(indexes[0], indexes[1]), + new MatrixBlock(1, 1, indexes[0] * 10d + indexes[1]))); + input.closeInput(); + OOCInstructionUtils.reduce(input, output, value -> new MatrixBlock(1, 1, 2 * value.getValue().get(0, 0)), + (left, right) -> new MatrixBlock(1, 1, left.get(0, 0) + right.get(0, 0)), + MatrixBlock::getExactSerializedSize, new StreamContext()); + + output.start(); + try(OOCStream.QueueCallback callback = output.dequeueCB()) { + Assert.assertEquals(204, callback.get().get(0, 0), 0); + } + Assert.assertNull(output.dequeueCB()); + } + @Test public void testGroupedReduceModes() { Assert.assertEquals(Map.of("1,1", 136d, "2,1", 166d), diff --git a/src/test/java/org/apache/sysds/test/component/ooc/StateTableUtilsTest.java b/src/test/java/org/apache/sysds/test/component/ooc/StateTableUtilsTest.java index 5ecbb070b48..7e75a4d52ec 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/StateTableUtilsTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/StateTableUtilsTest.java @@ -84,7 +84,7 @@ public void testCallbackPutOrTake() throws Exception { _producer.reserveBlocking(TILE_BYTES); StateTableUtils.Match referenced = StateTableUtils - .putOrTake(_table, 0, new InMemoryQueueCallback(tile(2.0), null, _producer, TILE_BYTES), _reader) + .putOrTake(_table, 0, new InMemoryQueueCallback<>(tile(2.0), null, _producer, TILE_BYTES), _reader) .get(WAIT_SECONDS, TimeUnit.SECONDS); Assert.assertNotNull(referenced); try(OOCStream.QueueCallback left = referenced.left(); @@ -95,7 +95,7 @@ public void testCallbackPutOrTake() throws Exception { _producer.reserveBlocking(TILE_BYTES); Assert.assertNull(StateTableUtils - .putOrTake(_table, 1, new InMemoryQueueCallback(tile(3.0), null, _producer, TILE_BYTES), _reader) + .putOrTake(_table, 1, new InMemoryQueueCallback<>(tile(3.0), null, _producer, TILE_BYTES), _reader) .get(WAIT_SECONDS, TimeUnit.SECONDS)); StateTableUtils.Match copied = StateTableUtils .putOrTake(_table, 1, new OOCStream.SimpleQueueCallback<>(tile(4.0), null), _reader) diff --git a/src/test/java/org/apache/sysds/test/component/ooc/memory/OOCMemoryAllowanceTest.java b/src/test/java/org/apache/sysds/test/component/ooc/memory/OOCMemoryAllowanceTest.java index 8660252e634..5986f6521db 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/memory/OOCMemoryAllowanceTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/memory/OOCMemoryAllowanceTest.java @@ -223,7 +223,7 @@ public long testNew(boolean optimal) { OOCStream leftStream = new SubscribableTaskQueue<>(); OOCStream rightStream = new SubscribableTaskQueue<>(); - OOCStream outStream = new SubscribableTaskQueue<>(); + OOCStream> outStream = new SubscribableTaskQueue<>(); long startMillis = System.currentTimeMillis(); @@ -248,31 +248,31 @@ public long testNew(boolean optimal) { rightStream.closeInput(); }).start(); - OOCStream leftStreamOut = new SubscribableTaskQueue<>(); - OOCStream leftStreamOutOut = new SubscribableTaskQueue<>(); - OOCStream rightStreamOut = new SubscribableTaskQueue<>(); + OOCStream> leftStreamOut = new SubscribableTaskQueue<>(); + OOCStream> leftStreamOutOut = new SubscribableTaskQueue<>(); + OOCStream> rightStreamOut = new SubscribableTaskQueue<>(); test.map(leftStream, leftStreamOut, i -> { var imv = new IndexedMatrixValue(new MatrixIndexes(i.longValue(), 1L), new MatrixBlock(1000, 1, 5.0)); - return new InMemoryQueueCallback(imv, null, leftAllowance, 8 * 1000); + return new InMemoryQueueCallback<>(imv, null, leftAllowance, 8 * 1000); }); test.map(leftStreamOut, leftStreamOutOut, cb -> { try(cb) { var imv = new IndexedMatrixValue(cb.get().getIndexes(), cb.get().getValue() .scalarOperations(new RightScalarOperator(Plus.getPlusFnObject(), 2.0), new MatrixBlock())); - return new InMemoryQueueCallback(imv, null, leftAllowance, 8 * 1000); + return new InMemoryQueueCallback<>(imv, null, leftAllowance, 8 * 1000); } }); test.map(rightStream, rightStreamOut, i -> { var imv = new IndexedMatrixValue(new MatrixIndexes(i.longValue(), 1L), new MatrixBlock(1000, 1, 3.0)); - return new InMemoryQueueCallback(imv, null, rightAllowance, 8 * 1000); + return new InMemoryQueueCallback<>(imv, null, rightAllowance, 8 * 1000); }); test.join(leftStreamOutOut, rightStreamOut, outStream, () -> joinAllowance.reserveBlocking(8 * 1000), cache, (l, r) -> { var imv = new IndexedMatrixValue(l.getIndexes(), ((MatrixBlock)l.getValue()).binaryOperations(new BinaryOperator( Plus.getPlusFnObject()), r.getValue())); - return new InMemoryQueueCallback(imv, null, joinAllowance, 8 * 1000); + return new InMemoryQueueCallback<>(imv, null, joinAllowance, 8 * 1000); }); CompletableFuture future = new CompletableFuture<>(); @@ -283,7 +283,7 @@ public long testNew(boolean optimal) { future.complete(null); return; } - InMemoryQueueCallback inner = cb.get(); + InMemoryQueueCallback inner = cb.get(); try(cb; inner) { ctr.incrementAndGet(); double checksum =((MatrixBlock)inner.get().getValue()).sum(); @@ -394,14 +394,15 @@ public CompletableFuture joinOOC(OOCStream l, OOCStrea return super.joinOOC(l, r, out, joinFn, IndexedMatrixValue::getIndexes); } - public CompletableFuture join(OOCStream l, OOCStream r, - OOCStream out, Runnable memoryReserver, CachedAllowance cache, - BiFunction joinFn) { + public CompletableFuture join(OOCStream> l, + OOCStream> r, + OOCStream> out, Runnable memoryReserver, CachedAllowance cache, + BiFunction> joinFn) { OOCStream, OOCStream.QueueCallback, Integer>> intermediate = createWritableStream(); new Thread(() -> { - InMemoryQueueCallback next; + InMemoryQueueCallback next; IndexedMatrixValue nextValue; boolean nextLeft = true; AtomicInteger pendingRequests = new AtomicInteger(1); From 88309f0dc62add9dd2a303fe2324c0b8dd6b8146 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:08:01 +0200 Subject: [PATCH 118/132] [SYSTEMDS-3891] Add NaryJoinOOCPrimitive and Generic Join Assisted-by: AI --- .../ooc/TernaryOOCInstruction.java | 32 ++ .../spark/data/IndexedMatrixValue.java | 6 + .../sysds/runtime/ooc/cache/OOCFuture.java | 52 +++ .../runtime/ooc/cache/io/SpillableObject.java | 1 + .../runtime/ooc/cache/packed/PackedBlock.java | 5 + .../ooc/primitives/JoinOOCPrimitive.java | 153 +++++---- .../ooc/primitives/NaryJoinOOCPrimitive.java | 307 ++++++++++++++++++ .../sysds/runtime/ooc/store/StateTable.java | 10 + .../runtime/ooc/util/OOCInstructionUtils.java | 37 ++- .../runtime/ooc/util/StateTableUtils.java | 78 +++-- .../test/component/ooc/OOCPrimitiveTest.java | 37 +++ .../test/functions/ooc/TernaryMatrixTest.java | 105 ++++++ .../scripts/functions/ooc/TernaryMatrix.dml | 36 ++ 13 files changed, 768 insertions(+), 91 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/primitives/NaryJoinOOCPrimitive.java create mode 100644 src/test/java/org/apache/sysds/test/functions/ooc/TernaryMatrixTest.java create mode 100644 src/test/scripts/functions/ooc/TernaryMatrix.dml diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/TernaryOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/TernaryOOCInstruction.java index 7b91b16d237..b0c8aaf27a5 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/TernaryOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/TernaryOOCInstruction.java @@ -26,6 +26,11 @@ import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; import org.apache.sysds.runtime.controlprogram.context.ExecutionContext; import org.apache.sysds.runtime.functionobjects.IfElse; +import org.apache.sysds.runtime.functionobjects.Minus; +import org.apache.sysds.runtime.functionobjects.MinusMultiply; +import org.apache.sysds.runtime.functionobjects.Multiply; +import org.apache.sysds.runtime.functionobjects.Plus; +import org.apache.sysds.runtime.functionobjects.PlusMultiply; import org.apache.sysds.runtime.instructions.InstructionUtils; import org.apache.sysds.runtime.instructions.cp.CPOperand; import org.apache.sysds.runtime.instructions.cp.ScalarObject; @@ -33,6 +38,7 @@ import org.apache.sysds.runtime.instructions.cp.StringObject; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.operators.BinaryOperator; import org.apache.sysds.runtime.matrix.operators.Operator; import org.apache.sysds.runtime.matrix.operators.TernaryOperator; import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; @@ -158,6 +164,32 @@ private void processThreeMatrixInstruction(ExecutionContext ec) { OOCStream qOut = createWritableStream(); ec.getMatrixObject(output).setStreamHandle(qOut); + if(m1.getDataCharacteristics().dimsKnown() && m2.getDataCharacteristics().dimsKnown() && + m3.getDataCharacteristics().dimsKnown()) { + TernaryOperator operator = (TernaryOperator) _optr; + if(operator.fn instanceof PlusMultiply || operator.fn instanceof MinusMultiply) { + OOCStream product = createWritableStream(); + BinaryOperator multiply = new BinaryOperator(Multiply.getMultiplyFnObject()); + BinaryOperator combine = operator.fn instanceof PlusMultiply ? new BinaryOperator( + Plus.getPlusFnObject()) : new BinaryOperator(Minus.getMinusFnObject()); + OOCInstructionUtils.equiJoin(m2.getStreamable(), m3.getStreamable(), product, + (left, right) -> left.binaryOperations(multiply, right, new MatrixBlock()), getContext()); + OOCInstructionUtils.equiJoin(m1.getStreamable(), product, qOut, + (left, right) -> left.binaryOperations(combine, right, new MatrixBlock()), getContext()); + return; + } + if(operator.fn instanceof IfElse) { + OOCInstructionUtils.naryEquiJoin(List.of(m1.getStreamable(), m2.getStreamable(), m3.getStreamable()), + qOut, + blocks -> new IndexedMatrixValue(blocks.get(0).getIndexes(), + ((MatrixBlock) blocks.get(0).getValue()).ternaryOperations(operator, + (MatrixBlock) blocks.get(1).getValue(), (MatrixBlock) blocks.get(2).getValue(), + new MatrixBlock())), + getContext()); + return; + } + } + List> streams = List.of( m1.getStreamHandle(), m2.getStreamHandle(), m3.getStreamHandle()); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java index bd96bbb614f..2f83caa5526 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java @@ -97,6 +97,12 @@ public boolean tryWrite(DataOutput dataOutput) throws IOException { return true; } + @Override + public long size() { + MatrixBlock block = (MatrixBlock) _value; + return Math.max(block.getExactSerializedSize(), block.getInMemorySize()); + } + @Override public void discard() { _value = null; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java index d6796e35a87..0e9504ce08f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java @@ -19,10 +19,16 @@ package org.apache.sysds.runtime.ooc.cache; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; @@ -51,6 +57,52 @@ public static OOCFuture failed(Throwable error) { return future; } + public static OOCFuture> allOf(List> futures, + Consumer failureCleanup) { + Objects.requireNonNull(futures); + Objects.requireNonNull(failureCleanup); + if(futures.isEmpty()) + return completed(List.of()); + OOCFuture> result = new OOCFuture<>(); + Object[] values = new Object[futures.size()]; + AtomicInteger remaining = new AtomicInteger(futures.size()); + AtomicReference firstError = new AtomicReference<>(); + for(int i = 0; i < futures.size(); i++) { + int index = i; + futures.get(i).whenComplete((value, error) -> { + values[index] = value; + if(error != null) + firstError.compareAndSet(null, error); + if(remaining.decrementAndGet() != 0) + return; + Throwable failure = firstError.get(); + List completed = new ArrayList<>(values.length); + for(Object item : values) { + @SuppressWarnings("unchecked") + T typed = (T) item; + completed.add(typed); + } + if(failure == null) { + result.complete(Collections.unmodifiableList(completed)); + return; + } + for(T item : completed) { + if(item == null) + continue; + try { + failureCleanup.accept(item); + } + catch(Throwable cleanupError) { + if(cleanupError != failure) + failure.addSuppressed(cleanupError); + } + } + result.completeExceptionally(failure); + }); + } + return result; + } + public boolean complete(T value) { return finish(value, null); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java index 434f70601d6..a93b1d18f2a 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java @@ -26,6 +26,7 @@ public interface SpillableObject { boolean tryWrite(DataOutput out) throws IOException; void read(DataInput in) throws IOException; + long size(); default void discard() { } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedBlock.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedBlock.java index 5727e2eea63..5cbb8976c33 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedBlock.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/packed/PackedBlock.java @@ -57,6 +57,11 @@ public boolean tryWrite(DataOutput out) throws IOException { return true; } + @Override + public long size() { + return totalSize; + } + @Override public void read(DataInput in) throws IOException { int count = in.readInt(); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/JoinOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/JoinOOCPrimitive.java index 0ec802b181c..bcc74cf21a7 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/primitives/JoinOOCPrimitive.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/JoinOOCPrimitive.java @@ -22,16 +22,18 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; +import java.util.function.ToIntFunction; +import java.util.function.ToLongFunction; import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.instructions.ooc.CachingStream; import org.apache.sysds.runtime.instructions.ooc.OOCStream; import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; -import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; -import org.apache.sysds.runtime.matrix.data.MatrixBlock; -import org.apache.sysds.runtime.ooc.cache.OOCFuture; import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; +import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; import org.apache.sysds.runtime.ooc.memory.ReservationBudget; import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; import org.apache.sysds.runtime.ooc.store.StateTable; @@ -40,24 +42,29 @@ import org.apache.sysds.runtime.ooc.util.OOCUtils; import org.apache.sysds.runtime.ooc.util.StateTableUtils; -public class JoinOOCPrimitive extends OOCPrimitive { - private final OOCStreamable _left; - private final OOCStreamable _right; - private final OOCStreamable _output; - private final BiFunction _operation; +public class JoinOOCPrimitive extends OOCPrimitive { + private final OOCStreamable _output; + private final ToIntFunction _leftKey; + private final ToIntFunction _rightKey; + private final ToLongFunction _outputSize; + private final BiFunction _operation; + private final long _taskBytes; private final AtomicInteger _pending = new AtomicInteger(1); private final AtomicInteger _unmatched = new AtomicInteger(); private final CompletableFuture _pendingCompletion = new CompletableFuture<>(); - private StateTable _table; + private StateTable _table; + private OOCStream _outputStream; - public JoinOOCPrimitive(OOCStreamable left, OOCStreamable right, - OOCStreamable output, BiFunction operation, - StreamContext context) { + public JoinOOCPrimitive(OOCStreamable left, OOCStreamable right, OOCStreamable output, + ToIntFunction leftKey, ToIntFunction rightKey, ToLongFunction outputSize, + BiFunction operation, long taskBytes, StreamContext context) { super(context, left, right); - _left = left; - _right = right; _output = output; + _leftKey = leftKey; + _rightKey = rightKey; + _outputSize = outputSize; _operation = operation; + _taskBytes = taskBytes; } @Override @@ -80,44 +87,34 @@ protected void requestPatternInternal(OOCAccessPattern accessPattern) { @Override protected void startExecution() { - OOCStream left = getInputReadStream(0); - OOCStream right = getInputReadStream(1); + OOCStream left = getInputReadStream(0); + OOCStream right = getInputReadStream(1); _table = new StateTable<>(OOCCacheManager.getGlobalCache(), CachingStream._streamSeq.getNextID()); - OOCStream output = _output.getWriteStream(); + _outputStream = _output.getWriteStream(); OOCStream matches = new SubscribableTaskQueue<>(); - long inputBytes = Math.max(OOCUtils.estimateOutputTileBytes(_left.getDataCharacteristics()), - OOCUtils.estimateOutputTileBytes(_right.getDataCharacteristics())); - long outputBytes = OOCUtils.estimateOutputTileBytes(_output.getDataCharacteristics()); - long taskBytes = outputBytes + 2 * inputBytes; - - getContext().addOutStream(output); - CompletableFuture processing = OOCInstructionUtils.submitCloseableOOCTasks(matches, (JoinWork work) -> { - IndexedMatrixValue mleft = work._left.get(); - IndexedMatrixValue mright = work._right.get(); - OOCUtils.enqueueExact(output, new IndexedMatrixValue(mleft.getIndexes(), - _operation.apply((MatrixBlock) mleft.getValue(), (MatrixBlock) mright.getValue())), work._budget); - }, getContext()); + + getContext().addOutStream(_outputStream); + CompletableFuture processing = OOCInstructionUtils.submitCloseableOOCTasks(matches, this::process, + getContext()); CompletableFuture.allOf(processing, _pendingCompletion).thenRun(() -> { try { _table.close(); onComplete(); } finally { - output.closeInput(); + _outputStream.closeInput(); } }); - OOCInstructionUtils.submitOOCTask(() -> drive(left, right, matches, taskBytes), - new StreamContext().addOutStream(output)); + OOCInstructionUtils.submitOOCTask(() -> drive(left, right, matches), + new StreamContext().addOutStream(_outputStream)); } - private void drive(OOCStream leftInput, OOCStream rightInput, - OOCStream matches, long taskBytes) { - long cols = _right.getDataCharacteristics().getNumColBlocks(); + private void drive(OOCStream leftInput, OOCStream rightInput, OOCStream matches) { try { while(true) { - OOCStream.QueueCallback left = leftInput.dequeueCB(); - OOCStream.QueueCallback right = rightInput.dequeueCB(); + OOCStream.QueueCallback left = leftInput.dequeueCB(); + OOCStream.QueueCallback right = rightInput.dequeueCB(); boolean leftEos = left == null || left.isEos(); boolean rightEos = right == null || right.isEos(); if(leftEos || rightEos) { @@ -129,8 +126,8 @@ private void drive(OOCStream leftInput, OOCStream leftInput, OOCStream callback, boolean left, long cols, long taskBytes, + @SuppressWarnings("unchecked") + private void accept(OOCStream.QueueCallback callback, boolean left, int key, OOCStream matches) { - if(callback == null) - return; - OOCStream.QueueCallback owned = null; ReservationBudget budget = null; boolean pending = false; + boolean handedOff = false; try { - owned = callback.keepOpen(); - callback.close(); - callback = null; - budget = OOCUtils.reserveBudget(_allowance, taskBytes); - IndexedMatrixValue value = owned.get(); - long row = value.getIndexes().getRowIndex() - 1; - long col = value.getIndexes().getColumnIndex() - 1; - int slot = Math.toIntExact(row * cols + col); + budget = OOCUtils.reserveBudget(_allowance, _taskBytes); _pending.incrementAndGet(); pending = true; - OOCFuture future = StateTableUtils.putOrTake(_table, slot, owned, budget); - owned = null; + OOCFuture> future = StateTableUtils.putOrTake(_table, key, + (OOCStream.QueueCallback) callback, budget); + handedOff = true; ReservationBudget pendingBudget = budget; budget = null; future.whenComplete((match, error) -> matchReady(match, left, pendingBudget, error, matches)); pending = false; } finally { + if(!handedOff) + callback.close(); if(pending) completePending(matches); - if(callback != null) - callback.close(); - if(owned != null) - owned.close(); if(budget != null) budget.close(); } } - private void matchReady(StateTableUtils.Match match, boolean left, ReservationBudget budget, Throwable error, - OOCStream matches) { + private void matchReady(StateTableUtils.Match match, boolean incomingLeft, + ReservationBudget budget, Throwable error, OOCStream matches) { JoinWork work = null; try { if(error != null) @@ -190,8 +178,7 @@ private void matchReady(StateTableUtils.Match match, boolean left, ReservationBu return; } _unmatched.decrementAndGet(); - work = left ? new JoinWork(match.left(), match.right(), budget) : new JoinWork(match.right(), match.left(), - budget); + work = new JoinWork(match.left(), match.right(), incomingLeft, budget); match = null; budget = null; matches.enqueue(work); @@ -213,6 +200,26 @@ private void matchReady(StateTableUtils.Match match, boolean left, ReservationBu } } + @SuppressWarnings("unchecked") + private void process(JoinWork work) { + SpillableObject incoming = work._incoming.get(); + SpillableObject existing = work._existing.get(); + L left = (L) (work._incomingLeft ? incoming : existing); + R right = (R) (work._incomingLeft ? existing : incoming); + O value = _operation.apply(left, right); + long bytes = _outputSize.applyAsLong(value); + work._budget.reserveBlocking(bytes); + OOCStream.QueueCallback callback = new InMemoryQueueCallback<>(value, null, work._budget, bytes); + try { + _outputStream.enqueue(callback); + callback = null; + } + finally { + if(callback != null) + callback.close(); + } + } + private void completePending(OOCStream matches) { if(_pending.decrementAndGet() != 0) return; @@ -233,22 +240,28 @@ private void completePending(OOCStream matches) { } } - private static final class JoinWork implements AutoCloseable { - private final OOCStream.QueueCallback _left; - private final OOCStream.QueueCallback _right; + private final class JoinWork implements AutoCloseable { + private final OOCStream.QueueCallback _incoming; + private final OOCStream.QueueCallback _existing; + private final boolean _incomingLeft; private final ReservationBudget _budget; - private JoinWork(OOCStream.QueueCallback left, - OOCStream.QueueCallback right, ReservationBudget budget) { - _left = left; - _right = right; + private JoinWork(OOCStream.QueueCallback incoming, + OOCStream.QueueCallback existing, boolean incomingLeft, ReservationBudget budget) { + _incoming = incoming; + _existing = existing; + _incomingLeft = incomingLeft; _budget = budget; } @Override public void close() { - try(_left; _right; _budget) { - // Release + try { + _incoming.close(); + _existing.close(); + } + finally { + _budget.close(); } } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/NaryJoinOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/NaryJoinOOCPrimitive.java new file mode 100644 index 00000000000..f239641f36e --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/NaryJoinOOCPrimitive.java @@ -0,0 +1,307 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.primitives; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.function.ToIntFunction; +import java.util.function.ToLongFunction; + +import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; +import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; +import org.apache.sysds.runtime.ooc.memory.ReservationBudget; +import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; +import org.apache.sysds.runtime.ooc.store.StateTable; +import org.apache.sysds.runtime.ooc.store.StoreLease; +import org.apache.sysds.runtime.ooc.stream.StreamContext; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; +import org.apache.sysds.runtime.ooc.util.OOCUtils; +import org.apache.sysds.runtime.ooc.util.StateTableUtils; + +public final class NaryJoinOOCPrimitive extends OOCPrimitive { + private final List> _inputs; + private final OOCStreamable _output; + private final ToIntFunction _key; + private final ToLongFunction _size; + private final Function, IndexedMatrixValue> _operation; + private final long _storeTaskBytes; + private final long _joinTaskBytes; + private final AtomicInteger _active = new AtomicInteger(1); + private final CompletableFuture _activeCompletion = new CompletableFuture<>(); + private StateTable _table; + private OOCStream _ready; + private OOCStream _outputStream; + + public NaryJoinOOCPrimitive(List> inputs, + OOCStreamable output, ToIntFunction key, + ToLongFunction size, Function, IndexedMatrixValue> operation, + long storeTaskBytes, long joinTaskBytes, StreamContext context) { + super(context, inputs.toArray(OOCStreamable[]::new)); + if(inputs.size() < 2) + throw new IllegalArgumentException("N-ary join requires at least two inputs."); + _inputs = inputs; + _output = output; + _key = key; + _size = size; + _operation = operation; + _storeTaskBytes = storeTaskBytes; + _joinTaskBytes = joinTaskBytes; + } + + @Override + protected void inferPatternsInternal() { + _pattern = OOCAccessPattern.ANY; + for(OOCPrimitive child : getChildren()) + _pattern = _pattern.fused(child.getAccessPattern()); + if(_pattern.isPlannable() && _pattern != OOCAccessPattern.ANY) + for(OOCPrimitive child : getChildren()) + child.requestPattern(_pattern); + inferParentPatterns(); + } + + @Override + protected void requestPatternInternal(OOCAccessPattern accessPattern) { + _pattern = accessPattern; + for(OOCPrimitive child : getChildren()) + child.requestPattern(accessPattern); + } + + @Override + protected void startExecution() { + List> inputs = new ArrayList<>(_inputs.size()); + for(int i = 0; i < _inputs.size(); i++) + inputs.add(getInputReadStream(i)); + + int groups = (int) OOCUtils.getNumBlocks(inputs.get(0).getDataCharacteristics()); + _table = new StateTable<>(groups * inputs.size()); + _outputStream = _output.getWriteStream(); + _ready = new SubscribableTaskQueue<>(); + getContext().addOutStream(_outputStream, _ready); + CompletableFuture processing = OOCInstructionUtils.submitCloseableOOCTasks(_ready, this::process, + getContext()); + CompletableFuture.allOf(processing, _activeCompletion).whenComplete((ignored, error) -> { + if(error != null) + fail(error); + try { + _outputStream.closeInput(); + } + catch(Throwable failure) { + fail(failure); + } + finally { + onComplete(); + } + }); + OOCInstructionUtils.submitOOCTask(() -> drive(inputs), new StreamContext().addOutStream(_outputStream)); + } + + private void drive(List> inputs) { + try { + byte[] groupCtr = new byte[(int) OOCUtils.getNumBlocks(inputs.get(0).getDataCharacteristics())]; + int n = inputs.size(); + int unmatchedGroups = 0; + while(true) { + List> callbacks = new ArrayList<>(n); + try { + int eos = 0; + for(OOCStream input : inputs) { + OOCStream.QueueCallback callback = input.dequeueCB(); + callbacks.add(callback); + if(callback == null || callback.isEos()) + eos++; + } + if(eos != 0) { + if(eos != n) + throw new DMLRuntimeException("Join inputs contain a different number of blocks"); + if(unmatchedGroups != 0) + throw new DMLRuntimeException("Join inputs contain unmatched blocks"); + break; + } + + for(int i = 0; i < n; i++) { + OOCStream.QueueCallback callback = callbacks.get(i); + int group = _key.applyAsInt(callback.get()); + int count = ++groupCtr[group]; + if(count == 1) + unmatchedGroups++; + if(count == n) { + unmatchedGroups--; + onJoinGroupAvailable(group, callback, i, n); + } + else { + ReservationBudget budget = OOCUtils.reserveBudget(_allowance, _storeTaskBytes); + try { + StateTableUtils.put(_table, group * n + i, callback, budget); + } + finally { + budget.close(); + } + } + } + } + finally { + for(OOCStream.QueueCallback callback : callbacks) + if(callback != null) + callback.close(); + } + } + } + catch(Throwable failure) { + fail(failure); + throw DMLRuntimeException.of(failure); + } + finally { + completeActive(); + } + } + + private void onJoinGroupAvailable(int group, OOCStream.QueueCallback callback, + int callbackIndex, int n) { + ReservationBudget budget = null; + OOCStream.QueueCallback anchor = null; + boolean active = false; + try { + budget = OOCUtils.reserveBudget(_allowance, _joinTaskBytes); + anchor = callback.keepOpen(); + _active.incrementAndGet(); + active = true; + List>> futures = new ArrayList<>(n - 1); + try { + for(int i = 0; i < n; i++) + if(i != callbackIndex) + futures.add(_table.take(group * n + i, budget).map(lease -> { + if(lease == null) + throw new DMLRuntimeException("Join input block is missing"); + return lease; + })); + } + catch(Throwable failure) { + futures.add(OOCFuture.failed(failure)); + } + OOCFuture>> leases = OOCFuture.allOf(futures, StoreLease::close); + OOCStream.QueueCallback pendingAnchor = anchor; + ReservationBudget pendingBudget = budget; + anchor = null; + budget = null; + active = false; + leases.whenComplete( + (values, error) -> onJoinReady(pendingAnchor, callbackIndex, values, pendingBudget, error)); + } + finally { + if(anchor != null) + anchor.close(); + if(budget != null) + budget.close(); + if(active) + completeActive(); + } + } + + private void onJoinReady(OOCStream.QueueCallback anchor, int anchorIndex, + List> leases, ReservationBudget budget, Throwable error) { + JoinWork work = null; + try { + if(error != null) + throw DMLRuntimeException.of(error); + work = new JoinWork(anchor, anchorIndex, leases, budget); + _ready.enqueue(work); + work = null; + } + catch(Throwable failure) { + fail(failure); + } + finally { + if(work != null) + work.close(); + if(error != null) { + anchor.close(); + budget.close(); + } + completeActive(); + } + } + + private void process(JoinWork work) { + List values = new ArrayList<>(work._leases.size() + 1); + int lease = 0; + for(int i = 0; i <= work._leases.size(); i++) + values.add(i == work._anchorIndex ? work._anchor.get() : work._leases.get(lease++).value()); + IndexedMatrixValue output = _operation.apply(values); + long bytes = _size.applyAsLong(output); + work._budget.reserveBlocking(bytes); + OOCStream.QueueCallback callback = new InMemoryQueueCallback<>(output, null, work._budget, + bytes); + try { + _outputStream.enqueue(callback); + callback = null; + } + finally { + if(callback != null) + callback.close(); + } + } + + private void completeActive() { + if(_active.decrementAndGet() != 0) + return; + try { + _table.close(); + try { + _ready.closeInput(); + } + catch(IllegalStateException ignored) { + } + } + finally { + _activeCompletion.complete(null); + } + } + + private static final class JoinWork implements AutoCloseable { + private final OOCStream.QueueCallback _anchor; + private final int _anchorIndex; + private final List> _leases; + private final ReservationBudget _budget; + + private JoinWork(OOCStream.QueueCallback anchor, int anchorIndex, + List> leases, ReservationBudget budget) { + _anchor = anchor; + _anchorIndex = anchorIndex; + _leases = leases; + _budget = budget; + } + + @Override + public void close() { + _anchor.close(); + for(StoreLease lease : _leases) + lease.close(); + _budget.close(); + } + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/store/StateTable.java b/src/main/java/org/apache/sysds/runtime/ooc/store/StateTable.java index b70a7f4988a..2abdd329667 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/store/StateTable.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/store/StateTable.java @@ -19,9 +19,11 @@ package org.apache.sysds.runtime.ooc.store; +import org.apache.sysds.runtime.instructions.ooc.CachingStream; import org.apache.sysds.runtime.ooc.cache.BlockEntry; import org.apache.sysds.runtime.ooc.cache.BlockKey; import org.apache.sysds.runtime.ooc.cache.OOCCache; +import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; import org.apache.sysds.runtime.ooc.cache.OOCFuture; import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; import org.apache.sysds.runtime.ooc.memory.ManagedPayload; @@ -49,6 +51,14 @@ public final class StateTable implements AutoCloseabl private volatile AtomicIntegerArray _generationSlots; private volatile boolean _closed; + public StateTable() { + this(OOCCacheManager.getGlobalCache(), CachingStream._streamSeq.getNextID()); + } + + public StateTable(int numSlots) { + this(OOCCacheManager.getGlobalCache(), CachingStream._streamSeq.getNextID(), numSlots); + } + public StateTable(OOCCache cache, long streamId) { this(cache, streamId, INITIAL_SLOTS); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java index 3eedb32b8e6..2d5fa50d787 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java @@ -40,13 +40,16 @@ import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; import org.apache.sysds.runtime.ooc.memory.ReservationBudget; import org.apache.sysds.runtime.ooc.primitives.BroadcastOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.GroupedReduceOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.JoinOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.MappingOOCPrimitive; +import org.apache.sysds.runtime.ooc.primitives.NaryJoinOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.PlannableDataGenOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.ReduceOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.TransposeOOCPrimitive; @@ -92,7 +95,39 @@ public static void transpose(OOCStreamable input, OOCStream< public static void equiJoin(OOCStreamable left, OOCStreamable right, OOCStream output, BiFunction operation, StreamContext context) { - output.assignPrimitive(new JoinOOCPrimitive(left, right, output, operation, context)); + long cols = right.getDataCharacteristics().getNumColBlocks(); + long inputBytes = Math.max(OOCUtils.estimateOutputTileBytes(left.getDataCharacteristics()), + OOCUtils.estimateOutputTileBytes(right.getDataCharacteristics())); + long outputBytes = OOCUtils.estimateOutputTileBytes(output.getDataCharacteristics()); + ToIntFunction key = value -> Math + .toIntExact((value.getIndexes().getRowIndex() - 1) * cols + value.getIndexes().getColumnIndex() - 1); + keyedJoin(left, right, output, key, key, value -> ((MatrixBlock) value.getValue()).getExactSerializedSize(), + (leftValue, rightValue) -> new IndexedMatrixValue(leftValue.getIndexes(), + operation.apply((MatrixBlock) leftValue.getValue(), (MatrixBlock) rightValue.getValue())), + inputBytes + OOCCacheManager.getGlobalCache().maxPhysicalPinBytes(inputBytes) + outputBytes, context); + } + + public static void keyedJoin(OOCStreamable left, + OOCStreamable right, OOCStream output, ToIntFunction leftKey, ToIntFunction rightKey, + ToLongFunction outputSize, BiFunction operation, long taskBytes, StreamContext context) { + output.assignPrimitive( + new JoinOOCPrimitive<>(left, right, output, leftKey, rightKey, outputSize, operation, taskBytes, context)); + } + + public static void naryEquiJoin(List> inputs, + OOCStream output, Function, IndexedMatrixValue> operation, + StreamContext context) { + long cols = inputs.get(0).getDataCharacteristics().getNumColBlocks(); + long inputBytes = inputs.stream().map(OOCStreamable::getDataCharacteristics) + .mapToLong(OOCUtils::estimateOutputTileBytes).max().orElse(0); + long outputBytes = OOCUtils.estimateOutputTileBytes(output.getDataCharacteristics()); + ToIntFunction key = value -> Math + .toIntExact((value.getIndexes().getRowIndex() - 1) * cols + value.getIndexes().getColumnIndex() - 1); + ToLongFunction size = value -> ((MatrixBlock) value.getValue()).getExactSerializedSize(); + long joinBytes = (inputs.size() - 1) * OOCCacheManager.getGlobalCache().maxPhysicalPinBytes(inputBytes) + + outputBytes; + output.assignPrimitive( + new NaryJoinOOCPrimitive(inputs, output, key, size, operation, inputBytes, joinBytes, context)); } public static void indexedBroadcastMap(OOCStreamable streamed, diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/StateTableUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/StateTableUtils.java index 641192c16a6..97978b7eaaa 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/util/StateTableUtils.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/StateTableUtils.java @@ -20,9 +20,8 @@ package org.apache.sysds.runtime.ooc.util; import org.apache.sysds.runtime.instructions.ooc.OOCStream; -import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; -import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; import org.apache.sysds.runtime.ooc.memory.InMemoryQueueCallback; import org.apache.sysds.runtime.ooc.memory.ManagedPayload; import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; @@ -31,24 +30,64 @@ import org.apache.sysds.runtime.ooc.store.StoreLease; public final class StateTableUtils { - public static OOCFuture putOrTake(StateTable table, int slot, - OOCStream.QueueCallback tile, MemoryAllowance allowance) { - if(tile instanceof MaterializedCallback pinned && pinned.pinnedEntry() != null) - return putReferenceOrTake(table, slot, pinned, allowance); - ManagedPayload payload; - if(tile instanceof InMemoryQueueCallback managed && managed.getManagedBytes() > 0) { + public static OOCFuture> take(StateTable table, int slot, + MemoryAllowance allowance) { + OOCFuture> future = table.take(slot, allowance); + OOCFuture> toReturn = new OOCFuture<>(); + future.whenComplete((l, err) -> { + if(err != null) + toReturn.completeExceptionally(err); + else + toReturn.complete(new MaterializedCallback<>(l)); + }); + return toReturn; + } + + public static void put(StateTable table, int slot, OOCStream.QueueCallback tile, + MemoryAllowance allowance) { + if(tile instanceof MaterializedCallback pinned && pinned.pinnedEntry() != null) { + table.putReference(slot, pinned.pinnedEntry()); + return; + } + ManagedPayload payload; + if(tile instanceof InMemoryQueueCallback managed && managed.getManagedBytes() > 0) + payload = managed.extractManagedPayload(); + else { + T value = tile.get(); + long bytes = value.size(); + allowance.reserveBlocking(bytes); + payload = new ManagedPayload<>(value, bytes, allowance); + } + try { + table.put(slot, payload); + } + catch(RuntimeException error) { + payload.release(); + throw error; + } + } + + public static OOCFuture> putOrTake(StateTable table, int slot, + OOCStream.QueueCallback tile, MemoryAllowance allowance) { + if(tile instanceof MaterializedCallback pinned && pinned.pinnedEntry() != null) { + MaterializedCallback retained = (MaterializedCallback) pinned.keepOpen(); + pinned.close(); + return putReferenceOrTake(table, slot, retained, allowance); + } + ManagedPayload payload; + if(tile instanceof InMemoryQueueCallback managed && managed.getManagedBytes() > 0) { payload = managed.extractManagedPayload(); managed.close(); } else { - IndexedMatrixValue value = tile.get(); - long bytes = ((MatrixBlock) value.getValue()).getExactSerializedSize(); + T value = tile.get(); + long bytes = value.size(); allowance.reserveBlocking(bytes); payload = new ManagedPayload<>(value, bytes, allowance); tile.close(); } - OOCFuture result = new OOCFuture<>(); - OOCFuture> matched; + OOCFuture> result = new OOCFuture<>(); + OOCFuture> matched; try { matched = table.putOrTake(slot, payload, allowance); } @@ -65,16 +104,16 @@ else if(lease == null) result.complete(null); else result.complete( - new Match(new MaterializedCallback<>(StoreLease.create(payload.value(), payload::release)), + new Match<>(new MaterializedCallback<>(StoreLease.create(payload.value(), payload::release)), new MaterializedCallback<>(lease))); }); return result; } - private static OOCFuture putReferenceOrTake(StateTable table, int slot, - MaterializedCallback pinned, MemoryAllowance allowance) { - OOCFuture result = new OOCFuture<>(); - OOCFuture> matched; + private static OOCFuture> putReferenceOrTake(StateTable table, int slot, + MaterializedCallback pinned, MemoryAllowance allowance) { + OOCFuture> result = new OOCFuture<>(); + OOCFuture> matched; try { matched = table.putReferenceOrTake(slot, pinned.pinnedEntry(), allowance); } @@ -92,12 +131,11 @@ else if(lease == null) { result.complete(null); } else - result.complete(new Match(pinned, new MaterializedCallback<>(lease))); + result.complete(new Match<>(pinned, new MaterializedCallback<>(lease))); }); return result; } - public record Match(OOCStream.QueueCallback left, - OOCStream.QueueCallback right) { + public record Match(OOCStream.QueueCallback left, OOCStream.QueueCallback right) { } } diff --git a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java index 336a7aecfd0..b408d3f1e9e 100644 --- a/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java +++ b/src/test/java/org/apache/sysds/test/component/ooc/OOCPrimitiveTest.java @@ -249,6 +249,43 @@ private static Map runGroupedReduce(GroupedReduceOOCPrimitive.Gr return values; } + @Test + public void testNaryJoinOutOfOrder() { + SubscribableTaskQueue first = new SubscribableTaskQueue<>(); + SubscribableTaskQueue second = new SubscribableTaskQueue<>(); + SubscribableTaskQueue third = new SubscribableTaskQueue<>(); + SubscribableTaskQueue output = new SubscribableTaskQueue<>(); + for(SubscribableTaskQueue stream : List.of(first, second, third, output)) + stream.setData(new MatrixObject(ValueType.FP64, "/dev/null", + new MetaDataFormat(new MatrixCharacteristics(1, 2, 1), FileFormat.BINARY))); + CachingStream cachedSecond = new CachingStream(second); + first.enqueue(new IndexedMatrixValue(new MatrixIndexes(1, 1), new MatrixBlock(1, 1, 10d))); + first.enqueue(new IndexedMatrixValue(new MatrixIndexes(1, 2), new MatrixBlock(1, 1, 20d))); + second.enqueue(new IndexedMatrixValue(new MatrixIndexes(1, 2), new MatrixBlock(1, 1, 2d))); + second.enqueue(new IndexedMatrixValue(new MatrixIndexes(1, 1), new MatrixBlock(1, 1, 1d))); + third.enqueue(new IndexedMatrixValue(new MatrixIndexes(1, 1), new MatrixBlock(1, 1, 100d))); + third.enqueue(new IndexedMatrixValue(new MatrixIndexes(1, 2), new MatrixBlock(1, 1, 200d))); + first.closeInput(); + second.closeInput(); + third.closeInput(); + + OOCInstructionUtils.naryEquiJoin(List.of(first, cachedSecond, third), output, + blocks -> new IndexedMatrixValue(blocks.get(0).getIndexes(), + new MatrixBlock(1, 1, blocks.get(0).getValue().get(0, 0) + 10 * blocks.get(1).getValue().get(0, 0) + + 100 * blocks.get(2).getValue().get(0, 0))), + new StreamContext()); + + output.start(); + Map values = new HashMap<>(); + OOCStream.QueueCallback callback; + while((callback = output.dequeueCB()) != null) + try(OOCStream.QueueCallback current = callback) { + values.put(current.get().getIndexes().getColumnIndex(), current.get().getValue().get(0, 0)); + } + Assert.assertEquals(Map.of(1L, 10020d, 2L, 20040d), values); + cachedSecond.scheduleDeletion(); + } + @Test public void testJoinOutOfOrder() { SubscribableTaskQueue left = new SubscribableTaskQueue<>(); diff --git a/src/test/java/org/apache/sysds/test/functions/ooc/TernaryMatrixTest.java b/src/test/java/org/apache/sysds/test/functions/ooc/TernaryMatrixTest.java new file mode 100644 index 00000000000..55791ec0160 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/ooc/TernaryMatrixTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.ooc; + +import java.io.IOException; + +import org.apache.sysds.common.Opcodes; +import org.apache.sysds.common.Types; +import org.apache.sysds.runtime.instructions.Instruction; +import org.apache.sysds.runtime.io.MatrixWriter; +import org.apache.sysds.runtime.io.MatrixWriterFactory; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.meta.MatrixCharacteristics; +import org.apache.sysds.runtime.util.DataConverter; +import org.apache.sysds.runtime.util.HDFSTool; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +public class TernaryMatrixTest extends AutomatedTestBase { + private static final String TEST_NAME = "TernaryMatrix"; + private static final String TEST_DIR = "functions/ooc/"; + private static final String TEST_CLASS_DIR = TEST_DIR + TernaryMatrixTest.class.getSimpleName() + "/"; + private static final int ROWS = 1200; + private static final int COLS = 1100; + private static final int BLOCK_SIZE = 1000; + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME)); + } + + @Test + public void testTernaryOperations() throws IOException { + Types.ExecMode oldPlatform = setExecMode(Types.ExecMode.SINGLE_NODE); + try { + getAndLoadTestConfiguration(TEST_NAME); + fullDMLScriptName = SCRIPT_DIR + TEST_DIR + TEST_NAME + ".dml"; + writeInput("A", MatrixBlock.randOperations(ROWS, COLS, 1, -1, 1, "uniform", 7)); + writeInput("B", MatrixBlock.randOperations(ROWS, COLS, 0.7, -2, 2, "uniform", 8)); + writeInput("C", MatrixBlock.randOperations(ROWS, COLS, 0.2, -3, 3, "uniform", 9)); + + String[] outputs = {"plus", "minus", "ifelse"}; + Opcodes[] opcodes = {Opcodes.PM, Opcodes.MINUSMULT, Opcodes.IFELSE}; + for(int i = 0; i < outputs.length; i++) { + programArgs = arguments(true, i + 1, outputs[i]); + runTest(true, false, null, -1); + Assert.assertTrue(heavyHittersContainsString(Instruction.OOC_INST_PREFIX + opcodes[i])); + + programArgs = arguments(false, i + 1, outputs[i] + "_target"); + runTest(true, false, null, -1); + MatrixBlock actual = DataConverter.readMatrixFromHDFS(output(outputs[i]), Types.FileFormat.BINARY, ROWS, + COLS, BLOCK_SIZE); + MatrixBlock expected = DataConverter.readMatrixFromHDFS(output(outputs[i] + "_target"), + Types.FileFormat.BINARY, ROWS, COLS, BLOCK_SIZE); + TestUtils.compareMatrices(actual, expected, 1e-8); + } + } + finally { + resetExecMode(oldPlatform); + } + } + + private String[] arguments(boolean ooc, int operation, String result) { + String[] args = new String[ooc ? 8 : 7]; + int offset = 0; + args[offset++] = "-stats"; + if(ooc) + args[offset++] = "-ooc"; + args[offset++] = "-args"; + args[offset++] = input("A"); + args[offset++] = input("B"); + args[offset++] = input("C"); + args[offset++] = Integer.toString(operation); + args[offset] = output(result); + return args; + } + + private void writeInput(String name, MatrixBlock value) throws IOException { + MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(Types.FileFormat.BINARY); + writer.writeMatrixToHDFS(value, input(name), ROWS, COLS, BLOCK_SIZE, value.getNonZeros()); + HDFSTool.writeMetaDataFile(input(name + ".mtd"), Types.ValueType.FP64, + new MatrixCharacteristics(ROWS, COLS, BLOCK_SIZE, value.getNonZeros()), Types.FileFormat.BINARY); + } +} diff --git a/src/test/scripts/functions/ooc/TernaryMatrix.dml b/src/test/scripts/functions/ooc/TernaryMatrix.dml new file mode 100644 index 00000000000..aef0b901973 --- /dev/null +++ b/src/test/scripts/functions/ooc/TernaryMatrix.dml @@ -0,0 +1,36 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +A = read($1); +B = read($2); +C = read($3); + +if($4 == 1) { + result = A + 2 * B; +} +else if($4 == 2) { + result = A - 2 * B; +} +else { + result = ifelse(A > 0, B, C); +} + +write(result, $5, format="binary"); From 9b0584ce84ec5174f1ba05acf0aba0f807c84fb2 Mon Sep 17 00:00:00 2001 From: Jannik Lindemann <52833175+janniklinde@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:43:51 +0200 Subject: [PATCH 119/132] [SYSTEMDS-3891] Add GeneralMMultOOCPrimitive Assisted-by: AI --- .../instructions/ooc/MMultOOCInstruction.java | 39 +- .../sysds/runtime/ooc/cache/OOCFuture.java | 5 + .../runtime/ooc/planning/OOCStoreLayout.java | 8 +- .../primitives/GeneralMMultOOCPrimitive.java | 499 ++++++++++++++++++ .../runtime/ooc/util/OOCInstructionUtils.java | 9 + .../sysds/runtime/ooc/util/OOCUtils.java | 18 + .../MatrixMatrixBinaryMultiplicationTest.java | 4 +- 7 files changed, 558 insertions(+), 24 deletions(-) create mode 100644 src/main/java/org/apache/sysds/runtime/ooc/primitives/GeneralMMultOOCPrimitive.java diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/MMultOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/MMultOOCInstruction.java index d176a0c4184..0890557c2fd 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/MMultOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/MMultOOCInstruction.java @@ -67,27 +67,30 @@ public void processInstruction( ExecutionContext ec ) { DataCharacteristics vdc = vin.getDataCharacteristics(); if(min != vin && mdc.getRows() > 0 && mdc.getCols() > 0 && vdc.getCols() > 0 && - mdc.getCols() == vdc.getRows() && vdc.getNumColBlocks() == 1) { - OOCStream partials = createWritableStream(); + mdc.getCols() == vdc.getRows() && mdc.getBlocksize() == vdc.getBlocksize()) { OOCStream out = createWritableStream(); - partials.setData(min); ec.getMatrixObject(output).setStreamHandle(out); - OOCInstructionUtils.indexedBroadcastMap(min.getStreamable(), vin.getStreamable(), partials, - left -> Math.toIntExact(left.getIndexes().getColumnIndex() - 1), - () -> new CountingLiveness(Math.toIntExact(vin.getDataCharacteristics().getNumRowBlocks()), - Math.toIntExact(min.getDataCharacteristics().getNumRowBlocks())), - (left, right) -> { - MatrixBlock leftBlock = (MatrixBlock) left.getValue(); - MatrixBlock rightBlock = (MatrixBlock) right.getValue(); - MatrixBlock partial = leftBlock.aggregateBinaryOperations(leftBlock, rightBlock, new MatrixBlock(), - (AggregateBinaryOperator) _optr); - MatrixIndexes indexes = left.getIndexes(); - return new IndexedMatrixValue(new MatrixIndexes(indexes.getRowIndex(), indexes.getColumnIndex()), - partial); - }, getContext()); BinaryOperator plus = InstructionUtils.parseBinaryOperator(Opcodes.PLUS.toString()); - OOCInstructionUtils.rowGroupedReduce(partials, out, - (left, right) -> left.binaryOperations(plus, right, new MatrixBlock()), getContext()); + if(vin.getDataCharacteristics().getNumColBlocks() == 1) { + OOCStream partials = createWritableStream(); + partials.setData(min); + OOCInstructionUtils.indexedBroadcastMap(min.getStreamable(), vin.getStreamable(), partials, + left -> Math.toIntExact(left.getIndexes().getColumnIndex() - 1), + () -> new CountingLiveness(Math.toIntExact(vin.getDataCharacteristics().getNumRowBlocks()), + Math.toIntExact(min.getDataCharacteristics().getNumRowBlocks())), + (left, right) -> { + MatrixBlock leftBlock = (MatrixBlock) left.getValue(); + MatrixBlock rightBlock = (MatrixBlock) right.getValue(); + MatrixBlock partial = leftBlock.aggregateBinaryOperations(leftBlock, rightBlock, + new MatrixBlock(), (AggregateBinaryOperator) _optr); + return new IndexedMatrixValue(left.getIndexes(), partial); + }, getContext()); + OOCInstructionUtils.rowGroupedReduce(partials, out, + (left, right) -> left.binaryOperations(plus, right, new MatrixBlock()), getContext()); + } + else + OOCInstructionUtils.matrixMultiply(min.getStreamable(), vin.getStreamable(), out, + (AggregateBinaryOperator) _optr, plus, getContext()); return; } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java index 0e9504ce08f..9e3494a7d48 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java @@ -57,6 +57,11 @@ public static OOCFuture failed(Throwable error) { return future; } + public static OOCFuture> allOf(List> futures) { + return allOf(futures, ignored -> { + }); + } + public static OOCFuture> allOf(List> futures, Consumer failureCleanup) { Objects.requireNonNull(futures); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCStoreLayout.java b/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCStoreLayout.java index bafd5ec79f7..cb4d949326e 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCStoreLayout.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/planning/OOCStoreLayout.java @@ -23,14 +23,14 @@ import org.apache.sysds.runtime.meta.DataCharacteristics; public enum OOCStoreLayout { - ROW_MAJOR; + ROW_MAJOR, COL_MAJOR; public int linearize(MatrixIndexes indexes, DataCharacteristics characteristics) { if(characteristics == null || !characteristics.dimsKnown() || characteristics.getBlocksize() <= 0) throw new IllegalArgumentException("Materialized store layout requires known dimensions and block size."); - long columns = characteristics.getNumColBlocks(); - long index = Math.addExact(Math.multiplyExact(indexes.getRowIndex() - 1, columns), - indexes.getColumnIndex() - 1); + long index = this == ROW_MAJOR ? (indexes.getRowIndex() - 1) * characteristics.getNumColBlocks() + + indexes.getColumnIndex() - + 1 : (indexes.getColumnIndex() - 1) * characteristics.getNumRowBlocks() + indexes.getRowIndex() - 1; return Math.toIntExact(index); } } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/primitives/GeneralMMultOOCPrimitive.java b/src/main/java/org/apache/sysds/runtime/ooc/primitives/GeneralMMultOOCPrimitive.java new file mode 100644 index 00000000000..2b3fee81cad --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/ooc/primitives/GeneralMMultOOCPrimitive.java @@ -0,0 +1,499 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.ooc.primitives; + +import java.util.List; +import java.util.concurrent.CompletionException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.sysds.runtime.instructions.ooc.OOCStream; +import org.apache.sysds.runtime.instructions.ooc.OOCStreamable; +import org.apache.sysds.runtime.instructions.ooc.SubscribableTaskQueue; +import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.matrix.operators.AggregateBinaryOperator; +import org.apache.sysds.runtime.matrix.operators.BinaryOperator; +import org.apache.sysds.runtime.meta.DataCharacteristics; +import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; +import org.apache.sysds.runtime.ooc.cache.OOCFuture; +import org.apache.sysds.runtime.ooc.memory.ManagedPayload; +import org.apache.sysds.runtime.ooc.memory.ReservationBudget; +import org.apache.sysds.runtime.ooc.planning.OOCAccessPattern; +import org.apache.sysds.runtime.ooc.planning.OOCStoreLayout; +import org.apache.sysds.runtime.ooc.store.CountingLiveness; +import org.apache.sysds.runtime.ooc.store.IndexedMaterializedStoreReader; +import org.apache.sysds.runtime.ooc.store.MaterializedStore; +import org.apache.sysds.runtime.ooc.store.StateTable; +import org.apache.sysds.runtime.ooc.store.StoreLease; +import org.apache.sysds.runtime.ooc.stream.StreamContext; +import org.apache.sysds.runtime.ooc.util.OOCInstructionUtils; +import org.apache.sysds.runtime.ooc.util.OOCUtils; + +public final class GeneralMMultOOCPrimitive extends OOCPrimitive { + private final OOCStreamable _output; + private final AggregateBinaryOperator _multiply; + private final BinaryOperator _plus; + private final AtomicBoolean _sourceComplete = new AtomicBoolean(); + private final AtomicInteger _active = new AtomicInteger(1); + private MaterializedStore _leftStore; + private MaterializedStore _rightStore; + private IndexedMaterializedStoreReader _leftReader; + private IndexedMaterializedStoreReader _rightReader; + private StateTable _accumulators; + private OOCStream _ready; + private OOCStream _outputStream; + private int _rowBlocks; + private int _innerBlocks; + private int _colBlocks; + private int _nextTask; + private int _numTasks; + private long _taskBytes; + + public GeneralMMultOOCPrimitive(OOCStreamable left, OOCStreamable right, + OOCStreamable output, AggregateBinaryOperator multiply, BinaryOperator plus, + StreamContext context) { + super(context, left, right); + _output = output; + _multiply = multiply; + _plus = plus; + } + + @Override + public List requiredMaterializedInputs() { + return List.of(new OOCMaterializedInputRequest(0, OOCStoreLayout.ROW_MAJOR, 1), + new OOCMaterializedInputRequest(1, OOCStoreLayout.COL_MAJOR, 1)); + } + + @Override + protected void inferPatternsInternal() { + _pattern = OOCAccessPattern.ANY; + inferParentPatterns(); + } + + @Override + protected void requestPatternInternal(OOCAccessPattern accessPattern) { + _pattern = _pattern.preferred(accessPattern); + OOCPrimitive left = getInputDependency(0); + OOCPrimitive right = getInputDependency(1); + if(left != null) + left.requestPattern(OOCAccessPattern.ROW_MAJOR); + if(right != null) + right.requestPattern(OOCAccessPattern.COL_MAJOR); + } + + @Override + protected void startExecution() { + DataCharacteristics left = getInput(0).getDataCharacteristics(); + DataCharacteristics right = getInput(1).getDataCharacteristics(); + _rowBlocks = Math.toIntExact(OOCUtils.getNumRowBlocks(left)); + _innerBlocks = Math.toIntExact(OOCUtils.getNumColBlocks(left)); + _colBlocks = Math.toIntExact(OOCUtils.getNumColBlocks(right)); + _numTasks = _rowBlocks * _innerBlocks * _colBlocks; + long leftBytes = OOCUtils.estimateFullTileBytes(left); + long rightBytes = OOCUtils.estimateFullTileBytes(right); + long outputBytes = OOCUtils.estimateFullTileBytes(_output.getDataCharacteristics()); + _taskBytes = OOCCacheManager.getGlobalCache().maxPhysicalPinBytes(leftBytes) + + OOCCacheManager.getGlobalCache().maxPhysicalPinBytes(rightBytes) + + OOCCacheManager.getGlobalCache().maxPhysicalPinBytes(outputBytes) + outputBytes * 3; + + _outputStream = _output.getWriteStream(); + _ready = new SubscribableTaskQueue<>(); + _accumulators = new StateTable<>(); + getContext().addOutStream(_outputStream, _ready); + OOCInstructionUtils.submitCloseableOOCTasks(_ready, this::process, getContext()) + .whenComplete((ignored, error) -> { + try { + if(error != null) + fail(error); + _outputStream.closeInput(); + } + catch(Throwable failure) { + fail(failure); + } + finally { + cleanup(); + } + }); + + OOCFuture.allOf(List.of(getMaterializedInput(0), getMaterializedInput(1)), MaterializedStore::close) + .whenComplete(this::storesReady); + } + + private void storesReady(List> stores, Throwable error) { + if(error != null) { + fail(error); + finishSource(); + return; + } + _leftStore = stores.get(0); + _rightStore = stores.get(1); + OOCFuture.allOf(List.of(_leftStore.completion(), _rightStore.completion())) + .whenComplete((ignored, completionError) -> { + if(completionError != null) { + fail(completionError); + finishSource(); + return; + } + _leftReader = _leftStore.openIndexedReader(new CountingLiveness(_leftStore.size(), _colBlocks)); + _rightReader = _rightStore.openIndexedReader(new CountingLiveness(_rightStore.size(), _rowBlocks)); + scheduleNext(); + }); + } + + private void scheduleNext() { + while(true) { + if(hasFailed() || _nextTask == _numTasks) { + finishSource(); + return; + } + + OOCFuture reservation = _allowance.reserveAsync(_taskBytes); + + if(!reservation.isDone()) { + // _nextTask ctr cannot be stale due to OOCFuture synchronization barrier + reservation.whenComplete((ignored, error) -> { + if(error != null) { + fail(error); + finishSource(); + return; + } + + startTask(); + scheduleNext(); + }); + return; + } + + try { + reservation.getNow(null); + } + catch(CompletionException ex) { + fail(ex.getCause()); + finishSource(); + return; + } + + startTask(); + } + } + + private void startTask() { + ReservationBudget budget = new ReservationBudget(_allowance, _taskBytes).enableReuse(); + + int task = _nextTask++; + _active.incrementAndGet(); + requestInputs(task, budget); + } + + private void requestInputs(int task, ReservationBudget budget) { + int inner = task % _innerBlocks; + int row; + int col; + if(_pattern == OOCAccessPattern.COL_MAJOR) { + row = task / _innerBlocks % _rowBlocks; + col = task / (_innerBlocks * _rowBlocks); + } + else { + col = task / _innerBlocks % _colBlocks; + row = task / (_innerBlocks * _colBlocks); + } + int leftIndex = row * _innerBlocks + inner; + int rightIndex = col * _innerBlocks + inner; + int outputSlot = row * _colBlocks + col; + try { + OOCFuture.allOf(List.of(_leftReader.request(leftIndex, budget), _rightReader.request(rightIndex, budget)), + StoreLease::close).whenComplete((inputs, error) -> { + if(error != null) { + budget.close(); + fail(error); + completeOne(); + return; + } + try { + _ready.enqueue(new MultiplyWork(inputs.get(0), inputs.get(1), outputSlot, budget)); + } + catch(Throwable failure) { + inputs.forEach(StoreLease::close); + budget.close(); + fail(failure); + completeOne(); + } + }); + } + catch(Throwable failure) { + budget.close(); + fail(failure); + completeOne(); + } + } + + private void process(AutoCloseable work) { + if(work instanceof MultiplyWork multiply) + multiply(multiply); + else + merge((MergeWork) work); + } + + private void multiply(MultiplyWork work) { + ReservationBudget budget = work.takeBudget(); + ManagedPayload partial = null; + try { + MatrixBlock left = (MatrixBlock) work._left.value().getValue(); + MatrixBlock right = (MatrixBlock) work._right.value().getValue(); + MatrixBlock block = left.aggregateBinaryOperations(left, right, new MatrixBlock(), _multiply); + partial = payload(work._outputSlot, 1, block, budget); + OOCFuture> released = work.releaseInputsAsync(); + ManagedPayload result = partial; + partial = null; + // wait for closure to not exceed reserved budget + released.whenComplete((ignored, error) -> { + if(error != null) { + result.release(); + budget.close(); + fail(error); + completeOne(); + } + else + reduce(work._outputSlot, result, budget); + }); + } + catch(Throwable failure) { + if(partial != null) + partial.release(); + budget.close(); + fail(failure); + completeOne(); + } + } + + private void reduce(int slot, ManagedPayload incoming, ReservationBudget budget) { + if(count(incoming.value()) == _innerBlocks) { + finalizeOutput(slot, incoming, budget); + return; + } + OOCFuture> match; + try { + match = _accumulators.putOrTake(slot, incoming, budget); + } + catch(Throwable failure) { + incoming.release(); + budget.close(); + fail(failure); + completeOne(); + return; + } + match.whenComplete((existing, error) -> { + if(error != null) { + incoming.release(); + budget.close(); + fail(error); + completeOne(); + } + else if(existing == null) { + budget.close(); + completeOne(); + } + else { + try { + _ready.enqueue(new MergeWork(slot, incoming, existing, budget)); + } + catch(Throwable failure) { + incoming.release(); + existing.close(); + budget.close(); + fail(failure); + completeOne(); + } + } + }); + } + + private void merge(MergeWork work) { + ReservationBudget budget = work.takeBudget(); + ManagedPayload merged = null; + try { + IndexedMatrixValue existing = work._existing.value(); + IndexedMatrixValue incoming = work._incoming.value(); + MatrixBlock block = ((MatrixBlock) existing.getValue()).binaryOperations(_plus, incoming.getValue(), + new MatrixBlock()); + merged = payload(work._slot, count(existing) + count(incoming), block, budget); + work.releaseIncoming(); + OOCFuture released = work.closeExistingAsync(); + ManagedPayload result = merged; + merged = null; + released.whenComplete((ignored, error) -> { + if(error != null) { + result.release(); + budget.close(); + fail(error); + completeOne(); + } + else + reduce(work._slot, result, budget); + }); + } + catch(Throwable failure) { + if(merged != null) + merged.release(); + budget.close(); + fail(failure); + completeOne(); + } + } + + private void finalizeOutput(int slot, ManagedPayload payload, ReservationBudget budget) { + try { + MatrixBlock block = (MatrixBlock) payload.value().getValue(); + payload.release(); + OOCUtils.enqueueExact(_outputStream, + new IndexedMatrixValue(new MatrixIndexes(slot / _colBlocks + 1L, slot % _colBlocks + 1L), block), + budget); + } + catch(Throwable failure) { + payload.release(); + budget.close(); + fail(failure); + } + completeOne(); + } + + private static ManagedPayload payload(int slot, int count, MatrixBlock block, + ReservationBudget budget) { + long bytes = block.getExactSerializedSize(); + budget.reserveBlocking(bytes); + return new ManagedPayload<>(new IndexedMatrixValue(new MatrixIndexes(slot + 1L, count), block), bytes, budget); + } + + private static int count(IndexedMatrixValue value) { + return Math.toIntExact(value.getIndexes().getColumnIndex()); + } + + private void finishSource() { + if(_sourceComplete.compareAndSet(false, true)) + completeOne(); + } + + private void completeOne() { + if(_active.decrementAndGet() != 0) + return; + try { + _ready.closeInput(); + } + catch(IllegalStateException ignored) { + } + } + + private void cleanup() { + if(_accumulators != null) + _accumulators.close(); + if(_leftReader != null) + _leftReader.close(); + if(_rightReader != null) + _rightReader.close(); + if(_leftStore != null) + _leftStore.close(); + if(_rightStore != null) + _rightStore.close(); + onComplete(); + } + + private static final class MultiplyWork implements AutoCloseable { + private final int _outputSlot; + private StoreLease _left; + private StoreLease _right; + private ReservationBudget _budget; + + private MultiplyWork(StoreLease left, StoreLease right, int outputSlot, + ReservationBudget budget) { + _left = left; + _right = right; + _outputSlot = outputSlot; + _budget = budget; + } + + private ReservationBudget takeBudget() { + ReservationBudget budget = _budget; + _budget = null; + return budget; + } + + private OOCFuture> releaseInputsAsync() { + OOCFuture left = _left.closeAsync(); + OOCFuture right = _right.closeAsync(); + _left = null; + _right = null; + return OOCFuture.allOf(List.of(left, right)); + } + + @Override + public void close() { + if(_left != null) + _left.close(); + if(_right != null) + _right.close(); + if(_budget != null) + _budget.close(); + } + } + + private static final class MergeWork implements AutoCloseable { + private final int _slot; + private ManagedPayload _incoming; + private StoreLease _existing; + private ReservationBudget _budget; + + private MergeWork(int slot, ManagedPayload incoming, + StoreLease existing, ReservationBudget budget) { + _slot = slot; + _incoming = incoming; + _existing = existing; + _budget = budget; + } + + private ReservationBudget takeBudget() { + ReservationBudget budget = _budget; + _budget = null; + return budget; + } + + private void releaseIncoming() { + _incoming.release(); + _incoming = null; + } + + private OOCFuture closeExistingAsync() { + OOCFuture released = _existing.closeAsync(); + _existing = null; + return released; + } + + @Override + public void close() { + if(_incoming != null) + _incoming.release(); + if(_existing != null) + _existing.close(); + if(_budget != null) + _budget.close(); + } + } +} diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java index 2d5fa50d787..a764fe03589 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCInstructionUtils.java @@ -40,12 +40,15 @@ import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.runtime.matrix.data.MatrixIndexes; +import org.apache.sysds.runtime.matrix.operators.AggregateBinaryOperator; +import org.apache.sysds.runtime.matrix.operators.BinaryOperator; import org.apache.sysds.runtime.ooc.cache.OOCCacheManager; import org.apache.sysds.runtime.ooc.cache.OOCFuture; import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; import org.apache.sysds.runtime.ooc.memory.MemoryAllowance; import org.apache.sysds.runtime.ooc.memory.ReservationBudget; import org.apache.sysds.runtime.ooc.primitives.BroadcastOOCPrimitive; +import org.apache.sysds.runtime.ooc.primitives.GeneralMMultOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.GroupedReduceOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.JoinOOCPrimitive; import org.apache.sysds.runtime.ooc.primitives.MappingOOCPrimitive; @@ -130,6 +133,12 @@ public static void naryEquiJoin(List> inputs, new NaryJoinOOCPrimitive(inputs, output, key, size, operation, inputBytes, joinBytes, context)); } + public static void matrixMultiply(OOCStreamable left, OOCStreamable right, + OOCStream output, AggregateBinaryOperator multiply, BinaryOperator plus, + StreamContext context) { + output.assignPrimitive(new GeneralMMultOOCPrimitive(left, right, output, multiply, plus, context)); + } + public static void indexedBroadcastMap(OOCStreamable streamed, OOCStreamable broadcast, OOCStream output, ToIntFunction lookup, Supplier liveness, diff --git a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java index c1a0e6439c1..2b070b9365f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/util/OOCUtils.java @@ -114,6 +114,24 @@ public static long getNumBlocks(DataCharacteristics dc) { return -1; } + public static long getNumRowBlocks(DataCharacteristics dc) { + if(dc != null && dc.dimsKnown() && dc.getBlocksize() > 0) { + if(dc.getCols() == 0 || dc.getRows() == 0) + return 0; + return dc.getNumRowBlocks(); + } + return -1; + } + + public static long getNumColBlocks(DataCharacteristics dc) { + if(dc != null && dc.dimsKnown() && dc.getBlocksize() > 0) { + if(dc.getCols() == 0 || dc.getRows() == 0) + return 0; + return dc.getNumColBlocks(); + } + return -1; + } + public static Iterable getAccessPattern(DataCharacteristics dc, OOCAccessPattern pattern) { long rows = dc.getRows() == 0 ? 0 : dc.getNumRowBlocks(); long cols = dc.getCols() == 0 ? 0 : dc.getNumColBlocks(); diff --git a/src/test/java/org/apache/sysds/test/functions/ooc/MatrixMatrixBinaryMultiplicationTest.java b/src/test/java/org/apache/sysds/test/functions/ooc/MatrixMatrixBinaryMultiplicationTest.java index 1e5233cc89c..5bf6de96754 100644 --- a/src/test/java/org/apache/sysds/test/functions/ooc/MatrixMatrixBinaryMultiplicationTest.java +++ b/src/test/java/org/apache/sysds/test/functions/ooc/MatrixMatrixBinaryMultiplicationTest.java @@ -63,7 +63,7 @@ public void testMVBinaryMultiplication1() { @Test public void testMVBinaryMultiplication2() { - runMatrixVectorMultiplicationTest(cols_skinny, false); + runMatrixVectorMultiplicationTest(cols_skinny, true); } private void runMatrixVectorMultiplicationTest(int cols, boolean sparse ) @@ -91,7 +91,7 @@ private void runMatrixVectorMultiplicationTest(int cols, boolean sparse ) A_data = null; A_mb = null; - double[][] x_data = getRandomMatrix(cols, rows, 0, 1, 1.0, 10); + double[][] x_data = getRandomMatrix(cols, rows, 0, 1, sparse ? sparsity2 : 1.0, 10); MatrixBlock x_mb = DataConverter.convertToMatrixBlock(x_data); writer.writeMatrixToHDFS(x_mb, input(INPUT_NAME2), cols, rows, 1000, x_mb.getNonZeros()); HDFSTool.writeMetaDataFile(input(INPUT_NAME2 + ".mtd"), Types.ValueType.FP64, From 3f6326a77f0ca8f3ffc50491bb4c6880c9845c00 Mon Sep 17 00:00:00 2001 From: Grigorii Turchenko Date: Mon, 24 Aug 2026 17:29:55 +0200 Subject: [PATCH 120/132] [SYSTEMDS-3960] Improve robustness against network delays in federated test environments Closes #2587. --- .../sysds/test/FederatedWorkerUtils.java | 44 +++++++++---------- .../federated/FederatedUrlParserTest.java | 31 +++++++++++++ 2 files changed, 53 insertions(+), 22 deletions(-) diff --git a/src/test/java/org/apache/sysds/test/FederatedWorkerUtils.java b/src/test/java/org/apache/sysds/test/FederatedWorkerUtils.java index d604d7dcab4..83cb9546745 100644 --- a/src/test/java/org/apache/sysds/test/FederatedWorkerUtils.java +++ b/src/test/java/org/apache/sysds/test/FederatedWorkerUtils.java @@ -27,9 +27,8 @@ /** * Test helpers that block until a federated worker is accepting TCP connections on its port. * - *

The federated worker opens its TCP port after Netty's {@code bind().sync()} returns; a successful - * TCP connect to that port therefore indicates that the worker is ready to accept requests. The methods - * here poll for that signal and throw {@link RuntimeException} on timeout or if the underlying + * The federated worker opens its TCP port after Netty's {@code bind().sync()} returns. The methods here poll for a + * successful TCP connection and throw {@link RuntimeException} on timeout or if the underlying * {@code Process}/{@code Thread} exits before the port becomes ready. */ public final class FederatedWorkerUtils { @@ -37,15 +36,14 @@ public final class FederatedWorkerUtils { /** Sleep between successive poll rounds, in milliseconds. */ private static final int POLL_INTERVAL_MS = 25; - /** Per-attempt {@link Socket#connect} timeout, in milliseconds. */ - private static final int CONNECT_TIMEOUT_MS = 25; + /** + * Per-attempt {@link Socket#connect} timeout in milliseconds, covering the full TCP handshake (round trip). + */ + private static final int CONNECT_TIMEOUT_MS = 2000; /** - * Minimum value applied to the caller-supplied {@code timeoutMs}. The wait returns as soon as the - * worker accepts a connection, so this only affects the upper bound used when a worker never becomes - * ready. Set to 60s to accommodate cold JVM startup on heavily contended CI runners: tests starting - * four workers in parallel can have all four still pending after 30s when the runner is CPU-starved, - * and burning a surefire retry costs more wall time than padding this clamp. + * Minimum value applied to the caller-supplied {@code timeoutMs}, returns as soon as the worker accepts a + * connection. */ private static final int MIN_TIMEOUT_MS = 60_000; @@ -76,7 +74,7 @@ public static void waitForWorker(int port, int timeoutMs, BooleanSupplier aliveC throw new RuntimeException( "Federated " + workerKind + " on port " + port + " died before becoming ready."); } - if(tryConnect(port)) { + if(tryConnect(port, deadline)) { return; } sleepQuietly(); @@ -96,9 +94,8 @@ public static void waitForWorker(Thread thread, int port, int timeoutMs) { } /** - * Block until every listed federated worker is accepting TCP connections. All ports are polled in - * one shared loop, so the wall-clock wait is bounded by the slowest worker rather than the sum of - * individual waits. + * Block until every listed federated worker is accepting TCP connections. All ports are polled in one shared loop, + * so the wall-clock wait is bounded by the slowest worker. * * @param ports ports the workers are expected to bind * @param timeoutMs upper bound on the wait, in ms; raised to {@link #MIN_TIMEOUT_MS} if smaller @@ -134,9 +131,8 @@ public static void waitForWorkers(Thread[] threads, int[] ports, int timeoutMs) } /** - * Bulk variant taking a per-index liveness predicate so callers can plug in either {@code Process} - * or {@code Thread} liveness. Each port flips to ready as soon as it accepts a connection; the loop - * yields between sweeps so a still-pending worker is not starved by repeated probes on the same CPU. + * Bulk variant taking a per-index liveness predicate so callers can plug in either {@code Process} or + * {@code Thread} liveness. Each port flips to ready as soon as it accepts a connection. */ public static void waitForWorkers(int[] ports, int timeoutMs, java.util.function.IntPredicate aliveCheck, String workerKind) { @@ -145,7 +141,8 @@ public static void waitForWorkers(int[] ports, int timeoutMs, java.util.function final boolean[] ready = new boolean[ports.length]; int remaining = ports.length; while(remaining > 0 && System.currentTimeMillis() < deadline) { - for(int i = 0; i < ports.length; i++) { + // recheck the deadline per port, a sweep can spend up to CONNECT_TIMEOUT_MS on each of them + for(int i = 0; i < ports.length && System.currentTimeMillis() < deadline; i++) { if(ready[i]) { continue; } @@ -153,7 +150,7 @@ public static void waitForWorkers(int[] ports, int timeoutMs, java.util.function throw new RuntimeException("Federated " + workerKind + " on port " + ports[i] + " died before becoming ready."); } - if(tryConnect(ports[i])) { + if(tryConnect(ports[i], deadline)) { ready[i] = true; remaining--; } @@ -174,12 +171,15 @@ public static void waitForWorkers(int[] ports, int timeoutMs, java.util.function } } - private static boolean tryConnect(int port) { + private static boolean tryConnect(int port, long deadline) { + final long remaining = deadline - System.currentTimeMillis(); + if(remaining <= 0) // out of time => connect reads a timeout of 0 as "infinite" + return false; try(Socket s = new Socket()) { - s.connect(new InetSocketAddress("localhost", port), CONNECT_TIMEOUT_MS); + s.connect(new InetSocketAddress("localhost", port), (int) Math.min(CONNECT_TIMEOUT_MS, remaining)); return true; } - catch(IOException e) { + catch(IOException e) { // closed port, or a handshake that outlasted the budget return false; } } diff --git a/src/test/java/org/apache/sysds/test/component/federated/FederatedUrlParserTest.java b/src/test/java/org/apache/sysds/test/component/federated/FederatedUrlParserTest.java index 10e1e6b549d..79971f965f1 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FederatedUrlParserTest.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FederatedUrlParserTest.java @@ -21,9 +21,14 @@ import org.apache.sysds.runtime.instructions.fed.InitFEDInstruction; import org.apache.sysds.conf.DMLConfig; +import org.apache.sysds.test.FederatedWorkerUtils; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.net.ServerSocket; import org.junit.Test; @@ -167,4 +172,30 @@ public void checkDefaultPortIsValid() { assertTrue(defaultPort <= IANA_limit); assertTrue(defaultPort > 0); } + + @Test + public void waitReturnsForAListeningPort() throws IOException { + try(ServerSocket listening = new ServerSocket(0)) { + // Return as soon as the port accepts, the timeout is only the upper bound. + FederatedWorkerUtils.waitForWorker(listening.getLocalPort(), 1000); + } + } + + @Test + public void waitFailsFastWhenTheWorkerDied() throws IOException { + final int port; + try(ServerSocket closed = new ServerSocket(0)) { + port = closed.getLocalPort(); + } + final long t0 = System.currentTimeMillis(); + try { + FederatedWorkerUtils.waitForWorker(port, 1000, () -> false, "worker"); + fail("expected the wait to report the dead worker"); + } + catch(RuntimeException e) { + assertTrue(e.getMessage(), e.getMessage().contains("died before becoming ready")); + // Must not sit out the timeout, which is clamped up to a minute. + assertTrue("the dead worker was not reported promptly", System.currentTimeMillis() - t0 < 10000); + } + } } From 781840d820e5197bf91965cb490cb5d085c02600 Mon Sep 17 00:00:00 2001 From: Matthias Boehm Date: Tue, 25 Aug 2026 12:21:47 +0200 Subject: [PATCH 121/132] [SYSTEMDS-3962] Fix memory check on recent windows11 versions On newer windows version, the check for physical memory via wmic runs into a "cannot run program" error on hard crashes. This patch adds a more robust version that checks the modern way, old way, and in all cases prevents any crash because this physical memory size is just used for warnings that SystemDS runs in a JVM with too little memory. --- .../apache/sysds/utils/SettingsChecker.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/apache/sysds/utils/SettingsChecker.java b/src/main/java/org/apache/sysds/utils/SettingsChecker.java index 62ab608b95e..c5f14a7043b 100644 --- a/src/main/java/org/apache/sysds/utils/SettingsChecker.java +++ b/src/main/java/org/apache/sysds/utils/SettingsChecker.java @@ -106,14 +106,25 @@ private static long maxMemMachineOSX() { } private static long maxMemMachineWin() { + //try modern powershell, otherwise wmic, log errors as warning but avoid crashes + long tmp = maxMemMachineWin(true); + if( tmp < 0 ) + tmp = maxMemMachineWin(false); + return tmp; + } + + private static long maxMemMachineWin(boolean modern) { + int startIx = modern ? 3 : 1; + String command = modern ? + "powershell Get-CimInstance -ClassName Win32_PhysicalMemory | Select-Object Capacity" : + "wmic memorychip get capacity"; //in bytes try { - String command = "wmic memorychip get capacity"; //in bytes Runtime rt = Runtime.getRuntime(); Process pr = rt.exec(command); String[] memStr = new String(pr.getInputStream().readAllBytes(), StandardCharsets.UTF_8).split("\n"); //skip header, and aggregate DIMM capacities long capacity = 0; - for( int i=1; i 0 ) capacity += Long.parseLong(tmp); @@ -121,7 +132,8 @@ private static long maxMemMachineWin() { return capacity; } catch(IOException e) { - throw new RuntimeException(e); + LOG.warn(e.getMessage()); + return -1; } } From c32f48a77c56c9d5e7c671cde04302d3077c2add Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:48:00 +0200 Subject: [PATCH 122/132] Bump docker/login-action from 4.5.2 to 4.6.0 (#2582) Bumps [docker/login-action](https://github.com/docker/login-action) from 4.5.2 to 4.6.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v4.5.2...v4.6.0) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.6.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker-cd.yml | 2 +- .github/workflows/docker-release.yml | 2 +- .github/workflows/docker-testImage.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-cd.yml b/.github/workflows/docker-cd.yml index 74f7ef8c8f5..3f240013fd4 100644 --- a/.github/workflows/docker-cd.yml +++ b/.github/workflows/docker-cd.yml @@ -57,7 +57,7 @@ jobs: # https://github.com/docker/login-action - name: Login to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@v4.5.2 + uses: docker/login-action@v4.6.0 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 9bdef4a8639..2706e53ee86 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -64,7 +64,7 @@ jobs: # https://github.com/docker/login-action - name: Login to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@v4.5.2 + uses: docker/login-action@v4.6.0 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/docker-testImage.yml b/.github/workflows/docker-testImage.yml index 30c91a6a5fc..f3f571ba1dc 100644 --- a/.github/workflows/docker-testImage.yml +++ b/.github/workflows/docker-testImage.yml @@ -55,7 +55,7 @@ jobs: # https://github.com/docker/login-action - name: Login to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@v4.5.2 + uses: docker/login-action@v4.6.0 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} From 044f96f308f655d1085ec61b6402efa261f5606a Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Wed, 26 Aug 2026 09:28:58 +0200 Subject: [PATCH 123/132] [SYSTEMDS-3887] Improve Scuro DAGs This patch improves the representation DAGs, reduces duplicates and finds good initial parameters for window operators. Additionally, it collects more statistics of the optimization runs. Assisted-by: AI --- .../scuro/drsearch/hyperparameter_tuner.py | 48 ++- .../systemds/scuro/drsearch/node_executor.py | 6 + .../scuro/drsearch/operator_registry.py | 25 +- .../scuro/drsearch/representation_dag.py | 92 ++++- .../scuro/drsearch/unimodal_optimizer.py | 259 +++++++++++--- .../systemds/scuro/modality/modality.py | 208 ++++++----- .../scuro/modality/unimodal_modality.py | 38 +- .../scuro/representations/aggregate.py | 32 +- .../aggregated_representation.py | 21 +- .../scuro/representations/representation.py | 10 + .../representations/window_aggregation.py | 332 +++++++++++------- src/main/python/tests/iotests/test_io_csv.py | 2 +- .../tests/iotests/test_io_pandas_systemds.py | 2 +- .../test_dense_numpy_matrix.py | 2 +- .../test_pandas_frame.py | 2 +- 15 files changed, 798 insertions(+), 281 deletions(-) diff --git a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py index a62b990fa99..094a4b9585e 100644 --- a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py +++ b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py @@ -75,7 +75,20 @@ def _param_values_to_spec( return None -def _expand_aggregation_param_specs(op_id: str, agg_cls: Any) -> List[Dict[str, Any]]: +def _window_input_stats(node_parameters: Optional[Dict[str, Any]]): + from systemds.scuro.representations.representation import RepresentationStats + + if not node_parameters: + return None + window_length = node_parameters.get("window_size") + if window_length is None: + return None + return RepresentationStats(1, (int(window_length),)) + + +def _expand_aggregation_param_specs( + op_id: str, agg_cls: Any, input_stats=None +) -> List[Dict[str, Any]]: if not inspect.isclass(agg_cls): return [] @@ -98,6 +111,10 @@ def _expand_aggregation_param_specs(op_id: str, agg_cls: Any) -> List[Dict[str, nested_values = search_template.get(nested_name) if nested_values is None: continue + if input_stats is not None and isinstance(nested_values, list): + narrow = getattr(instance, "filter_parameter_domain", None) + if narrow is not None: + nested_values = narrow(nested_name, nested_values, input_stats) full_name = f"{op_id}-aggregation_function_{nested_name}" spec = _param_values_to_spec(full_name, nested_values) if spec is not None: @@ -251,7 +268,7 @@ def setup_mm(self, optimize_unimodal): for task in self.tasks: self.results[task.model.name] = {"mm_results": []} - def get_k_best_results(self, modality, task, performance_metric_name): + def get_k_best_dags(self, modality, task): results = self.results[task.model.name][modality.modality_id] dags = [] for result in results: @@ -269,6 +286,10 @@ def get_k_best_results(self, modality, task, performance_metric_name): ) dags.append(dag_with_best_params.build(prev_node_id)) + return results, dags + + def get_k_best_results(self, modality, task, performance_metric_name): + results, dags = self.get_k_best_dags(modality, task) representations = [list(dag.execute([modality]).values())[-1] for dag in dags] return results, representations @@ -488,7 +509,11 @@ def visit_node(node_id): if not hyperparams: all_results = [baseline] else: - param_specs = self._build_param_specs(hyperparams) + node_parameters = { + node_id: (dag.get_node_by_id(node_id).parameters or {}) + for node_id in hyperparams + } + param_specs = self._build_param_specs(hyperparams, node_parameters) discrete_size = self._estimate_discrete_search_size(param_specs) n_calls = min(discrete_size, max_evals) if max_evals else discrete_size all_results = self._search_best_configs( @@ -575,7 +600,12 @@ def __get_params_for_node(self, node: RepresentationNode) -> Dict[str, Any]: if node.parameters: if inspect.isclass(node.parameters.get("aggregation_function")): params["aggregation_function"] = node.parameters["aggregation_function"] - for fixed_key in ("target_dimensions", "self_contained"): + for fixed_key in ( + "target_dimensions", + "self_contained", + "aggregate_leading", + "preserve_leading_axis", + ): if fixed_key in node.parameters: params[fixed_key] = node.parameters[fixed_key] @@ -587,13 +617,19 @@ def __get_params_for_node(self, node: RepresentationNode) -> Dict[str, Any]: return params def _build_param_specs( - self, hyperparams: Dict[str, Dict[str, Any]] + self, + hyperparams: Dict[str, Dict[str, Any]], + node_parameters: Optional[Dict[str, Dict[str, Any]]] = None, ) -> List[Dict[str, Any]]: param_specs = [] + node_parameters = node_parameters or {} for op_id, op_params in hyperparams.items(): + input_stats = _window_input_stats(node_parameters.get(op_id)) for param_name, param_values in op_params.items(): if param_name == "aggregation_function": - expanded = _expand_aggregation_param_specs(op_id, param_values) + expanded = _expand_aggregation_param_specs( + op_id, param_values, input_stats + ) if expanded: param_specs.extend(expanded) continue diff --git a/src/main/python/systemds/scuro/drsearch/node_executor.py b/src/main/python/systemds/scuro/drsearch/node_executor.py index ec5d9f40c36..3400e6c6c50 100644 --- a/src/main/python/systemds/scuro/drsearch/node_executor.py +++ b/src/main/python/systemds/scuro/drsearch/node_executor.py @@ -381,6 +381,7 @@ def __init__( result_path: Optional[str] = None, enable_checkpointing: bool = False, worker_pool: Optional[PersistentWorkerPool] = None, + search_start: Optional[float] = None, ): self.enable_checkpointing = enable_checkpointing available_total_cpu = cpu_memory_budget_bytes() @@ -411,6 +412,11 @@ def __init__( ) self._memory_usage_data: Dict[str, Any] = {} self.statistics = {"worker_stats": {}, "node_stats": {}} + self._eval_counter = 0 + self._nodes_executed = 0 + self._search_start = ( + search_start if search_start is not None else time.perf_counter() + ) self._node_attempts: Dict[str, int] = {} diff --git a/src/main/python/systemds/scuro/drsearch/operator_registry.py b/src/main/python/systemds/scuro/drsearch/operator_registry.py index 4c5641fdd91..7a80aafa913 100644 --- a/src/main/python/systemds/scuro/drsearch/operator_registry.py +++ b/src/main/python/systemds/scuro/drsearch/operator_registry.py @@ -47,7 +47,7 @@ def __new__(cls): def set_fusion_operators(self, fusion_operators): if isinstance(fusion_operators, list): - self._context_operators = fusion_operators + self._fusion_operators = fusion_operators else: self._fusion_operators = [fusion_operators] @@ -155,7 +155,7 @@ def get_representation_by_name(self, representation_name, modality_type): return None, False def get_context_representations(self, modality_type): - return self._context_representation_operators[modality_type] + return self._context_representation_operators.get(modality_type, []) def get_context_lenghts_for_modality(self, modality_type, statistics): if modality_type == ModalityType.AUDIO: @@ -176,7 +176,7 @@ def get_context_lenghts_for_modality(self, modality_type, statistics): modality_type == ModalityType.TIMESERIES or modality_type == ModalityType.PHYSIOLOGICAL ): - window_lengths = [0.05, 0.1, 0.5, 0.75, 1, 2, 5, 10, 30, 60] # seconds + window_lengths = [0.5, 0.75, 1, 2, 5, 10, 30, 60] # seconds if modality_type == ModalityType.VIDEO: window_lengths = [0.5, 1, 2, 5, 10] # seconds @@ -198,7 +198,9 @@ def get_context_lenghts_for_modality(self, modality_type, statistics): math.ceil(statistics.avg_length / length) for length in effective_window_lenghts ] - return effective_window_lenghts, num_windows + return self._drop_windows_below_min_count( + effective_window_lenghts, num_windows + ) if modality_type == ModalityType.VIDEO: max_length_in_seconds = statistics.max_length / statistics.fps @@ -213,7 +215,22 @@ def get_context_lenghts_for_modality(self, modality_type, statistics): math.ceil(statistics.avg_length / length) for length in effective_window_lenghts ] + return self._drop_windows_below_min_count( + effective_window_lenghts, num_windows + ) + + MIN_TUNABLE_NUM_WINDOWS = 5 + + def _drop_windows_below_min_count(self, effective_window_lenghts, num_windows): + filtered = [ + (length, count) + for length, count in zip(effective_window_lenghts, num_windows) + if count >= self.MIN_TUNABLE_NUM_WINDOWS and length >= 1 + ] + if not filtered: return effective_window_lenghts, num_windows + lengths, counts = zip(*filtered) + return list(lengths), list(counts) def register_representation(modalities: Union[ModalityType, List[ModalityType]]): diff --git a/src/main/python/systemds/scuro/drsearch/representation_dag.py b/src/main/python/systemds/scuro/drsearch/representation_dag.py index a19c44396fd..ad7f174acfa 100644 --- a/src/main/python/systemds/scuro/drsearch/representation_dag.py +++ b/src/main/python/systemds/scuro/drsearch/representation_dag.py @@ -19,6 +19,8 @@ # # ------------------------------------------------------------- import copy +import hashlib +import json from dataclasses import dataclass, field from typing import List, Dict, Set, Tuple, Union, Any, Hashable, Optional from systemds.scuro.modality.modality import Modality @@ -112,6 +114,65 @@ def filter_connected_nodes(self, nodes): return [node for node in nodes if node.node_id in visited] + def to_spec(self) -> Dict[str, Any]: + order = self._topological_order() + relabel = {node_id: f"n{i}" for i, node_id in enumerate(order)} + + nodes = [] + for node_id in order: + node = self.get_node_by_id(node_id) + operation = getattr(node, "operation", None) + nodes.append( + { + "id": relabel[node_id], + "op": ( + getattr(operation, "__name__", None) + if operation is not None + else None + ), + "params": { + k: _spec_safe(v) + for k, v in sorted((node.parameters or {}).items()) + }, + "inputs": [relabel[i] for i in node.inputs if i in relabel], + "modality_id": node.modality_id, + "representation_index": node.representation_index, + "aggregation": ( + type(node.aggregation).__name__ + if node.aggregation is not None + else None + ), + } + ) + + return { + "root": relabel.get(self.root_node_id), + "nodes": nodes, + "representation_names": self.get_represntation_names(), + } + + def pipeline_id(self) -> str: + blob = json.dumps(self.to_spec(), sort_keys=True, separators=(",", ":")) + return hashlib.sha1(blob.encode()).hexdigest()[:16] + + def _topological_order(self) -> List[str]: + node_map = {node.node_id: node for node in self.nodes} + visited = [] + seen = set() + + def visit(node_id): + if node_id in seen or node_id not in node_map: + return + seen.add(node_id) + for input_id in sorted(node_map[node_id].inputs or []): + visit(input_id) + visited.append(node_id) + + visit(self.root_node_id) + for node_id in sorted(node_map): + visit(node_id) + return visited + def get_leaf_nodes(self) -> List[str]: leaf_nodes = [] for node in self.nodes: @@ -414,6 +475,26 @@ def get_leaf_node_id(self) -> str: return None +def _spec_safe(value): + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, (list, tuple)): + return [_spec_safe(v) for v in value] + if isinstance(value, dict): + return { + str(k): _spec_safe(v) + for k, v in sorted(value.items(), key=lambda kv: str(kv[0])) + } + if hasattr(value, "item") and hasattr(value, "dtype"): + try: + return value.item() + except Exception: + pass + if isinstance(value, type): + return value.__name__ + return type(value).__name__ + + def get_modality_by_id_and_instance_id( modalities: List[Modality], modality_id: int, instance_id: int ): @@ -565,12 +646,13 @@ def __init__(self): self.node_to_signature: Dict[str, Hashable] = {} self.node_counter = 0 self.dag_counter = 0 + self._dag_by_root: Dict[str, RepresentationDag] = {} def _compute_node_signature( self, operation: Any, inputs: List[str], parameters: Dict[str, Any] = None ) -> Hashable: ip = [self.node_to_signature[inp] for inp in inputs] - input_sigs = tuple(sorted(ip)) if inputs else () + input_sigs = tuple(sorted(ip, key=repr)) if inputs else () op_cls = operation().name params_items = tuple(sorted((parameters or {}).items())) return ("op", op_cls, params_items, input_sigs) @@ -644,6 +726,11 @@ def create_operation_node( ) def build(self, root_node_id: str, dag_id: int = None) -> RepresentationDag: + if dag_id is None: + memoized = self._dag_by_root.get(root_node_id) + if memoized is not None: + return memoized + dag = RepresentationDag( nodes=self.global_nodes, root_node_id=root_node_id, @@ -652,6 +739,9 @@ def build(self, root_node_id: str, dag_id: int = None) -> RepresentationDag: self.dag_counter += 1 if not dag.validate(): raise ValueError("Invalid DAG construction") + + if dag_id is None: + self._dag_by_root[root_node_id] = dag return dag def get_node(self, node_id: str) -> Optional[RepresentationNode]: diff --git a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py index a632c97e973..1b2227b773c 100644 --- a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py +++ b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py @@ -19,23 +19,23 @@ # # ------------------------------------------------------------- import copy +import math import pickle -import csv -from pathlib import Path import time from concurrent.futures import ProcessPoolExecutor, as_completed -from dataclasses import dataclass import multiprocessing as mp from typing import List, Any, Optional, Dict from functools import lru_cache -from systemds.scuro import ModalityType +from systemds.scuro.modality.type import ModalityType from systemds.scuro.drsearch.node_executor import NodeExecutor, ResultEntry +from systemds.scuro.representations.representation import RepresentationStats from systemds.scuro.drsearch.ranking import rank_by_tradeoff from systemds.scuro.drsearch.task import PerformanceMeasure from systemds.scuro.representations.concatenation import Concatenation from systemds.scuro.representations.hadamard import Hadamard from systemds.scuro.representations.sum import Sum +from systemds.scuro.representations.average import Average from systemds.scuro.representations.aggregated_representation import ( AggregatedRepresentation, ) @@ -65,10 +65,11 @@ def __init__( checkpoint_every: Optional[int] = 1, resume: bool = False, max_num_workers: int = -1, - enable_checkpointing: bool = True, - enable_execution_profile: bool = False, - execution_profile_path: Optional[str] = None, + enable_checkpointing: bool = False, + window_combination_chains: int = 1, ): + self._node_stats: Dict[str, Any] = {} + self.window_combination_chains = window_combination_chains self.enable_checkpointing = enable_checkpointing self.modalities = modalities self.tasks = tasks @@ -92,13 +93,13 @@ def __init__( } self.debug = debug + self._search_start = time.perf_counter() + self._search_start_unix = time.time() self.operator_registry = Registry() self.operator_performance = UnimodalResults( modalities, tasks, debug, True, k, self.metric_name ) - self.enable_execution_profile = enable_execution_profile - self.execution_profile_path = execution_profile_path self._tasks_require_same_dims = True self.expected_dimensions = tasks[0].expected_dim @@ -140,6 +141,22 @@ def store_results(self, file_name=None): with open(file_name, "wb") as f: pickle.dump(self.operator_performance.results, f) + stats_file_name = file_name.replace(".pkl", "_exec_stats.pkl") + if stats_file_name == file_name: + stats_file_name = file_name + "_exec_stats.pkl" + with open(stats_file_name, "wb") as f: + pickle.dump( + { + "worker_stats": self.operator_performance.worker_stats, + "node_stats": self.operator_performance.node_stats, + "reuse_stats": self.operator_performance.reuse_stats, + "wall_clock_s": self.operator_performance.wall_clock_s, + "search_start_unix": self._search_start_unix, + "max_num_workers": self.max_num_workers, + }, + f, + ) + def store_cache(self, file_name=None): if file_name is None: import time @@ -339,6 +356,7 @@ def _process_modality(self, modality, skip_remaining: int = 0, scheduler=None): max_num_workers=self.max_num_workers, result_path=self.result_path, enable_checkpointing=self.enable_checkpointing, + search_start=self._search_start, ) start_time = time.perf_counter() exec_out = node_executor.run() @@ -348,10 +366,11 @@ def _process_modality(self, modality, skip_remaining: int = 0, scheduler=None): for task_result in task_results: local_results.add_task_result(task_result, dags) statistics = exec_out["statistics"] - for worker_stat in statistics["worker_stats"]: - local_results.add_worker_stat(worker_stat, modality.modality_id) - for node_stat in statistics["node_stats"]: - local_results.add_node_stat(node_stat, modality.modality_id) + + local_results.add_worker_stat(statistics["worker_stats"], modality.modality_id) + local_results.add_node_stat(statistics["node_stats"], modality.modality_id) + local_results.add_reuse_stat(statistics.get("reuse", {}), modality.modality_id) + local_results.wall_clock_s[modality.modality_id] = end_time - start_time if self.save_all_results: timestr = time.strftime("%Y%m%d-%H%M%S") @@ -361,12 +380,29 @@ def _process_modality(self, modality, skip_remaining: int = 0, scheduler=None): return local_results, end_time - start_time + def _window_input_stats(self, modality: Modality, window_length: int): + modality_stats = modality.get_output_stats() + return RepresentationStats( + modality_stats.num_instances, + (int(window_length),), + output_shape_is_known=modality_stats.output_shape_is_known, + dtype=getattr(modality_stats, "dtype", None), + sampling_rate=getattr(modality_stats, "sampling_rate", None), + ) + + @staticmethod + def _effective_window_length(context_operator, window_size, num_window, signal_len): + if context_operator.granularity_kind == "count": + return max(1, int(math.ceil(signal_len / max(1, int(num_window))))) + return max(1, int(window_size)) + def _build_execution_dags_for_modality( self, modality: Modality, skip_remaining: int = 0 ) -> tuple: modality_specific_operators = self._get_modality_operators( modality.modality_type ) + self._node_stats = {} dags = [] for operator in modality_specific_operators: dags.extend(self._build_modality_dag(modality, operator())) @@ -401,6 +437,18 @@ def _merge_results(self, local_results): self.operator_performance.results[modality_id][task_name].extend( local_results.results[modality_id][task_name] ) + self.operator_performance.add_worker_stat( + local_results.worker_stats[modality_id], modality_id + ) + self.operator_performance.add_node_stat( + local_results.node_stats[modality_id], modality_id + ) + self.operator_performance.add_reuse_stat( + local_results.reuse_stats.get(modality_id, {}), modality_id + ) + self.operator_performance.wall_clock_s[modality_id] = ( + local_results.wall_clock_s.get(modality_id, 0.0) + ) def add_dimensionality_reduction_operators(self, builder, current_node_id): dags = [] @@ -483,7 +531,10 @@ def _build_modality_dag( not_self_contained_reps = [ rep for rep in not_self_contained_reps if rep != operator.__class__ ] - rep_id = current_node_id + chain_tips = { + combination.__class__: current_node_id + for combination in self._combination_operators + } for rep in not_self_contained_reps: other_rep_id = builder.create_operation_node( @@ -492,9 +543,10 @@ def _build_modality_dag( for combination in self._combination_operators: combine_id = builder.create_operation_node( combination.__class__, - [rep_id, other_rep_id], + [chain_tips[combination.__class__], other_rep_id], combination.get_current_parameters(), ) + chain_tips[combination.__class__] = combine_id rep_dag = builder.build(combine_id) dags.append(rep_dag) if modality.modality_type in [ @@ -507,15 +559,6 @@ def _build_modality_dag( modality, builder, leaf_id, rep_dag, False ) ) - elif modality.modality_type == ModalityType.TIMESERIES: - dags.extend( - self.temporal_context_operators( - modality, - builder, - leaf_id, - ) - ) - rep_id = combine_id if rep_dag.nodes[-1].operation().output_modality_type in [ ModalityType.EMBEDDING @@ -528,22 +571,35 @@ def _build_modality_dag( return dags - def _aggregation_needed(self, dag: RepresentationDag) -> bool: - input_stats = {} + def _node_output_stats(self, dag: RepresentationDag) -> Dict[str, Any]: + stats = self._node_stats for modality in self.modalities: if modality.modality_id == dag.nodes[0].modality_id: - input_stats[dag.nodes[0].node_id] = modality.stats + stats.setdefault(dag.nodes[0].node_id, modality.stats) break for node in dag.nodes[1:]: + if node.node_id in stats or node.operation is None: + continue previous_stats = [ - input_stats.get(input_node_id, None) for input_node_id in node.inputs + stats.get(input_node_id, None) for input_node_id in node.inputs ] - current_stats = node.operation(params=node.parameters).get_output_stats( + stats[node.node_id] = node.operation( + params=node.parameters + ).get_output_stats( previous_stats if len(previous_stats) > 1 else previous_stats[0] ) - input_stats[node.node_id] = current_stats - return len(input_stats.get(dag.root_node_id, None).output_shape) > 1 + return stats + + def _dag_output_length(self, dag: RepresentationDag) -> Optional[int]: + stats = self._node_output_stats(dag).get(dag.root_node_id, None) + output_shape = getattr(stats, "output_shape", None) + if not output_shape: + return None + return int(output_shape[0]) + + def _aggregation_needed(self, dag: RepresentationDag) -> bool: + return len(self._node_output_stats(dag)[dag.root_node_id].output_shape) > 1 def add_aggregation_operator(self, builder, dags): new_dags = [] @@ -586,19 +642,49 @@ def default_context_operators( ) dags.append(builder.build(context_node_id)) - context_operators = self._get_context_operators( - rep_dag.nodes[-1].operation().output_modality_type - ) - for context_op in context_operators: - context_node_id = builder.create_operation_node( - context_op, - [rep_dag.nodes[-1].node_id], - context_op().get_current_parameters(), + if self._representations_keep_time_axis(modality.modality_type): + rep_root = rep_dag.get_node_by_id(rep_dag.root_node_id) + context_operators = self._get_context_operators( + rep_root.operation().output_modality_type ) - dags.append(builder.build(context_node_id)) + output_length = self._dag_output_length(rep_dag) + for context_op in context_operators: + context_operator_instance = context_op() + if not self._size_context_operator( + context_operator_instance, output_length + ): + continue + context_node_id = builder.create_operation_node( + context_op, + [rep_root.node_id], + context_operator_instance.get_current_parameters(), + ) + dags.append(builder.build(context_node_id)) return dags + def _size_context_operator(self, context_operator_instance, output_length) -> bool: + parameter = getattr(context_operator_instance, "granularity_parameter", None) + kind = getattr(context_operator_instance, "granularity_kind", None) + if parameter is None or kind not in ("length", "count"): + return True + if output_length is None or output_length < 4: + return False + current = int(getattr(context_operator_instance, parameter)) + setattr( + context_operator_instance, + parameter, + max(2, min(current, output_length // 2)), + ) + return True + + @staticmethod + def _representations_keep_time_axis(modality_type) -> bool: + return modality_type not in ( + ModalityType.TIMESERIES, + ModalityType.PHYSIOLOGICAL, + ) + def temporal_context_operators(self, modality, builder, leaf_id): aggregators = self.operator_registry.get_context_representations( modality.modality_type @@ -610,23 +696,81 @@ def temporal_context_operators(self, modality, builder, leaf_id): ) ) dags = [] - for agg in aggregators: - for context_operator in context_operators: - for window_size, num_window in zip(window_lengths, num_windows): + for context_operator in context_operators: + for window_size, num_window in zip(window_lengths, num_windows): + window_node_ids = [] + for agg in aggregators: context_operator_instance = context_operator(agg()) - if hasattr(context_operator_instance, "num_windows"): - context_operator_instance.num_windows = num_window - elif hasattr(context_operator_instance, "window_size"): - context_operator_instance.window_size = window_size + self._apply_granularity( + context_operator_instance, window_size, num_window + ) context_node_id = builder.create_operation_node( context_operator, [leaf_id], context_operator_instance.get_current_parameters(), ) + window_node_ids.append(context_node_id) dags.append(builder.build(context_node_id)) + dags.extend( + self.combine_windowed_representations(builder, window_node_ids) + ) + return dags + @staticmethod + def _apply_granularity(context_operator_instance, window_size, num_window): + parameter = getattr(context_operator_instance, "granularity_parameter", None) + kind = getattr(context_operator_instance, "granularity_kind", None) + if parameter is None or kind not in ("length", "count"): + raise ValueError( + f"{type(context_operator_instance).__name__} is registered as a " + "context operator but does not declare granularity_parameter / " + "granularity_kind, so the window-length search cannot vary it." + ) + value = window_size if kind == "length" else num_window + setattr(context_operator_instance, parameter, int(value)) + + def combine_windowed_representations(self, builder, window_node_ids): + dags = [] + num_chains = min(self.window_combination_chains, len(window_node_ids)) + if len(window_node_ids) < 2 or num_chains < 1: + return dags + + for start in range(num_chains): + ordered = window_node_ids[start:] + window_node_ids[:start] + for combination in self._combination_operators: + parameters = combination.get_current_parameters() + if "preserve_leading_axis" not in parameters: + continue + parameters["preserve_leading_axis"] = True + + chain_tip = ordered[0] + for next_node_id in ordered[1:]: + chain_tip = builder.create_operation_node( + combination.__class__, + [chain_tip, next_node_id], + parameters, + ) + summary_id = self._summarize_windows(builder, chain_tip) + dags.append( + builder.build(chain_tip if summary_id is None else summary_id) + ) + return dags + + def _summarize_windows(self, builder, node_id): + if not (self._tasks_require_same_dims and self.expected_dimensions == 1): + return None + + agg_operator = AggregatedRepresentation( + target_dimensions=self.expected_dimensions, aggregate_leading=True + ) + return builder.create_operation_node( + agg_operator.__class__, + [node_id], + agg_operator.get_current_parameters(), + ) + class UnimodalResults: def __init__( @@ -651,13 +795,22 @@ def __init__( self.cache[modality] = {task_name: [] for task_name in self.task_names} self.worker_stats = {} self.node_stats = {} + self.reuse_stats = {} + self.wall_clock_s = {} + self._eval_counter = 0 + self._search_start = time.perf_counter() + self._dag_index = None + self._dag_index_source = None def add_task_result(self, task_result: ResultEntry, dags: List[RepresentationDag]): dag_id = task_result.dag.dag_id task_name = self.task_names[ task_result.dag.nodes[-1].parameters.get("_task_idx", 0) ] - task_result.dag = get_dag_by_id(dags, dag_id) + if self._dag_index_source is not dags: + self._dag_index = {dag.dag_id: dag for dag in dags} + self._dag_index_source = dags + task_result.dag = self._dag_index.get(dag_id) self.results[task_result.dag.nodes[0].modality_id][task_name].append( task_result ) @@ -692,10 +845,17 @@ def add_result( train_score=scores[0].average_scores, val_score=scores[1].average_scores, test_score=scores[2].average_scores, + train_fold_scores=scores[0].fold_scores(), + val_fold_scores=scores[1].fold_scores(), + test_fold_scores=scores[2].fold_scores(), representation_time=transform_time, task_time=task_time, dag=dag, + eval_index=self._eval_counter, + t_since_search_start_s=time.perf_counter() - self._search_start, + t_eval_end_unix=time.time(), ) + self._eval_counter += 1 scores = [ -item.val_score[self.metric_name] @@ -785,6 +945,9 @@ def get_k_best_results( def add_worker_stat(self, worker_stats, modality_id): self.worker_stats[modality_id] = worker_stats + def add_reuse_stat(self, reuse_stats, modality_id): + self.reuse_stats[modality_id] = reuse_stats + def add_node_stat(self, node_stats, modality_id): self.node_stats[modality_id] = node_stats diff --git a/src/main/python/systemds/scuro/modality/modality.py b/src/main/python/systemds/scuro/modality/modality.py index 477e4e45f35..a0d1e36377d 100644 --- a/src/main/python/systemds/scuro/modality/modality.py +++ b/src/main/python/systemds/scuro/modality/modality.py @@ -84,13 +84,19 @@ def update_metadata(self): """ Updates the metadata of the modality (i.e.: updates timestamps) """ - if ( - not self.has_metadata() - or not self.has_data() - or len(self.data) < len(self.metadata) - ): + if not self.has_metadata() or not self.has_data(): + return + + num_instances = len(self.data) + if num_instances < len(self.metadata): return + while len(self.metadata) < num_instances: + template = ( + selective_copy_metadata(self.metadata[0]) if self.metadata else {} + ) + self.metadata.append(template) + for i, md_v in enumerate(self.metadata): md_v = selective_copy_metadata(md_v) updated_md = self.modality_type.update_metadata(md_v, self.data[i]) @@ -132,88 +138,118 @@ def flatten(self, padding=False): self.data = np.array(data) return self - def pad(self, value=0, max_len=None): - try: - if max_len is None: - result = np.array(self.data) - elif isinstance(self.data, np.ndarray) and self.data.shape[1] == max_len: - result = self.data - else: - raise "Needs padding to max_len" - except: - first = self.data[0] - if isinstance(first, np.ndarray) and first.ndim == 3: - maxlen = ( - max([seq.shape[0] for seq in self.data]) - if max_len is None - else max_len - ) - tail_shape = first.shape[1:] - result = np.full( - (len(self.data), maxlen, *tail_shape), - value, - dtype=self.data_type or first.dtype, - ) - for i, seq in enumerate(self.data): - data = seq[:maxlen] - result[i, : len(data), ...] = data - if self.has_metadata(): - attention_mask = np.zeros(maxlen, dtype=np.int8) - attention_mask[: len(data)] = 1 - if "attention_mask" in self.metadata[i]: - self.metadata[i]["attention_mask"] = attention_mask - else: - self.metadata[i].update({"attention_mask": attention_mask}) - elif ( - isinstance(first, list) - and len(first) > 0 - and isinstance(first[0], np.ndarray) - and first[0].ndim == 2 - ): - maxlen = ( - max([len(seq) for seq in self.data]) if max_len is None else max_len - ) - row_dim, col_dim = first[0].shape - result = np.full( - (len(self.data), maxlen, row_dim, col_dim), - value, - dtype=self.data_type or first[0].dtype, - ) - for i, seq in enumerate(self.data): - data = seq[:maxlen] - # stack list of 2D arrays into 3D then assign - if len(data) > 0: - result[i, : len(data), :, :] = np.stack(data, axis=0) - if self.has_metadata(): - attention_mask = np.zeros(maxlen, dtype=np.int8) - attention_mask[: len(data)] = 1 - if "attention_mask" in self.metadata[i]: - self.metadata[i]["attention_mask"] = attention_mask - else: - self.metadata[i].update({"attention_mask": attention_mask}) - else: - maxlen = ( - max([len(seq) for seq in self.data]) if max_len is None else max_len - ) - result = np.full((len(self.data), maxlen), value, dtype=self.data_type) - for i, seq in enumerate(self.data): - data = seq[:maxlen] - try: - result[i, : len(data)] = data - except: - print(f"Error padding data for modality {self.modality_id}") - print(f"Data shape: {data.shape}") - print(f"Result shape: {result.shape}") - raise Exception("Error padding data") - if self.has_metadata(): - attention_mask = np.zeros(result.shape[1], dtype=np.int8) - attention_mask[: len(data)] = 1 - if "attention_mask" in self.metadata[i]: - self.metadata[i]["attention_mask"] = attention_mask - else: - self.metadata[i].update({"attention_mask": attention_mask}) - # TODO: this might need to be a new modality (otherwise we loose the original data) + def _set_attention_mask(self, index, attention_mask): + if not self.has_metadata() or index >= len(self.metadata): + return + if "attention_mask" in self.metadata[index]: + self.metadata[index]["attention_mask"] = attention_mask + else: + self.metadata[index].update({"attention_mask": attention_mask}) + + def _reshape_single_instance_embedding(self, arr): + if arr.ndim != 1: + return arr + if self.has_metadata() and len(self.metadata) == 1: + return arr.reshape(1, -1) + if not self.has_metadata() or len(self.metadata) <= 1: + return arr.reshape(1, -1) + return arr + + def _pad_embedding_matrix(self, value, max_len): + arr = np.asarray(self.data) + if arr.dtype == object: + return False + + arr = self._reshape_single_instance_embedding(arr) + if arr.ndim != 2: + return False + + if arr.shape[1] == max_len: + self.data = arr.copy() + return True + + result = np.full( + (arr.shape[0], max_len), + value, + dtype=self.data_type or arr.dtype, + ) + copy_width = min(arr.shape[1], max_len) + result[:, :copy_width] = arr[:, :copy_width] self.data = result + return True + + def _pad_variable_length_sequences(self, value, max_len): + first = self.data[0] + if isinstance(first, np.ndarray) and first.ndim == 3: + maxlen = ( + max([seq.shape[0] for seq in self.data]) if max_len is None else max_len + ) + tail_shape = first.shape[1:] + result = np.full( + (len(self.data), maxlen, *tail_shape), + value, + dtype=self.data_type or first.dtype, + ) + for i, seq in enumerate(self.data): + data = seq[:maxlen] + result[i, : len(data), ...] = data + attention_mask = np.zeros(maxlen, dtype=np.int8) + attention_mask[: len(data)] = 1 + self._set_attention_mask(i, attention_mask) + elif ( + isinstance(first, list) + and len(first) > 0 + and isinstance(first[0], np.ndarray) + and first[0].ndim == 2 + ): + maxlen = ( + max([len(seq) for seq in self.data]) if max_len is None else max_len + ) + row_dim, col_dim = first[0].shape + result = np.full( + (len(self.data), maxlen, row_dim, col_dim), + value, + dtype=self.data_type or first[0].dtype, + ) + for i, seq in enumerate(self.data): + data = seq[:maxlen] + if len(data) > 0: + result[i, : len(data), :, :] = np.stack(data, axis=0) + attention_mask = np.zeros(maxlen, dtype=np.int8) + attention_mask[: len(data)] = 1 + self._set_attention_mask(i, attention_mask) + else: + maxlen = ( + max([len(seq) for seq in self.data]) if max_len is None else max_len + ) + result = np.full((len(self.data), maxlen), value, dtype=self.data_type) + for i, seq in enumerate(self.data): + data = seq[:maxlen] + try: + result[i, : len(data)] = data + except Exception as exc: + raise ValueError( + f"Error padding data for modality {self.modality_id}: " + f"data shape {getattr(data, 'shape', None)}, " + f"result shape {result.shape}" + ) from exc + attention_mask = np.zeros(result.shape[1], dtype=np.int8) + attention_mask[: len(data)] = 1 + self._set_attention_mask(i, attention_mask) + self.data = result + + def pad(self, value=0, max_len=None): + if not self.has_data(): + return + + if max_len is None: + self.data = np.array(self.data) + return + + if self._pad_embedding_matrix(value, max_len): + return + + self._pad_variable_length_sequences(value, max_len) def get_data_layout(self): if self.has_metadata(): diff --git a/src/main/python/systemds/scuro/modality/unimodal_modality.py b/src/main/python/systemds/scuro/modality/unimodal_modality.py index 0535c64bcee..e1bba7df9da 100644 --- a/src/main/python/systemds/scuro/modality/unimodal_modality.py +++ b/src/main/python/systemds/scuro/modality/unimodal_modality.py @@ -27,7 +27,11 @@ from systemds.scuro.modality.modality import Modality from systemds.scuro.modality.joined import JoinedModality from systemds.scuro.modality.transformed import TransformedModality -from systemds.scuro.representations.representation import RepresentationStats +from systemds.scuro.representations.representation import ( + CONTAINER_ARRAY, + RepresentationStats, + stats_bytes, +) from systemds.scuro.utils.identifier import Identifier @@ -64,21 +68,36 @@ def get_metadata_at_position(self, position: int): return self.metadata[position] def get_stats(self): + if self.stats is not None and getattr(self.stats, "dtype", None) is None: + try: + self.stats.dtype = np.dtype(self.data_loader.data_type) + except (AttributeError, TypeError): + pass return self.stats def get_output_stats(self): - return RepresentationStats(self.stats.num_instances, self.stats.output_shape) + stats = self.get_stats() + return RepresentationStats( + stats.num_instances, + stats.output_shape, + output_shape_is_known=getattr(stats, "output_shape_is_known", True), + dtype=getattr(stats, "dtype", None), + container=getattr(stats, "container", CONTAINER_ARRAY), + shape_variance=getattr(stats, "shape_variance", 0.0), + sampling_rate=getattr(stats, "sampling_rate", None), + ) def estimate_memory_bytes(self): - memory_bytes = 1 - for i in self.stats.output_shape: - memory_bytes *= i - - return ( - self.stats.num_instances * memory_bytes * 4 - ) # TODO: check how to meausure str size + return stats_bytes(self.get_stats()) def estimate_peak_memory_bytes(self): + loader_estimate = getattr(self.data_loader, "estimate_peak_memory_bytes", None) + if callable(loader_estimate): + estimate = loader_estimate() + return { + "cpu_peak_bytes": float(estimate["cpu_peak_bytes"]), + "gpu_peak_bytes": float(estimate.get("gpu_peak_bytes", 0.0)), + } return {"cpu_peak_bytes": self.estimate_memory_bytes(), "gpu_peak_bytes": 0.0} def extract_raw_data(self): @@ -149,6 +168,7 @@ def apply_representations(self, representations, aggregation=None, parallel=Fals for representation in representations: transformed_modality = TransformedModality(self, representation.name) transformed_modality.data = [] + transformed_modality.metadata = [] transformed_modalities_per_representation[representation.name] = ( transformed_modality ) diff --git a/src/main/python/systemds/scuro/representations/aggregate.py b/src/main/python/systemds/scuro/representations/aggregate.py index e8a44faa34c..68f42d14e98 100644 --- a/src/main/python/systemds/scuro/representations/aggregate.py +++ b/src/main/python/systemds/scuro/representations/aggregate.py @@ -41,11 +41,38 @@ def _min_agg(data, aggregate_dim=0): def _sum_agg(data, aggregate_dim=0): return np.sum(data, axis=aggregate_dim) + @staticmethod + def _median_agg(data, aggregate_dim=0): + return np.median(data, axis=Aggregation._normalize_axis(aggregate_dim)) + + @staticmethod + def _mode_agg(data, aggregate_dim=0): + axis = Aggregation._normalize_axis(aggregate_dim) + arr = np.asarray(data) + if arr.ndim == 1: + values, counts = np.unique(arr, return_counts=True) + return values[np.argmax(counts)] + moved = np.moveaxis(arr, axis, -1) + flat = moved.reshape(-1, moved.shape[-1]) + modes = np.empty(flat.shape[0], dtype=arr.dtype) + for i in range(flat.shape[0]): + values, counts = np.unique(flat[i], return_counts=True) + modes[i] = values[np.argmax(counts)] + return modes.reshape(moved.shape[:-1]) + + @staticmethod + def _normalize_axis(aggregate_dim): + if isinstance(aggregate_dim, tuple): + return aggregate_dim[0] if aggregate_dim else 0 + return aggregate_dim + _aggregation_function = { "mean": _mean_agg.__func__, "max": _max_agg.__func__, "min": _min_agg.__func__, "sum": _sum_agg.__func__, + "median": _median_agg.__func__, + "mode": _mode_agg.__func__, } def __init__(self, aggregation_function="mean", pad_modality=True, params=None): @@ -71,7 +98,7 @@ def get_current_parameters(self): "pad_modality": self.pad_modality, } - def execute(self, modality, aggregate_dim=(0,)): + def execute(self, modality, aggregate_dim=(0,), squeeze_singleton=True): data = [] max_len = 0 for i, instance in enumerate(modality.data): @@ -83,7 +110,8 @@ def execute(self, modality, aggregate_dim=(0,)): ) and instance.ndim > 2: aggregated_data = instance.flatten() elif ( - isinstance(instance, np.ndarray) + squeeze_singleton + and isinstance(instance, np.ndarray) and instance.ndim == 2 and instance.shape[1] == 1 ): diff --git a/src/main/python/systemds/scuro/representations/aggregated_representation.py b/src/main/python/systemds/scuro/representations/aggregated_representation.py index 85744f9209a..b37b1eceda9 100644 --- a/src/main/python/systemds/scuro/representations/aggregated_representation.py +++ b/src/main/python/systemds/scuro/representations/aggregated_representation.py @@ -30,7 +30,13 @@ class AggregatedRepresentation(Representation): - def __init__(self, aggregation="mean", target_dimensions=None, params=None): + def __init__( + self, + aggregation="mean", + target_dimensions=None, + params=None, + aggregate_leading=False, + ): if params is not None: if "aggregation_function_aggregation_function" in params: aggregation = params["aggregation_function_aggregation_function"] @@ -40,6 +46,7 @@ def __init__(self, aggregation="mean", target_dimensions=None, params=None): aggregation = params["aggregation"] if "target_dimensions" in params: target_dimensions = params["target_dimensions"] + aggregate_leading = params.get("aggregate_leading", aggregate_leading) parameters = { "aggregation": list(Aggregation().get_aggregation_functions()), } @@ -48,13 +55,14 @@ def __init__(self, aggregation="mean", target_dimensions=None, params=None): self.aggregation = Aggregation(aggregation) self.self_contained = True self.target_dimensions = target_dimensions + self.aggregate_leading = bool(aggregate_leading) self.data_type = np.float32 def get_output_stats(self, input_stats: RepresentationStats) -> RepresentationStats: input_shape = list(copy.deepcopy(input_stats.output_shape)) if self.target_dimensions is not None: while len(input_shape) > self.target_dimensions: - input_shape.pop() + input_shape.pop(0 if self.aggregate_leading else -1) out_shape = tuple(input_shape) self.stats = RepresentationStats( input_stats.num_instances, @@ -99,6 +107,10 @@ def transform(self, modality): if len(input_dimensions) == self.target_dimensions: return modality + elif self.aggregate_leading: + aggregate_dim = tuple( + range(len(input_dimensions) - self.target_dimensions) + ) else: i = len(input_dimensions) - 1 aggregate_dim = () @@ -107,7 +119,9 @@ def transform(self, modality): i -= 1 input_dimensions = input_dimensions[:-1] - aggregated_data = self.aggregation.execute(modality, aggregate_dim) + aggregated_data = self.aggregation.execute( + modality, aggregate_dim, squeeze_singleton=not self.aggregate_leading + ) aggregated_modality.data = aggregated_data end = time.perf_counter() @@ -122,6 +136,7 @@ def get_current_parameters(self): current_params[f"aggregation_function_{key}"] = value current_params["self_contained"] = self.self_contained current_params["target_dimensions"] = self.target_dimensions + current_params["aggregate_leading"] = self.aggregate_leading return current_params def assert_output_stats(self, aggregated_data): diff --git a/src/main/python/systemds/scuro/representations/representation.py b/src/main/python/systemds/scuro/representations/representation.py index c7b6d69d730..ae7df5e4477 100644 --- a/src/main/python/systemds/scuro/representations/representation.py +++ b/src/main/python/systemds/scuro/representations/representation.py @@ -42,6 +42,7 @@ class RepresentationStats: dtype: Optional[Any] = None container: str = CONTAINER_ARRAY shape_variance: float = 0.0 + sampling_rate: Optional[float] = None def stats_dtype(stats) -> np.dtype: @@ -180,6 +181,15 @@ def set_parameters(self, parameters): for parameter in parameters: setattr(self, parameter, parameters[parameter]) + def check_preconditions(self, input_stats) -> Optional[str]: + return None + + def configure_for_input(self, input_stats) -> None: + return None + + def filter_parameter_domain(self, name, values, input_stats): + return values + def estimate_memory_bytes(self, input_stats): output_memory_bytes = self.estimate_output_memory_bytes(input_stats) return output_memory_bytes diff --git a/src/main/python/systemds/scuro/representations/window_aggregation.py b/src/main/python/systemds/scuro/representations/window_aggregation.py index 52a17401959..a9a1f1eb41b 100644 --- a/src/main/python/systemds/scuro/representations/window_aggregation.py +++ b/src/main/python/systemds/scuro/representations/window_aggregation.py @@ -29,8 +29,10 @@ from systemds.scuro.representations.aggregate import Aggregation from systemds.scuro.representations.context import Context from systemds.scuro.representations.representation import ( + NDARRAY_OBJECT_OVERHEAD_BYTES, Representation, RepresentationStats, + stats_itemsize, ) @@ -65,7 +67,59 @@ def instantiate_nested_aggregation(agg_cls, nested): return agg_cls(**filtered) +def _pad_stack(arrays): + arrays = [np.asarray(a) for a in arrays] + if len({a.shape for a in arrays}) == 1: + return np.stack(arrays) + + ndim = max(a.ndim for a in arrays) + arrays = [a.reshape((1,) * (ndim - a.ndim) + a.shape) for a in arrays] + target_shape = tuple(max(a.shape[d] for a in arrays) for d in range(ndim)) + stacked = np.zeros((len(arrays), *target_shape), dtype=arrays[0].dtype) + for i, a in enumerate(arrays): + slices = tuple(slice(0, s) for s in a.shape) + stacked[(i, *slices)] = a + return stacked + + +def _append_tail_row(full_result, tail_result): + full_result = np.asarray(full_result) + tail_result = np.asarray(tail_result) + target_shape = full_result.shape[1:] + if tail_result.shape == target_shape: + tail_row = tail_result + else: + tail_row = np.zeros(target_shape, dtype=full_result.dtype) + slices = tuple( + slice(0, min(d, s)) for d, s in zip(target_shape, tail_result.shape) + ) + tail_row[slices] = tail_result[slices] + return np.concatenate([full_result, tail_row[None, ...]]) + + +def resolve_aggregation_function(aggregation_function, params): + if params is None: + return aggregation_function + if isinstance(params.get("aggregation_function"), (Aggregation, Representation)): + return params["aggregation_function"] + + nested_agg = { + key[len("aggregation_function_") :]: value + for key, value in params.items() + if key.startswith("aggregation_function_") + } + agg_value = params.get("aggregation_function") + if nested_agg and inspect.isclass(agg_value): + return instantiate_nested_aggregation(agg_value, nested_agg) + if inspect.isclass(agg_value): + return agg_value() + return params.get("aggregation_function", aggregation_function) + + class Window(Context): + granularity_parameter = None + granularity_kind = None # "length" | "count" + def __init__(self, name, aggregation_function): self.aggregation_function = aggregation_function parameters = {} @@ -143,11 +197,26 @@ def _shape_numel(shape): def _rest_numel(shape): return int(np.prod(shape[1:])) if len(shape) > 1 else 1 + def _per_window_feature_shape(self, approx_window_size): + windowed_input_stats = RepresentationStats(1, (approx_window_size,)) + feat_shape = self.aggregation_function.get_output_stats( + windowed_input_stats + ).output_shape + return () if self._shape_numel(feat_shape) <= 1 else tuple(feat_shape) + @register_context_operator( - [ModalityType.TIMESERIES, ModalityType.AUDIO, ModalityType.EMBEDDING] + [ + ModalityType.TIMESERIES, + ModalityType.PHYSIOLOGICAL, + ModalityType.AUDIO, + ModalityType.EMBEDDING, + ] ) class WindowAggregation(Window): + granularity_parameter = "window_size" + granularity_kind = "length" + def __init__( self, aggregation_function="mean", @@ -163,30 +232,12 @@ def __init__( window_size_set = True if params is not None: - if isinstance( - params.get("aggregation_function"), (Aggregation, Representation) - ): - aggregation_function = params["aggregation_function"] - if hasattr(aggregation_function, "window_size"): - window_size = aggregation_function.window_size - window_size_set = True - else: - nested_agg = { - key[len("aggregation_function_") :]: value - for key, value in params.items() - if key.startswith("aggregation_function_") - } - agg_value = params.get("aggregation_function") - if nested_agg and inspect.isclass(agg_value): - aggregation_function = instantiate_nested_aggregation( - agg_value, nested_agg - ) - elif inspect.isclass(agg_value): - aggregation_function = agg_value() - else: - aggregation_function = params.get( - "aggregation_function", aggregation_function - ) + aggregation_function = resolve_aggregation_function( + aggregation_function, params + ) + if hasattr(aggregation_function, "window_size"): + window_size = aggregation_function.window_size + window_size_set = True window_size = params["window_size"] if not window_size_set else window_size pad = params.get("pad", True) @@ -197,13 +248,8 @@ def __init__( def get_output_stats(self, input_stats: RepresentationStats) -> tuple: if not isinstance(self.aggregation_function, Aggregation): - windowed_input_stats = RepresentationStats( - input_stats.num_instances, (self.window_size,) - ) - in_shape = self.aggregation_function.get_output_stats( - windowed_input_stats - ).output_shape - in_shape = (input_stats.output_shape[0], *in_shape) + feat_shape = self._per_window_feature_shape(self.window_size) + in_shape = (input_stats.output_shape[0], *feat_shape) else: in_shape = tuple(int(s) for s in input_stats.output_shape) if len(in_shape) == 1: @@ -226,40 +272,35 @@ def estimate_output_memory_bytes(self, input_stats: RepresentationStats) -> int: out_seq_len = math.ceil(in_shape[0] / self.window_size) output_bytes = out_seq_len * self._rest_numel(in_shape) - return ( - input_stats.num_instances * output_bytes * np.dtype(self.data_type).itemsize - ) + return input_stats.num_instances * output_bytes * stats_itemsize(input_stats) def estimate_peak_memory_bytes(self, input_stats: RepresentationStats) -> dict: in_shape = tuple(int(s) for s in input_stats.output_shape) if len(in_shape) == 0: return {"cpu_peak_bytes": 0, "gpu_peak_bytes": 0} - out_stats = self.get_output_stats(input_stats) - out_shape = out_stats.output_shape - output_bytes = ( - input_stats.num_instances - * np.prod(out_shape) - * np.dtype(self.data_type).itemsize - ) - effective_seq_len = in_shape[0] in_numel = effective_seq_len * self._rest_numel(in_shape) output_bytes = self.estimate_output_memory_bytes(input_stats) - one_instance_bytes = in_numel * np.dtype(self.data_type).itemsize + one_instance_bytes = in_numel * stats_itemsize(input_stats) input_bytes = one_instance_bytes * input_stats.num_instances - output_transient = output_bytes + list_bytes = output_bytes + input_stats.num_instances * ( + NDARRAY_OBJECT_OVERHEAD_BYTES + ) - pad_overhead = 0 + pad_bytes = 0 if getattr(self, "pad", False): out_seq_len = math.ceil(in_shape[0] / self.window_size) - pad_overhead = int(input_stats.num_instances * out_seq_len * 8) + padded_elems = ( + input_stats.num_instances * out_seq_len * self._rest_numel(in_shape) + ) + pad_bytes = int( + padded_elems * np.dtype(np.float64).itemsize + + padded_elems * stats_itemsize(input_stats) + ) - cpu_peak = int( - (input_bytes + output_bytes + output_transient + pad_overhead) * 1.15 - + 16 * 1024 * 1024 - ) + cpu_peak = int((input_bytes + list_bytes + pad_bytes) * 1.15 + 16 * 1024 * 1024) return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} def execute(self, modality): @@ -317,34 +358,51 @@ def window_aggregate_single_level(self, instance, new_length): arr = np.asarray(instance) cut_length = (new_length - 1) * self.window_size - + tail = arr[cut_length:] + sig = inspect.signature(self.aggregation_function.compute_feature) + if new_length <= 1: + if not tail.size: + raise ValueError( + "Cannot window-aggregate an empty instance " + f"(window_size={self.window_size})." + ) + if tail.shape[0] < self.window_size: + pad_len = self.window_size - tail.shape[0] + if tail.ndim == 1: + tail = np.pad(tail, (0, pad_len), mode="constant") + else: + pad_width = [(0, 0)] * tail.ndim + pad_width[0] = (0, pad_len) + tail = np.pad(tail, pad_width=pad_width, mode="constant") + if "axis" in sig.parameters: + return np.array([self.aggregation_function.compute_feature(tail)]) + tail_result = self.aggregation_function.compute_feature(tail) + return ( + tail_result[None, :] + if tail_result.ndim > 0 + else np.array([tail_result]) + ) full_batches = arr[:cut_length].reshape( new_length - 1, self.window_size, *arr.shape[1:] ) - tail = arr[cut_length:] - sig = inspect.signature(self.aggregation_function.compute_feature) if "axis" in sig.parameters: full_result = self.aggregation_function.compute_feature( full_batches, axis=1 ) if tail.size: tail_result = self.aggregation_function.compute_feature(tail) - full_result = np.concatenate([full_result, np.array([tail_result])]) + full_result = _append_tail_row(full_result, tail_result) else: - full_result = self.aggregation_function.compute_feature(full_batches) + full_result = np.stack( + [ + self.aggregation_function.compute_feature(full_batches[i]) + for i in range(full_batches.shape[0]) + ] + ) if tail.size: tail_result = self.aggregation_function.compute_feature(tail) - if tail_result.shape == full_result.shape[1:]: - tail_row = tail_result - else: - tail_row = np.zeros_like(full_result[0]) - slices = tuple( - slice(0, min(d, s)) - for d, s in zip(tail_row.shape, tail_result.shape) - ) - tail_row[slices] = tail_result[slices] - full_result = np.concatenate([full_result, tail_row[None, :]]) + full_result = _append_tail_row(full_result, tail_result) return full_result @@ -359,27 +417,42 @@ def window_aggregate_nested_level(self, instance, new_length): @register_context_operator( - [ModalityType.TIMESERIES, ModalityType.AUDIO, ModalityType.EMBEDDING] + [ + ModalityType.TIMESERIES, + ModalityType.PHYSIOLOGICAL, + ModalityType.AUDIO, + ModalityType.EMBEDDING, + ] ) class StaticWindow(Window): + granularity_parameter = "num_windows" + granularity_kind = "count" + def __init__(self, aggregation_function="mean", num_windows=100, params=None): - super().__init__("StaticWindow", aggregation_function) if params is not None: - num_windows = params.get("num_windows", 100) + aggregation_function = resolve_aggregation_function( + aggregation_function, params + ) + num_windows = params.get("num_windows", num_windows) - self.parameters["num_windows"] = (5, num_windows) + super().__init__("StaticWindow", aggregation_function) + self.parameters["num_windows"] = (min(5, num_windows), max(5, num_windows)) self.num_windows = int(num_windows) + def _feature_shape(self, in_shape): + if isinstance(self.aggregation_function, Aggregation): + return in_shape[1:] + approx_window_size = ( + max(1, int(in_shape[0] / self.num_windows)) if in_shape else 1 + ) + return self._per_window_feature_shape(approx_window_size) + def get_output_stats(self, input_stats: RepresentationStats) -> tuple: in_shape = tuple(int(s) for s in input_stats.output_shape) - if len(in_shape) <= 1: - self.stats = RepresentationStats( - input_stats.num_instances, (self.num_windows,) - ) - else: - self.stats = RepresentationStats( - input_stats.num_instances, (self.num_windows, *in_shape[1:]) - ) + feat_shape = self._feature_shape(in_shape) + self.stats = RepresentationStats( + input_stats.num_instances, (self.num_windows, *feat_shape) + ) self.stats.output_shape_is_known = input_stats.output_shape_is_known return self.stats @@ -389,11 +462,9 @@ def estimate_output_memory_bytes(self, input_stats: RepresentationStats) -> int: if len(in_shape) == 0: return 0 - out_seq_len = self.num_windows - output_bytes = out_seq_len * self._rest_numel(in_shape) - return ( - input_stats.num_instances * output_bytes * np.dtype(self.data_type).itemsize - ) + out_shape = self.get_output_stats(input_stats).output_shape + out_numel = int(np.prod(out_shape)) if len(out_shape) > 0 else 1 + return input_stats.num_instances * out_numel * stats_itemsize(input_stats) def estimate_peak_memory_bytes(self, input_stats: RepresentationStats) -> dict: in_shape = tuple(int(s) for s in input_stats.output_shape) @@ -401,7 +472,7 @@ def estimate_peak_memory_bytes(self, input_stats: RepresentationStats) -> dict: return {"cpu_peak_bytes": 0, "gpu_peak_bytes": 0} effective_seq_len = in_shape[0] in_numel = effective_seq_len * self._rest_numel(in_shape) - one_instance_bytes = in_numel * np.dtype(self.data_type).itemsize + one_instance_bytes = in_numel * stats_itemsize(input_stats) input_bytes = one_instance_bytes * input_stats.num_instances output_bytes = self.estimate_output_memory_bytes(input_stats) output_transient = output_bytes @@ -428,34 +499,71 @@ def execute(self, modality): if "axis" in sig.parameters: f = self.aggregation_function.compute_feature(full_batches, axis=1) else: - f = self.aggregation_function.compute_feature(full_batches) + f = np.stack( + [ + self.aggregation_function.compute_feature(full_batches[i]) + for i in range(full_batches.shape[0]) + ] + ) windowed_data.append(f) - windowed_data = np.array(windowed_data) + windowed_data = _pad_stack(windowed_data) return windowed_data @register_context_operator( - [ModalityType.TIMESERIES, ModalityType.AUDIO, ModalityType.EMBEDDING] + [ + ModalityType.TIMESERIES, + ModalityType.PHYSIOLOGICAL, + ModalityType.AUDIO, + ModalityType.EMBEDDING, + ] ) class DynamicWindow(Window): + granularity_parameter = "num_windows" + granularity_kind = "count" + def __init__(self, aggregation_function="mean", num_windows=100, params=None): - super().__init__("DynamicWindow", aggregation_function) if params is not None: - num_windows = params.get("num_windows", 100) - self.parameters["num_windows"] = (5, num_windows) + aggregation_function = resolve_aggregation_function( + aggregation_function, params + ) + num_windows = params.get("num_windows", num_windows) + super().__init__("DynamicWindow", aggregation_function) + self.parameters["num_windows"] = (min(5, num_windows), max(5, num_windows)) self.num_windows = int(num_windows) + def _effective_num_windows(self, signal_length: int) -> int: + if signal_length <= 0: + return max(1, self.num_windows) + return max(1, min(self.num_windows, int(signal_length))) + + def _window_sizes(self, signal_length: int) -> np.ndarray: + num_windows = self._effective_num_windows(signal_length) + length = max(int(signal_length), num_windows) + weights = np.geomspace(4, 256, num=num_windows) + weights = weights / np.sum(weights) + + sizes = 1 + (weights * (length - num_windows)).astype(int) + sizes[-1] += length - int(sizes.sum()) + return sizes + + def _feature_shape(self, in_shape): + if isinstance(self.aggregation_function, Aggregation): + return in_shape[1:] + length = in_shape[0] if in_shape else 0 + approx_window_size = ( + max(1, int(length / self._effective_num_windows(length))) if in_shape else 1 + ) + return self._per_window_feature_shape(approx_window_size) + def get_output_stats(self, input_stats: RepresentationStats) -> tuple: in_shape = tuple(int(s) for s in input_stats.output_shape) - if len(in_shape) <= 1: - self.stats = RepresentationStats( - input_stats.num_instances, (self.num_windows,) - ) - else: - self.stats = RepresentationStats( - input_stats.num_instances, (self.num_windows, *in_shape[1:]) - ) + feat_shape = self._feature_shape(in_shape) + num_windows = self._effective_num_windows(in_shape[0] if in_shape else 0) + self.stats = RepresentationStats( + input_stats.num_instances, (num_windows, *feat_shape) + ) self.stats.output_shape_is_known = input_stats.output_shape_is_known return self.stats @@ -464,11 +572,9 @@ def estimate_output_memory_bytes(self, input_stats: RepresentationStats) -> int: if len(in_shape) == 0: return 0 - out_seq_len = self.num_windows - output_bytes = out_seq_len * self._rest_numel(in_shape) - return ( - input_stats.num_instances * output_bytes * np.dtype(self.data_type).itemsize - ) + out_shape = self.get_output_stats(input_stats).output_shape + out_numel = int(np.prod(out_shape)) if len(out_shape) > 0 else 1 + return input_stats.num_instances * out_numel * stats_itemsize(input_stats) def estimate_peak_memory_bytes(self, input_stats: RepresentationStats) -> dict: in_shape = tuple(int(s) for s in input_stats.output_shape) @@ -477,7 +583,7 @@ def estimate_peak_memory_bytes(self, input_stats: RepresentationStats) -> dict: effective_seq_len = in_shape[0] in_numel = effective_seq_len * self._rest_numel(in_shape) output_bytes = self.estimate_output_memory_bytes(input_stats) - one_instance_bytes = in_numel * np.dtype(self.data_type).itemsize + one_instance_bytes = in_numel * stats_itemsize(input_stats) cpu_peak = ( output_bytes * 2 + one_instance_bytes * input_stats.num_instances @@ -489,26 +595,16 @@ def execute(self, modality): windowed_data = [] for instance in modality.data: - N = len(instance) - weights = np.geomspace(4, 256, num=self.num_windows) - weights = weights / np.sum(weights) - window_sizes = (weights * N).astype(int) - window_sizes[-1] += N - np.sum(window_sizes) - indices = np.cumsum(window_sizes) + indices = np.cumsum(self._window_sizes(len(instance))) output = [] start = 0 for end in indices: window = instance[start:end] window.setflags(write=False) - val = ( - self.aggregation_function.compute_feature(window) - if len(window) > 0 - else np.zeros_like(instance[0]) - ) - output.append(val) + output.append(self.aggregation_function.compute_feature(window)) start = end - windowed_data.append(output) - windowed_data = np.array(windowed_data) + windowed_data.append(_pad_stack(output)) + windowed_data = _pad_stack(windowed_data) self.assert_output_stats(windowed_data) return windowed_data diff --git a/src/main/python/tests/iotests/test_io_csv.py b/src/main/python/tests/iotests/test_io_csv.py index 042e7e308a4..9d6860454dc 100644 --- a/src/main/python/tests/iotests/test_io_csv.py +++ b/src/main/python/tests/iotests/test_io_csv.py @@ -31,7 +31,7 @@ class TestReadCSV(unittest.TestCase): sds: SystemDSContext = None - temp_dir: str = "tests/iotests/temp_write_csv/" + temp_dir: str = "tests/iotests/temp_write_csv_read/" n_cols = 3 n_rows = 100 diff --git a/src/main/python/tests/iotests/test_io_pandas_systemds.py b/src/main/python/tests/iotests/test_io_pandas_systemds.py index 214bc7475ad..91482da7619 100644 --- a/src/main/python/tests/iotests/test_io_pandas_systemds.py +++ b/src/main/python/tests/iotests/test_io_pandas_systemds.py @@ -42,7 +42,7 @@ def create_dataframe(n_rows, n_cols, mixed=True): class TestPandasFromToSystemds(unittest.TestCase): sds: SystemDSContext = None - temp_dir: str = "tests/iotests/temp_write_csv/" + temp_dir: str = "tests/iotests/temp_write_csv_pandas/" @classmethod def setUpClass(cls): diff --git a/src/main/python/tests/python_java_data_transfer/test_dense_numpy_matrix.py b/src/main/python/tests/python_java_data_transfer/test_dense_numpy_matrix.py index fcfe683dc7f..5d84d2a0666 100644 --- a/src/main/python/tests/python_java_data_transfer/test_dense_numpy_matrix.py +++ b/src/main/python/tests/python_java_data_transfer/test_dense_numpy_matrix.py @@ -32,7 +32,7 @@ class TestMatrixBlockConverterUnixPipe(unittest.TestCase): sds: SystemDSContext = None - temp_dir: str = "tests/iotests/temp_write_csv/" + temp_dir: str = "tests/iotests/temp_write_csv_matrix/" @classmethod def setUpClass(cls): diff --git a/src/main/python/tests/python_java_data_transfer/test_pandas_frame.py b/src/main/python/tests/python_java_data_transfer/test_pandas_frame.py index a841795363a..28982bc4e0a 100644 --- a/src/main/python/tests/python_java_data_transfer/test_pandas_frame.py +++ b/src/main/python/tests/python_java_data_transfer/test_pandas_frame.py @@ -32,7 +32,7 @@ class TestFrameConverterUnixPipe(unittest.TestCase): sds: SystemDSContext = None - temp_dir: str = "tests/iotests/temp_write_csv/" + temp_dir: str = "tests/iotests/temp_write_csv_frame/" @classmethod def setUpClass(cls): From 072d50aeaa146023818a90ea07a68eed864a03eb Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Wed, 26 Aug 2026 10:05:25 +0200 Subject: [PATCH 124/132] [SYSTEMDS-3835] Add physiological representations to Scuro This patch adds new representations for physiological data modalities, including new aggregators and context operations. Additionally, it refines existing operators with additional batch functionality and memory estimations. Assisted-by: AI --- .../scuro/dataloader/tabular_loader.py | 78 +++ .../scuro/representations/aggregate.py | 2 +- .../systemds/scuro/representations/average.py | 37 +- .../systemds/scuro/representations/bert.py | 2 + .../systemds/scuro/representations/bow.py | 16 +- .../systemds/scuro/representations/clip.py | 12 +- .../scuro/representations/concatenation.py | 115 +++- .../systemds/scuro/representations/fusion.py | 76 ++- .../systemds/scuro/representations/glove.py | 4 +- .../scuro/representations/hadamard.py | 37 +- .../systemds/scuro/representations/mfcc.py | 2 +- .../scuro/representations/mlp_averaging.py | 16 +- .../physiological_representations.py | 641 ++++++++++++++++++ .../representations/physiological_window.py | 260 +++++++ .../systemds/scuro/representations/sum.py | 41 +- .../scuro/representations/tabular_features.py | 59 ++ .../text_context_with_indices.py | 2 + .../systemds/scuro/representations/tfidf.py | 5 +- .../timeseries_representations.py | 248 ++++++- .../systemds/scuro/representations/utils.py | 23 + .../systemds/scuro/representations/wav2vec.py | 32 +- .../representations/window_aggregation.py | 114 +++- .../scuro/representations/word2vec.py | 4 +- .../tests/scuro/test_operator_registry.py | 12 + 24 files changed, 1705 insertions(+), 133 deletions(-) create mode 100644 src/main/python/systemds/scuro/dataloader/tabular_loader.py create mode 100644 src/main/python/systemds/scuro/representations/physiological_representations.py create mode 100644 src/main/python/systemds/scuro/representations/physiological_window.py create mode 100644 src/main/python/systemds/scuro/representations/tabular_features.py diff --git a/src/main/python/systemds/scuro/dataloader/tabular_loader.py b/src/main/python/systemds/scuro/dataloader/tabular_loader.py new file mode 100644 index 00000000000..6652ba2b937 --- /dev/null +++ b/src/main/python/systemds/scuro/dataloader/tabular_loader.py @@ -0,0 +1,78 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- +from dataclasses import dataclass +import numpy as np +from typing import List, Optional, Union + +from systemds.scuro.dataloader.base_loader import BaseLoader +from systemds.scuro.modality.type import ModalityType + + +@dataclass +class TabularStats: + num_instances: int + num_features: int + output_shape: tuple + output_shape_is_known: bool = True + + +class TabularLoader(BaseLoader): + def __init__( + self, + source_path: str, + indices: List[str], + feature_names: Optional[List[str]] = None, + data_type: Union[np.dtype, str] = np.float32, + chunk_size: Optional[int] = None, + normalize: bool = False, + file_format: str = "npy", + modality_type: Optional[ModalityType] = ModalityType.EMBEDDING, + ): + super().__init__(source_path, indices, data_type, chunk_size, modality_type) + self.feature_names = feature_names + self.normalize = normalize + self.file_format = file_format.lower() + if self.file_format != "npy": + raise ValueError(f"Unsupported file format: {self.file_format}") + self.stats = self.get_stats(source_path) + + def extract(self, file: str, index: Optional[Union[str, List[str]]] = None): + self.file_sanity_check(file) + data = np.load(file).astype(self._data_type, copy=False).reshape(-1) + + if self.normalize: + mean = np.mean(data) + std = np.std(data) + data = (data - mean) / (std + 1e-8) + + self.metadata.append(self.modality_type.create_metadata(data)) + self.data.append(data) + + def get_stats(self, source_path: str) -> TabularStats: + num_instances = 0 + num_features = 0 + for file_name in self.indices: + file = source_path + file_name + "." + self.file_format + self.file_sanity_check(file) + data = np.load(file) + num_features = max(num_features, int(np.prod(data.shape))) + num_instances += 1 + return TabularStats(num_instances, num_features, (num_features,)) diff --git a/src/main/python/systemds/scuro/representations/aggregate.py b/src/main/python/systemds/scuro/representations/aggregate.py index 68f42d14e98..afbd6b346ea 100644 --- a/src/main/python/systemds/scuro/representations/aggregate.py +++ b/src/main/python/systemds/scuro/representations/aggregate.py @@ -89,7 +89,7 @@ def __init__(self, aggregation_function="mean", pad_modality=True, params=None): self.aggregation_function_name = aggregation_function self.parameters = { - "aggregation_function": self._aggregation_function.keys(), + "aggregation_function": list(self._aggregation_function.keys()), } def get_current_parameters(self): diff --git a/src/main/python/systemds/scuro/representations/average.py b/src/main/python/systemds/scuro/representations/average.py index f58ba0b6802..06c435d6580 100644 --- a/src/main/python/systemds/scuro/representations/average.py +++ b/src/main/python/systemds/scuro/representations/average.py @@ -18,12 +18,12 @@ # under the License. # # ------------------------------------------------------------- -import copy from typing import List import numpy as np from systemds.scuro.modality.modality import Modality +from systemds.scuro.representations.representation import RepresentationStats from systemds.scuro.representations.utils import pad_sequences from systemds.scuro.representations.fusion import Fusion @@ -41,11 +41,38 @@ def __init__(self, params=None): self.associative = True self.commutative = True - def execute(self, modalities: List[Modality], labels=None): - data = np.asarray(copy.deepcopy(modalities[0].data), dtype=float) + def execute(self, modalities: List[Modality]): + data = np.array(modalities[0].data, dtype=np.float64) for i in range(1, len(modalities)): - data += np.asarray(modalities[i].data, dtype=float) + data += np.asarray(modalities[i].data, dtype=np.float64) data /= len(modalities) - return np.array(data) + return data + + def get_output_stats(self, input_stats_list) -> RepresentationStats: + stats_list = self._fusion_input_stats(input_stats_list) + if not stats_list: + return RepresentationStats(0, (0,)) + + num_instances = max(s.num_instances for s in stats_list) + rank = len(stats_list[0].output_shape) + if rank > 0 and all(len(s.output_shape) == rank for s in stats_list): + output_shape = tuple( + max(s.output_shape[d] for s in stats_list) for d in range(rank) + ) + else: + output_shape = max(stats_list, key=self._stats_num_elements).output_shape + output_shape_is_known = all(s.output_shape_is_known for s in stats_list) + return RepresentationStats(num_instances, output_shape, output_shape_is_known) + + def estimate_peak_memory_bytes(self, input_stats) -> dict: + stats_list = self._as_stats_list(input_stats) + input_bytes = sum(self._stats_bytes(s) for s in stats_list) + output_bytes = self._stats_bytes(self.get_output_stats(input_stats)) + + raw_bytes = self._raw_input_bytes(input_stats) + cpu_peak = ( + int((raw_bytes + input_bytes + 2 * output_bytes) * 1.15) + 8 * 1024 * 1024 + ) + return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} diff --git a/src/main/python/systemds/scuro/representations/bert.py b/src/main/python/systemds/scuro/representations/bert.py index 245466afb43..9e60f843416 100644 --- a/src/main/python/systemds/scuro/representations/bert.py +++ b/src/main/python/systemds/scuro/representations/bert.py @@ -102,6 +102,7 @@ def get_output_stats(self, input_stats) -> RepresentationStats: input_stats.num_instances, (self.max_seq_length, 768), aggregate_dim=(0,), + dtype=self.data_type, ) else: self.stats = RepresentationStats( @@ -111,6 +112,7 @@ def get_output_stats(self, input_stats) -> RepresentationStats: 0, 1, ), + dtype=self.data_type, ) if self.params and "_pushdown_aggregation" in self.params: output_shape = (768,) diff --git a/src/main/python/systemds/scuro/representations/bow.py b/src/main/python/systemds/scuro/representations/bow.py index 9e55add5de0..9a7766106c1 100644 --- a/src/main/python/systemds/scuro/representations/bow.py +++ b/src/main/python/systemds/scuro/representations/bow.py @@ -18,6 +18,8 @@ # under the License. # # ------------------------------------------------------------- +import os + import numpy as np from sklearn.feature_extraction.text import CountVectorizer @@ -30,6 +32,8 @@ from systemds.scuro.drsearch.operator_registry import register_representation from systemds.scuro.dataloader.text_loader import TextStats +_MAX_VOCAB_FEATURES = int(os.environ.get("SCURO_BOW_MAX_FEATURES", "100000")) + @register_representation(ModalityType.TEXT) class BoW(UnimodalRepresentation): @@ -43,14 +47,17 @@ def __init__(self, ngram_range=2, min_df=2, output_file=None, params=None): def get_output_stats(self, input_stats: TextStats) -> RepresentationStats: vocab_estimate = min( - 100000, + _MAX_VOCAB_FEATURES, max( 1000, input_stats.num_instances * input_stats.max_length * self.ngram_range, ), ) return RepresentationStats( - input_stats.num_instances, (vocab_estimate,), output_shape_is_known=False + input_stats.num_instances, + (vocab_estimate,), + output_shape_is_known=False, + dtype=self.data_type, ) def estimate_output_memory_bytes(self, input_stats: TextStats) -> int: @@ -73,7 +80,10 @@ def estimate_peak_memory_bytes(self, input_stats: TextStats) -> dict: def transform(self, modality, aggregation=None): transformed_modality = TransformedModality(modality, self) vectorizer = CountVectorizer( - ngram_range=(1, self.ngram_range), min_df=self.min_df + ngram_range=(1, self.ngram_range), + min_df=self.min_df, + max_features=_MAX_VOCAB_FEATURES, + dtype=np.float32, ) X = ( diff --git a/src/main/python/systemds/scuro/representations/clip.py b/src/main/python/systemds/scuro/representations/clip.py index 2c880686b11..c4e28404466 100644 --- a/src/main/python/systemds/scuro/representations/clip.py +++ b/src/main/python/systemds/scuro/representations/clip.py @@ -109,13 +109,17 @@ def get_output_stats(self, input_stats) -> RepresentationStats: input_stats.max_length, 512, ), + dtype=self.data_type, ) elif not isinstance(input_stats, RepresentationStats): - return RepresentationStats(input_stats.num_instances, (512,)) + return RepresentationStats( + input_stats.num_instances, (512,), dtype=self.data_type + ) else: return RepresentationStats( input_stats.num_instances, (input_stats.output_shape[0], 512), + dtype=self.data_type, ) def estimate_peak_memory_bytes(self, input_stats) -> dict: @@ -394,7 +398,10 @@ def _get_parameters(self): def get_output_stats(self, input_stats) -> RepresentationStats: if not isinstance(input_stats, RepresentationStats): self.stats = RepresentationStats( - input_stats.num_instances, (512,), aggregate_dim=(0,) + input_stats.num_instances, + (512,), + aggregate_dim=(0,), + dtype=self.data_type, ) else: self.stats = RepresentationStats( @@ -404,6 +411,7 @@ def get_output_stats(self, input_stats) -> RepresentationStats: 0, 1, ), + dtype=self.data_type, ) if self.params and "_pushdown_aggregation" in self.params: output_shape = (512,) diff --git a/src/main/python/systemds/scuro/representations/concatenation.py b/src/main/python/systemds/scuro/representations/concatenation.py index 3bdfdb28b1f..cee30382963 100644 --- a/src/main/python/systemds/scuro/representations/concatenation.py +++ b/src/main/python/systemds/scuro/representations/concatenation.py @@ -34,11 +34,70 @@ @register_fusion_operator() class Concatenation(Fusion): - def __init__(self, params=None): - """ - Combines modalities using concatenation - """ + def __init__(self, params=None, preserve_leading_axis=False): super().__init__("Concatenation") + if params is not None: + preserve_leading_axis = params.get( + "preserve_leading_axis", preserve_leading_axis + ) + + self.preserve_leading_axis = bool(preserve_leading_axis) + self.preserves_leading_axis = self.preserve_leading_axis + + def get_current_parameters(self): + current_params = super().get_current_parameters() + current_params["preserve_leading_axis"] = self.preserve_leading_axis + return current_params + + @staticmethod + def _as_dense(modality): + dtype = modality.metadata[0]["data_layout"]["type"] + data = modality.data + arr = ( + np.asarray(data, dtype=dtype) if not isinstance(data, np.ndarray) else data + ) + if arr.dtype == object: + instances = [np.asarray(instance, dtype=dtype) for instance in data] + rest = tuple( + max(i.shape[d] for i in instances) for d in range(instances[0].ndim) + ) + arr = np.zeros((len(instances), *rest), dtype=dtype) + for i, instance in enumerate(instances): + arr[(i, *(slice(0, s) for s in instance.shape))] = instance + return arr + + @staticmethod + def _to_window_feature_matrix(arr): + if arr.ndim == 1: + return arr[:, None, None] + if arr.ndim == 2: + return arr[:, :, None] + return arr.reshape(arr.shape[0], arr.shape[1], -1) + + @staticmethod + def _flatten_feature_shape(shape): + if len(shape) == 0: + return (1, 1) + if len(shape) == 1: + return (shape[0], 1) + return (shape[0], int(np.prod(shape[1:]))) + + def _concat_on_leading_axis(self, modalities: List[Modality]): + arrays = [ + self._to_window_feature_matrix(self._as_dense(modality)) + for modality in modalities + ] + + num_windows = max(arr.shape[1] for arr in arrays) + aligned = [] + for arr in arrays: + if arr.shape[1] < num_windows: + pad_width = [(0, 0)] * arr.ndim + pad_width[1] = (0, num_windows - arr.shape[1]) + arr = np.pad(arr, pad_width=pad_width, mode="constant") + aligned.append(arr) + + return np.concatenate(aligned, axis=-1) def execute(self, modalities: List[Modality]): if len(modalities) == 1: @@ -47,6 +106,9 @@ def execute(self, modalities: List[Modality]): dtype=modalities[0].metadata[0]["data_layout"]["type"], ) + if self.preserve_leading_axis: + return self._concat_on_leading_axis(modalities) + max_emb_size = self.get_max_embedding_size(modalities) size = len(modalities[0].data) @@ -71,35 +133,32 @@ def execute(self, modalities: List[Modality]): return np.array(data) def get_output_stats(self, input_stats_list) -> RepresentationStats: - if isinstance(input_stats_list, RepresentationStats): - return input_stats_list - - stats_list = list(input_stats_list) + stats_list = self._fusion_input_stats(input_stats_list) if not stats_list: return RepresentationStats(0, (0,)) - num_instances = stats_list[0].num_instances - total_dim = sum(s.output_shape[-1] for s in stats_list) - output_shape = (total_dim,) - - return RepresentationStats(num_instances, output_shape) + num_instances = max(s.num_instances for s in stats_list) + shapes = [tuple(s.output_shape) for s in stats_list] + if self.preserve_leading_axis: + shapes = [self._flatten_feature_shape(shape) for shape in shapes] + rank = len(shapes[0]) - def estimate_peak_memory_bytes(self, input_stats_list) -> dict: - elem_size = np.dtype(np.float32).itemsize - - def stats_bytes(s: RepresentationStats) -> int: - numel = int(np.prod(s.output_shape)) if len(s.output_shape) > 0 else 1 - return int(s.num_instances * numel * elem_size) + if rank >= 1 and all(len(shape) == rank for shape in shapes): + leading = tuple(max(shape[d] for shape in shapes) for d in range(rank - 1)) + output_shape = (*leading, sum(shape[-1] for shape in shapes)) + else: + output_shape = max(stats_list, key=self._stats_num_elements).output_shape - current_output = 0 - peak = 0 - for s in input_stats_list: - chunk = stats_bytes(s) - new_output = current_output + chunk + output_shape_is_known = all(s.output_shape_is_known for s in stats_list) + return RepresentationStats(num_instances, output_shape, output_shape_is_known) - step_peak = current_output + chunk + new_output + chunk - peak = max(peak, step_peak) - current_output = new_output + def estimate_peak_memory_bytes(self, input_stats) -> dict: + stats_list = self._as_stats_list(input_stats) + input_bytes = sum(self._stats_bytes(s) for s in stats_list) + output_bytes = self._stats_bytes(self.get_output_stats(input_stats)) - cpu_peak = int(peak * 1.15 + 16 * 1024 * 1024) + raw_bytes = self._raw_input_bytes(input_stats) + cpu_peak = ( + int((raw_bytes + 2 * input_bytes + output_bytes) * 1.1) + 8 * 1024 * 1024 + ) return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} diff --git a/src/main/python/systemds/scuro/representations/fusion.py b/src/main/python/systemds/scuro/representations/fusion.py index 1426797f00b..e56eb465bc6 100644 --- a/src/main/python/systemds/scuro/representations/fusion.py +++ b/src/main/python/systemds/scuro/representations/fusion.py @@ -30,7 +30,14 @@ from systemds.scuro.modality.transformed import TransformedModality from systemds.scuro.modality.modality import Modality -from systemds.scuro.representations.representation import Representation +from systemds.scuro.representations.representation import ( + CONTAINER_ARRAY, + Representation, + RepresentationStats, + derive_stats, + stats_bytes, + stats_num_elements, +) from systemds.scuro.utils.schema_helpers import get_shape @@ -46,7 +53,42 @@ def __init__(self, name, parameters=None): self.needs_alignment = False self.needs_training = False self.needs_instance_alignment = False + self.preserves_leading_axis = False self.output_modality_type = ModalityType.EMBEDDING + self.data_type = np.float32 + + @staticmethod + def _as_stats_list(input_stats) -> List[RepresentationStats]: + if isinstance(input_stats, RepresentationStats): + return [input_stats] + return list(input_stats) + + def _pre_aggregated_stats(self, stats: RepresentationStats) -> RepresentationStats: + shape = tuple(stats.output_shape) + if len(shape) > 1 and not self.preserves_leading_axis: + shape = shape[1:] + + return derive_stats( + stats, + output_shape=shape, + dtype=self.data_type, + container=CONTAINER_ARRAY, + ) + + def _fusion_input_stats(self, input_stats) -> List[RepresentationStats]: + return [self._pre_aggregated_stats(s) for s in self._as_stats_list(input_stats)] + + def _raw_input_bytes(self, input_stats) -> int: + return sum( + stats_bytes(s, quantile=0.95) for s in self._as_stats_list(input_stats) + ) + + @staticmethod + def _stats_num_elements(stats: RepresentationStats) -> int: + return stats_num_elements(stats) + + def _stats_bytes(self, stats: RepresentationStats) -> int: + return stats_bytes(stats) def transform(self, modalities: List[Modality]): """ @@ -58,12 +100,14 @@ def transform(self, modalities: List[Modality]): mods = [] for modality in modalities: agg_modality = None - if get_shape(modality.metadata) > 1: + if not self.preserves_leading_axis and get_shape(modality.metadata) > 1: agg_operator = AggregatedRepresentation() agg_modality = agg_operator.transform(modality) mods.append(agg_modality if agg_modality else modality) if self.needs_alignment: + for modality in mods: + self._normalize_for_fusion(modality) max_len = self.get_max_embedding_size(mods) for modality in mods: modality.pad(max_len=max_len) @@ -124,6 +168,20 @@ def apply_representation(self, modalities: List[Modality]): else: return self.execute(modalities) + @staticmethod + def _normalize_for_fusion(modality: Modality): + if not modality.has_data(): + return + + arr = np.asarray(modality.data) + if arr.dtype == object or arr.ndim != 1: + return + + if modality.has_metadata() and len(modality.metadata) == 1: + modality.data = arr.reshape(1, -1).copy() + elif not modality.has_metadata() or len(modality.metadata) <= 1: + modality.data = arr.reshape(1, -1).copy() + def get_max_embedding_size(self, modalities: List[Modality]): """ Computes the maximum embedding size from a given list of modalities @@ -137,9 +195,19 @@ def get_max_embedding_size(self, modalities: List[Modality]): if isinstance(data, memoryview): data = np.array(data) arr = np.asarray(data) - if arr.ndim < 2: + if arr.dtype == object: + continue + if arr.ndim == 1: + if m.has_metadata() and len(m.metadata) == 1: + emb_size = arr.shape[0] + elif not m.has_metadata() or len(m.metadata) <= 1: + emb_size = arr.shape[0] + else: + continue + elif arr.ndim >= 2: + emb_size = arr.shape[1] + else: continue - emb_size = arr.shape[1] if emb_size > max_size: max_size = emb_size return max_size diff --git a/src/main/python/systemds/scuro/representations/glove.py b/src/main/python/systemds/scuro/representations/glove.py index b45213ae19a..acf7aff522b 100644 --- a/src/main/python/systemds/scuro/representations/glove.py +++ b/src/main/python/systemds/scuro/representations/glove.py @@ -59,7 +59,9 @@ def __init__(self, output_file=None, params=None): self.embedding_dim = 100 def get_output_stats(self, input_stats: TextStats) -> RepresentationStats: - return RepresentationStats(input_stats.num_instances, (self.embedding_dim,)) + return RepresentationStats( + input_stats.num_instances, (self.embedding_dim,), dtype=self.data_type + ) def estimate_output_memory_bytes(self, input_stats: TextStats) -> int: output_bytes = 1 diff --git a/src/main/python/systemds/scuro/representations/hadamard.py b/src/main/python/systemds/scuro/representations/hadamard.py index fc053f9c6dc..0697a9f67ea 100644 --- a/src/main/python/systemds/scuro/representations/hadamard.py +++ b/src/main/python/systemds/scuro/representations/hadamard.py @@ -48,29 +48,28 @@ def execute(self, modalities: List[Modality], train_indices=None): return fused_data def get_output_stats(self, input_stats_list) -> RepresentationStats: - if isinstance(input_stats_list, RepresentationStats): - return input_stats_list - - stats_list = list(input_stats_list) + stats_list = self._fusion_input_stats(input_stats_list) if not stats_list: return RepresentationStats(0, (0,)) - max_dim = max([stats.output_shape[-1] for stats in stats_list]) - return RepresentationStats(stats_list[0].num_instances, (max_dim,)) - - def estimate_peak_memory_bytes(self, input_stats_list) -> dict: - elem_size = np.dtype(np.float64).itemsize + num_instances = max(s.num_instances for s in stats_list) + rank = len(stats_list[0].output_shape) + if rank > 0 and all(len(s.output_shape) == rank for s in stats_list): + output_shape = tuple( + max(s.output_shape[d] for s in stats_list) for d in range(rank) + ) + else: + output_shape = max(stats_list, key=self._stats_num_elements).output_shape + output_shape_is_known = all(s.output_shape_is_known for s in stats_list) + return RepresentationStats(num_instances, output_shape, output_shape_is_known) - def stats_payload_bytes(s: RepresentationStats) -> int: - numel = int(np.prod(s.output_shape)) if len(s.output_shape) > 0 else 1 - return int(s.num_instances * numel * elem_size) + def estimate_peak_memory_bytes(self, input_stats) -> dict: + stats_list = self._as_stats_list(input_stats) + input_bytes = sum(self._stats_bytes(s) for s in stats_list) + output_bytes = self._stats_bytes(self.get_output_stats(input_stats)) - stacked_input_bytes = sum(stats_payload_bytes(s) for s in input_stats_list) - out_stats = self.get_output_stats(input_stats_list) - output_bytes = stats_payload_bytes(out_stats) - reduction_workspace_bytes = output_bytes - cpu_peak = int( - (stacked_input_bytes + output_bytes + reduction_workspace_bytes) * 1.15 - + 8 * 1024 * 1024 + raw_bytes = self._raw_input_bytes(input_stats) + cpu_peak = ( + int((raw_bytes + 2 * input_bytes + output_bytes) * 1.15) + 8 * 1024 * 1024 ) return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} diff --git a/src/main/python/systemds/scuro/representations/mfcc.py b/src/main/python/systemds/scuro/representations/mfcc.py index 483ae3eef81..804dc04f1f2 100644 --- a/src/main/python/systemds/scuro/representations/mfcc.py +++ b/src/main/python/systemds/scuro/representations/mfcc.py @@ -58,7 +58,7 @@ def __init__( "n_fft": [1024, 2048, 4096], } - super().__init__("MFCC", ModalityType.TIMESERIES, parameters, False) + super().__init__("MFCC", ModalityType.TIMESERIES, parameters, True) if params is not None: n_mfcc = params.get("n_mfcc", n_mfcc) diff --git a/src/main/python/systemds/scuro/representations/mlp_averaging.py b/src/main/python/systemds/scuro/representations/mlp_averaging.py index fb71424d738..1150a149ccd 100644 --- a/src/main/python/systemds/scuro/representations/mlp_averaging.py +++ b/src/main/python/systemds/scuro/representations/mlp_averaging.py @@ -24,13 +24,11 @@ from torch.utils.data import DataLoader, TensorDataset import numpy as np -import warnings +import logging from systemds.scuro.modality.type import ModalityType from systemds.scuro.representations.representation import RepresentationStats from systemds.scuro.utils.static_variables import ( - compute_batch_size, get_device, - get_device_for_model, ) from systemds.scuro.utils.utils import set_random_seeds from systemds.scuro.drsearch.operator_registry import ( @@ -40,6 +38,8 @@ DimensionalityReduction, ) +logger = logging.getLogger(__name__) + @register_dimensionality_reduction_operator(ModalityType.EMBEDDING) class MLPAveraging(DimensionalityReduction): @@ -144,8 +144,14 @@ def execute(self, data): input_dim = data.shape[1] if input_dim <= self.output_dim: - warnings.warn( - f"Input dimension {input_dim} is smaller than output dimension {self.output_dim}. Returning original data." + # Expected outcome, not a defect: the search offers MLPAveraging + # every output_dim in its parameter grid, so a narrow input hits + # this on most of them. A warning per call buried the run log, so + # it is reported at debug level instead. + logger.debug( + "Input dimension %d is smaller than output dimension %d. Returning original data.", + input_dim, + self.output_dim, ) # TODO: this should be pruned as possible representation, could add output_dim as parameter to reps if possible return data diff --git a/src/main/python/systemds/scuro/representations/physiological_representations.py b/src/main/python/systemds/scuro/representations/physiological_representations.py new file mode 100644 index 00000000000..669732855fa --- /dev/null +++ b/src/main/python/systemds/scuro/representations/physiological_representations.py @@ -0,0 +1,641 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- +import numpy as np +from scipy.signal import find_peaks +from scipy.spatial.distance import pdist + +from systemds.scuro.modality.type import ModalityType +from systemds.scuro.modality.transformed import TransformedModality +from systemds.scuro.representations.representation import RepresentationStats +from systemds.scuro.representations.unimodal import UnimodalRepresentation +from systemds.scuro.drsearch.operator_registry import ( + register_representation, + register_context_representation_operator, +) + + +def _as_float1d(signal): + return np.asarray(signal, dtype=np.float64).reshape(-1) + + +def _successive_diffs(intervals): + intervals = _as_float1d(intervals) + if intervals.size < 2: + return np.array([], dtype=np.float64) + return np.diff(intervals) + + +def _segment_duration(intervals): + intervals = _as_float1d(intervals) + if intervals.size == 0: + return 0.0 + return float(np.sum(intervals)) + + +def _interpolate_tachogram(nn_intervals, fs=4.0): + nn = _as_float1d(nn_intervals) + if nn.size < 2: + return nn + times = np.cumsum(nn) + times = times - times[0] + duration = times[-1] + if duration <= 0.0: + return nn + t_uniform = np.arange(0.0, duration, 1.0 / fs) + if t_uniform.size < 2: + return nn + return np.interp(t_uniform, times, nn) + + +def _bandpower(signal, fs, f1, f2): + x = _as_float1d(signal) + if x.size < 4: + return 0.0 + x = x - np.mean(x) + freqs = np.fft.rfftfreq(x.size, d=1.0 / fs) + psd = np.abs(np.fft.rfft(x)) ** 2 + mask = (freqs >= f1) & (freqs < f2) + return float(np.sum(psd[mask])) + + +def _detect_ecg_r_peaks(signal, fs, min_rr_s=0.3): + x = _as_float1d(signal) + min_distance = max(1, int(min_rr_s * fs)) + + if x.size <= min_distance: + return np.array([], dtype=int) + # 99th percentile instead of max to stay robust to outlier spikes. + height = np.quantile(x, 0.99) / 2.0 + peaks, _ = find_peaks(x, distance=min_distance, height=height) + return peaks + + +def _ecg_nn_intervals(signal, fs): + peaks = _detect_ecg_r_peaks(signal, fs) + if peaks.size < 2: + return np.array([], dtype=np.float64) + return np.diff(peaks) / float(fs) + + +def _count_matches_vectorized(x, template_len, r): + n = x.size + n_templates = n - template_len + if n_templates < 2: + return 0 + templates = np.lib.stride_tricks.sliding_window_view(x, template_len)[:n_templates] + dists = pdist(templates, metric="chebyshev") + return int(np.sum(dists <= r)) + + +def _sample_entropy(intervals, m=2, r_factor=0.2): + x = _as_float1d(intervals) + n = x.size + if n <= m + 1: + return 0.0 + r = r_factor * np.std(x) + if r <= 0.0: + return 0.0 + + a = _count_matches_vectorized(x, m + 1, r) + b = _count_matches_vectorized(x, m, r) + if b == 0 or a == 0: + return 0.0 + return float(-np.log(a / b)) + + +def _fluctuation_for_scale(y, scale, n_segments): + segments = y[: n_segments * scale].reshape(n_segments, scale) + t = np.arange(scale, dtype=np.float64) + t_mean = t.mean() + t_centered = t - t_mean + denom = np.sum(t_centered**2) + if denom == 0.0: + return None + seg_mean = segments.mean(axis=1, keepdims=True) + slope = (segments * t_centered[None, :]).sum(axis=1, keepdims=True) / denom + intercept = seg_mean - slope * t_mean + trend = intercept + slope * t[None, :] + rms = np.sqrt(np.mean((segments - trend) ** 2, axis=1)) + return float(rms.mean()) + + +def _dfa_alpha(signal, min_box=4, max_box=None): + x = _as_float1d(signal) + n = x.size + if n < min_box * 2: + return 0.0 + y = np.cumsum(x - np.mean(x)) + if max_box is None: + max_box = max(min_box + 1, n // 4) + if max_box <= min_box: + return 0.0 + + scales = np.unique( + np.logspace(np.log10(min_box), np.log10(max_box), num=10, dtype=int) + ) + fluctuations = [] + for scale in scales: + if scale < min_box: + continue + n_segments = n // scale + if n_segments < 1: + continue + fluctuation = _fluctuation_for_scale(y, int(scale), n_segments) + if fluctuation is not None: + fluctuations.append((scale, fluctuation)) + + fluctuations = [(s, f) for s, f in fluctuations if f > 0.0] + if len(fluctuations) < 2: + return 0.0 + scales, f_vals = zip(*fluctuations) + alpha = np.polyfit(np.log(scales), np.log(f_vals), 1)[0] + return float(alpha) + + +def _detect_scr_peaks(signal, fs, min_distance_s=1.0, prominence_factor=0.05): + x = _as_float1d(signal) + min_distance = max(1, int(min_distance_s * fs)) + + if x.size <= min_distance: + return ( + np.array([], dtype=int), + np.array([], dtype=float), + np.array([], dtype=float), + ) + + prominence = max(prominence_factor * (np.max(x) - np.min(x)), 1e-12) + peaks, props = find_peaks(x, distance=min_distance, prominence=prominence) + + if peaks.size == 0: + return peaks, np.array([], dtype=np.float64), np.array([], dtype=np.float64) + + left = np.maximum(0, peaks - min_distance) + right = np.minimum(x.size - 1, peaks + min_distance) + window = min_distance + 1 + offsets = np.arange(window) + + k_max_left = peaks - left + idx_left = np.clip(peaks[:, None] - offsets[None, :], 0, x.size - 1) + vals_left = x[idx_left] + valid_left = offsets[None, :] <= k_max_left[:, None] + + baseline = np.min(np.where(valid_left, vals_left, np.inf), axis=1) + amplitudes = x[peaks] - baseline + half = baseline + 0.5 * amplitudes + + below_left = (vals_left <= half[:, None]) & valid_left + has_below_left = below_left.any(axis=1) + first_below_left = np.argmax(below_left, axis=1) + li = np.maximum( + left, peaks - np.where(has_below_left, first_below_left, k_max_left) + ) + + m_max = right - peaks + idx_right = np.clip(peaks[:, None] + offsets[None, :], 0, x.size - 1) + vals_right = x[idx_right] + valid_right = offsets[None, :] <= m_max[:, None] + + below_right = (vals_right <= half[:, None]) & valid_right + has_below_right = below_right.any(axis=1) + first_below_right = np.argmax(below_right, axis=1) + ri = np.minimum(right, peaks + np.where(has_below_right, first_below_right, m_max)) + + durations = (ri - li) / fs + + return ( + peaks, + amplitudes.astype(np.float64), + durations.astype(np.float64), + ) + + +class PhysiologicalRepresentation(UnimodalRepresentation): + def __init__(self, name, parameters=None, params=None): + if params is None: + params = {} + super().__init__(name, ModalityType.EMBEDDING, parameters, False) + + def compute_feature(self, signal): + raise NotImplementedError("Subclasses should implement this method.") + + def transform(self, modality, aggregation=None): + transformed_modality = TransformedModality( + modality, self, self.output_modality_type + ) + result = [] + for signal in modality.data: + result.append(self.compute_feature(signal)) + + maxlen = max(r.size for r in result) + padded_result = [ + np.pad(r, (0, maxlen - r.size), mode="constant", constant_values=0.0) + for r in result + ] + dtype = modality.metadata[0]["data_layout"]["type"] + transformed_modality.data = np.vstack(np.asarray(padded_result)).astype(dtype) + return transformed_modality + + def get_output_stats(self, input_stats): + return RepresentationStats( + input_stats.num_instances, (1,), input_stats.output_shape_is_known + ) + + @staticmethod + def _num_elements(shape) -> int: + n = 1 + for d in shape: + n *= int(d) + return n + + def estimate_output_memory_bytes(self, input_stats): + out_stats = self.get_output_stats(input_stats) + return ( + int(out_stats.num_instances) + * self._num_elements(out_stats.output_shape) + * np.dtype(np.float32).itemsize + ) + + def estimate_peak_memory_bytes(self, input_stats): + n_per_instance = self._num_elements(input_stats.output_shape) + input_bytes = ( + int(input_stats.num_instances) + * n_per_instance + * np.dtype(np.float32).itemsize + ) + transient_bytes = n_per_instance * np.dtype(np.float64).itemsize + output_bytes = self.estimate_output_memory_bytes(input_stats) + cpu_peak = ( + int((input_bytes + 2 * transient_bytes + output_bytes) * 1.15) + + 4 * 1024 * 1024 + ) + return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class SDNN(PhysiologicalRepresentation): + def __init__(self, fs=500.0, params=None): + super().__init__("SDNN") + if params is not None: + fs = params.get("fs", fs) + self.fs = fs + + def compute_feature(self, signal): + nn = _ecg_nn_intervals(signal, self.fs) + if nn.size < 2: + return np.array(0.0) + return np.array(np.std(nn, ddof=1)) + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class RMSSD(PhysiologicalRepresentation): + def __init__(self, fs=500.0, params=None): + super().__init__("RMSSD") + if params is not None: + fs = params.get("fs", fs) + self.fs = fs + + def compute_feature(self, signal): + nn = _ecg_nn_intervals(signal, self.fs) + diffs = _successive_diffs(nn) + if diffs.size == 0: + return np.array(0.0) + return np.array(np.sqrt(np.mean(diffs**2))) + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class pNN(PhysiologicalRepresentation): + def __init__(self, threshold_ms=50, fs=500.0, params=None): + super().__init__("pNN", parameters={"threshold_ms": [20, 50]}) + if params is not None: + threshold_ms = params.get("threshold_ms", threshold_ms) + fs = params.get("fs", fs) + self.threshold_ms = threshold_ms + self.fs = fs + + def compute_feature(self, signal): + nn = _ecg_nn_intervals(signal, self.fs) + diffs = _successive_diffs(nn) + if diffs.size == 0: + return np.array(0.0) + threshold = self.threshold_ms / 1000.0 + return np.array(100.0 * np.mean(np.abs(diffs) > threshold)) + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class RRPerMinute(PhysiologicalRepresentation): + def __init__(self, fs=500.0, params=None): + super().__init__("RRPerMinute") + if params is not None: + fs = params.get("fs", fs) + self.fs = fs + + def compute_feature(self, signal): + nn = _ecg_nn_intervals(signal, self.fs) + duration = _segment_duration(nn) + if duration <= 0.0 or nn.size == 0: + return np.array(0.0) + return np.array(60.0 * nn.size / duration) + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class HRVBandPower(PhysiologicalRepresentation): + def __init__(self, fs=4.0, f1=0.04, f2=0.15, signal_fs=500.0, params=None): + super().__init__( + "HRVBandPower", + parameters={ + "fs": [2.0, 4.0], + "f1": [0.003, 0.04, 0.15], + "f2": [0.04, 0.15, 0.4], + }, + ) + if params is not None: + fs = params.get("fs", fs) + f1 = params.get("f1", f1) + f2 = params.get("f2", f2) + signal_fs = params.get("signal_fs", signal_fs) + self.fs = fs + self.f1 = f1 + self.f2 = f2 + self.signal_fs = signal_fs + + def compute_feature(self, signal): + nn = _ecg_nn_intervals(signal, self.signal_fs) + tach = _interpolate_tachogram(nn, fs=self.fs) + return np.array(_bandpower(tach, self.fs, self.f1, self.f2)) + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class HRVVLF(HRVBandPower): + def __init__(self, fs=4.0, signal_fs=500.0, params=None): + super().__init__(fs=fs, f1=0.003, f2=0.04, signal_fs=signal_fs, params=params) + self.name = "HRVVLF" + self._parameters = {"fs": [2.0, 4.0]} + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class HRVLF(HRVBandPower): + def __init__(self, fs=4.0, signal_fs=500.0, params=None): + super().__init__(fs=fs, f1=0.04, f2=0.15, signal_fs=signal_fs, params=params) + self.name = "HRVLF" + self._parameters = {"fs": [2.0, 4.0]} + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class HRVHF(HRVBandPower): + def __init__(self, fs=4.0, signal_fs=500.0, params=None): + super().__init__(fs=fs, f1=0.15, f2=0.40, signal_fs=signal_fs, params=params) + self.name = "HRVHF" + self._parameters = {"fs": [2.0, 4.0]} + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class HRVLFHF(PhysiologicalRepresentation): + def __init__(self, fs=4.0, signal_fs=500.0, params=None): + super().__init__("HRVLFHF", parameters={"fs": [2.0, 4.0]}) + if params is not None: + fs = params.get("fs", fs) + signal_fs = params.get("signal_fs", signal_fs) + self.fs = fs + self.signal_fs = signal_fs + + def compute_feature(self, signal): + lf = HRVLF(fs=self.fs, signal_fs=self.signal_fs).compute_feature(signal)[()] + hf = HRVHF(fs=self.fs, signal_fs=self.signal_fs).compute_feature(signal)[()] + if hf <= 0.0: + return np.array(0.0) + return np.array(lf / hf) + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class PoincareSD1(PhysiologicalRepresentation): + def __init__(self, fs=500.0, params=None): + super().__init__("PoincareSD1") + if params is not None: + fs = params.get("fs", fs) + self.fs = fs + + def compute_feature(self, signal): + nn = _ecg_nn_intervals(signal, self.fs) + diffs = _successive_diffs(nn) + if diffs.size < 2: + return np.array(0.0) + return np.array(np.std(diffs, ddof=1) / np.sqrt(2.0)) + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class PoincareSD2(PhysiologicalRepresentation): + def __init__(self, fs=500.0, params=None): + super().__init__("PoincareSD2") + if params is not None: + fs = params.get("fs", fs) + self.fs = fs + + def compute_feature(self, signal): + nn = _ecg_nn_intervals(signal, self.fs) + if nn.size < 3: + return np.array(0.0) + summed = nn[:-1] + nn[1:] + return np.array(np.std(summed, ddof=1) / np.sqrt(2.0)) + + +# @register_representation([ModalityType.PHYSIOLOGICAL]) +# @register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +# class SampleEntropy(PhysiologicalRepresentation): +# def __init__(self, m=2, r_factor=0.2, params=None): +# super().__init__( +# "SampleEntropy", parameters={"m": [2, 3], "r_factor": [0.15, 0.2, 0.25]} +# ) +# if params is not None: +# m = params.get("m", m) +# r_factor = params.get("r_factor", r_factor) +# self.m = m +# self.r_factor = r_factor + +# def compute_feature(self, nn): +# return np.array(_sample_entropy(nn, m=self.m, r_factor=self.r_factor)) + + +# @register_representation([ModalityType.PHYSIOLOGICAL]) +# @register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +# class DFAAlpha(PhysiologicalRepresentation): +# def __init__(self, params=None): +# super().__init__("DFAAlpha") + +# def compute_feature(self, nn): +# return np.array(_dfa_alpha(nn)) + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class SCLSlope(PhysiologicalRepresentation): + def __init__(self, params=None): + super().__init__("SCLSlope") + + def compute_feature(self, scl): + x = _as_float1d(scl) + if x.size < 2: + return np.array(0.0) + t = np.arange(x.size, dtype=np.float64) + return np.array(np.polyfit(t, x, 1)[0]) + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class SCLDynamicRange(PhysiologicalRepresentation): + def __init__(self, params=None): + super().__init__("SCLDynamicRange") + + def compute_feature(self, scl): + x = _as_float1d(scl) + if x.size == 0: + return np.array(0.0) + return np.array(np.max(x) - np.min(x)) + + +class _SCRFeature(PhysiologicalRepresentation): + def __init__(self, name, fs=4.0, parameters=None, params=None): + params_dict = parameters or {"fs": [2.0, 4.0, 8.0]} + super().__init__(name, parameters=params_dict, params=params) + self.fs = fs + + def _scr_stats(self, scr): + duration = _as_float1d(scr).size / self.fs + peaks, amplitudes, durations = _detect_scr_peaks(scr, self.fs) + return duration, peaks, amplitudes, durations + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class SCRPeaksPerMinute(_SCRFeature): + def __init__(self, fs=4.0, params=None): + super().__init__("SCRPeaksPerMinute", fs=fs, params=params) + + def compute_feature(self, scr): + duration, peaks, _, _ = self._scr_stats(scr) + if duration <= 0.0: + return np.array(0.0) + return np.array(60.0 * peaks.size / duration) + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class SCRAverageAmplitude(_SCRFeature): + def __init__(self, fs=4.0, params=None): + super().__init__("SCRAverageAmplitude", fs=fs, params=params) + + def compute_feature(self, scr): + _, _, amplitudes, _ = self._scr_stats(scr) + if amplitudes.size == 0: + return np.array(0.0) + return np.array(np.mean(amplitudes)) + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class SCRAverageDuration(_SCRFeature): + def __init__(self, fs=4.0, params=None): + super().__init__("SCRAverageDuration", fs=fs, params=params) + + def compute_feature(self, scr): + _, _, _, durations = self._scr_stats(scr) + if durations.size == 0: + return np.array(0.0) + return np.array(np.mean(durations)) + + +def _detect_resp_extrema(signal, fs, min_breath_period_s=1.5): + x = _as_float1d(signal) + min_distance = max(1, int(min_breath_period_s * fs)) + if x.size <= min_distance: + empty = np.array([], dtype=int) + return empty, empty + height = np.std(x) * 0.25 + peaks, _ = find_peaks(x, distance=min_distance, height=height) + troughs, _ = find_peaks(-x, distance=min_distance, height=height) + return peaks, troughs + + +def _resp_breath_intervals(signal, fs): + peaks, _ = _detect_resp_extrema(signal, fs) + if peaks.size < 2: + return np.array([], dtype=np.float64) + return np.diff(peaks) / float(fs) + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class BreathingRate(PhysiologicalRepresentation): + def __init__(self, fs=500.0, params=None): + super().__init__("BreathingRate") + if params is not None: + fs = params.get("fs", fs) + self.fs = fs + + def compute_feature(self, signal): + intervals = _resp_breath_intervals(signal, self.fs) + if intervals.size == 0: + return np.array(0.0) + return np.array(60.0 / np.mean(intervals)) + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class BreathIntervalRMSSD(PhysiologicalRepresentation): + def __init__(self, fs=500.0, params=None): + super().__init__("BreathIntervalRMSSD") + if params is not None: + fs = params.get("fs", fs) + self.fs = fs + + def compute_feature(self, signal): + intervals = _resp_breath_intervals(signal, self.fs) + diffs = _successive_diffs(intervals) + if diffs.size == 0: + return np.array(0.0) + return np.array(np.sqrt(np.mean(diffs**2))) + + +@register_representation([ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class BreathAmplitude(PhysiologicalRepresentation): + def __init__(self, fs=500.0, params=None): + super().__init__("BreathAmplitude") + if params is not None: + fs = params.get("fs", fs) + self.fs = fs + + def compute_feature(self, signal): + x = _as_float1d(signal) + peaks, troughs = _detect_resp_extrema(x, self.fs) + if peaks.size == 0 or troughs.size == 0: + return np.array(0.0) + return np.array(np.mean(x[peaks]) - np.mean(x[troughs])) diff --git a/src/main/python/systemds/scuro/representations/physiological_window.py b/src/main/python/systemds/scuro/representations/physiological_window.py new file mode 100644 index 00000000000..338cfdd2201 --- /dev/null +++ b/src/main/python/systemds/scuro/representations/physiological_window.py @@ -0,0 +1,260 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- +import math +import numpy as np + +from systemds.scuro.modality.type import ModalityType +from systemds.scuro.drsearch.operator_registry import register_context_operator +from systemds.scuro.representations.aggregate import Aggregation +from systemds.scuro.representations.representation import RepresentationStats +from systemds.scuro.representations.window_aggregation import ( + Window, + resolve_aggregation_function, + _pad_stack, +) + + +def _estimate_windowed_output_stats( + window_obj, input_stats, estimated_num_windows, representative_window_length +): + in_shape = tuple(int(s) for s in input_stats.output_shape) + if not isinstance(window_obj.aggregation_function, Aggregation): + windowed_input_stats = RepresentationStats( + input_stats.num_instances, (representative_window_length,) + ) + feat_shape = window_obj.aggregation_function.get_output_stats( + windowed_input_stats + ).output_shape + else: + feat_shape = in_shape[1:] + window_obj.stats = RepresentationStats( + input_stats.num_instances, + (estimated_num_windows, *feat_shape), + output_shape_is_known=False, + ) + return window_obj.stats + + +def _estimate_windowed_memory_bytes(window_obj, input_stats): + out_shape = window_obj.get_output_stats(input_stats).output_shape + out_numel = int(np.prod(out_shape)) if len(out_shape) > 0 else 1 + return ( + input_stats.num_instances * out_numel * np.dtype(window_obj.data_type).itemsize + ) + + +def _estimate_windowed_peak_memory_bytes(window_obj, input_stats): + in_shape = tuple(int(s) for s in input_stats.output_shape) + in_numel = int(np.prod(in_shape)) if len(in_shape) > 0 else 1 + input_bytes = ( + input_stats.num_instances * in_numel * np.dtype(window_obj.data_type).itemsize + ) + output_bytes = _estimate_windowed_memory_bytes(window_obj, input_stats) + cpu_peak = int((input_bytes + output_bytes * 2) * 1.2 + 8 * 1024 * 1024) + return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} + + +@register_context_operator([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) +class AdaptiveWindow(Window): + granularity_parameter = "base_window_size" + granularity_kind = "length" + + def __init__( + self, + aggregation_function="mean", + base_window_size=256, + overlap=0.5, + min_window_size=64, + params=None, + ): + if params is not None: + aggregation_function = resolve_aggregation_function( + aggregation_function, params + ) + base_window_size = params.get("base_window_size", base_window_size) + overlap = params.get("overlap", overlap) + min_window_size = params.get("min_window_size", min_window_size) + super().__init__("AdaptiveWindow", aggregation_function) + base_window_size = max(1, int(base_window_size)) + min_window_size = max(1, min(int(min_window_size), base_window_size)) + self.parameters.update( + { + "base_window_size": (min(4, base_window_size), base_window_size), + "overlap": [0.25, 0.5, 0.75], + "min_window_size": (min(2, min_window_size), min_window_size), + } + ) + self.base_window_size = base_window_size + self.overlap = overlap + self.min_window_size = min_window_size + + def _estimate_num_windows(self, signal_length): + if signal_length <= 0: + return 1 + step = max(1, int(self.base_window_size * (1 - self.overlap))) + return max(1, math.ceil(signal_length / step)) + + def get_output_stats(self, input_stats: RepresentationStats) -> RepresentationStats: + in_shape = tuple(int(s) for s in input_stats.output_shape) + signal_length = in_shape[0] if in_shape else 0 + return _estimate_windowed_output_stats( + self, + input_stats, + self._estimate_num_windows(signal_length), + self.base_window_size, + ) + + def estimate_output_memory_bytes(self, input_stats: RepresentationStats) -> int: + return _estimate_windowed_memory_bytes(self, input_stats) + + def estimate_peak_memory_bytes(self, input_stats: RepresentationStats) -> dict: + return _estimate_windowed_peak_memory_bytes(self, input_stats) + + def execute(self, modality): + windowed_data = [] + + for signal in modality.data: + local_var = np.array( + [ + np.var(signal[i : i + self.min_window_size]) + for i in range( + 0, len(signal) - self.min_window_size, self.min_window_size + ) + ] + ) + + if local_var.size == 0: + window_sizes = np.array([self.base_window_size]) + else: + norm_var = (local_var - np.min(local_var)) / ( + np.max(local_var) - np.min(local_var) + 1e-6 + ) + window_sizes = np.clip( + self.base_window_size * (1 - 0.5 * norm_var), + self.min_window_size, + self.base_window_size, + ).astype(int) + + windows = [] + start = 0 + while start < len(signal): + current_size = window_sizes[ + min(len(window_sizes) - 1, start // self.min_window_size) + ] + end = min(start + current_size, len(signal)) + window = signal[start:end] + if len(window) > 0: + windows.append(window) + start += max(1, int(current_size * (1 - self.overlap))) + + processed_windows = _pad_stack( + [self.aggregation_function.compute_feature(w) for w in windows] + ) + windowed_data.append(processed_windows) + + return _pad_stack(windowed_data) + + +@register_context_operator([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) +class PhysiologicalEventWindow(Window): + granularity_parameter = "min_distance" + granularity_kind = "length" + + def __init__( + self, + aggregation_function="mean", + event_threshold=0.5, + min_distance=100, + params=None, + ): + if params is not None: + aggregation_function = resolve_aggregation_function( + aggregation_function, params + ) + event_threshold = params.get("event_threshold", event_threshold) + min_distance = params.get("min_distance", min_distance) + super().__init__("PhysiologicalEventWindow", aggregation_function) + min_distance = max(1, int(min_distance)) + self.parameters.update( + { + "event_threshold": [0.3, 0.5, 0.7], + "min_distance": (min(4, min_distance), min_distance), + } + ) + self.event_threshold = event_threshold + self.min_distance = min_distance + + def _estimate_num_windows(self, signal_length): + if signal_length <= 0: + return 1 + return max(1, math.ceil(signal_length / max(1, self.min_distance))) + + def get_output_stats(self, input_stats: RepresentationStats) -> RepresentationStats: + in_shape = tuple(int(s) for s in input_stats.output_shape) + signal_length = in_shape[0] if in_shape else 0 + return _estimate_windowed_output_stats( + self, + input_stats, + self._estimate_num_windows(signal_length), + self.min_distance, + ) + + def estimate_output_memory_bytes(self, input_stats: RepresentationStats) -> int: + return _estimate_windowed_memory_bytes(self, input_stats) + + def estimate_peak_memory_bytes(self, input_stats: RepresentationStats) -> dict: + return _estimate_windowed_peak_memory_bytes(self, input_stats) + + def execute(self, modality): + windowed_data = [] + + for signal in modality.data: + normalized = (signal - np.mean(signal)) / (np.std(signal) + 1e-6) + + peaks = [] + last_peak = -self.min_distance + for i in range(1, len(normalized) - 1): + if ( + normalized[i] > self.event_threshold + and normalized[i] > normalized[i - 1] + and normalized[i] > normalized[i + 1] + and i - last_peak >= self.min_distance + ): + peaks.append(i) + last_peak = i + + windows = [signal[peaks[i] : peaks[i + 1]] for i in range(len(peaks) - 1)] + + if not windows: + windows = [ + w + for w in np.array_split( + signal, max(1, len(signal) // self.min_distance) + ) + if len(w) > 0 + ] + + processed_windows = _pad_stack( + [self.aggregation_function.compute_feature(w) for w in windows] + ) + windowed_data.append(processed_windows) + + return _pad_stack(windowed_data) diff --git a/src/main/python/systemds/scuro/representations/sum.py b/src/main/python/systemds/scuro/representations/sum.py index 4f658020f1e..58cac2e7210 100644 --- a/src/main/python/systemds/scuro/representations/sum.py +++ b/src/main/python/systemds/scuro/representations/sum.py @@ -41,7 +41,7 @@ def __init__(self, params=None): self.needs_alignment = True def execute(self, modalities: List[Modality]): - data = np.asarray( + data = np.array( modalities[0].data, dtype=modalities[0].metadata[0]["data_layout"]["type"], ) @@ -54,31 +54,28 @@ def execute(self, modalities: List[Modality]): return data def get_output_stats(self, input_stats_list) -> RepresentationStats: - if isinstance(input_stats_list, RepresentationStats): - return input_stats_list - - stats_list = list(input_stats_list) + stats_list = self._fusion_input_stats(input_stats_list) if not stats_list: return RepresentationStats(0, (0,)) - max_dim = max([stats.output_shape[-1] for stats in stats_list]) - return RepresentationStats(stats_list[0].num_instances, (max_dim,)) - - def estimate_peak_memory_bytes(self, input_stats_list) -> dict: - elem_size = np.dtype(np.float64).itemsize - - def stats_payload_bytes(s: RepresentationStats) -> int: - numel = int(np.prod(s.output_shape)) if len(s.output_shape) > 0 else 1 - return int(s.num_instances * numel * elem_size) + num_instances = max(s.num_instances for s in stats_list) + rank = len(stats_list[0].output_shape) + if rank > 0 and all(len(s.output_shape) == rank for s in stats_list): + output_shape = tuple( + max(s.output_shape[d] for s in stats_list) for d in range(rank) + ) + else: + output_shape = max(stats_list, key=self._stats_num_elements).output_shape + output_shape_is_known = all(s.output_shape_is_known for s in stats_list) + return RepresentationStats(num_instances, output_shape, output_shape_is_known) - first_bytes = stats_payload_bytes(input_stats_list[0]) - max_other_bytes = 0 - if len(input_stats_list) > 1: - max_other_bytes = max(stats_payload_bytes(s) for s in input_stats_list[1:]) + def estimate_peak_memory_bytes(self, input_stats) -> dict: + stats_list = self._as_stats_list(input_stats) + input_bytes = sum(self._stats_bytes(s) for s in stats_list) + output_bytes = self._stats_bytes(self.get_output_stats(input_stats)) - ufunc_workspace_bytes = int(0.1 * max(first_bytes, max_other_bytes)) - cpu_peak = int( - (first_bytes + max_other_bytes + ufunc_workspace_bytes) * 1.15 - + 8 * 1024 * 1024 + raw_bytes = self._raw_input_bytes(input_stats) + cpu_peak = ( + int((raw_bytes + input_bytes + output_bytes) * 1.15) + 8 * 1024 * 1024 ) return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} diff --git a/src/main/python/systemds/scuro/representations/tabular_features.py b/src/main/python/systemds/scuro/representations/tabular_features.py new file mode 100644 index 00000000000..9734faa39db --- /dev/null +++ b/src/main/python/systemds/scuro/representations/tabular_features.py @@ -0,0 +1,59 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- +import numpy as np + +from systemds.scuro.dataloader.tabular_loader import TabularStats +from systemds.scuro.modality.transformed import TransformedModality +from systemds.scuro.modality.type import ModalityType +from systemds.scuro.drsearch.operator_registry import register_representation +from systemds.scuro.representations.representation import RepresentationStats +from systemds.scuro.representations.unimodal import UnimodalRepresentation + + +@register_representation(ModalityType.EMBEDDING) +class TabularFeatures(UnimodalRepresentation): + def __init__(self, params=None): + super().__init__("TabularFeatures", ModalityType.EMBEDDING, None) + self.data_type = np.float32 + + def get_output_stats(self, input_stats: TabularStats) -> RepresentationStats: + return RepresentationStats( + input_stats.num_instances, input_stats.output_shape, dtype=self.data_type + ) + + def estimate_output_memory_bytes(self, input_stats: TabularStats) -> int: + return ( + input_stats.num_instances + * input_stats.num_features + * np.dtype(self.data_type).itemsize + ) + + def estimate_peak_memory_bytes(self, input_stats: TabularStats) -> dict: + return { + "cpu_peak_bytes": self.estimate_output_memory_bytes(input_stats) * 2, + "gpu_peak_bytes": 0, + } + + def transform(self, modality, params=None): + transformed_modality = TransformedModality(modality, self) + transformed_modality.data_type = self.data_type + transformed_modality.data = np.array(modality.data, dtype=self.data_type) + return transformed_modality diff --git a/src/main/python/systemds/scuro/representations/text_context_with_indices.py b/src/main/python/systemds/scuro/representations/text_context_with_indices.py index 4de53698d7a..6d8964c1d14 100644 --- a/src/main/python/systemds/scuro/representations/text_context_with_indices.py +++ b/src/main/python/systemds/scuro/representations/text_context_with_indices.py @@ -177,6 +177,7 @@ def get_output_stats(self, input_stats: TextStats) -> RepresentationStats: ), self.max_words, ), + dtype=self.data_type, ) def estimate_output_memory_bytes(self, input_stats: TextStats) -> int: @@ -318,6 +319,7 @@ def get_output_stats(self, input_stats: TextStats) -> RepresentationStats: ), self.max_words, ), + dtype=self.data_type, ) def estimate_output_memory_bytes(self, input_stats: TextStats) -> int: diff --git a/src/main/python/systemds/scuro/representations/tfidf.py b/src/main/python/systemds/scuro/representations/tfidf.py index 3c3d894c173..bea18a56024 100644 --- a/src/main/python/systemds/scuro/representations/tfidf.py +++ b/src/main/python/systemds/scuro/representations/tfidf.py @@ -47,7 +47,10 @@ def get_output_stats(self, input_stats: TextStats) -> RepresentationStats: 100_000, max(1000, input_stats.num_instances * input_stats.max_length) ) return RepresentationStats( - input_stats.num_instances, (vocab_estimate,), output_shape_is_known=False + input_stats.num_instances, + (vocab_estimate,), + output_shape_is_known=False, + dtype=self.data_type, ) def estimate_output_memory_bytes(self, input_stats: TextStats) -> int: diff --git a/src/main/python/systemds/scuro/representations/timeseries_representations.py b/src/main/python/systemds/scuro/representations/timeseries_representations.py index 6bf7f38d132..2c7fbbe7404 100644 --- a/src/main/python/systemds/scuro/representations/timeseries_representations.py +++ b/src/main/python/systemds/scuro/representations/timeseries_representations.py @@ -23,28 +23,75 @@ from systemds.scuro.modality.type import ModalityType from systemds.scuro.modality.transformed import TransformedModality -from systemds.scuro.representations.representation import RepresentationStats +from systemds.scuro.representations.representation import ( + CONTAINER_ARRAY, + CONTAINER_LIST, + RepresentationStats, +) from systemds.scuro.representations.unimodal import UnimodalRepresentation +from systemds.scuro.representations.utils import dense_instance_batch from systemds.scuro.drsearch.operator_registry import ( register_representation, register_context_representation_operator, ) +import warnings + +warnings.filterwarnings( + "ignore", + message=r"Precision loss occurred in moment calculation", + category=RuntimeWarning, +) + class TimeSeriesRepresentation(UnimodalRepresentation): - def __init__(self, name, parameters=None, params=None): + + def __init__( + self, + name, + parameters=None, + params=None, + self_contained=False, + min_input_length=1, + ): if params is None: params = {} - - super().__init__(name, ModalityType.EMBEDDING, parameters, False) + self.min_input_length = min_input_length + super().__init__(name, ModalityType.EMBEDDING, parameters, self_contained) + + @staticmethod + def _input_length(input_stats) -> int: + shape = getattr(input_stats, "output_shape", ()) or () + return int(shape[0]) if shape else 0 + + def check_preconditions(self, input_stats): + length = self._input_length(input_stats) + if length < self.min_input_length: + return ( + f"{self.name} needs >= {self.min_input_length} samples, " + f"input provides {length}" + ) + return None def compute_feature(self, signal): raise NotImplementedError("Subclasses should implement this method.") + def compute_features_batched(self, data): + return np.asarray(self.compute_feature(data, axis=-1)) + def transform(self, modality, aggregation=None): transformed_modality = TransformedModality( modality, self, self.output_modality_type ) + dtype = modality.metadata[0]["data_layout"]["type"] + batch = dense_instance_batch(modality.data) + if batch is not None: + features = self.compute_features_batched(batch) + if features.ndim == 1: + features = features[:, None] + transformed_modality.data = features.astype(dtype) + return transformed_modality + result = [] for signal in modality.data: @@ -56,23 +103,45 @@ def transform(self, modality, aggregation=None): np.pad(r, (0, maxlen - r.size), mode="constant", constant_values=0.0) for r in result ] - dtype = modality.metadata[0]["data_layout"]["type"] transformed_modality.data = np.vstack(np.asarray(padded_result)).astype(dtype) return transformed_modality def get_output_stats(self, input_stats): - return RepresentationStats(input_stats.num_instances, (1,)) + return RepresentationStats( + input_stats.num_instances, (1,), input_stats.output_shape_is_known + ) + + @staticmethod + def _num_elements(shape) -> int: + n = 1 + for d in shape: + n *= int(d) + return n def estimate_output_memory_bytes(self, input_stats): - # TODO: adapt this to the actual output shapes and transformations - return input_stats.num_instances * 4 + out_stats = self.get_output_stats(input_stats) + return ( + int(out_stats.num_instances) + * self._num_elements(out_stats.output_shape) + * np.dtype(np.float32).itemsize + ) def estimate_peak_memory_bytes(self, input_stats): - # TODO: adapt this to the actual output shapes and transformations - return { - "cpu_peak_bytes": self.estimate_output_memory_bytes(input_stats), - "gpu_peak_bytes": 0, - } + input_bytes = ( + int(input_stats.num_instances) + * self._num_elements(input_stats.output_shape) + * np.dtype(np.float32).itemsize + ) + output_bytes = self.estimate_output_memory_bytes(input_stats) + batch_bytes = ( + input_bytes + if getattr(input_stats, "container", CONTAINER_ARRAY) == CONTAINER_LIST + else 0 + ) + cpu_peak = ( + int((input_bytes + batch_bytes + 3 * output_bytes) * 1.15) + 4 * 1024 * 1024 + ) + return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} @register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) @@ -126,7 +195,7 @@ def compute_feature(self, signal, axis=-1): @register_context_representation_operator(ModalityType.AUDIO) class Std(TimeSeriesRepresentation): def __init__(self, params=None): - super().__init__("Std") + super().__init__("Std", min_input_length=2) def compute_feature(self, signal, axis=-1): return np.array(np.std(signal, axis=axis)) @@ -138,7 +207,7 @@ def compute_feature(self, signal, axis=-1): @register_context_representation_operator(ModalityType.AUDIO) class Skew(TimeSeriesRepresentation): def __init__(self, params=None): - super().__init__("Skew") + super().__init__("Skew", min_input_length=3) def compute_feature(self, signal, axis=-1): return np.array(stats.skew(signal, axis=axis)) @@ -152,17 +221,36 @@ def __init__(self, quantile=0.9, params=None): super().__init__( "Qunatile", {"quantile": [0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99]} ) + if params is not None: + quantile = params.get("quantile", quantile) self.quantile = quantile def compute_feature(self, signal, axis=-1): return np.array(np.quantile(signal, self.quantile, axis=axis)) + def compute_features_batched(self, data): + features = np.asarray(np.quantile(data, self.quantile, axis=-1)) + if np.ndim(self.quantile) == 0: + return features + + return np.moveaxis(features, 0, -1) + + def get_output_stats(self, input_stats): + n_quantiles = np.atleast_1d(self.quantile).size + return RepresentationStats( + input_stats.num_instances, + (n_quantiles,), + input_stats.output_shape_is_known, + ) + @register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) @register_context_representation_operator(ModalityType.TIMESERIES) @register_context_representation_operator(ModalityType.PHYSIOLOGICAL) @register_context_representation_operator(ModalityType.AUDIO) class Kurtosis(TimeSeriesRepresentation): + min_input_length = 4 # the fourth moment is undefined below four samples + def __init__(self, params=None): super().__init__("Kurtosis") @@ -187,22 +275,89 @@ def compute_feature(self, signal, axis=-1): @register_context_representation_operator(ModalityType.PHYSIOLOGICAL) class ZeroCrossingRate(TimeSeriesRepresentation): def __init__(self, params=None): - super().__init__("ZeroCrossingRate") + super().__init__("ZeroCrossingRate", min_input_length=2) def compute_feature(self, signal, axis=-1): return np.array(np.sum(np.diff(np.signbit(signal), axis=axis) != 0, axis=axis)) +@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.TIMESERIES) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class LastValue(TimeSeriesRepresentation): + def __init__(self, params=None): + super().__init__("LastValue") + + def compute_feature(self, signal, axis=-1): + return np.take(signal, -1, axis=axis) + + +@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.TIMESERIES) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class TransitionCount(TimeSeriesRepresentation): + def __init__(self, threshold=0.0, params=None): + super().__init__( + "TransitionCount", + parameters={"threshold": [0.0, 1.0, 5.0, 10.0]}, + min_input_length=2, + ) + if params is not None: + threshold = params.get("threshold", threshold) + self.threshold = threshold + + def compute_feature(self, signal, axis=-1): + return np.array( + np.sum(np.abs(np.diff(signal, axis=axis)) > self.threshold, axis=axis) + ) + + +@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) +@register_context_representation_operator(ModalityType.TIMESERIES) +@register_context_representation_operator(ModalityType.PHYSIOLOGICAL) +class ObservationDensity(TimeSeriesRepresentation): + def __init__(self, params=None): + super().__init__("ObservationDensity", min_input_length=2) + + def compute_feature(self, signal, axis=-1): + n = signal.shape[axis] + transitions = np.sum(np.diff(signal, axis=axis) != 0, axis=axis) + return np.array((transitions + 1) / n) + + @register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) @register_context_representation_operator(ModalityType.TIMESERIES) @register_context_representation_operator(ModalityType.PHYSIOLOGICAL) class ACF(TimeSeriesRepresentation): def __init__(self, k=1, params=None): - super().__init__("ACF", {"k": [1, 2, 5, 10, 20, 25, 50, 100, 200, 500]}) + super().__init__( + "ACF", {"k": [1, 2, 5, 10, 20, 25, 50, 100, 200, 500]}, min_input_length=2 + ) if params is not None: k = params.get("k", k) self.k = k + def filter_parameter_domain(self, name, values, input_stats): + if name != "k": + return values + length = self._input_length(input_stats) + if length <= 1: + return values + usable = [k for k in values if 0 < int(k) < length] + return usable or [1] + + def check_preconditions(self, input_stats): + failure = super().check_preconditions(input_stats) + if failure: + return failure + length = self._input_length(input_stats) + if int(self.k) >= length: + return ( + f"ACF lag k={int(self.k)} needs > {int(self.k)} samples, " + f"input provides {length}" + ) + return None + def compute_feature(self, signal, axis=-1): x = np.asarray(signal, dtype=np.float64) x = x - np.mean(x, axis=axis, keepdims=True) @@ -235,22 +390,39 @@ def get_k_values(self, max_length, percent=0.2, num=10, log=False): @register_context_representation_operator(ModalityType.TIMESERIES) @register_context_representation_operator(ModalityType.PHYSIOLOGICAL) class FrequencyMagnitude(TimeSeriesRepresentation): - def __init__(self, params=None): - super().__init__("FrequencyMagnitude") + def __init__(self, params=None, self_contained=True): + super().__init__("FrequencyMagnitude", min_input_length=2) def compute_feature(self, signal, axis=-1): return np.array(np.abs(np.fft.rfft(signal, axis=axis))) + def get_output_stats(self, input_stats): + n = self._num_elements(input_stats.output_shape) + out_len = n // 2 + 1 if n > 0 else 0 + return RepresentationStats( + input_stats.num_instances, (out_len,), input_stats.output_shape_is_known + ) + @register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) @register_context_representation_operator(ModalityType.PHYSIOLOGICAL) @register_context_representation_operator(ModalityType.TIMESERIES) class SpectralCentroid(TimeSeriesRepresentation): def __init__(self, fs=1.0, params=None): - super().__init__("SpectralCentroid", parameters={"fs": [0.5, 1.0, 2.0]}) + super().__init__("SpectralCentroid", min_input_length=2) if params is not None: fs = params.get("fs", fs) - self.fs = fs + self.fs = float(fs) + + def get_current_parameters(self): + current_params = super().get_current_parameters() + current_params["fs"] = self.fs + return current_params + + def configure_for_input(self, input_stats): + sampling_rate = getattr(input_stats, "sampling_rate", None) + if sampling_rate: + self.fs = float(sampling_rate) def compute_feature(self, signal, axis=-1): signal = np.asarray(signal, dtype=np.float64) @@ -270,22 +442,42 @@ def compute_feature(self, signal, axis=-1): @register_context_representation_operator(ModalityType.PHYSIOLOGICAL) @register_context_representation_operator(ModalityType.TIMESERIES) class BandpowerFFT(TimeSeriesRepresentation): - def __init__(self, fs=1.0, f1=0.0, f2=0.5, params=None): + def __init__(self, fs=1.0, band_low=0.0, band_width=0.5, params=None): super().__init__( "BandpowerFFT", - parameters={"fs": [0.5, 1.0], "f1": [0.0, 1.0], "f2": [0.5, 1.0]}, + parameters={ + "band_low": [0.0, 0.25, 0.5], + "band_width": [0.25, 0.5, 1.0], + }, + min_input_length=2, ) if params is not None: fs = params.get("fs", fs) - f1 = params.get("f1", f1) - f2 = params.get("f2", f2) - self.fs = fs - self.f1 = f1 - self.f2 = f2 + band_low = params.get("band_low", band_low) + band_width = params.get("band_width", band_width) + self.fs = float(fs) + self.band_low = float(band_low) + self.band_width = float(band_width) + + @property + def band_high(self) -> float: + return min(1.0, self.band_low + self.band_width) + + def get_current_parameters(self): + current_params = super().get_current_parameters() + current_params["fs"] = self.fs # bound to the data, not searched + return current_params + + def configure_for_input(self, input_stats): + sampling_rate = getattr(input_stats, "sampling_rate", None) + if sampling_rate: + self.fs = float(sampling_rate) def compute_feature(self, signal, axis=-1): signal = np.asarray(signal, dtype=np.float64) n = signal.shape[axis] + nyquist = self.fs / 2.0 + self.f1, self.f2 = self.band_low * nyquist, self.band_high * nyquist frequency_magnitude = FrequencyMagnitude().compute_feature(signal, axis=axis) frequencies = np.fft.rfftfreq(n, d=1.0 / self.fs) diff --git a/src/main/python/systemds/scuro/representations/utils.py b/src/main/python/systemds/scuro/representations/utils.py index 7551c6cb2bf..5041e18770c 100644 --- a/src/main/python/systemds/scuro/representations/utils.py +++ b/src/main/python/systemds/scuro/representations/utils.py @@ -24,6 +24,29 @@ import numpy as np +def dense_instance_batch(data): + if isinstance(data, np.ndarray): + if data.ndim == 2 and np.issubdtype(data.dtype, np.number): + return data + return None + + if not isinstance(data, (list, tuple)) or len(data) == 0: + return None + + first = data[0] + if ( + not isinstance(first, np.ndarray) + or first.ndim != 1 + or not np.issubdtype(first.dtype, np.number) + ): + return None + for instance in data: + if not isinstance(instance, np.ndarray) or instance.shape != first.shape: + return None + + return np.asarray(data) + + def pad_sequences(sequences, maxlen=None, dtype="float32", value=0): if maxlen is None: maxlen = max([len(seq) for seq in sequences]) diff --git a/src/main/python/systemds/scuro/representations/wav2vec.py b/src/main/python/systemds/scuro/representations/wav2vec.py index 5e03baf8bc4..c9f4025579a 100644 --- a/src/main/python/systemds/scuro/representations/wav2vec.py +++ b/src/main/python/systemds/scuro/representations/wav2vec.py @@ -37,14 +37,34 @@ @register_representation(ModalityType.AUDIO) class Wav2Vec(UnimodalRepresentation): + cache_in_worker = True + instance_parallel = True + + MODEL_NAME = "facebook/wav2vec2-base-960h" + def __init__(self, params=None): super().__init__("Wav2Vec", ModalityType.TIMESERIES, {}) - self.processor = Wav2Vec2Processor.from_pretrained( - "facebook/wav2vec2-base-960h" - ) - self.model = Wav2Vec2Model.from_pretrained( - "facebook/wav2vec2-base-960h" - ).float() + self._processor = None + self._model = None + + @staticmethod + def _from_pretrained(loader_cls, name): + try: + return loader_cls.from_pretrained(name, local_files_only=True) + except Exception: + return loader_cls.from_pretrained(name) + + @property + def processor(self): + if self._processor is None: + self._processor = self._from_pretrained(Wav2Vec2Processor, self.MODEL_NAME) + return self._processor + + @property + def model(self): + if self._model is None: + self._model = self._from_pretrained(Wav2Vec2Model, self.MODEL_NAME).float() + return self._model def transform(self, modality, aggregation=None): transformed_modality = TransformedModality( diff --git a/src/main/python/systemds/scuro/representations/window_aggregation.py b/src/main/python/systemds/scuro/representations/window_aggregation.py index a9a1f1eb41b..713a398312a 100644 --- a/src/main/python/systemds/scuro/representations/window_aggregation.py +++ b/src/main/python/systemds/scuro/representations/window_aggregation.py @@ -29,11 +29,25 @@ from systemds.scuro.representations.aggregate import Aggregation from systemds.scuro.representations.context import Context from systemds.scuro.representations.representation import ( + CONTAINER_ARRAY, + CONTAINER_LIST, NDARRAY_OBJECT_OVERHEAD_BYTES, Representation, RepresentationStats, stats_itemsize, ) +from systemds.scuro.representations.utils import dense_instance_batch + +_ACCEPTS_AXIS_CACHE = {} + + +def _accepts_axis(compute_feature): + func = getattr(compute_feature, "__func__", compute_feature) + accepts = _ACCEPTS_AXIS_CACHE.get(func) + if accepts is None: + accepts = "axis" in inspect.signature(compute_feature).parameters + _ACCEPTS_AXIS_CACHE[func] = accepts + return accepts def nested_aggregation_param_names(agg_cls): @@ -97,6 +111,23 @@ def _append_tail_row(full_result, tail_result): return np.concatenate([full_result, tail_row[None, ...]]) +def _append_tail_rows(full_result, tail_result): + full_result = np.asarray(full_result) + tail_result = np.asarray(tail_result) + target_shape = full_result.shape[2:] + if tail_result.shape[1:] == target_shape: + tail_rows = tail_result + else: + tail_rows = np.zeros( + (full_result.shape[0], *target_shape), dtype=full_result.dtype + ) + slices = tuple( + slice(0, min(d, s)) for d, s in zip(target_shape, tail_result.shape[1:]) + ) + tail_rows[(slice(None), *slices)] = tail_result[(slice(None), *slices)] + return np.concatenate([full_result, tail_rows[:, None, ...]], axis=1) + + def resolve_aggregation_function(aggregation_function, params): if params is None: return aggregation_function @@ -300,10 +331,32 @@ def estimate_peak_memory_bytes(self, input_stats: RepresentationStats) -> dict: + padded_elems * stats_itemsize(input_stats) ) - cpu_peak = int((input_bytes + list_bytes + pad_bytes) * 1.15 + 16 * 1024 * 1024) + batch_bytes = ( + input_bytes + if getattr(input_stats, "container", CONTAINER_ARRAY) == CONTAINER_LIST + else 0 + ) + + cpu_peak = int( + (input_bytes + batch_bytes + list_bytes + pad_bytes) * 1.15 + + 16 * 1024 * 1024 + ) return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} def execute(self, modality): + batch = self._dense_batch(modality) + if batch is not None: + windowed_data = self.window_aggregate_single_level_batched(batch) + if windowed_data is not None: + if self.pad: + data_type = modality.metadata[0]["data_layout"]["type"] + if data_type != "str": + windowed_data = windowed_data.astype(data_type) + else: + windowed_data = list(windowed_data) + self.assert_output_stats(windowed_data) + return windowed_data + windowed_data = [] original_lengths = [] for instance in modality.data: @@ -352,6 +405,56 @@ def execute(self, modality): self.assert_output_stats(windowed_data) return windowed_data + def _dense_batch(self, modality): + if modality.get_data_layout() != DataLayout.SINGLE_LEVEL: + return None + if not _accepts_axis(self.aggregation_function.compute_feature): + return None + + batch = dense_instance_batch(modality.data) + if batch is None: + return None + + batch = batch.view() + batch.setflags(write=False) + return batch + + def window_aggregate_single_level_batched(self, data): + num_instances, length = data.shape + new_length = math.ceil(length / self.window_size) + cut_length = (new_length - 1) * self.window_size + tail = data[:, cut_length:] + compute_feature = self.aggregation_function.compute_feature + + if new_length <= 1: + if not tail.shape[1]: + raise ValueError( + "Cannot window-aggregate an empty instance " + f"(window_size={self.window_size})." + ) + if tail.shape[1] < self.window_size: + pad_len = self.window_size - tail.shape[1] + tail = np.pad(tail, ((0, 0), (0, pad_len)), mode="constant") + result = np.asarray(compute_feature(tail, axis=1)) + if result.shape[0] != num_instances: + return None + return result[:, None, ...] + + full_batches = data[:, :cut_length].reshape( + num_instances, new_length - 1, self.window_size + ) + full_result = np.asarray(compute_feature(full_batches, axis=2)) + if full_result.shape[:2] != (num_instances, new_length - 1): + return None + + if tail.shape[1]: + tail_result = np.asarray(compute_feature(tail, axis=1)) + if tail_result.shape[0] != num_instances: + return None + full_result = _append_tail_rows(full_result, tail_result) + + return full_result + def window_aggregate_single_level(self, instance, new_length): if isinstance(instance, str): return instance @@ -359,7 +462,7 @@ def window_aggregate_single_level(self, instance, new_length): arr = np.asarray(instance) cut_length = (new_length - 1) * self.window_size tail = arr[cut_length:] - sig = inspect.signature(self.aggregation_function.compute_feature) + takes_axis = _accepts_axis(self.aggregation_function.compute_feature) if new_length <= 1: if not tail.size: raise ValueError( @@ -374,7 +477,7 @@ def window_aggregate_single_level(self, instance, new_length): pad_width = [(0, 0)] * tail.ndim pad_width[0] = (0, pad_len) tail = np.pad(tail, pad_width=pad_width, mode="constant") - if "axis" in sig.parameters: + if takes_axis: return np.array([self.aggregation_function.compute_feature(tail)]) tail_result = self.aggregation_function.compute_feature(tail) return ( @@ -386,7 +489,7 @@ def window_aggregate_single_level(self, instance, new_length): new_length - 1, self.window_size, *arr.shape[1:] ) - if "axis" in sig.parameters: + if takes_axis: full_result = self.aggregation_function.compute_feature( full_batches, axis=1 ) @@ -495,8 +598,7 @@ def execute(self, modality): self.num_windows, window_size, *instance.shape[1:] ) - sig = inspect.signature(self.aggregation_function.compute_feature) - if "axis" in sig.parameters: + if _accepts_axis(self.aggregation_function.compute_feature): f = self.aggregation_function.compute_feature(full_batches, axis=1) else: f = np.stack( diff --git a/src/main/python/systemds/scuro/representations/word2vec.py b/src/main/python/systemds/scuro/representations/word2vec.py index bc1c8791f20..fd1e148e117 100644 --- a/src/main/python/systemds/scuro/representations/word2vec.py +++ b/src/main/python/systemds/scuro/representations/word2vec.py @@ -58,7 +58,9 @@ def __init__(self, vector_size=150, min_count=1, output_file=None, params=None): self.data_type = np.float32 def get_output_stats(self, input_stats: TextStats) -> RepresentationStats: - return RepresentationStats(input_stats.num_instances, (self.vector_size,)) + return RepresentationStats( + input_stats.num_instances, (self.vector_size,), dtype=self.data_type + ) def estimate_output_memory_bytes(self, input_stats: TextStats) -> int: return ( diff --git a/src/main/python/tests/scuro/test_operator_registry.py b/src/main/python/tests/scuro/test_operator_registry.py index 443cc039d6b..93afba342b0 100644 --- a/src/main/python/tests/scuro/test_operator_registry.py +++ b/src/main/python/tests/scuro/test_operator_registry.py @@ -62,6 +62,9 @@ Quantile, ZeroCrossingRate, FrequencyMagnitude, + LastValue, + TransitionCount, + ObservationDensity, ) from systemds.scuro.modality.type import ModalityType from systemds.scuro.representations.average import Average @@ -74,6 +77,10 @@ from systemds.scuro.representations.hadamard import Hadamard from systemds.scuro.representations.resnet import ResNet from systemds.scuro.representations.multimodal_attention_fusion import AttentionFusion +from systemds.scuro.representations.physiological_window import ( + AdaptiveWindow, + PhysiologicalEventWindow, +) class TestOperatorRegistry(unittest.TestCase): @@ -113,6 +120,9 @@ def test_timeseries_representations_in_registry(self): Kurtosis, RMS, ZeroCrossingRate, + LastValue, + TransitionCount, + ObservationDensity, ACF, FrequencyMagnitude, SpectralCentroid, @@ -132,6 +142,8 @@ def test_context_operator_in_registry(self): WindowAggregation, StaticWindow, DynamicWindow, + AdaptiveWindow, + PhysiologicalEventWindow, ] assert registry.get_context_operators(ModalityType.TEXT) == [ SentenceBoundarySplitIndices, From b8c8aa8c2382fb0cc773ad83173ac74e263beacb Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Mon, 31 Aug 2026 15:22:36 +0200 Subject: [PATCH 125/132] [SYSTEMDS-3835] Improve efficiency of Representations in Scuro This patch adds improvements to multiple text, visual and audio representations. It includes a LazyLoading approach for visual modalities where the raw data is read when needed in the CustomDataloader. It adds two new representations: OpenFace and ImageBind. Assisted-by: AI --- .github/workflows/python.yml | 7 +- src/main/python/systemds/scuro/__init__.py | 63 ++ .../systemds/scuro/dataloader/base_loader.py | 77 ++- .../systemds/scuro/dataloader/image_loader.py | 51 +- .../systemds/scuro/dataloader/video_loader.py | 206 +++++-- .../scuro/drsearch/dag_group_executor.py | 2 +- .../scuro/drsearch/dag_group_scheduler.py | 4 +- .../scuro/drsearch/modality_shared_memory.py | 6 + .../scuro/drsearch/multimodal_optimizer.py | 6 +- .../systemds/scuro/drsearch/node_executor.py | 143 ++++- .../systemds/scuro/drsearch/node_scheduler.py | 173 +++++- .../scuro/drsearch/representation_dag.py | 50 +- .../python/systemds/scuro/drsearch/task.py | 26 +- .../scuro/drsearch/unimodal_optimizer.py | 64 +- .../systemds/scuro/drsearch/worker_pool.py | 149 ++++- .../python/systemds/scuro/modality/joined.py | 54 +- .../systemds/scuro/modality/modality.py | 24 + .../scuro/modality/unimodal_modality.py | 158 +++-- .../python/systemds/scuro/models/model.py | 2 + .../systemds/scuro/representations/bert.py | 202 +++--- .../systemds/scuro/representations/bow.py | 1 + .../systemds/scuro/representations/clip.py | 279 +++++---- .../scuro/representations/color_histogram.py | 7 +- .../scuro/representations/image_bind.py | 364 +++++++++-- .../scuro/representations/mel_spectrogram.py | 5 +- .../systemds/scuro/representations/mfcc.py | 9 +- .../scuro/representations/openface.py | 577 ++++++++++++++++++ .../scuro/representations/optical_flow.py | 8 +- .../systemds/scuro/representations/resnet.py | 135 ++-- .../representations/swin_video_transformer.py | 104 ++-- .../systemds/scuro/representations/tfidf.py | 1 + .../timeseries_representations.py | 4 +- .../systemds/scuro/representations/utils.py | 289 +++++++++ .../systemds/scuro/representations/vgg.py | 161 ++--- .../systemds/scuro/representations/wav2vec.py | 103 +++- .../representations/window_aggregation.py | 105 +++- .../scuro/representations/word2vec.py | 4 +- .../systemds/scuro/representations/x3d.py | 293 ++++++--- .../systemds/scuro/utils/checkpointing.py | 2 +- .../systemds/scuro/utils/memory_utility.py | 22 +- src/main/python/tests/scuro/data_generator.py | 47 +- .../scuro/test_chunked_leaf_execution.py | 352 +++++++++++ .../tests/scuro/test_lazy_visual_loading.py | 177 ++++++ .../python/tests/scuro/test_modality_pad.py | 105 ++++ .../scuro/test_neural_encoder_batching.py | 156 +++++ .../tests/scuro/test_operator_registry.py | 7 + .../test_transformer_text_aggregation.py | 353 +++++++++++ .../tests/scuro/test_unimodal_optimizer.py | 201 +++++- .../scuro/test_unimodal_representations.py | 512 +++++++++++++++- .../tests/scuro/test_window_operations.py | 398 ++++++++++++ .../test_window_representation_batching.py | 141 +++++ 51 files changed, 5504 insertions(+), 885 deletions(-) create mode 100644 src/main/python/systemds/scuro/representations/openface.py create mode 100644 src/main/python/tests/scuro/test_chunked_leaf_execution.py create mode 100644 src/main/python/tests/scuro/test_lazy_visual_loading.py create mode 100644 src/main/python/tests/scuro/test_modality_pad.py create mode 100644 src/main/python/tests/scuro/test_neural_encoder_batching.py create mode 100644 src/main/python/tests/scuro/test_transformer_text_aggregation.py create mode 100644 src/main/python/tests/scuro/test_window_representation_batching.py diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index d55f9adc6c0..d679aa74a92 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -167,7 +167,7 @@ jobs: pip install --upgrade pip wheel setuptools # Use CUDA 12.1 wheels to avoid slow/source builds pip install --extra-index-url https://download.pytorch.org/whl/cu121 \ - torch==2.4.1 torchvision==0.19.1 + torch==2.4.1 torchvision==0.19.1 torchaudio==2.4.1 pip install \ transformers \ opencv-python \ @@ -179,7 +179,10 @@ jobs: fvcore \ scikit-optimize \ flair \ - optuna + optuna \ + openface-test \ + imagebind \ + "pytorchvideo @ git+https://github.com/facebookresearch/pytorchvideo.git@eb04d1b" kill $KA cd src/main/python python -m unittest discover -s tests/scuro -p 'test_*.py' -v diff --git a/src/main/python/systemds/scuro/__init__.py b/src/main/python/systemds/scuro/__init__.py index 168f036b1e3..c8ba06e72d9 100644 --- a/src/main/python/systemds/scuro/__init__.py +++ b/src/main/python/systemds/scuro/__init__.py @@ -24,6 +24,7 @@ from systemds.scuro.dataloader.video_loader import VideoLoader from systemds.scuro.dataloader.text_loader import TextLoader from systemds.scuro.dataloader.json_loader import JSONLoader +from systemds.scuro.dataloader.tabular_loader import TabularLoader from systemds.scuro.representations.representation import Representation from systemds.scuro.representations.aggregate import Aggregation from systemds.scuro.representations.aggregated_representation import ( @@ -71,9 +72,23 @@ from systemds.scuro.representations.representation_dataloader import JSON from systemds.scuro.representations.representation_dataloader import Pickle from systemds.scuro.representations.resnet import ResNet +from systemds.scuro.representations.openface import OpenFace + +try: + from systemds.scuro.representations.image_bind import ImageBind +except ImportError as _imagebind_import_error: # pragma: no cover + import warnings as _warnings + + _warnings.warn( + f"ImageBind representation unavailable, it is excluded from the search " + f"({_imagebind_import_error})." + ) + ImageBind = None + from systemds.scuro.representations.spectrogram import Spectrogram from systemds.scuro.representations.sum import Sum from systemds.scuro.representations.swin_video_transformer import SwinVideoTransformer +from systemds.scuro.representations.tabular_features import TabularFeatures from systemds.scuro.representations.tfidf import TfIdf from systemds.scuro.representations.unimodal import UnimodalRepresentation from systemds.scuro.representations.wav2vec import Wav2Vec @@ -82,6 +97,10 @@ DynamicWindow, StaticWindow, ) +from systemds.scuro.representations.physiological_window import ( + AdaptiveWindow, + PhysiologicalEventWindow, +) from systemds.scuro.representations.word2vec import W2V from systemds.scuro.representations.x3d import X3D from systemds.scuro.representations.color_histogram import ColorHistogram @@ -124,6 +143,28 @@ MLPLearnedDimReduction, ) + +from systemds.scuro.representations.physiological_representations import ( + SDNN, + RMSSD, + pNN, + RRPerMinute, + HRVBandPower, + HRVVLF, + HRVLF, + HRVHF, + HRVLFHF, + PoincareSD1, + PoincareSD2, + # SampleEntropy, + # DFAAlpha, + SCLSlope, + SCLDynamicRange, + SCRPeaksPerMinute, + SCRAverageAmplitude, + SCRAverageDuration, +) + __all__ = [ "BaseLoader", "ImageLoader", @@ -156,6 +197,8 @@ "JSON", "Pickle", "ResNet", + "OpenFace", + "ImageBind", "Spectrogram", "Sum", "BoW", @@ -187,6 +230,8 @@ "AttentionFusion", "DynamicWindow", "StaticWindow", + "AdaptiveWindow", + "PhysiologicalEventWindow", "Min", "Max", "Mean", @@ -211,4 +256,22 @@ "MLPAveraging", "MLPLearnedDimReduction", "DimensionalityReduction", + "MultimodalGAPymooOptimizer", + "SDNN", + "RMSSD", + "pNN", + "RRPerMinute", + "HRVBandPower", + "HRVVLF", + "HRVLF", + "HRVHF", + "HRVLFHF", + "PoincareSD1", + "PoincareSD2", + # "DFAAlpha", + "SCLSlope", + "SCLDynamicRange", + "SCRPeaksPerMinute", + "SCRAverageAmplitude", + "SCRAverageDuration", ] diff --git a/src/main/python/systemds/scuro/dataloader/base_loader.py b/src/main/python/systemds/scuro/dataloader/base_loader.py index 9b89c773942..e58a755838c 100644 --- a/src/main/python/systemds/scuro/dataloader/base_loader.py +++ b/src/main/python/systemds/scuro/dataloader/base_loader.py @@ -20,12 +20,33 @@ # ------------------------------------------------------------- import os from abc import ABC, abstractmethod -from typing import Iterator, List, Optional, Tuple, Union +from collections.abc import Sequence +from typing import Callable, Iterator, List, Optional, Tuple, Union import math +from numbers import Integral import numpy as np +class LazyFileSequence(Sequence): + """List-like file references decoded only when a sample is requested.""" + + def __init__(self, file_names: List[str], decoder: Callable[[str], object]): + self.file_names = tuple(file_names) + self.decoder = decoder + + def __getitem__(self, index): + if isinstance(index, slice): + return [self[i] for i in range(*index.indices(len(self)))] + return self.decoder(self.file_names[index]) + + def __len__(self): + return len(self.file_names) + + def subset(self, indices): + return type(self)([self.file_names[i] for i in indices], self.decoder) + + class BaseLoader(ABC): def __init__( self, @@ -54,8 +75,7 @@ def __init__( self._data_type = data_type self._ext = ext self.stats = None - if chunk_size: - self.chunk_size = chunk_size + self.chunk_size = chunk_size @property def chunk_size(self): @@ -63,8 +83,28 @@ def chunk_size(self): @chunk_size.setter def chunk_size(self, value): - self._chunk_size = value - self._num_chunks = int(math.ceil(len(self.indices) / self._chunk_size)) + if value is None: + self._chunk_size = None + self._num_chunks = 1 + else: + if isinstance(value, bool) or not isinstance(value, Integral) or value <= 0: + raise ValueError("chunk_size must be None or a positive integer") + self._chunk_size = int(value) + self._num_chunks = int(math.ceil(len(self.indices) / self._chunk_size)) + + stats = getattr(self, "stats", None) + if stats is not None and hasattr(stats, "num_instances"): + stats.num_instances = ( + len(self.indices) + if self._chunk_size is None + else min(len(self.indices), self._chunk_size) + ) + if stats is not None and hasattr(stats, "num_total_instances"): + stats.num_total_instances = len(self.indices) + + @property + def is_chunked(self): + return self._chunk_size is not None @property def num_chunks(self): @@ -91,7 +131,7 @@ def load(self): """ Takes care of loading the raw data either chunk wise (if chunk size is defined) or all at once """ - if self._chunk_size: + if self.is_chunked: return self._load_next_chunk() return self._load(self.indices) @@ -102,8 +142,8 @@ def iter_loaded_chunks( if reset: self.reset() - if not self._chunk_size: - data, metadata = self._load(self.indices) + if not self.is_chunked: + data, metadata = self.load() yield data, metadata, self.indices return @@ -115,24 +155,21 @@ def iter_loaded_chunks( yield data, metadata, chunk_indices def update_chunk_sizes(self, other): - if not self._chunk_size and not other.chunk_size: + sizes = [ + size for size in (self.chunk_size, other.chunk_size) if size is not None + ] + if not sizes: return - - if ( - self._chunk_size - and not other.chunk_size - or self._chunk_size < other.chunk_size - ): - other.chunk_size = self.chunk_size - else: - self.chunk_size = other.chunk_size + shared_size = min(sizes) + self.chunk_size = shared_size + other.chunk_size = shared_size def _load_next_chunk(self): """ Loads the next chunk of data """ self.data = [] - # TODO: Handle metadata correctly + self.metadata = [] next_chunk_indices = self.indices[ self._next_chunk * self._chunk_size : (self._next_chunk + 1) @@ -161,7 +198,7 @@ def get_file_names(self, indices=None): if self._ext is None: _, self._ext = os.path.splitext(os.listdir(self.source_path)[0]) for index in self.indices if indices is None else indices: - file_names.append(self.source_path + index + self._ext) + file_names.append(os.path.join(self.source_path, index + self._ext)) return file_names else: return self.source_path diff --git a/src/main/python/systemds/scuro/dataloader/image_loader.py b/src/main/python/systemds/scuro/dataloader/image_loader.py index 25e8690cf5a..8da151a113e 100644 --- a/src/main/python/systemds/scuro/dataloader/image_loader.py +++ b/src/main/python/systemds/scuro/dataloader/image_loader.py @@ -24,7 +24,7 @@ import numpy as np -from systemds.scuro.dataloader.base_loader import BaseLoader +from systemds.scuro.dataloader.base_loader import BaseLoader, LazyFileSequence import cv2 from systemds.scuro.modality.type import ModalityType @@ -55,13 +55,28 @@ def __init__( source_path, indices, data_type, chunk_size, ModalityType.IMAGE, ext ) self.load_data_from_file = load + self._all_metadata = [] self.stats = self.get_stats(source_path) - def extract(self, file: str, index: Optional[Union[str, List[str]]] = None): - self.file_sanity_check(file) + def load(self): + if self.chunk_size: + return super().load() + self.data = LazyFileSequence( + self.get_file_names(self.indices), self._decode_file + ) + self.metadata = self._all_metadata.copy() + return self.data, self.metadata + + def _decode_file(self, file: str): + self.file_sanity_check(file) image = cv2.imread(file, cv2.IMREAD_COLOR) - image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + if image is None: + raise ValueError(f"Could not read image at path: {file}") + return cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.uint8, copy=False) + + def extract(self, file: str, index: Optional[Union[str, List[str]]] = None): + image = self._decode_file(file) if image.ndim == 2: height, width = image.shape @@ -69,8 +84,6 @@ def extract(self, file: str, index: Optional[Union[str, List[str]]] = None): else: height, width, channels = image.shape - image = image.astype(np.uint8, copy=False) - self.metadata.append( self.modality_type.create_metadata(width, height, channels) ) @@ -78,6 +91,7 @@ def extract(self, file: str, index: Optional[Union[str, List[str]]] = None): self.data.append(image) def get_stats(self, source_path: str): + self._all_metadata = [] max_width = 0 max_height = 0 max_channels = 0 @@ -87,18 +101,15 @@ def get_stats(self, source_path: str): average_channels = 0 for file in self.indices: path = os.path.join(source_path, f"{file}{self._ext}") - # if self.chunk_size is None: - # self.extract(path) - # md = self.metadata[path] - # max_width = max(max_width, md["width"]) - # max_height = max(max_height, md["height"]) - # max_channels = max(max_channels, md["num_channels"]) - # num_instances += 1 - # else: self.file_sanity_check(path) image = cv2.imread(path, cv2.IMREAD_COLOR) + if image is None: + raise ValueError(f"Could not read image at path: {path}") image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) height, width, channels = image.shape + self._all_metadata.append( + self.modality_type.create_metadata(width, height, channels) + ) max_width = max(max_width, width) max_height = max(max_height, height) max_channels = max(max_channels, channels) @@ -119,3 +130,15 @@ def get_stats(self, source_path: str): average_height, average_channels, ) + + def estimate_peak_memory_bytes(self) -> dict: + n = self.chunk_size if self.chunk_size is not None else 1 + per_instance = ( + self.stats.average_width + * self.stats.average_height + * self.stats.average_channels + ) + return { + "cpu_peak_bytes": int(n * per_instance * np.dtype(np.uint8).itemsize), + "gpu_peak_bytes": 0, + } diff --git a/src/main/python/systemds/scuro/dataloader/video_loader.py b/src/main/python/systemds/scuro/dataloader/video_loader.py index bf7bdd846c7..e2ee7b72d24 100644 --- a/src/main/python/systemds/scuro/dataloader/video_loader.py +++ b/src/main/python/systemds/scuro/dataloader/video_loader.py @@ -19,12 +19,12 @@ # # ------------------------------------------------------------- from dataclasses import dataclass -import os -from typing import List, Optional, Union +import math +from typing import List, Optional, Tuple, Union import numpy as np -from systemds.scuro.dataloader.base_loader import BaseLoader +from systemds.scuro.dataloader.base_loader import BaseLoader, LazyFileSequence import cv2 from systemds.scuro.modality.type import ModalityType @@ -39,6 +39,7 @@ class VideoStats: max_channels: int num_instances: int num_total_instances: int + shape_variance: float = 0.0 @property def output_shape(self): @@ -52,6 +53,15 @@ def output_shape(self): """ return (self.max_length, self.max_height, self.max_width, self.max_channels) + @property + def avg_output_shape(self): + return ( + max(1, int(round(self.avg_length))), + self.max_height, + self.max_width, + self.max_channels, + ) + class VideoLoader(BaseLoader): def __init__( @@ -62,109 +72,181 @@ def __init__( chunk_size: Optional[int] = None, load=True, fps=None, + target_size: Optional[Tuple[int, int]] = None, ): super().__init__( source_path, indices, data_type, chunk_size, ModalityType.VIDEO ) self.load_data_from_file = load self.fps = fps + self.target_size = tuple(int(v) for v in target_size) if target_size else None + self._all_metadata = [] self.stats = self.get_stats(source_path) - def extract(self, file: str, index: Optional[Union[str, List[str]]] = None): + def load(self): + if self.chunk_size: + return super().load() + + self.data = LazyFileSequence( + self.get_file_names(self.indices), self._decode_data + ) + self.metadata = self._all_metadata.copy() + return self.data, self.metadata + + def _decode_data(self, file: str): + return self._decode_file(file)[0] + + def _frame_interval(self, source_fps: float) -> int: + if self.fps and source_fps and self.fps < source_fps: + return max(1, int(round(source_fps / self.fps))) + return 1 + + def _stored_length(self, source_length: int, source_fps: float) -> int: + interval = self._frame_interval(source_fps) + return int(math.ceil(source_length / interval)) if source_length > 0 else 0 + + def _stored_frame_size(self, width: int, height: int) -> Tuple[int, int]: + return self.target_size if self.target_size else (width, height) + + def _fit_frame(self, frame: np.ndarray) -> np.ndarray: + if self.target_size is None: + return frame + + target_w, target_h = self.target_size + height, width = frame.shape[:2] + if (width, height) == (target_w, target_h): + return frame + + scale = max(target_w / width, target_h / height) + new_w = max(target_w, int(round(width * scale))) + new_h = max(target_h, int(round(height * scale))) + interpolation = cv2.INTER_AREA if scale < 1.0 else cv2.INTER_LINEAR + frame = cv2.resize(frame, (new_w, new_h), interpolation=interpolation) + + left = (new_w - target_w) // 2 + top = (new_h - target_h) // 2 + return frame[top : top + target_h, left : left + target_w] + + def _decode_file(self, file: str): self.file_sanity_check(file) cap = cv2.VideoCapture(file) if not cap.isOpened(): - raise f"Could not read video at path: {file}" - - orig_fps = cap.get(cv2.CAP_PROP_FPS) - frame_interval = 1 - if self.fps is not None and self.fps < orig_fps: - frame_interval = int(round(orig_fps / self.fps)) - else: - self.fps = orig_fps - - length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) - width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) - height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) - num_channels = 3 - - self.metadata.append( - self.modality_type.create_metadata( - self.fps, length, width, height, num_channels - ) + raise ValueError(f"Could not read video at path: {file}") + + try: + source_fps = cap.get(cv2.CAP_PROP_FPS) + frame_interval = self._frame_interval(source_fps) + stored_fps = source_fps / frame_interval if source_fps else source_fps + + scale_denominator = np.dtype(self._data_type).type(255.0) + frames = [] + idx = 0 + while True: + ret, frame = cap.read() + if not ret: + break + if idx % frame_interval == 0: + frame = self._fit_frame(frame) + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + frame = frame.astype(self._data_type) / scale_denominator + frames.append(frame) + idx += 1 + finally: + cap.release() + + if not frames: + raise ValueError(f"No frames could be decoded from {file}") + + data = np.stack(frames) + + num_frames, height, width = data.shape[0], data.shape[1], data.shape[2] + metadata = self.modality_type.create_metadata( + stored_fps, num_frames, width, height, data.shape[3] ) + return data, metadata - frames = [] - idx = 0 - while cap.isOpened(): - ret, frame = cap.read() - - if not ret: - break - if idx % frame_interval == 0: - frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - frame = frame.astype(self._data_type, copy=False) / 255.0 - frames.append(frame) - idx += 1 - - self.data.append(np.stack(frames)) + def extract(self, file: str, index: Optional[Union[str, List[str]]] = None): + data, metadata = self._decode_file(file) + self.metadata.append(metadata) + self.data.append(data) def get_stats(self, source_path: str): + self._all_metadata = [] self.file_sanity_check(source_path) - fps = 0 max_length = 0 max_width = 0 max_height = 0 max_num_channels = 0 num_instances = 0 - avg_length = 0 - for file in os.listdir(source_path): - file_name = file.split(".")[0] - if file_name not in self.indices: - continue - self.file_sanity_check(source_path + file) - cap = cv2.VideoCapture(source_path + file) - - length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) - width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) - height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + stored_lengths = [] + stored_fps = [] + + for file in self.get_file_names(self.indices): + self.file_sanity_check(file) + cap = cv2.VideoCapture(file) + if not cap.isOpened(): + raise ValueError(f"Could not read video at path: {file}") + try: + source_length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + source_fps = cap.get(cv2.CAP_PROP_FPS) + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + finally: + cap.release() + + length = self._stored_length(source_length, source_fps) + width, height = self._stored_frame_size(width, height) num_channels = 3 + stored_frequency = ( + source_fps / self._frame_interval(source_fps) if source_fps else 0 + ) + self._all_metadata.append( + self.modality_type.create_metadata( + stored_frequency, length, width, height, num_channels + ) + ) + max_length = max(max_length, length) - avg_length += length max_width = max(max_width, width) max_height = max(max_height, height) max_num_channels = max(max_num_channels, num_channels) + stored_lengths.append(length) + if source_fps: + stored_fps.append(stored_frequency) num_instances += 1 + num_total_instances = num_instances + avg_length = float(np.mean(stored_lengths)) if stored_lengths else 0.0 + shape_variance = ( + float(np.std(stored_lengths) / avg_length) + if stored_lengths and avg_length > 0 + else 0.0 + ) num_instances = ( min(num_instances, self.chunk_size) if self.chunk_size is not None else num_instances ) return VideoStats( - fps, + float(np.mean(stored_fps)) if stored_fps else 0, max_length, - avg_length / num_instances, + avg_length, max_width, max_height, max_num_channels, num_instances, num_total_instances, + shape_variance, ) def estimate_peak_memory_bytes(self) -> dict: - s = self.stats - if self.chunk_size is not None: - n = self.chunk_size - else: - n = s.num_instances + stats = self.stats + n = self.chunk_size if self.chunk_size is not None else 1 + n = min(n, stats.num_total_instances) + per_instance = int(np.prod(stats.avg_output_shape)) + itemsize = np.dtype(self._data_type).itemsize return { - "cpu_peak_bytes": n - * s.output_shape[0] - * s.output_shape[1] - * s.output_shape[2] - * s.output_shape[3] - * 4, + "cpu_peak_bytes": int(n * per_instance * itemsize), "gpu_peak_bytes": 0, } diff --git a/src/main/python/systemds/scuro/drsearch/dag_group_executor.py b/src/main/python/systemds/scuro/drsearch/dag_group_executor.py index 57007cb06be..5181f934f4e 100644 --- a/src/main/python/systemds/scuro/drsearch/dag_group_executor.py +++ b/src/main/python/systemds/scuro/drsearch/dag_group_executor.py @@ -25,7 +25,7 @@ import time from typing import Any, Dict, List, Optional -from systemds.scuro import Modality +from systemds.scuro.modality.modality import Modality from systemds.scuro.drsearch.representation_dag import ( LRUCache, RepresentationDag, diff --git a/src/main/python/systemds/scuro/drsearch/dag_group_scheduler.py b/src/main/python/systemds/scuro/drsearch/dag_group_scheduler.py index def12219f54..23799f78a0c 100644 --- a/src/main/python/systemds/scuro/drsearch/dag_group_scheduler.py +++ b/src/main/python/systemds/scuro/drsearch/dag_group_scheduler.py @@ -28,10 +28,10 @@ def get_peak_memory_from_dag_group( dag_group: List[RepresentationDag], modality: Modality -) -> tuple[float, float]: +) -> Tuple[float, float]: peak_memory_cpu = 0.0 peak_memory_gpu = 0.0 - leaf_memory_bytes = modality.estimate_memory_bytes() + leaf_memory_bytes = modality.estimate_peak_memory_bytes()["cpu_peak_bytes"] for dag in dag_group: prev_stats = modality.get_stats() for node in dag.nodes[1:]: diff --git a/src/main/python/systemds/scuro/drsearch/modality_shared_memory.py b/src/main/python/systemds/scuro/drsearch/modality_shared_memory.py index e68d7195a2d..57a5048cda3 100644 --- a/src/main/python/systemds/scuro/drsearch/modality_shared_memory.py +++ b/src/main/python/systemds/scuro/drsearch/modality_shared_memory.py @@ -275,6 +275,12 @@ def __getstate__(self): state["_buffer"] = None return state + def __reduce_ex__(self, protocol): + return ( + type(self), + (self.shm_name, self.dtype_str, self.offsets, self.total_elems), + ) + def _is_string_list_shared_memory_candidate(data: Any) -> bool: if not isinstance(data, list) or not data: diff --git a/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py index 596cab0237b..42369e423be 100644 --- a/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py +++ b/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py @@ -33,7 +33,6 @@ from systemds.scuro.representations.aggregated_representation import ( AggregatedRepresentation, ) -from systemds.scuro.representations.aggregate import Aggregation from systemds.scuro.drsearch.operator_registry import Registry from systemds.scuro.utils.schema_helpers import get_shape @@ -69,10 +68,9 @@ def _evaluate_dag_worker(dag_pickle, task_pickle, modalities_pickle, debug=False from systemds.scuro.representations.aggregated_representation import ( AggregatedRepresentation, ) - from systemds.scuro.representations.aggregate import Aggregation if task.expected_dim == 1 and get_shape(final_representation.metadata) > 1: - agg_operator = AggregatedRepresentation(Aggregation()) + agg_operator = AggregatedRepresentation() final_representation = agg_operator.transform(final_representation) eval_start = time.time() @@ -424,7 +422,7 @@ def _evaluate_dag(self, dag: RepresentationDag, task: Task) -> "OptimizationResu return None if task.expected_dim == 1 and get_shape(fused_representation.metadata) > 1: - agg_operator = AggregatedRepresentation(Aggregation()) + agg_operator = AggregatedRepresentation() fused_representation = agg_operator.transform(fused_representation) eval_start = time.time() diff --git a/src/main/python/systemds/scuro/drsearch/node_executor.py b/src/main/python/systemds/scuro/drsearch/node_executor.py index 3400e6c6c50..c56a6f4ca1f 100644 --- a/src/main/python/systemds/scuro/drsearch/node_executor.py +++ b/src/main/python/systemds/scuro/drsearch/node_executor.py @@ -26,7 +26,7 @@ import torch -from systemds.scuro import Modality +from systemds.scuro.modality.modality import Modality from systemds.scuro.drsearch.modality_result_cache import RefCountResultCache from systemds.scuro.drsearch.modality_shared_memory import ( add_shared_memory_candidate, @@ -66,6 +66,24 @@ _MAX_NODE_RETRIES = int(os.environ.get("SCURO_MAX_NODE_RETRIES", "3")) +def _place_on_device(obj: Any, gpu_id: Optional[int]) -> None: + """Apply scheduler placement even when an object selected CUDA itself.""" + device = torch.device("cpu" if gpu_id is None else f"cuda:{gpu_id}") + if hasattr(obj, "gpu_id"): + try: + obj.gpu_id = gpu_id + except (AttributeError, RuntimeError): + pass + if hasattr(obj, "device"): + try: + obj.device = device + except (AttributeError, RuntimeError): + pass + model = getattr(obj, "model", None) + if model is not None and hasattr(model, "to"): + obj.model = model.to(device) + + def _run_gpu_op(fn, gpu_id: Optional[int]): if gpu_id is None or not torch.cuda.is_available(): return fn() @@ -143,13 +161,11 @@ def _execute_node_worker(node, input_mods: List[Any], gpu_id: Optional[int]): torch.cuda.reset_peak_memory_stats(device) node_operation = _instantiate_operation(node) + _place_on_device(node_operation, gpu_id) operation_name = node_operation.name if DEBUG: print(f"Executing node {node.node_id} {operation_name} on GPU {gpu_id}") - if gpu_id is not None and hasattr(node_operation, "gpu_id"): - node_operation.gpu_id = gpu_id - def _run_node_op(): if len(input_mods) == 1: if isinstance(node_operation, Context): @@ -227,8 +243,7 @@ def _execute_task_worker( torch.cuda.set_device(device) torch.cuda.reset_peak_memory_stats(device) - if gpu_id is not None and hasattr(task, "model") and hasattr(task.model, "device"): - task.model.device = torch.device(f"cuda:{gpu_id}") + _place_on_device(getattr(task, "model", task), gpu_id) def _run_task(): start = time.perf_counter() @@ -271,17 +286,27 @@ def _run_task(): def _execute_leaf_batch_worker(nodes: List[Any], modality: Any, gpu_id: Optional[int]): - node_id_by_representation = {} - def _run(): representations = [] + representation_keys = [] + aggregations = [] for node in nodes: operation = node.operation(params=node.parameters) - if hasattr(operation, "gpu_id"): - operation.gpu_id = gpu_id + _place_on_device(operation, gpu_id) representations.append(operation) - node_id_by_representation[operation.name] = node.node_id - return modality.apply_representations(representations, parallel=True) + representation_keys.append(node.node_id) + pushdown_config = node.parameters.get("_pushdown_aggregation") + aggregations.append( + AggregatedRepresentation(params=pushdown_config) + if pushdown_config is not None + else None + ) + return modality.apply_representations( + representations, + parallel=True, + representation_keys=representation_keys, + aggregations=aggregations, + ) modality_results = _run_gpu_op(_run, gpu_id) shm_info = {} @@ -297,8 +322,13 @@ def _run(): } return { "results": modality_results, - "node_id_by_representation": node_id_by_representation, "shm_info": shm_info, + "failed_nodes": { + node_id: f"{type(error).__name__}: {error}" + for node_id, error in getattr( + modality, "failed_representations", {} + ).items() + }, } @@ -309,7 +339,7 @@ def _load_leaf_worker(modality: Any) -> Dict[str, Any]: data = modality.data resident_bytes = 0 try: - resident_bytes = modality.estimate_memory_bytes() + resident_bytes = modality.estimate_peak_memory_bytes()["cpu_peak_bytes"] except Exception: resident_bytes = 0 @@ -324,7 +354,14 @@ def _load_leaf_worker(modality: Any) -> Dict[str, Any]: def _dispatch_node(payload, gpu_id): node, input_mods = payload - return _execute_node_worker(node, input_mods, gpu_id) + try: + return _execute_node_worker(node, input_mods, gpu_id) + finally: + # CSE executes each node once; retaining its model after completion + # makes scheduler reservations lie about persistent worker memory. + _WORKER_OP_CACHE.clear() + if gpu_id is not None: + cleanup_gpu(gpu_id) def _dispatch_task(payload, gpu_id): @@ -434,8 +471,19 @@ def __init__( _WORKER_DISPATCH, ctx=create_mp_context(), threads_per_worker=threads_per_worker, + gpu_devices=list(getattr(self.scheduler, "gpu_devices", [])), + gpu_slots_per_device=int( + os.environ.get("SCURO_GPU_SLOTS_PER_DEVICE", "1") + ), + gpu_demand_fraction=self.scheduler.gpu_demand_fraction(), ) self._pool = worker_pool + restrict_devices = getattr(self.scheduler, "restrict_gpu_devices", None) + if callable(restrict_devices): + restrict_devices( + getattr(worker_pool, "gpu_worker_devices", []), + getattr(worker_pool, "gpu_slots_per_device", 1), + ) def _requeue_or_give_up(self, node_id: str, reason: str) -> bool: attempts = self._node_attempts.get(node_id, 0) + 1 @@ -474,6 +522,8 @@ def _retain_for_submit(self, parent_ids: List[str], payload: Any) -> List[str]: def _load_leaf_modalities(self) -> None: for modality in self._modalities: + if self._loads_in_chunks(modality): + continue if getattr(modality, "has_data", None) and modality.has_data(): continue attempts = 0 @@ -494,12 +544,19 @@ def _load_leaf_modalities(self) -> None: if shm_name is not None: self._leaf_shm_names.append(shm_name) + @staticmethod + def _loads_in_chunks(modality: Any) -> bool: + data_loader = getattr(modality, "data_loader", None) + return getattr(data_loader, "chunk_size", None) is not None + def _cleanup_leaf_shared_memory(self) -> None: for shm_name in self._leaf_shm_names: unlink_shm(shm_name) self._leaf_shm_names = [] - def _submit_node(self, node_id: str) -> None: + def _submit_node( + self, node_id: str, allow_gpu_worker_for_cpu: bool = False + ) -> None: node = self.scheduler.mapping[node_id] gpu_id = node.gpu_id parent_ids = self.scheduler.get_valid_parents(node_id) @@ -523,18 +580,26 @@ def _submit_node(self, node_id: str) -> None: "task", (node_id, self._tasks[task_idx], payload, node.aggregation), gpu_id=gpu_id, + allow_gpu_worker_for_cpu=allow_gpu_worker_for_cpu, ) else: payload = self._modalities if parent_results is None else parent_results retained = self._retain_for_submit(parent_ids, payload) self.scheduler.begin_execution(node_id) self.scheduler.move_to_running(node_id) - job_id = self._pool.submit("node", (node, payload), gpu_id=gpu_id) + job_id = self._pool.submit( + "node", + (node, payload), + gpu_id=gpu_id, + allow_gpu_worker_for_cpu=allow_gpu_worker_for_cpu, + ) self._job_units[job_id] = _NodeUnit(node_id) self._job_retained_shm[job_id] = retained - def _submit_leaf_batch(self, node_ids: List[str]) -> None: + def _submit_leaf_batch( + self, node_ids: List[str], allow_gpu_worker_for_cpu: bool = False + ) -> None: nodes = [self.scheduler.mapping[nid] for nid in node_ids] gpu_id = nodes[0].gpu_id retained = self._retain_for_submit([], self._modalities[0].data) @@ -542,22 +607,42 @@ def _submit_leaf_batch(self, node_ids: List[str]) -> None: self.scheduler.begin_execution(nid) self.scheduler.move_to_running(node_ids) job_id = self._pool.submit( - "leaf_batch", (nodes, self._modalities[0]), gpu_id=gpu_id + "leaf_batch", + (nodes, self._modalities[0]), + gpu_id=gpu_id, + allow_gpu_worker_for_cpu=allow_gpu_worker_for_cpu, ) self._job_units[job_id] = _BatchUnit(node_ids) self._job_retained_shm[job_id] = retained def _fill_pipeline(self) -> None: ready = self.scheduler.get_runnable().copy() + + def _gpu_id(entry): + node_id = entry[0] if isinstance(entry, list) else entry + return self.scheduler.mapping[node_id].gpu_id + + ready.sort(key=lambda entry: _gpu_id(entry) is None) for entry in ready: - if not self._pool.has_idle_worker: - break + gpu_id = _gpu_id(entry) + allow_gpu_worker_for_cpu = gpu_id is None + if not self._pool.has_idle_worker_for( + gpu_id, + allow_gpu_worker_for_cpu=allow_gpu_worker_for_cpu, + ): + continue if isinstance(entry, list): - self._submit_leaf_batch(entry) + self._submit_leaf_batch( + entry, + allow_gpu_worker_for_cpu=allow_gpu_worker_for_cpu, + ) else: if not self.scheduler.can_start_now(entry): continue - self._submit_node(entry) + self._submit_node( + entry, + allow_gpu_worker_for_cpu=allow_gpu_worker_for_cpu, + ) def _record_stats(self, node_id: str, pid: int, start_time: float, end_time: float): node_stats = self.statistics["node_stats"] @@ -648,23 +733,25 @@ def _handle_node_success(self, node_id: str, value: Dict[str, Any]) -> None: def _handle_batch_success(self, value: Dict[str, Any]) -> None: results = value["results"] - node_id_by_representation = value["node_id_by_representation"] shm_info = value.get("shm_info", {}) - for representation, transformed_modality in results.items(): - node_id = node_id_by_representation[representation] - info = shm_info.get(representation, {}) + for node_id, transformed_modality in results.items(): + info = shm_info.get(node_id, {}) self._handle_modality_result( transformed_modality, node_id, None, None, - representation, + self.scheduler.mapping[node_id].operation.__name__, actual_stats=info.get("actual_stats"), shm_name=info.get("shm_name"), resident_bytes=info.get("resident_bytes"), shm_bytes=info.get("shm_bytes", 0), ) + for node_id, reason in value.get("failed_nodes", {}).items(): + if not self._requeue_or_give_up(node_id, reason): + self._release_parents(node_id) + def _handle_modality_result( self, transformed_modality: Any, diff --git a/src/main/python/systemds/scuro/drsearch/node_scheduler.py b/src/main/python/systemds/scuro/drsearch/node_scheduler.py index 1ca681e88ad..8398c82a81b 100644 --- a/src/main/python/systemds/scuro/drsearch/node_scheduler.py +++ b/src/main/python/systemds/scuro/drsearch/node_scheduler.py @@ -70,6 +70,14 @@ def __init__( self.nodes = self._get_nodes_from_dags(nodes) self.unresolved_parents = self._get_unresolved_parents() self.node_resources = self._estimate_node_resources() + self.node_costs = self._estimate_node_costs() + self.node_priorities = self._compute_upward_ranks() + self.gpu_devices = list(self.memory_budget["gpu"]) + self.gpu_slots_per_device = max( + 1, int(os.environ.get("SCURO_GPU_SLOTS_PER_DEVICE", "1")) + ) + self.gpu_slots_in_use = {gpu_id: 0 for gpu_id in self.gpu_devices} + self._gpu_slot_nodes: Dict[str, int] = {} self.success = False self.deadlock = False self.ready_nodes = [] @@ -91,9 +99,7 @@ def __init__( for node_id in self.topo_order if node_id not in self.leaves and self.unresolved_parents[node_id] == 0 } - self.n_gpu = ( - torch.cuda.device_count() if torch and torch.cuda.is_available() else 0 - ) + self.n_gpu = len(self.gpu_devices) leaf_cached = sum(self.node_resources[node][0] for node in self.leaves) self.memory_stats = { "cpu_cached": leaf_cached, @@ -134,18 +140,25 @@ def get_runnable(self) -> List[RepresentationNode]: runnable_nodes = self._get_runnable_nodes() admitted_bytes = self._pending_admitted_cpu_bytes() + pending_gpu = {gpu_id: 0 for gpu_id in self.gpu_devices} + pending_slots = {gpu_id: 0 for gpu_id in self.gpu_devices} for node in runnable_nodes: if node in self._ready_set: continue - ok, gpu_id = self._check_memory_constraints(node, admitted_bytes) + ok, gpu_id = self._check_memory_constraints( + node, admitted_bytes, pending_gpu, pending_slots + ) if ok: admitted_bytes += self.node_resources[node][0] + if gpu_id is not None: + pending_gpu[gpu_id] += self.node_resources[node][1] + pending_slots[gpu_id] += 1 self.mapping[node].gpu_id = gpu_id self._candidates.discard(node) self.ready_nodes.append(node) self._ready_set.add(node) - contains_leaf = [] + chunked_leaf_by_device = defaultdict(list) for node in self.ready_nodes: if isinstance(node, list): continue @@ -156,14 +169,16 @@ def get_runnable(self) -> List[RepresentationNode]: == self.mapping[self.mapping[node].inputs[0]].modality_id ): if mod.data_loader.chunk_size is not None: - contains_leaf.append(node) + chunked_leaf_by_device[self.mapping[node].gpu_id].append( + node + ) break - for node in contains_leaf: - self.ready_nodes.remove(node) + for nodes_for_device in chunked_leaf_by_device.values(): + for node in nodes_for_device: + self.ready_nodes.remove(node) - if len(contains_leaf) > 0: - self.ready_nodes.append(contains_leaf) + self.ready_nodes.extend(chunked_leaf_by_device.values()) return self.ready_nodes def _get_runnable_nodes(self) -> List[str]: @@ -177,11 +192,64 @@ def _score(node_id: str): and self.remaining_children.get(parent_id, 0) == 1 ): release_bytes += self.node_resources[parent_id][0] - return (-release_bytes, node_id not in self.roots, node_id) + return ( + -self.node_priorities.get(node_id, 0.0), + -release_bytes, + node_id not in self.roots, + node_id, + ) runnable_nodes.sort(key=_score) return runnable_nodes + def _estimate_node_costs(self) -> Dict[str, float]: + costs: Dict[str, float] = {} + for node_id in self.topo_order: + if node_id in self.leaves: + costs[node_id] = 1.0 + continue + if node_id in self.roots: + parent_ids = list(self.parents.get(node_id, set())) + parent_stats = [ + self.node_stats.get(parent_id) for parent_id in parent_ids + ] + input_stats = ( + parent_stats[0] if len(parent_stats) == 1 else parent_stats + ) + task_idx = self.mapping[node_id].parameters.get("_task_idx", 0) + try: + costs[node_id] = max( + 1.0, + float(self.tasks[task_idx].estimate_relative_cost(input_stats)), + ) + except Exception: + costs[node_id] = 1.0 + else: + cpu_bytes, gpu_bytes = self.node_resources.get(node_id, (0, 0)) + costs[node_id] = max(1.0, float(cpu_bytes + gpu_bytes) / (1024**2)) + return costs + + def _compute_upward_ranks(self) -> Dict[str, float]: + ranks: Dict[str, float] = {} + for node_id in reversed(self.topo_order): + downstream = [ranks[child] for child in self.children.get(node_id, set())] + ranks[node_id] = float(self.node_costs.get(node_id, 1.0)) + ( + max(downstream) if downstream else 0.0 + ) + return ranks + + def gpu_demand_fraction(self) -> float: + total = 0.0 + gpu_total = 0.0 + for node_id, resources in self.node_resources.items(): + weight = float(self.node_costs.get(node_id, 1.0)) + if weight <= 0: + weight = 1.0 + total += weight + if resources[1] > 0: + gpu_total += weight + return gpu_total / total if total else 0.0 + def add_failed_node(self, node_id: str, reason: str = "unknown failure"): self.failed_nodes.append(node_id) self.failed_node_reasons[node_id] = reason @@ -196,8 +264,10 @@ def requeue_node(self, node_id: str) -> None: def begin_execution(self, node_id: str) -> None: gpu_id = self.mapping[node_id].gpu_id cpu_mem, gpu_mem = self.node_resources[node_id] - if gpu_id is not None and gpu_mem > 0: + if gpu_id is not None and gpu_mem > 0 and node_id not in self._gpu_slot_nodes: self.memory_stats["gpu_in_use"][gpu_id] += gpu_mem + self.gpu_slots_in_use[gpu_id] += 1 + self._gpu_slot_nodes[node_id] = gpu_id if cpu_mem > 0 and node_id not in self._cpu_reserved_nodes: self._cpu_reserved_nodes[node_id] = int(cpu_mem) self.memory_stats["cpu_in_flight"] += int(cpu_mem) @@ -382,7 +452,13 @@ def not_enough_memory(self) -> bool: return True return False - def _check_memory_constraints(self, node_id: str, pending_bytes: int = 0) -> bool: + def _check_memory_constraints( + self, + node_id: str, + pending_bytes: int = 0, + pending_gpu: Optional[Dict[int, int]] = None, + pending_slots: Optional[Dict[int, int]] = None, + ) -> bool: cpu_mem, gpu_mem = self.node_resources[node_id] gpu_id = None if ( @@ -400,9 +476,13 @@ def _check_memory_constraints(self, node_id: str, pending_bytes: int = 0) -> boo return False, None if gpu_mem > 0.0 and self.n_gpu > 0: - gpu_id = self._gpu_with_most_free_memory(gpu_mem) + gpu_id = self._gpu_with_most_free_memory( + gpu_mem, pending_gpu, pending_slots + ) if gpu_id is None: + if self._waiting_for_gpu_slot(gpu_mem, pending_gpu, pending_slots): + return False, None attempts = self.gpu_wait_attempts.get(node_id, 0) + 1 self.gpu_wait_attempts[node_id] = attempts if attempts > _MAX_GPU_SCHEDULE_ATTEMPTS: @@ -421,25 +501,68 @@ def _check_memory_constraints(self, node_id: str, pending_bytes: int = 0) -> boo return True, gpu_id - def _gpu_with_most_free_memory(self, memory_needed): - free_memory = [] - for i in range(self.n_gpu): - free_memory.append( - self.memory_budget["gpu"][i] - self.memory_stats["gpu_in_use"][i] - ) - - if max(free_memory) < memory_needed: + def _gpu_with_most_free_memory( + self, + memory_needed: int, + pending_gpu: Optional[Dict[int, int]] = None, + pending_slots: Optional[Dict[int, int]] = None, + ) -> Optional[int]: + pending_gpu = pending_gpu or {} + pending_slots = pending_slots or {} + candidates = [] + for gpu_id in self.gpu_devices: + used_slots = self.gpu_slots_in_use.get(gpu_id, 0) + used_slots += pending_slots.get(gpu_id, 0) + if used_slots >= self.gpu_slots_per_device: + continue + free = self.memory_budget["gpu"][gpu_id] + free -= self.memory_stats["gpu_in_use"].get(gpu_id, 0) + free -= pending_gpu.get(gpu_id, 0) + if free >= memory_needed: + candidates.append((free, -used_slots, -gpu_id, gpu_id)) + if not candidates: return None + return max(candidates)[-1] + + def _waiting_for_gpu_slot( + self, + memory_needed: int, + pending_gpu: Optional[Dict[int, int]] = None, + pending_slots: Optional[Dict[int, int]] = None, + ) -> bool: + pending_gpu = pending_gpu or {} + pending_slots = pending_slots or {} + for gpu_id in self.gpu_devices: + free = self.memory_budget["gpu"][gpu_id] + free -= self.memory_stats["gpu_in_use"].get(gpu_id, 0) + free -= pending_gpu.get(gpu_id, 0) + if free >= memory_needed: + return True + return False - return free_memory.index(max(free_memory)) + def restrict_gpu_devices( + self, devices: List[int], slots_per_device: int = 1 + ) -> None: + self.gpu_devices = [ + gpu_id for gpu_id in devices if gpu_id in self.memory_budget["gpu"] + ] + self.n_gpu = len(self.gpu_devices) + self.gpu_slots_per_device = max(1, int(slots_per_device)) + self.gpu_slots_in_use = {gpu_id: 0 for gpu_id in self.gpu_devices} def _get_pending_nodes(self) -> List[str]: return list(self._candidates) def _release_execution_memory(self, node_id: str, gpu_id: int) -> None: _, gpu_mem = self.node_resources[node_id] - if gpu_id is not None and gpu_mem > 0: - self.memory_stats["gpu_in_use"][gpu_id] -= gpu_mem + reserved_gpu_id = self._gpu_slot_nodes.pop(node_id, None) + if reserved_gpu_id is not None and gpu_mem > 0: + self.memory_stats["gpu_in_use"][reserved_gpu_id] = max( + 0, self.memory_stats["gpu_in_use"][reserved_gpu_id] - gpu_mem + ) + self.gpu_slots_in_use[reserved_gpu_id] = max( + 0, self.gpu_slots_in_use[reserved_gpu_id] - 1 + ) reserved = self._cpu_reserved_nodes.pop(node_id, 0) if reserved: self.memory_stats["cpu_in_flight"] = max( diff --git a/src/main/python/systemds/scuro/drsearch/representation_dag.py b/src/main/python/systemds/scuro/drsearch/representation_dag.py index ad7f174acfa..6315e853dda 100644 --- a/src/main/python/systemds/scuro/drsearch/representation_dag.py +++ b/src/main/python/systemds/scuro/drsearch/representation_dag.py @@ -577,15 +577,6 @@ def get_consumer_count(dags: List[RepresentationDag]) -> Dict[str, int]: def pushdown_aggregation(dag_group: List[RepresentationDag]) -> List[RepresentationDag]: - consumer_count: Dict[str, int] = defaultdict(int) - - for dag in dag_group: - for node in dag.nodes: - for inp in node.inputs: - consumer_count[inp] += 1 - - processed_agg_ids: Set[str] = set() - for dag in dag_group: agg_nodes = [ n @@ -593,10 +584,6 @@ def pushdown_aggregation(dag_group: List[RepresentationDag]) -> List[Representat if n.operation and issubclass(n.operation, AggregatedRepresentation) ] for agg_node in agg_nodes: - if agg_node.node_id in processed_agg_ids: - continue - processed_agg_ids.add(agg_node.node_id) - if len(agg_node.inputs) != 1: print( f"Aggregation node {agg_node.node_id} has {len(agg_node.inputs)} inputs, skipping (SHOULD NOT HAPPEN)" @@ -604,37 +591,22 @@ def pushdown_aggregation(dag_group: List[RepresentationDag]) -> List[Representat continue input_id = agg_node.inputs[0] - - processed_agg_ids.add(input_id) - if consumer_count[input_id] != 1: - continue - - input_node = None - for d in dag_group: - input_node = d.get_node_by_id(input_id) - if input_node is not None: - break + input_node = dag.get_node_by_id(input_id) if not input_node or not input_node.operation: continue - op_instance = input_node.operation(params=input_node.parameters) - if op_instance.__class__.__bases__[0].__name__ != "BertFamily": + if not getattr( + input_node.operation, "supports_aggregation_pushdown", False + ): continue - input_node.parameters["_pushdown_aggregation"] = agg_node.parameters - - for d in dag_group: - for node in d.nodes: - node.inputs = [ - input_id if inp == agg_node.node_id else inp - for inp in node.inputs - ] - - if d.root_node_id == agg_node.node_id: - d.root_node_id = input_id - - d.nodes = [n for n in d.nodes if n.node_id != agg_node.node_id] + aggregation_parameters = copy.deepcopy(agg_node.parameters) + agg_node.operation = input_node.operation + agg_node.inputs = list(input_node.inputs) + agg_node.parameters = copy.deepcopy(input_node.parameters) + agg_node.parameters["_pushdown_aggregation"] = aggregation_parameters + dag.nodes = dag.filter_connected_nodes(dag.nodes) return dag_group @@ -758,7 +730,7 @@ def group_dags_by_dependencies( return [] unique_dags: List[RepresentationDag] = [] - seen_signatures: set[Hashable] = set() + seen_signatures: Set[Hashable] = set() for dag in dags: dag_sig = dag.compute_full_node_signature(dag.root_node_id) diff --git a/src/main/python/systemds/scuro/drsearch/task.py b/src/main/python/systemds/scuro/drsearch/task.py index e74e791794c..7977c628c12 100644 --- a/src/main/python/systemds/scuro/drsearch/task.py +++ b/src/main/python/systemds/scuro/drsearch/task.py @@ -19,6 +19,7 @@ # # ------------------------------------------------------------- import copy +import os import time from typing import List from systemds.scuro.models.model import Model @@ -26,6 +27,10 @@ from sklearn.model_selection import train_test_split from systemds.scuro.representations.representation import RepresentationStats +_GPU_CONTEXT_FLOOR_BYTES = ( + int(os.environ.get("SCURO_GPU_CONTEXT_FLOOR_MB", "512")) * 1024 * 1024 +) + class PerformanceMeasure: def __init__(self, name, metrics, higher_is_better=True): @@ -170,13 +175,32 @@ def estimate_peak_memory_bytes(self, input_stats): self.model.estimate_peak_memory_bytes(feature_dim, n_train) ) + uses_gpu = getattr(self.model, "uses_gpu", None) + if uses_gpu is None: + uses_gpu = model_peak_memory_gpu > 0 + + gpu_peak_bytes = int(model_peak_memory_gpu * 1.4) if uses_gpu else 0 + if uses_gpu and gpu_peak_bytes <= 0: + gpu_peak_bytes = _GPU_CONTEXT_FLOOR_BYTES + return { "cpu_peak_bytes": int( model_peak_memory_cpu * 1.5 + scheduler_side_cpu_bytes * 2 ), - "gpu_peak_bytes": int(model_peak_memory_gpu) * 1.4, + "gpu_peak_bytes": gpu_peak_bytes, } + def estimate_relative_cost(self, input_stats) -> float: + feature_dim = ( + int(np.prod(input_stats.output_shape)) if input_stats.output_shape else 0 + ) + fold_sizes = [len(fold) for fold in self.cv_train_indices] + n_fold_train = max(fold_sizes, default=len(self.train_indices or [])) + passes = max(1, int(getattr(self.model, "epochs", 1) or 1)) + return float( + max(1, self.kfold) * passes * max(1, n_fold_train) * max(1, feature_dim) + ) + def _create_cv_splits(self): train_labels = [self.labels[i] for i in self.train_indices] train_labels_array = np.array(train_labels) diff --git a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py index 1b2227b773c..b185bd45dfb 100644 --- a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py +++ b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py @@ -69,6 +69,7 @@ def __init__( window_combination_chains: int = 1, ): self._node_stats: Dict[str, Any] = {} + self.pruned = [] self.window_combination_chains = window_combination_chains self.enable_checkpointing = enable_checkpointing self.modalities = modalities @@ -483,7 +484,11 @@ def _build_modality_dag( ) current_node_id = rep_node_id rep_dag = builder.build(current_node_id) - dags.append(rep_dag) + requires_dimensionality_reduction = getattr( + operator, "requires_dimensionality_reduction", False + ) + if not requires_dimensionality_reduction: + dags.append(rep_dag) dimensionality_reduction_dags = self.add_dimensionality_reduction_operators( builder, current_node_id @@ -515,7 +520,9 @@ def _build_modality_dag( operator.get_current_parameters(), ) - agg_operator = AggregatedRepresentation(target_dimensions=1) + agg_operator = AggregatedRepresentation( + target_dimensions=1, aggregate_leading=True + ) context_agg_node_id = builder.create_operation_node( agg_operator.__class__, [context_rep_node_id], @@ -560,9 +567,9 @@ def _build_modality_dag( ) ) - if rep_dag.nodes[-1].operation().output_modality_type in [ - ModalityType.EMBEDDING - ]: + if not requires_dimensionality_reduction and rep_dag.nodes[ + -1 + ].operation().output_modality_type in [ModalityType.EMBEDDING]: dags.extend( self.default_context_operators( modality, builder, leaf_id, rep_dag, True @@ -695,12 +702,57 @@ def temporal_context_operators(self, modality, builder, leaf_id): modality.modality_type, modality.stats ) ) + if not window_lengths: + for context_operator in context_operators: + self.pruned.append( + { + "operation": context_operator.__name__, + "reason": ("no configured window length fits the input signal"), + } + ) + return [] dags = [] for context_operator in context_operators: for window_size, num_window in zip(window_lengths, num_windows): window_node_ids = [] for agg in aggregators: - context_operator_instance = context_operator(agg()) + aggregation_instance = agg() + effective_length = self._effective_window_length( + context_operator(), + window_size, + num_window, + modality.stats.max_length, + ) + input_stats = self._window_input_stats(modality, effective_length) + for parameter, values in ( + aggregation_instance.parameters or {} + ).items(): + if not isinstance(values, list): + continue + accepted = aggregation_instance.filter_parameter_domain( + parameter, values, input_stats + ) + for value in values: + if value not in accepted: + self.pruned.append( + { + "operation": aggregation_instance.name, + "window_length": effective_length, + "parameters": {parameter: value}, + "reason": "outside the valid input domain", + } + ) + failure = aggregation_instance.check_preconditions(input_stats) + if failure is not None: + self.pruned.append( + { + "operation": aggregation_instance.name, + "window_length": effective_length, + "reason": failure, + } + ) + continue + context_operator_instance = context_operator(aggregation_instance) self._apply_granularity( context_operator_instance, window_size, num_window ) diff --git a/src/main/python/systemds/scuro/drsearch/worker_pool.py b/src/main/python/systemds/scuro/drsearch/worker_pool.py index 7e78862a104..752efdb6eed 100644 --- a/src/main/python/systemds/scuro/drsearch/worker_pool.py +++ b/src/main/python/systemds/scuro/drsearch/worker_pool.py @@ -24,7 +24,7 @@ import os import signal from dataclasses import dataclass -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Tuple import torch @@ -86,15 +86,25 @@ class _JobResult: def _worker_main( - job_q, result_q, dispatch: Dict[str, Callable], num_threads: int + job_q, + result_q, + dispatch: Dict[str, Callable], + num_threads: int, + physical_gpu_id: Optional[int], ) -> None: + os.environ["CUDA_VISIBLE_DEVICES"] = ( + "" if physical_gpu_id is None else str(physical_gpu_id) + ) _worker_initializer(num_threads) while True: job = job_q.get() if job is None: return try: - value = dispatch[job.kind](job.payload, job.gpu_id) + local_gpu_id = ( + 0 if physical_gpu_id is not None and job.gpu_id is not None else None + ) + value = dispatch[job.kind](job.payload, local_gpu_id) result_q.put(_JobResult(job.job_id, True, os.getpid(), value=value)) except Exception as e: result_q.put( @@ -141,44 +151,143 @@ def __init__( dispatch: Dict[str, Callable], ctx=None, threads_per_worker: int = 1, + gpu_devices: Optional[List[int]] = None, + gpu_slots_per_device: int = 1, + gpu_demand_fraction: float = 1.0, ): self._ctx = ctx or create_mp_context() self._dispatch = dispatch self._threads_per_worker = max(1, int(threads_per_worker)) + self.gpu_devices = list(dict.fromkeys(gpu_devices or [])) + self.gpu_slots_per_device = max(1, int(gpu_slots_per_device)) self._result_q = self._ctx.Queue() self._job_counter = itertools.count() self._workers: Dict[int, Dict[str, Any]] = {} self._idle_pids: List[int] = [] + self._idle_gpu_pids: Dict[int, List[int]] = { + gpu_id: [] for gpu_id in self.gpu_devices + } self._running: Dict[int, tuple] = {} - for _ in range(max(1, n_workers)): - self._spawn_worker() - def _spawn_worker(self) -> None: + n_workers = max(1, int(n_workers)) + gpu_capacity = len(self.gpu_devices) * self.gpu_slots_per_device + gpu_workers = 0 + if gpu_capacity: + gpu_workers = min( + n_workers, + gpu_capacity, + max(1, int(round(n_workers * float(gpu_demand_fraction)))), + ) + self._cpu_worker_count = n_workers - gpu_workers + for worker_index in range(gpu_workers): + gpu_id = self.gpu_devices[worker_index % len(self.gpu_devices)] + self._spawn_worker(gpu_id) + for _ in range(self._cpu_worker_count): + self._spawn_worker(None) + + @property + def gpu_worker_devices(self) -> List[int]: + return list( + dict.fromkeys( + worker["gpu_id"] + for worker in self._workers.values() + if worker["gpu_id"] is not None + ) + ) + + def _spawn_worker(self, physical_gpu_id: Optional[int]) -> None: set_thread_env_before_spawn(self._threads_per_worker) job_q = self._ctx.Queue() + previous_visible = os.environ.get("CUDA_VISIBLE_DEVICES") + os.environ["CUDA_VISIBLE_DEVICES"] = ( + "" if physical_gpu_id is None else str(physical_gpu_id) + ) p = self._ctx.Process( target=_worker_main, - args=(job_q, self._result_q, self._dispatch, self._threads_per_worker), + args=( + job_q, + self._result_q, + self._dispatch, + self._threads_per_worker, + physical_gpu_id, + ), daemon=True, ) p.start() - self._workers[p.pid] = {"process": p, "job_q": job_q} - self._idle_pids.append(p.pid) + if previous_visible is None: + os.environ.pop("CUDA_VISIBLE_DEVICES", None) + else: + os.environ["CUDA_VISIBLE_DEVICES"] = previous_visible + self._workers[p.pid] = { + "process": p, + "job_q": job_q, + "gpu_id": physical_gpu_id, + } + self._mark_idle(p.pid) + + def _mark_idle(self, pid: int) -> None: + worker = self._workers.get(pid) + if worker is None: + return + gpu_id = worker["gpu_id"] + idle = self._idle_pids if gpu_id is None else self._idle_gpu_pids[gpu_id] + if pid not in idle: + idle.append(pid) @property def has_idle_worker(self) -> bool: - return len(self._idle_pids) > 0 + return bool(self._idle_pids) or any(self._idle_gpu_pids.values()) + + @property + def has_idle_cpu_worker(self) -> bool: + return bool(self._idle_pids) + + def has_idle_worker_for( + self, gpu_id: Optional[int], allow_gpu_worker_for_cpu: bool = False + ) -> bool: + if gpu_id is not None: + if self._idle_gpu_pids.get(gpu_id): + return True + return not self.gpu_devices and bool(self._idle_pids) + if self._idle_pids: + return True + return (allow_gpu_worker_for_cpu or self._cpu_worker_count == 0) and any( + self._idle_gpu_pids.values() + ) @property def num_in_flight(self) -> int: return len(self._running) - def submit(self, kind: str, payload: tuple, gpu_id: Optional[int] = None) -> int: - if not self._idle_pids: + def _take_worker( + self, gpu_id: Optional[int], allow_gpu_worker_for_cpu: bool + ) -> Tuple[int, Optional[int]]: + if gpu_id is not None: + gpu_idle = self._idle_gpu_pids.get(gpu_id, []) + if gpu_idle: + return gpu_idle.pop(), gpu_id + if not self.gpu_devices and self._idle_pids: + return self._idle_pids.pop(), None + elif self._idle_pids: + return self._idle_pids.pop(), None + elif allow_gpu_worker_for_cpu or self._cpu_worker_count == 0: + for lane_gpu_id in self.gpu_devices: + if self._idle_gpu_pids[lane_gpu_id]: + return self._idle_gpu_pids[lane_gpu_id].pop(), None + raise RuntimeError("submit() called with no compatible idle worker") + + def submit( + self, + kind: str, + payload: tuple, + gpu_id: Optional[int] = None, + allow_gpu_worker_for_cpu: bool = False, + ) -> int: + if not self.has_idle_worker_for(gpu_id, allow_gpu_worker_for_cpu): raise RuntimeError("submit() called with no idle worker available") job_id = next(self._job_counter) - job = _Job(job_id, kind, payload, gpu_id) - pid = self._idle_pids.pop() + pid, dispatched_gpu_id = self._take_worker(gpu_id, allow_gpu_worker_for_cpu) + job = _Job(job_id, kind, payload, dispatched_gpu_id) self._running[job_id] = (pid, job) self._workers[pid]["job_q"].put(job) return job_id @@ -197,7 +306,7 @@ def wait(self) -> _JobResult: if entry is not None: pid, _job = entry if pid in self._workers: - self._idle_pids.append(pid) + self._mark_idle(pid) return jr for r in ready: dead_pid = sentinel_to_pid.get(r) @@ -211,6 +320,12 @@ def _replace_dead_worker(self, pid: int) -> Optional[_JobResult]: w = self._workers.pop(pid, None) if w is None: return None + physical_gpu_id = w.get("gpu_id") + gpu_idle = self._idle_gpu_pids.get(physical_gpu_id, []) + try: + gpu_idle.remove(pid) + except ValueError: + pass try: if pid in self._idle_pids: self._idle_pids.remove(pid) @@ -235,7 +350,7 @@ def _replace_dead_worker(self, pid: int) -> Optional[_JobResult]: if failed_job_id is not None: self._running.pop(failed_job_id, None) - self._spawn_worker() + self._spawn_worker(physical_gpu_id) if failed_job_id is None: return None @@ -274,4 +389,6 @@ def shutdown(self) -> None: pass self._workers.clear() self._idle_pids.clear() + for idle in self._idle_gpu_pids.values(): + idle.clear() self._running.clear() diff --git a/src/main/python/systemds/scuro/modality/joined.py b/src/main/python/systemds/scuro/modality/joined.py index 124c7952fd4..9b4bdc4791b 100644 --- a/src/main/python/systemds/scuro/modality/joined.py +++ b/src/main/python/systemds/scuro/modality/joined.py @@ -18,6 +18,7 @@ # under the License. # # ------------------------------------------------------------- +import copy import importlib import sys @@ -256,7 +257,8 @@ def _apply_representation_chunked( ) def _apply_representation(self, modality, representation): - transformed = representation.transform(modality) + normalized = self._normalize_representation_input(modality) + transformed = representation.transform(normalized) # if self.aggregation: # aggregated_data_left = self.aggregation.execute(transformed) # transformed = Modality( @@ -266,3 +268,53 @@ def _apply_representation(self, modality, representation): # transformed.data = aggregated_data_left return transformed + + @staticmethod + def _normalize_representation_input(modality): + data = [] + changed = False + + def collect_samples(value): + try: + array = np.asarray(value) + except ValueError: + array = np.asarray(value, dtype=object) + + numeric = array.dtype != object and np.issubdtype(array.dtype, np.number) + if numeric and array.ndim == 3 and array.shape[-1] <= 4: + return [array], False + if numeric and array.ndim == 2: + return [np.repeat(array[..., np.newaxis], 3, axis=-1)], True + if array.ndim == 0: + return None + + samples = [] + for entry in value: + result = collect_samples(entry) + if result is None: + return None + entry_samples, _ = result + samples.extend(entry_samples) + return samples, True + + for instance in modality.data: + result = collect_samples(instance) + if result is None: + return modality + samples, instance_changed = result + data.append(samples) + changed = changed or instance_changed + + if not changed: + return modality + + normalized = Modality( + modality.modality_type, + modality.modality_id, + copy.deepcopy(modality.metadata), + modality.data_type, + modality.transform_time, + ) + normalized.data = data + normalized.stats = modality.stats + return normalized diff --git a/src/main/python/systemds/scuro/modality/modality.py b/src/main/python/systemds/scuro/modality/modality.py index a0d1e36377d..6d9af9394b7 100644 --- a/src/main/python/systemds/scuro/modality/modality.py +++ b/src/main/python/systemds/scuro/modality/modality.py @@ -257,6 +257,30 @@ def get_data_layout(self): return None + def subset(self, indices): + indices = list(indices) + subset_modality = self.copy_from_instance() + + if self.has_metadata(): + metadata = [selective_copy_metadata(self.metadata[i]) for i in indices] + else: + metadata = [] + + subset_modality.metadata = metadata + + if self.has_data(): + data = self.data + if hasattr(data, "subset"): + subset_modality._data = data.subset(indices) + elif isinstance(data, np.ndarray): + subset_modality.data = data[indices] + else: + subset_modality.data = [data[i] for i in indices] + + subset_modality.data_type = self.data_type + subset_modality.modality_id = self.modality_id + return subset_modality + def has_data(self): return self.data is not None and len(self.data) != 0 diff --git a/src/main/python/systemds/scuro/modality/unimodal_modality.py b/src/main/python/systemds/scuro/modality/unimodal_modality.py index e1bba7df9da..c23faa06e09 100644 --- a/src/main/python/systemds/scuro/modality/unimodal_modality.py +++ b/src/main/python/systemds/scuro/modality/unimodal_modality.py @@ -18,13 +18,14 @@ # under the License. # # ------------------------------------------------------------- +import copy from concurrent.futures import ThreadPoolExecutor, as_completed import gc import time import numpy as np -from systemds.scuro import ModalityType from systemds.scuro.dataloader.base_loader import BaseLoader from systemds.scuro.modality.modality import Modality +from systemds.scuro.modality.type import ModalityType from systemds.scuro.modality.joined import JoinedModality from systemds.scuro.modality.transformed import TransformedModality from systemds.scuro.representations.representation import ( @@ -58,13 +59,27 @@ def copy_from_instance(self): new_instance.metadata = self.metadata.copy() return new_instance - def get_metadata_at_position(self, position: int): - if self.data_loader.chunk_size: - return self.metadata[ - (self.data_loader.next_chunk - 1) * self.data_loader.chunk_size - + position - ] + def subset(self, indices): + if self.data_loader.chunk_size is None: + return super().subset(indices) + + indices = list(indices) + subset_loader = copy.copy(self.data_loader) + subset_loader.indices = [self.data_loader.indices[i] for i in indices] + subset_loader.stats = copy.copy(self.data_loader.stats) + if hasattr(subset_loader.stats, "num_instances"): + subset_loader.stats.num_instances = len(indices) + subset_loader.reset() + # Re-run the setter after replacing indices so num_chunks reflects the + # subset rather than the original dataset. + subset_loader.chunk_size = self.data_loader.chunk_size + subset_modality = type(self)(subset_loader) + subset_modality.modality_id = self.modality_id + subset_modality.transform_time = self.transform_time + return subset_modality + + def get_metadata_at_position(self, position: int): return self.metadata[position] def get_stats(self): @@ -105,13 +120,15 @@ def extract_raw_data(self): Uses the data loader to read the raw data from a specified location and stores the data in the data location. """ - self.data, self.metadata = self.data_loader.load() + data, metadata = self.data_loader.load() + self._data = data + self.metadata = metadata def iter_raw_data_chunks(self, reset: bool = True): for data, metadata, chunk_indices in self.data_loader.iter_loaded_chunks( reset=reset ): - self.data = data + self._data = data self.metadata = metadata yield chunk_indices @@ -155,74 +172,117 @@ def aggregate(self, aggregation_function): if self.data is None: raise Exception("Data is None") - def apply_representations(self, representations, aggregation=None, parallel=False): + def apply_representations( + self, + representations, + aggregation=None, + parallel=False, + representation_keys=None, + aggregations=None, + ): """ Applies a list of representations to the modality. Specifically, it applies the representations to the modality in a chunked manner. :param representations: List of representations to apply :return: List of transformed modalities """ + if representation_keys is None: + representation_keys = [ + representation.name for representation in representations + ] + if len(representation_keys) != len(representations): + raise ValueError("representation_keys must match representations") + if len(set(representation_keys)) != len(representation_keys): + raise ValueError("representation_keys must be unique") + + if aggregations is None: + aggregations = [aggregation] * len(representations) + if len(aggregations) != len(representations): + raise ValueError("aggregations must match representations") + + representation_specs = list( + zip(representation_keys, representations, aggregations) + ) transformed_modalities_per_representation = {} padding_per_representation = {} original_lengths_per_representation = {} + failed_representations = {} - for representation in representations: - transformed_modality = TransformedModality(self, representation.name) + for representation_key, representation, _ in representation_specs: + transformed_modality = TransformedModality( + self, representation.name, representation.output_modality_type + ) transformed_modality.data = [] transformed_modality.metadata = [] - transformed_modalities_per_representation[representation.name] = ( + transformed_modalities_per_representation[representation_key] = ( transformed_modality ) - padding_per_representation[representation.name] = False - original_lengths_per_representation[representation.name] = [] + padding_per_representation[representation_key] = False + original_lengths_per_representation[representation_key] = [] start = ( time.time() ) # TODO: should be repalced in unimodal_representation.transform - if self.data_loader.chunk_size: - with ThreadPoolExecutor( - max_workers=len(representations) if parallel else 1 - ) as executor: - time_s = time.time() - for _ in self.iter_raw_data_chunks(reset=True): - representations_futures = {} - for representation in representations: - future = executor.submit(representation.transform, self) - representations_futures[future] = representation.name - for future in as_completed(representations_futures.keys()): - representation_name = representations_futures.get(future) + with ThreadPoolExecutor( + max_workers=len(representations) if parallel else 1 + ) as executor: + time_s = time.time() + for _ in self.iter_raw_data_chunks(reset=True): + representations_futures = {} + for ( + representation_key, + representation, + rep_aggregation, + ) in representation_specs: + if representation_key in failed_representations: + continue + future = executor.submit( + representation.transform, self, rep_aggregation + ) + representations_futures[future] = representation_key + for future in as_completed(representations_futures.keys()): + representation_key = representations_futures.get(future) + try: transformed_chunk = future.result() - transformed_modalities_per_representation[ - representation_name - ].data.extend(transformed_chunk.data) - transformed_modalities_per_representation[ - representation_name - ].metadata.extend(transformed_chunk.metadata) - for d in transformed_chunk.data: + except Exception as e: + failed_representations[representation_key] = e + continue + transformed_modalities_per_representation[ + representation_key + ].data.extend(transformed_chunk.data) + transformed_modalities_per_representation[ + representation_key + ].metadata.extend(transformed_chunk.metadata) + for d in transformed_chunk.data: + shape = getattr(d, "shape", ()) + if shape: original_lengths_per_representation[ - representation_name - ].append(d.shape[0]) + representation_key + ].append(int(shape[0])) + if self.data_loader.is_chunked: print(f"Time for transforming data chunks: {time.time() - time_s}") - else: - if not self.has_data(): - self.extract_raw_data() - new_modality = representation.transform(self) - transformed_modalities_per_representation[representation.name] = ( - new_modality - ) - for representation in representations: + for representation_name in failed_representations: + transformed_modalities_per_representation.pop(representation_name, None) + + if representations and not transformed_modalities_per_representation: + raise next(iter(failed_representations.values())) + + for representation_key, representation, _ in representation_specs: + if representation_key in failed_representations: + continue self._apply_padding( - transformed_modalities_per_representation[representation.name], - original_lengths_per_representation[representation.name], - padding_per_representation[representation.name], + transformed_modalities_per_representation[representation_key], + original_lengths_per_representation[representation_key], + padding_per_representation[representation_key], ) transformed_modalities_per_representation[ - representation.name + representation_key ].transform_time += (time.time() - start) transformed_modalities_per_representation[ - representation.name + representation_key ].self_contained = representation.self_contained gc.collect() + self.failed_representations = failed_representations return transformed_modalities_per_representation def apply_representation(self, representation, aggregation=None): diff --git a/src/main/python/systemds/scuro/models/model.py b/src/main/python/systemds/scuro/models/model.py index 22d1bbeccfd..fcd4150cab2 100644 --- a/src/main/python/systemds/scuro/models/model.py +++ b/src/main/python/systemds/scuro/models/model.py @@ -21,6 +21,8 @@ class Model: + uses_gpu = None + def __init__(self, name: str): """ Parent class for models used to perform a given task diff --git a/src/main/python/systemds/scuro/representations/bert.py b/src/main/python/systemds/scuro/representations/bert.py index 9e60f843416..394da666ccd 100644 --- a/src/main/python/systemds/scuro/representations/bert.py +++ b/src/main/python/systemds/scuro/representations/bert.py @@ -25,7 +25,17 @@ from systemds.scuro.representations.unimodal import UnimodalRepresentation import torch from transformers import AutoTokenizer, AutoModel -from systemds.scuro.representations.utils import save_embeddings +from systemds.scuro.representations.utils import ( + LengthBucketBatchSampler, + OwnerAccumulator, + OwnedSequenceDataset, + flatten_owned_sequences, + move_batch_to_device, + pin_memory_for, + pool_transformer_output, + save_embeddings, + transformer_inference_context, +) from systemds.scuro.modality.type import ModalityType from systemds.scuro.drsearch.operator_registry import register_representation from systemds.scuro.utils.memory_utility import ( @@ -39,6 +49,9 @@ class BertFamily(UnimodalRepresentation): + supports_aggregation_pushdown = True + cache_in_worker = True + def __init__( self, representation_name, @@ -69,6 +82,10 @@ def __init__( self.data_type = torch.float32 self.aggregation = aggregation self.params = params + self.model = None + self.tokenizer = None + self.bert_output = None + self._activation_hook = None if params is not None: self.layer = params.get("layer", self.layer) self.batch_size = int(params.get("batch_size", self.batch_size)) @@ -100,18 +117,15 @@ def get_output_stats(self, input_stats) -> RepresentationStats: if not isinstance(input_stats, RepresentationStats): self.stats = RepresentationStats( input_stats.num_instances, - (self.max_seq_length, 768), - aggregate_dim=(0,), + (768,), + aggregate_dim=None, dtype=self.data_type, ) else: self.stats = RepresentationStats( input_stats.num_instances, - (input_stats.output_shape[0], self.max_seq_length, 768), - aggregate_dim=( - 0, - 1, - ), + (input_stats.output_shape[0], 768), + aggregate_dim=(0,), dtype=self.data_type, ) if self.params and "_pushdown_aggregation" in self.params: @@ -181,12 +195,15 @@ def estimate_peak_memory_bytes(self, input_stats): def transform(self, modality, aggregation=None): transformed_modality = TransformedModality(modality, self) - tokenizer = AutoTokenizer.from_pretrained( - self.model_name, clean_up_tokenization_spaces=True - ) - self.model = AutoModel.from_pretrained(self.model_name) + if self.tokenizer is None: + self.tokenizer = AutoTokenizer.from_pretrained( + self.model_name, clean_up_tokenization_spaces=True + ) + if self.model is None: + self.model = AutoModel.from_pretrained(self.model_name) self.model = self.model.to(self.device) + self.model.eval() self.bert_output = None def get_activation(name): @@ -202,26 +219,29 @@ def hook(model, input, output): aggregate_dim = (0,) if self.layer != "cls": - for name, layer in self.model.named_modules(): - if name == self.layer: - layer.register_forward_hook(get_activation(name)) - break + if self._activation_hook is None: + for name, layer in self.model.named_modules(): + if name == self.layer: + self._activation_hook = layer.register_forward_hook( + get_activation(name) + ) + break if ModalityType.TEXT.has_field(modality.metadata, "text_spans"): - dataset = TextSpanDataset(modality.data, modality.metadata) - embeddings = [] - aggregate_dim = (0, 1) - for text in dataset: - embedding = self.create_embeddings( - text, self.model, tokenizer, aggregation - ) - embeddings.append( - aggregation.execute(embedding) - if aggregation is not None - else embedding - ) + chunk_groups = list(TextSpanDataset(modality.data, modality.metadata)) + chunks, owner_ids = flatten_owned_sequences(chunk_groups) + aggregate_dim = None if aggregation is not None else (0,) + embeddings = self.create_embeddings( + chunks, + self.model, + self.tokenizer, + aggregation, + owner_ids=owner_ids, + num_owners=len(chunk_groups), + grouped=aggregation is None, + ) else: embeddings = self.create_embeddings( - modality.data, self.model, tokenizer, aggregation + modality.data, self.model, self.tokenizer ) if self.output_file is not None: save_embeddings(embeddings, self.output_file) @@ -235,69 +255,97 @@ def hook(model, input, output): def assert_output_stats(self, transformed_modality): if self.stats: assert len(transformed_modality.data) == self.stats.num_instances - if len(self.stats.output_shape) == 3: - assert ( - transformed_modality.data[0].shape[0] <= self.stats.output_shape[0] - ), f"Output shape: {transformed_modality.data[0].shape}, Expected shape: {self.stats.output_shape}" + actual_shape = np.asarray(transformed_modality.data[0]).shape + if len(self.stats.output_shape) == 2: assert ( - transformed_modality.data[0].shape[1] == self.stats.output_shape[1] - ), f"Output shape: {transformed_modality.data[0].shape}, Expected shape: {self.stats.output_shape}" + actual_shape[0] <= self.stats.output_shape[0] + ), f"Output shape: {actual_shape}, Expected shape: {self.stats.output_shape}" assert ( - transformed_modality.data[0].shape[2] == self.stats.output_shape[2] - ), f"Output shape: {transformed_modality.data[0].shape}, Expected shape: {self.stats.output_shape}" + actual_shape[1] == self.stats.output_shape[1] + ), f"Output shape: {actual_shape}, Expected shape: {self.stats.output_shape}" else: assert ( - transformed_modality.data[0].shape[0] == self.stats.output_shape[0] - ), f"Output shape: {transformed_modality.data[0].shape}, Expected shape: {self.stats.output_shape}" - assert ( - transformed_modality.data[0].shape[1] == self.stats.output_shape[1] - ), f"Output shape: {transformed_modality.data[0].shape}, Expected shape: {self.stats.output_shape}" + actual_shape == self.stats.output_shape + ), f"Output shape: {actual_shape}, Expected shape: {self.stats.output_shape}" - def create_embeddings(self, data, model, tokenizer, aggregation=None): - dataset = TextDataset(data) - dataloader = DataLoader( - dataset, batch_size=self.batch_size, shuffle=False, collate_fn=None + def create_embeddings( + self, + data, + model, + tokenizer, + aggregation=None, + owner_ids=None, + num_owners=None, + grouped=False, + ): + texts = list(TextDataset(data)) + single_owner = owner_ids is None and aggregation is not None + if owner_ids is None: + owner_ids = [0] * len(texts) if single_owner else range(len(texts)) + if num_owners is None: + num_owners = 1 if single_owner else len(texts) + dataset = OwnedSequenceDataset(texts, owner_ids) + + length_encoding = tokenizer( + texts, + padding=False, + truncation=True, + max_length=self.max_seq_length, + return_attention_mask=True, ) - cls_embeddings = [] - for batch in dataloader: + attention_mask = length_encoding.get("attention_mask") + if isinstance(attention_mask, torch.Tensor): + lengths = attention_mask.sum(dim=1).tolist() + else: + lengths = [sum(mask) for mask in attention_mask] + + def collate(samples): + batch_texts, batch_owner_ids, chunk_ids = zip(*samples) inputs = tokenizer( - batch, + list(batch_texts), return_offsets_mapping=True, return_tensors="pt", - padding="max_length", + padding=True, return_attention_mask=True, truncation=True, - max_length=self.max_seq_length, # TODO: make this dynamic with parameter to tune + max_length=self.max_seq_length, + ) + inputs = dict(inputs) + inputs.pop("offset_mapping", None) + return ( + inputs, + torch.tensor(batch_owner_ids, dtype=torch.long), + torch.tensor(chunk_ids, dtype=torch.long), ) - inputs.to(self.device) - # ModalityType.TEXT.add_field_for_instances( - # modality.metadata, - # "token_to_character_mapping", - # inputs.data["offset_mapping"].tolist(), - # ) - # - # ModalityType.TEXT.add_field_for_instances( - # modality.metadata, - # "attention_masks", - # inputs.data["attention_mask"].tolist(), - # ) - del inputs.data["offset_mapping"] - - with torch.no_grad(): + + dataloader = DataLoader( + dataset, + batch_sampler=LengthBucketBatchSampler(lengths, self.batch_size), + collate_fn=collate, + pin_memory=pin_memory_for(self.device), + ) + accumulator = OwnerAccumulator(num_owners, len(dataset), aggregation) + with transformer_inference_context(self.device): + for inputs, batch_owner_ids, chunk_ids in dataloader: + inputs = move_batch_to_device(inputs, self.device) outputs = model(**inputs) if self.layer == "cls": - cls_embedding = outputs.last_hidden_state.detach().cpu().numpy() + hidden_state = outputs.last_hidden_state else: - cls_embedding = self.bert_output.cpu().numpy() - if ( - aggregation is not None - and self.layer != "pooler" - and self.layer != "pooler.activation" - ): - cls_embedding = aggregation.execute(cls_embedding) - cls_embeddings.extend(cls_embedding) - - return cls_embeddings + hidden_state = self.bert_output + pooled = pool_transformer_output( + hidden_state, + inputs["attention_mask"], + use_cls=self.layer == "cls", + ) + accumulator.update(pooled, batch_owner_ids, chunk_ids) + + embeddings = accumulator.finalize(grouped=grouped) + if single_owner: + return embeddings[0] + if aggregation is None and not grouped: + return list(embeddings) + return embeddings @register_representation(ModalityType.TEXT) diff --git a/src/main/python/systemds/scuro/representations/bow.py b/src/main/python/systemds/scuro/representations/bow.py index 9a7766106c1..12fc2e4eb69 100644 --- a/src/main/python/systemds/scuro/representations/bow.py +++ b/src/main/python/systemds/scuro/representations/bow.py @@ -44,6 +44,7 @@ def __init__(self, ngram_range=2, min_df=2, output_file=None, params=None): self.min_df = int(min_df) self.output_file = output_file self.data_type = np.float32 + self.requires_dimensionality_reduction = True def get_output_stats(self, input_stats: TextStats) -> RepresentationStats: vocab_estimate = min( diff --git a/src/main/python/systemds/scuro/representations/clip.py b/src/main/python/systemds/scuro/representations/clip.py index c4e28404466..adffc0f39cc 100644 --- a/src/main/python/systemds/scuro/representations/clip.py +++ b/src/main/python/systemds/scuro/representations/clip.py @@ -19,14 +19,25 @@ # # ------------------------------------------------------------- import numpy as np +import torch from torchvision import transforms from systemds.scuro.dataloader.video_loader import VideoStats from systemds.scuro.modality.transformed import TransformedModality from systemds.scuro.representations.representation import RepresentationStats from systemds.scuro.representations.unimodal import UnimodalRepresentation -import torch -from systemds.scuro.representations.utils import save_embeddings +from systemds.scuro.representations.utils import ( + LengthBucketBatchSampler, + OwnerAccumulator, + OwnedSequenceDataset, + flatten_owned_sequences, + get_sequence_lengths, + move_batch_to_device, + pin_memory_for, + pool_transformer_output, + save_embeddings, + transformer_inference_context, +) from systemds.scuro.modality.type import ModalityType from systemds.scuro.drsearch.operator_registry import register_representation from transformers import CLIPProcessor, CLIPModel @@ -49,9 +60,14 @@ @register_representation([ModalityType.VIDEO, ModalityType.IMAGE]) class CLIPVisual(UnimodalRepresentation): + supports_aggregation_pushdown = True + cache_in_worker = True + def __init__(self, output_file=None, batch_size=32, layer_name="", params=None): parameters = self._get_parameters() super().__init__("CLIPVisual", ModalityType.EMBEDDING, parameters) + self.params = params + self._activation_hook = None self.model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32") self.processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") if params is not None: @@ -99,9 +115,17 @@ def _get_parameters(self): return parameters def estimate_output_memory_bytes(self, input_stats) -> int: - return input_stats.num_instances * 512 * self.data_type.itemsize + shape = self.get_output_stats(input_stats).output_shape + return int(input_stats.num_instances * np.prod(shape) * self.data_type.itemsize) def get_output_stats(self, input_stats) -> RepresentationStats: + if self.params and "_pushdown_aggregation" in self.params: + return RepresentationStats( + input_stats.num_instances, + (512,), + aggregate_dim=None, + dtype=self.data_type, + ) if isinstance(input_stats, VideoStats): return RepresentationStats( input_stats.num_instances, @@ -219,32 +243,38 @@ def transform(self, modality, aggregation=None): self.model = self.model.to(self.data_type) self.model = self.model.to(self.device) + self.model.eval() self.clip_output = None def get_activation(name): def hook(model, input, output): - self.clip_output = ( - output[0].detach() if isinstance(output, tuple) else output.detach() - ) + self.clip_output = output[0] if isinstance(output, tuple) else output return hook - if self.layer_name != "": + if self.layer_name != "" and self._activation_hook is None: for name, layer in self.model.vision_model.named_modules(): if name == self.layer_name: - layer.register_forward_hook(get_activation(name)) + self._activation_hook = layer.register_forward_hook( + get_activation(name) + ) break - embeddings = self.create_visual_embeddings(modality) + embeddings = self.create_visual_embeddings(modality, aggregation) if self.output_file is not None: save_embeddings(embeddings, self.output_file) + transformed_modality.data_type = np.float32 + transformed_modality.aggregate_dim = ( + None + if aggregation is not None or modality.modality_type == ModalityType.IMAGE + else (0,) + ) transformed_modality.data = embeddings return transformed_modality - def create_visual_embeddings(self, modality): - + def create_visual_embeddings(self, modality, aggregation=None): clip_transform = transforms.Compose( [ transforms.ToPILImage(), @@ -254,74 +284,48 @@ def create_visual_embeddings(self, modality): transforms.ConvertImageDtype(dtype=self.data_type), ] ) - dataset = CustomDataset(modality.data, self.data_type, "cpu", tf=clip_transform) - - embeddings = {} - if modality.modality_type == ModalityType.IMAGE: - embeddings = [] - for batch in torch.utils.data.DataLoader( - dataset, batch_size=self.batch_size - ): - images = batch["data"] - inputs = self.processor( - images=images, return_tensors="pt", do_rescale=False - ) - inputs.to(self.device) - - with torch.no_grad(): - if self.layer_name != "": - _ = self.model.vision_model(**inputs) - output = self.clip_output - else: - output = self.model.get_image_features(**inputs) - - output = self._pool_visual_output(output) - - embeddings.extend( - torch.flatten(output, 1) - .detach() - .cpu() - .float() - .numpy() - .astype(np.float32) - ) - return embeddings - - for instance in torch.utils.data.DataLoader(dataset): - id = int(instance["id"][0]) - frames = instance["data"][0] - embeddings[id] = [] - batch_size = self.batch_size + is_image = modality.modality_type == ModalityType.IMAGE + if is_image: + samples = modality.data + owner_ids = list(range(len(samples))) + else: + lengths = get_sequence_lengths(modality.data, modality.metadata) + samples, owner_ids = flatten_owned_sequences(modality.data, lengths) - for start_index in range(0, len(frames), batch_size): - end_index = min(start_index + batch_size, len(frames)) - frame_ids_range = range(start_index, end_index) - frame_batch = frames[frame_ids_range] + dataset = CustomDataset(samples, self.data_type, "cpu", tf=clip_transform) + dataloader = DataLoader( + dataset, + batch_size=self.batch_size, + shuffle=False, + pin_memory=pin_memory_for(self.device), + ) + owner_by_chunk = torch.tensor(owner_ids, dtype=torch.long) + accumulator = OwnerAccumulator(len(modality.data), len(dataset), aggregation) + with transformer_inference_context(self.device): + for batch in dataloader: + chunk_ids = batch["id"].long() inputs = self.processor( - images=frame_batch, return_tensors="pt", do_rescale=False + images=batch["data"], return_tensors="pt", do_rescale=False ) - inputs.to(self.device) - with torch.no_grad(): - if self.layer_name != "": - _ = self.model.vision_model(**inputs) - output = self.clip_output - else: - output = self.model.get_image_features(**inputs) + inputs = move_batch_to_device(dict(inputs), self.device) + if self.layer_name != "": + _ = self.model.vision_model(**inputs) + output = self.clip_output + else: + output = self.model.get_image_features(**inputs) output = self._pool_visual_output(output) - - embeddings[id].extend( - torch.flatten(output, 1) - .detach() - .cpu() - .float() - .numpy() - .astype(np.float32) + accumulator.update( + torch.flatten(output, 1), + owner_by_chunk.index_select(0, chunk_ids), + chunk_ids, ) - embeddings[id] = np.array(embeddings[id]) - return list(embeddings.values()) + embeddings = accumulator.finalize(grouped=not is_image and aggregation is None) + if is_image and aggregation is None: + return list(embeddings) + return embeddings def _pool_visual_output(self, output: torch.Tensor) -> torch.Tensor: if output.ndim == 4: @@ -336,6 +340,9 @@ def _pool_visual_output(self, output: torch.Tensor) -> torch.Tensor: @register_representation(ModalityType.TEXT) class CLIPText(UnimodalRepresentation): + supports_aggregation_pushdown = True + cache_in_worker = True + def __init__(self, output_file=None, batch_size=32, layer_name="", params=None): if params is not None: self.batch_size = int(params.get("batch_size", batch_size)) @@ -356,6 +363,7 @@ def __init__(self, output_file=None, batch_size=32, layer_name="", params=None): self.gpu_id = None self.device = get_device() self.params = params + self._activation_hook = None @property def gpu_id(self): @@ -400,17 +408,14 @@ def get_output_stats(self, input_stats) -> RepresentationStats: self.stats = RepresentationStats( input_stats.num_instances, (512,), - aggregate_dim=(0,), + aggregate_dim=None, dtype=self.data_type, ) else: self.stats = RepresentationStats( input_stats.num_instances, (input_stats.output_shape[0], 512), - aggregate_dim=( - 0, - 1, - ), + aggregate_dim=(0,), dtype=self.data_type, ) if self.params and "_pushdown_aggregation" in self.params: @@ -480,72 +485,120 @@ def transform(self, modality, aggregation=None): transformed_modality = TransformedModality( modality, self, self.output_modality_type ) - self.processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") - self.model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32") + if self.processor is None: + self.processor = CLIPProcessor.from_pretrained( + "openai/clip-vit-base-patch32" + ) + if self.model is None: + self.model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32") self.model = self.model.to(self.device) + self.model.eval() self.clip_output = None def get_activation(name): def hook(model, input, output): - self.clip_output = ( - output[0].detach() if isinstance(output, tuple) else output.detach() - ) + self.clip_output = output[0] if isinstance(output, tuple) else output return hook - if self.layer_name != "": + if self.layer_name != "" and self._activation_hook is None: for name, layer in self.model.text_model.named_modules(): if name == self.layer_name: - layer.register_forward_hook(get_activation(name)) + self._activation_hook = layer.register_forward_hook( + get_activation(name) + ) break + aggregate_dim = None if ModalityType.TEXT.has_field(modality.metadata, "text_spans"): - dataset = TextSpanDataset(modality.data, modality.metadata) - embeddings = [] - for text_chunks in dataset: - embedding = self.create_text_embeddings( - text_chunks, self.model, aggregation - ) - embeddings.append(embedding) - else: + chunk_groups = list(TextSpanDataset(modality.data, modality.metadata)) + chunks, owner_ids = flatten_owned_sequences(chunk_groups) + aggregate_dim = None if aggregation is not None else (0,) embeddings = self.create_text_embeddings( - modality.data, self.model, aggregation + chunks, + self.model, + aggregation, + owner_ids=owner_ids, + num_owners=len(chunk_groups), + grouped=aggregation is None, ) + else: + embeddings = self.create_text_embeddings(modality.data, self.model) if self.output_file is not None: save_embeddings(embeddings, self.output_file) + transformed_modality.data_type = np.float32 + transformed_modality.aggregate_dim = aggregate_dim transformed_modality.data = embeddings return transformed_modality - def create_text_embeddings(self, data, model, aggregation=None): - dataset = TextDataset(data) - dataloader = DataLoader( - dataset, batch_size=self.batch_size, shuffle=False, collate_fn=None + def create_text_embeddings( + self, + data, + model, + aggregation=None, + owner_ids=None, + num_owners=None, + grouped=False, + ): + texts = list(TextDataset(data)) + single_owner = owner_ids is None and aggregation is not None + if owner_ids is None: + owner_ids = [0] * len(texts) if single_owner else range(len(texts)) + if num_owners is None: + num_owners = 1 if single_owner else len(texts) + dataset = OwnedSequenceDataset(texts, owner_ids) + + length_encoding = self.processor( + text=texts, + padding=False, + truncation=True, + max_length=self.max_seq_length, ) - embeddings = [] - for batch in dataloader: + attention_mask = length_encoding.get("attention_mask") + if isinstance(attention_mask, torch.Tensor): + lengths = attention_mask.sum(dim=1).tolist() + else: + lengths = [sum(mask) for mask in attention_mask] + + def collate(samples): + batch_texts, batch_owner_ids, chunk_ids = zip(*samples) inputs = self.processor( - text=batch, + text=list(batch_texts), return_tensors="pt", padding=True, truncation=True, - max_length=77, + max_length=self.max_seq_length, + ) + return ( + dict(inputs), + torch.tensor(batch_owner_ids, dtype=torch.long), + torch.tensor(chunk_ids, dtype=torch.long), ) - inputs.to(self.device) - with torch.no_grad(): + + dataloader = DataLoader( + dataset, + batch_sampler=LengthBucketBatchSampler(lengths, self.batch_size), + collate_fn=collate, + pin_memory=pin_memory_for(self.device), + ) + accumulator = OwnerAccumulator(num_owners, len(dataset), aggregation) + with transformer_inference_context(self.device): + for inputs, batch_owner_ids, chunk_ids in dataloader: + inputs = move_batch_to_device(inputs, self.device) if self.layer_name != "": _ = model.text_model(**inputs) - - batch_np = self.clip_output.cpu().float().numpy() - if batch_np.ndim == 3: - batch_np = batch_np.mean(axis=1) + pooled = pool_transformer_output( + self.clip_output, inputs["attention_mask"] + ) else: - batch_np = model.get_text_features(**inputs).cpu().float().numpy() - - if aggregation is not None: - batch_np = aggregation.execute(batch_np) - - embeddings.extend(batch_np) - + pooled = model.get_text_features(**inputs) + accumulator.update(pooled, batch_owner_ids, chunk_ids) + + embeddings = accumulator.finalize(grouped=grouped) + if single_owner: + return embeddings[0] + if aggregation is None and not grouped: + return list(embeddings) return embeddings diff --git a/src/main/python/systemds/scuro/representations/color_histogram.py b/src/main/python/systemds/scuro/representations/color_histogram.py index 993b179fb8b..c159b285893 100644 --- a/src/main/python/systemds/scuro/representations/color_histogram.py +++ b/src/main/python/systemds/scuro/representations/color_histogram.py @@ -35,7 +35,7 @@ ) -@register_representation(ModalityType.IMAGE) +@register_representation([ModalityType.IMAGE, ModalityType.VIDEO]) class ColorHistogram(UnimodalRepresentation): def __init__( self, @@ -70,6 +70,11 @@ def _get_parameters(self): } def compute_histogram(self, image): + if np.issubdtype(image.dtype, np.floating): + if image.size and image.min() >= 0 and image.max() <= 1: + image = image * 255 + image = np.clip(image, 0, 255).astype(np.uint8) + if self.color_space == "HSV": img = cv2.cvtColor(image, cv2.COLOR_RGB2HSV) channels = [0, 1, 2] diff --git a/src/main/python/systemds/scuro/representations/image_bind.py b/src/main/python/systemds/scuro/representations/image_bind.py index a96cffeb6f8..0cd30af1a70 100644 --- a/src/main/python/systemds/scuro/representations/image_bind.py +++ b/src/main/python/systemds/scuro/representations/image_bind.py @@ -18,83 +18,323 @@ # under the License. # # ------------------------------------------------------------- -import torch -import imagebind.data as data +import math -from imagebind.models.imagebind_model import ModalityType as IBModalityType +import numpy as np +import torch +from pytorchvideo import transforms as pv_transforms +from pytorchvideo.data.clip_sampling import ConstantClipsPerVideoSampler +import torchaudio +from torchvision import transforms -from imagebind.models import imagebind_model +from systemds.scuro.drsearch.operator_registry import register_representation from systemds.scuro.modality.transformed import TransformedModality -from systemds.scuro.representations.unimodal import UnimodalRepresentation -from systemds.scuro.representations.utils import save_embeddings - from systemds.scuro.modality.type import ModalityType -from systemds.scuro.drsearch.operator_registry import register_representation - -if torch.backends.mps.is_available(): - DEVICE = torch.device("mps") -# elif torch.cuda.is_available(): -# DEVICE = torch.device("cuda") -else: - DEVICE = torch.device("cpu") +from systemds.scuro.representations.representation import RepresentationStats +from systemds.scuro.representations.unimodal import UnimodalRepresentation +from systemds.scuro.representations.utils import ( + OwnerAccumulator, + flatten_owned_sequences, + inference_context, + save_embeddings, +) +from systemds.scuro.utils.memory_utility import get_device +from systemds.scuro.utils.torch_dataset import TextDataset, TextSpanDataset -# @register_representation([ModalityType.TEXT, ModalityType.AUDIO, ModalityType.VIDEO]) +@register_representation([ModalityType.VIDEO, ModalityType.AUDIO, ModalityType.TEXT]) class ImageBind(UnimodalRepresentation): - def __init__(self): - parameters = {} + _EMBEDDING_DIM = 1024 + _MODEL_PARAMETER_COUNT = 1_200_000_000 + _CLIPS_PER_VIDEO = 5 + _SPATIAL_CROPS = 3 + _FRAMES_PER_CLIP = 2 + _CROP_SIZE = 224 + supports_aggregation_pushdown = True + cache_in_worker = True + + def __init__(self, output_file=None, batch_size=8, params=None): + parameters = {"batch_size": [1, 2, 4, 8, 16, 32]} super().__init__("ImageBind", ModalityType.EMBEDDING, parameters) - self.model = imagebind_model.imagebind_huge(pretrained=True) - for param in self.model.parameters(): - param.requires_grad = False + self.params = params + self.output_file = output_file + self.batch_size = batch_size + if params is not None: + self.batch_size = int(params.get("batch_size", batch_size)) + self.output_file = params.get("output_file", output_file) + self.data_type = torch.float32 + self.model = None + self.device = get_device() + self._gpu_id = self.device.index + + @property + def gpu_id(self): + return self._gpu_id + + @gpu_id.setter + def gpu_id(self, gpu_id): + self._gpu_id = gpu_id + self.device = get_device(gpu_id) + if self.model is not None: + self.model = self.model.to(self.device) + + def get_output_stats(self, input_stats) -> RepresentationStats: + return RepresentationStats( + input_stats.num_instances, + (self._EMBEDDING_DIM,), + aggregate_dim=None, + dtype=self.data_type, + ) + + def estimate_output_memory_bytes(self, input_stats) -> int: + return input_stats.num_instances * self._EMBEDDING_DIM * self.data_type.itemsize + + def estimate_peak_memory_bytes(self, input_stats) -> dict: + num_instances = max(getattr(input_stats, "num_instances", 1), 1) + + model_bytes = self._MODEL_PARAMETER_COUNT * self.data_type.itemsize + + clip_bytes = ( + self._CLIPS_PER_VIDEO + * self._SPATIAL_CROPS + * self._FRAMES_PER_CLIP + * 3 + * self._CROP_SIZE + * self._CROP_SIZE + * self.data_type.itemsize + ) + preprocessed_bytes = num_instances * clip_bytes + + batch_activation_bytes = self.batch_size * clip_bytes * 4 + + output_bytes = self.estimate_output_memory_bytes(input_stats) + + decoded_bytes = ( + getattr(input_stats, "max_length", 0) + * getattr(input_stats, "max_channels", 3) + * getattr(input_stats, "max_height", self._CROP_SIZE) + * getattr(input_stats, "max_width", self._CROP_SIZE) + * self.data_type.itemsize + ) + + safety_margin_bytes = 512 * 1024 * 1024 + + gpu_peak = ( + model_bytes + preprocessed_bytes + batch_activation_bytes + output_bytes + ) + cpu_peak = ( + model_bytes + decoded_bytes + preprocessed_bytes + output_bytes + ) + safety_margin_bytes + return {"cpu_peak_bytes": int(cpu_peak), "gpu_peak_bytes": int(gpu_peak)} + + def _ensure_model(self): + global data, imagebind_model, IBModalityType + try: + import imagebind.data as data + from imagebind.models import imagebind_model + from imagebind.models.imagebind_model import ModalityType as IBModalityType + except ImportError as error: + raise ImportError( + "ImageBind requires the optional 'imagebind' package" + ) from error + if self.model is None: + self.model = imagebind_model.imagebind_huge(pretrained=True) + for param in self.model.parameters(): + param.requires_grad = False + self.model = self.model.to(self.device) self.model.eval() - self.model.to(DEVICE) - def transform(self, modality, aggregation=None): - transformed_modality = TransformedModality( - modality, self, ModalityType.EMBEDDING + @staticmethod + def _metadata_for_sample(modality, index): + # Scuro loaders retain metadata across chunks, so resolve the current + # chunk through the modality instead of indexing the raw list directly. + get_metadata = getattr(modality, "get_metadata_at_position", None) + if callable(get_metadata): + return get_metadata(index) + return modality.metadata[index] + + @staticmethod + def _sampling_rate(metadata, modality_type): + sampling_rate = metadata.get("frequency") + if sampling_rate is None or sampling_rate <= 0: + raise ValueError( + f"ImageBind requires a positive sampling frequency for {modality_type}" + ) + return sampling_rate + + def _transform_audio_data(self, samples, metadata): + """Adapt ImageBind audio preprocessing to Scuro-loaded waveforms. + + The clip sampling, mel conversion, and normalization match ImageBind's + load_and_transform_audio_data; only path-based loading is replaced. + """ + sample_rate = 16000 + clip_duration = 2 + clip_sampler = ConstantClipsPerVideoSampler( + clip_duration=clip_duration, clips_per_video=3 ) + normalize = transforms.Normalize(mean=-4.268, std=9.138) + audio_outputs = [] - result = [] - if modality.modality_type == ModalityType.TEXT: - for i, instance in enumerate(modality.data): - text_inputs = data.load_and_transform_text(instance, DEVICE) - text_embeddings = self.model({IBModalityType.TEXT: text_inputs})[ - IBModalityType.TEXT - ] - result.append(text_embeddings.mean(axis=0).cpu().detach().numpy()) - if modality.modality_type == ModalityType.AUDIO: - audio_inputs = data.load_and_transform_audio_data( - list(modality.metadata)[ - (modality.data_loader.next_chunk - 1) - * (modality.data_loader.chunk_size) : ( - modality.data_loader.next_chunk - 1 - ) - * (modality.data_loader.chunk_size) - + (modality.data_loader.chunk_size) - ], - DEVICE, + for sample, sample_metadata in zip(samples, metadata): + # ImageBind uses torchaudio.load(path). Scuro already supplies the + # decoded waveform, and cloning prevents in-place mean centering in + # waveform2melspec from modifying the modality data. + waveform = torch.as_tensor(np.asarray(sample)).clone().float() + if waveform.ndim == 1: + waveform = waveform.unsqueeze(0) + elif waveform.ndim != 2: + raise ValueError( + "ImageBind audio samples must have shape (samples,) or " + "(channels, samples)" + ) + + original_rate = self._sampling_rate(sample_metadata, ModalityType.AUDIO) + if sample_metadata.get("length") == waveform.shape[0]: + waveform = waveform.transpose(0, 1) + if original_rate != sample_rate: + waveform = torchaudio.functional.resample( + waveform, orig_freq=original_rate, new_freq=sample_rate + ) + + timepoints = data.get_clip_timepoints( + clip_sampler, waveform.size(1) / sample_rate ) - audio_embeddings = self.model({IBModalityType.AUDIO: audio_inputs})[ - IBModalityType.AUDIO + clips = [] + for start, end in timepoints: + waveform_clip = waveform[ + :, int(start * sample_rate) : int(end * sample_rate) + ] + mel_spectrogram = data.waveform2melspec( + waveform_clip, + sample_rate, + num_mel_bins=128, + target_length=204, + ) + clips.append(normalize(mel_spectrogram)) + audio_outputs.append(torch.stack(clips)) + + return torch.stack(audio_outputs).to(self.device) + + def _transform_video_data(self, samples, metadata): + """Adapt ImageBind video preprocessing to Scuro-loaded frame arrays. + + ImageBind's temporal sampling, spatial transforms, normalization, and + crop expansion are retained; file decoding is replaced by array slicing. + """ + clip_duration = 2 + clip_sampler = ConstantClipsPerVideoSampler( + clip_duration=clip_duration, clips_per_video=5 + ) + frame_sampler = pv_transforms.UniformTemporalSubsample( + num_samples=clip_duration + ) + video_transform = transforms.Compose( + [ + pv_transforms.ShortSideScale(224), + data.NormalizeVideo( + mean=(0.48145466, 0.4578275, 0.40821073), + std=(0.26862954, 0.26130258, 0.27577711), + ), ] - result.extend(audio_embeddings.cpu().detach().numpy()) - if modality.modality_type == ModalityType.VIDEO: - video_inputs = data.load_and_transform_video_data( - list(modality.metadata)[ - (modality.data_loader.next_chunk - 1) - * (modality.data_loader.chunk_size) : ( - modality.data_loader.next_chunk - 1 - ) - * (modality.data_loader.chunk_size) - + (modality.data_loader.chunk_size) - ], - DEVICE, + ) + video_outputs = [] + + for sample, sample_metadata in zip(samples, metadata): + # ImageBind's decoder returns (C, T, H, W). Scuro stores decoded + # video as (T, H, W, C), already scaled when it has a float dtype. + frames = np.asarray(sample) + if frames.ndim != 4 or frames.shape[-1] != 3: + raise ValueError( + "ImageBind video samples must have shape " + "(frames, height, width, 3)" + ) + + video = torch.as_tensor(frames).permute(3, 0, 1, 2).float() + if np.issubdtype(frames.dtype, np.integer): + video = video / 255.0 + + sampling_rate = self._sampling_rate(sample_metadata, ModalityType.VIDEO) + timepoints = data.get_clip_timepoints( + clip_sampler, video.shape[1] / sampling_rate ) - video_embeddings = self.model({IBModalityType.VISION: video_inputs})[ - IBModalityType.VISION + clips = [] + for start, end in timepoints: + # Match the ceil-based [start, end) indexing used by the + # Decord-backed EncodedVideo loader in ImageBind. + start_frame = math.ceil(sampling_rate * start) + end_frame = min(math.ceil(sampling_rate * end), video.shape[1]) + video_clip = frame_sampler(video[:, start_frame:end_frame]) + clips.append(video_transform(video_clip)) + + clips = data.SpatialCrop(224, num_crops=3)(clips) + video_outputs.append(torch.stack(clips)) + + return torch.stack(video_outputs).to(self.device) + + def _prepare_inputs(self, samples, metadata, modality_type): + if modality_type == ModalityType.TEXT: + # ImageBind's text loader already accepts decoded strings, so its + # tokenizer can be reused without a Scuro-specific adapter. + return IBModalityType.TEXT, data.load_and_transform_text( + samples, self.device + ) + if modality_type == ModalityType.AUDIO: + return IBModalityType.AUDIO, self._transform_audio_data(samples, metadata) + if modality_type == ModalityType.VIDEO: + return IBModalityType.VISION, self._transform_video_data(samples, metadata) + raise ValueError(f"ImageBind does not support {modality_type}") + + def transform(self, modality, aggregation=None): + self._ensure_model() + grouped = False + if modality.modality_type == ModalityType.TEXT and ModalityType.TEXT.has_field( + modality.metadata, "text_spans" + ): + groups = list(TextSpanDataset(modality.data, modality.metadata)) + samples, owner_ids = flatten_owned_sequences(groups) + num_owners = len(groups) + grouped = aggregation is None + else: + if modality.modality_type == ModalityType.TEXT: + samples = list(TextDataset(modality.data)) + else: + samples = modality.data + owner_ids = list(range(len(samples))) + num_owners = len(samples) + + if modality.modality_type == ModalityType.TEXT: + metadata = [None] * len(samples) + else: + metadata = [ + self._metadata_for_sample(modality, index) + for index in range(len(samples)) ] - result.extend(video_embeddings.cpu().detach().numpy()) - transformed_modality.data = result + accumulator = OwnerAccumulator(num_owners, len(samples), aggregation) + with inference_context(self.device): + for start in range(0, len(samples), self.batch_size): + end = min(start + self.batch_size, len(samples)) + modality_key, inputs = self._prepare_inputs( + samples[start:end], + metadata[start:end], + modality.modality_type, + ) + output = self.model({modality_key: inputs})[modality_key] + chunk_ids = torch.arange(start, end, device=output.device) + batch_owner_ids = torch.as_tensor( + owner_ids[start:end], device=output.device, dtype=torch.long + ) + accumulator.update(torch.flatten(output, 1), batch_owner_ids, chunk_ids) + + embeddings = accumulator.finalize(grouped=grouped) + if self.output_file is not None: + save_embeddings(embeddings, self.output_file) + + transformed_modality = TransformedModality( + modality, self, self.output_modality_type + ) + transformed_modality.data_type = np.float32 + transformed_modality.aggregate_dim = (0,) if grouped else None + transformed_modality.data = embeddings return transformed_modality diff --git a/src/main/python/systemds/scuro/representations/mel_spectrogram.py b/src/main/python/systemds/scuro/representations/mel_spectrogram.py index 3cf25b44c4a..5683b8f2117 100644 --- a/src/main/python/systemds/scuro/representations/mel_spectrogram.py +++ b/src/main/python/systemds/scuro/representations/mel_spectrogram.py @@ -93,7 +93,10 @@ def compute_feature(self, instance, sr=None): if instance.ndim == 1: return S.T - return S.transpose(0, 2, 1) + return np.swapaxes(S, -2, -1) + + def compute_features_batched(self, data, sr=None): + return self.compute_feature(np.asarray(data), sr=sr) def get_output_stats(self, input_stats) -> RepresentationStats: num_instances = getattr(input_stats, "num_instances", 0) diff --git a/src/main/python/systemds/scuro/representations/mfcc.py b/src/main/python/systemds/scuro/representations/mfcc.py index 804dc04f1f2..2994a37efeb 100644 --- a/src/main/python/systemds/scuro/representations/mfcc.py +++ b/src/main/python/systemds/scuro/representations/mfcc.py @@ -102,15 +102,18 @@ def compute_feature(self, instance, sr=None): mean = mfcc.mean(keepdims=True) std = mfcc.std(keepdims=True) else: - mean = mfcc.mean(axis=(1, 2), keepdims=True) - std = mfcc.std(axis=(1, 2), keepdims=True) + mean = mfcc.mean(axis=(-2, -1), keepdims=True) + std = mfcc.std(axis=(-2, -1), keepdims=True) mfcc -= mean mfcc /= np.maximum(std, 1e-8) if instance.ndim == 1: return mfcc.T - return mfcc.transpose(0, 2, 1) + return np.swapaxes(mfcc, -2, -1) + + def compute_features_batched(self, data, sr=None): + return self.compute_feature(np.asarray(data), sr=sr) def get_output_stats(self, input_stats) -> RepresentationStats: num_instances = getattr(input_stats, "num_instances", 0) diff --git a/src/main/python/systemds/scuro/representations/openface.py b/src/main/python/systemds/scuro/representations/openface.py new file mode 100644 index 00000000000..7715e85951f --- /dev/null +++ b/src/main/python/systemds/scuro/representations/openface.py @@ -0,0 +1,577 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- +import os +import tempfile +from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path + +import cv2 +import numpy as np +import torch + +from systemds.scuro.dataloader.video_loader import VideoStats +from systemds.scuro.drsearch.operator_registry import register_representation +from systemds.scuro.modality.transformed import TransformedModality +from systemds.scuro.modality.type import ModalityType +from systemds.scuro.representations.representation import ( + CONTAINER_LIST, + RepresentationStats, +) +from systemds.scuro.representations.unimodal import UnimodalRepresentation +from systemds.scuro.representations.utils import get_sequence_lengths +from systemds.scuro.utils.static_variables import ( + NP_ARRAY_HEADER_BYTES, + PY_LIST_HEADER_BYTES, + PY_LIST_SLOT_BYTES, + get_device, +) + +_retinaface_pretrain_patched = False +_star_dirs_patched = False + + +def _patch_openface_package_defaults(needs_landmarks: bool) -> None: + global _retinaface_pretrain_patched, _star_dirs_patched + + if not _retinaface_pretrain_patched: + try: + from openface.Pytorch_Retinaface.data.config import cfg_mnet, cfg_re50 + except ImportError: + pass + else: + cfg_mnet["pretrain"] = False + cfg_re50["pretrain"] = False + _retinaface_pretrain_patched = True + + if needs_landmarks and not _star_dirs_patched: + import openface.STAR.conf.alignment as star_alignment + + star_work_dir = Path(tempfile.gettempdir()) / "scuro_openface_star" + original_init = star_alignment.Alignment.__init__ + + def patched_init(self, args): + original_init(self, args) + self.ckpt_dir = str(star_work_dir) + self.work_dir = os.path.join( + self.ckpt_dir, self.data_definition, self.folder + ) + self.model_dir = os.path.join(self.work_dir, "model") + self.log_dir = os.path.join(self.work_dir, "log") + + star_alignment.Alignment.__init__ = patched_init + _star_dirs_patched = True + + +@register_representation([ModalityType.IMAGE, ModalityType.VIDEO]) +class OpenFace(UnimodalRepresentation): + supports_aggregation_pushdown = True + cache_in_worker = True + + MODEL_REPOSITORY = "nutPace/openface_weights" + FACE_MODEL_FILENAME = "Alignment_RetinaFace.pth" + MULTITASK_MODEL_FILENAME = "MTL_backbone.pth" + LANDMARK_MODEL_FILENAME = "Landmark_98.pkl" + + FEATURE_SETS = ("landmarks", "behavioral", "multitask", "backbone", "all") + DEFAULT_FEATURE_SET = "landmarks" + NUM_LANDMARKS = 98 + BACKBONE_DIM = 1280 + + ACTION_UNIT_INTENSITIES = ("01", "06", "17", "25", "26", "02", "12", "15") + BEHAVIORAL_COLUMNS = ( + "gaze_yaw", + "gaze_pitch", + *(f"AU{action_unit}_r" for action_unit in ACTION_UNIT_INTENSITIES), + ) + EMOTION_COLUMNS = ( + "emotion_neutral", + "emotion_happy", + "emotion_sad", + "emotion_surprise", + "emotion_fear", + "emotion_disgust", + "emotion_anger", + "emotion_contempt", + ) + MULTITASK_COLUMNS = BEHAVIORAL_COLUMNS + EMOTION_COLUMNS + LANDMARK_COLUMNS = tuple( + coordinate + for landmark_id in range(NUM_LANDMARKS) + for coordinate in (f"landmark_{landmark_id}_x", f"landmark_{landmark_id}_y") + ) + DETECTION_COLUMNS = ( + "face_x1", + "face_y1", + "face_x2", + "face_y2", + "face_confidence", + *( + coordinate + for landmark_id in range(5) + for coordinate in ( + f"retinaface_landmark_{landmark_id}_x", + f"retinaface_landmark_{landmark_id}_y", + ) + ), + ) + BACKBONE_COLUMNS = tuple( + f"backbone_{feature_id}" for feature_id in range(BACKBONE_DIM) + ) + + FEATURE_SET_DIMS = { + "behavioral": len(BEHAVIORAL_COLUMNS), + "multitask": len(MULTITASK_COLUMNS), + "landmarks": len(MULTITASK_COLUMNS) + len(LANDMARK_COLUMNS), + "backbone": len(BACKBONE_COLUMNS), + "all": ( + len(MULTITASK_COLUMNS) + + len(DETECTION_COLUMNS) + + len(LANDMARK_COLUMNS) + + len(BACKBONE_COLUMNS) + ), + } + FEATURE_COLUMNS = MULTITASK_COLUMNS + LANDMARK_COLUMNS + FEATURE_DIM = len(FEATURE_COLUMNS) + + FACE_MODEL_MEMORY_BYTES = 8 * 1024 * 1024 + MULTITASK_MODEL_MEMORY_BYTES = 128 * 1024 * 1024 + LANDMARK_MODEL_MEMORY_BYTES = 192 * 1024 * 1024 + CPU_RUNTIME_OVERHEAD_BYTES = 128 * 1024 * 1024 + GPU_RUNTIME_OVERHEAD_BYTES = 64 * 1024 * 1024 + + def __init__( + self, + feature_set=DEFAULT_FEATURE_SET, + confidence_threshold=0.02, + nms_threshold=0.4, + vis_threshold=0.5, + params=None, + ): + if params is not None: + feature_set = params.get("feature_set", feature_set) + confidence_threshold = params.get( + "confidence_threshold", confidence_threshold + ) + nms_threshold = params.get("nms_threshold", nms_threshold) + vis_threshold = params.get("vis_threshold", vis_threshold) + + parameters = { + "feature_set": list(self.FEATURE_SETS), + "confidence_threshold": [0.01, 0.02, 0.05], + "nms_threshold": [0.3, 0.4, 0.5], + "vis_threshold": [0.4, 0.5, 0.6], + } + super().__init__("OpenFace", ModalityType.EMBEDDING, parameters) + self.feature_set = feature_set + self.confidence_threshold = float(confidence_threshold) + self.nms_threshold = float(nms_threshold) + self.vis_threshold = float(vis_threshold) + self.params = params + self.data_type = np.float32 + self._gpu_id = None + self.device = get_device() + self._face_detector = None + self._multitask_predictor = None + self._landmark_detector = None + self._backbone_hook = None + self._backbone_output = None + + @property + def feature_set(self): + return self._feature_set + + @feature_set.setter + def feature_set(self, feature_set): + if feature_set not in self.FEATURE_SETS: + raise ValueError( + f"Unknown OpenFace feature set '{feature_set}'. " + f"Expected one of: {', '.join(self.FEATURE_SETS)}" + ) + self._feature_set = feature_set + + @property + def feature_dim(self): + return self.FEATURE_SET_DIMS[self.feature_set] + + @property + def gpu_id(self): + return self._gpu_id + + @gpu_id.setter + def gpu_id(self, gpu_id): + self._gpu_id = gpu_id + self.device = get_device(gpu_id) + if self._backbone_hook is not None: + self._backbone_hook.remove() + self._face_detector = None + self._multitask_predictor = None + self._landmark_detector = None + self._backbone_hook = None + self._backbone_output = None + + def get_output_stats(self, input_stats) -> RepresentationStats: + if self.params and "_pushdown_aggregation" in self.params: + return RepresentationStats( + input_stats.num_instances, + (self.feature_dim,), + aggregate_dim=None, + dtype=self.data_type, + ) + + if isinstance(input_stats, VideoStats): + return RepresentationStats( + input_stats.num_instances, + (input_stats.max_length, self.feature_dim), + dtype=self.data_type, + container=CONTAINER_LIST, + ) + + if isinstance(input_stats, RepresentationStats): + return RepresentationStats( + input_stats.num_instances, + (input_stats.output_shape[0], self.feature_dim), + dtype=self.data_type, + container=CONTAINER_LIST, + ) + + return RepresentationStats( + input_stats.num_instances, + (self.feature_dim,), + aggregate_dim=None, + dtype=self.data_type, + container=CONTAINER_LIST, + ) + + def estimate_output_memory_bytes(self, input_stats) -> int: + stats = self.get_output_stats(input_stats) + payload = int( + input_stats.num_instances + * np.prod(stats.output_shape) + * np.dtype(self.data_type).itemsize + ) + return int( + PY_LIST_HEADER_BYTES + + input_stats.num_instances * (NP_ARRAY_HEADER_BYTES + PY_LIST_SLOT_BYTES) + + payload + ) + + def estimate_peak_memory_bytes(self, input_stats) -> dict: + max_height = int(getattr(input_stats, "max_height", 224)) + max_width = int(getattr(input_stats, "max_width", 224)) + max_channels = int(getattr(input_stats, "max_channels", 3)) + frame_bytes = ( + max_height * max_width * max_channels * np.dtype(np.float32).itemsize + ) + model_bytes = self.FACE_MODEL_MEMORY_BYTES + self.MULTITASK_MODEL_MEMORY_BYTES + if self.feature_set in ("landmarks", "all"): + model_bytes += self.LANDMARK_MODEL_MEMORY_BYTES + + output_bytes = self.estimate_output_memory_bytes(input_stats) + return { + "cpu_peak_bytes": int( + output_bytes + + frame_bytes + + model_bytes + + self.CPU_RUNTIME_OVERHEAD_BYTES + ), + "gpu_peak_bytes": int( + model_bytes + 2 * frame_bytes + self.GPU_RUNTIME_OVERHEAD_BYTES + ), + } + + def transform(self, modality, aggregation=None): + if modality.modality_type not in (ModalityType.IMAGE, ModalityType.VIDEO): + raise ValueError("OpenFace supports only image and video modalities") + + is_image = modality.modality_type == ModalityType.IMAGE + if is_image: + embeddings = self._extract_image_features(modality.data) + else: + embeddings = [] + lengths = get_sequence_lengths(modality.data, modality.metadata) + for owner_id, (frames, length) in enumerate(zip(modality.data, lengths)): + features = self._extract_video_features(frames[:length], owner_id) + embeddings.append( + self._aggregate(features, aggregation) + if aggregation is not None + else features + ) + + if aggregation is not None: + embeddings = np.stack(embeddings).astype(np.float32, copy=False) + + if is_image and aggregation is not None: + embeddings = np.stack( + [ + self._aggregate(feature[None, :], aggregation) + for feature in embeddings + ] + ).astype(np.float32, copy=False) + + transformed_modality = TransformedModality( + modality, self, self.output_modality_type + ) + transformed_modality.data_type = np.float32 + transformed_modality.aggregate_dim = ( + None if aggregation is not None or is_image else (0,) + ) + transformed_modality.data = embeddings + return transformed_modality + + def _extract_image_features(self, images): + return [self._extract_features(image) for image in images] + + def _extract_video_features(self, frames, owner_id): + if len(frames) == 0: + raise ValueError(f"Video instance {owner_id} contains no frames") + return np.stack([self._extract_features(frame) for frame in frames]) + + def _extract_features(self, image): + self._load_models() + image_bgr = self._to_bgr(image) + face, detections = self._face_detector.get_face(image_bgr) + if face is None or face.size == 0: + return np.zeros(self.feature_dim, dtype=np.float32) + + needs_backbone = self.feature_set in ("backbone", "all") + if needs_backbone: + self._ensure_backbone_hook() + self._backbone_output = None + + emotion_output, gaze_output, action_unit_output = ( + self._multitask_predictor.predict(face) + ) + behavioral = self._behavioral_features(gaze_output, action_unit_output) + emotion = self._checked_vector( + emotion_output, len(self.EMOTION_COLUMNS), "emotion" + ) + multitask = np.concatenate((behavioral, emotion)) + + if self.feature_set == "behavioral": + return behavioral + if self.feature_set == "multitask": + return multitask.astype(np.float32, copy=False) + + backbone = None + if needs_backbone: + backbone = self._checked_vector( + self._backbone_output, self.BACKBONE_DIM, "backbone" + ) + if self.feature_set == "backbone": + return backbone + + landmarks = self._landmark_features(image_bgr, detections) + if self.feature_set == "landmarks": + return np.concatenate((multitask, landmarks)).astype(np.float32, copy=False) + + detection = self._detection_features(detections, image_bgr.shape) + return np.concatenate((multitask, detection, landmarks, backbone)).astype( + np.float32, copy=False + ) + + def _behavioral_features(self, gaze_output, action_unit_output): + gaze = self._checked_vector(gaze_output, 2, "gaze") + action_units = self._checked_vector( + action_unit_output, len(self.ACTION_UNIT_INTENSITIES), "action-unit" + ) + return np.concatenate((gaze, action_units)).astype(np.float32, copy=False) + + def _landmark_features(self, image_bgr, detections): + with redirect_stdout(StringIO()): + landmarks = self._landmark_detector.detect_landmarks( + image_bgr, + detections[:1], + confidence_threshold=self.vis_threshold, + ) + if not landmarks: + return np.zeros(len(self.LANDMARK_COLUMNS), dtype=np.float32) + + points = np.asarray(landmarks[0], dtype=np.float32) + if points.shape != (self.NUM_LANDMARKS, 2): + raise RuntimeError( + "OpenFace 3.0 returned unexpected landmark dimensions: " + f"{points.shape}" + ) + height, width = image_bgr.shape[:2] + points = points.copy() + points[:, 0] /= width + points[:, 1] /= height + return points.reshape(-1) + + @classmethod + def _detection_features(cls, detections, image_shape): + detection = np.asarray(detections[0], dtype=np.float32) + if detection.size < len(cls.DETECTION_COLUMNS): + raise RuntimeError( + "OpenFace 3.0 returned unexpected face-detection dimensions: " + f"{detection.size}" + ) + + detection = detection[: len(cls.DETECTION_COLUMNS)].copy() + height, width = image_shape[:2] + detection[[0, 2, 5, 7, 9, 11, 13]] /= width + detection[[1, 3, 6, 8, 10, 12, 14]] /= height + return detection + + def _ensure_backbone_hook(self): + if self._backbone_hook is not None: + return + + def capture_backbone(_module, _inputs, output): + self._backbone_output = output + + self._backbone_hook = ( + self._multitask_predictor.model.base_model.register_forward_hook( + capture_backbone + ) + ) + + def _load_models(self): + needs_landmarks = self.feature_set in ("landmarks", "all") + models_ready = ( + self._face_detector is not None + and self._multitask_predictor is not None + and (not needs_landmarks or self._landmark_detector is not None) + ) + if models_ready: + return + + try: + from huggingface_hub import snapshot_download + from openface.face_detection import FaceDetector + from openface.multitask_model import MultitaskPredictor + + if needs_landmarks: + from openface.landmark_detection import LandmarkDetector + except ImportError as error: + raise ImportError( + "OpenFace 3.0 is required for this representation. " + "Install it with 'pip install openface-test'." + ) from error + + _patch_openface_package_defaults(needs_landmarks) + + weight_files = [self.FACE_MODEL_FILENAME, self.MULTITASK_MODEL_FILENAME] + if needs_landmarks: + weight_files.append(self.LANDMARK_MODEL_FILENAME) + weights_directory = Path( + snapshot_download( + repo_id=self.MODEL_REPOSITORY, + allow_patterns=weight_files, + ) + ) + + class ArrayFaceDetector(FaceDetector): + def preprocess_image(self, image, resize=1.0): + image_raw = np.ascontiguousarray(image) + detector_input = np.float32(image_raw) + if resize != 1: + detector_input = cv2.resize( + detector_input, + None, + fx=resize, + fy=resize, + interpolation=cv2.INTER_LINEAR, + ) + detector_input -= (104, 117, 123) + detector_input = detector_input.transpose(2, 0, 1) + detector_input = ( + torch.from_numpy(detector_input).unsqueeze(0).to(self.device) + ) + return detector_input, image_raw + + device = str(self.device) + if self._face_detector is None: + self._face_detector = ArrayFaceDetector( + model_path=str(weights_directory / self.FACE_MODEL_FILENAME), + device=device, + confidence_threshold=self.confidence_threshold, + nms_threshold=self.nms_threshold, + vis_threshold=self.vis_threshold, + ) + if self._multitask_predictor is None: + self._multitask_predictor = MultitaskPredictor( + model_path=str(weights_directory / self.MULTITASK_MODEL_FILENAME), + device=device, + ) + if needs_landmarks and self._landmark_detector is None: + landmark_device = self.device.type + device_ids = ( + [-1] + if landmark_device == "cpu" + else [self.device.index if self.device.index is not None else 0] + ) + self._landmark_detector = LandmarkDetector( + model_path=str(weights_directory / self.LANDMARK_MODEL_FILENAME), + device=landmark_device, + device_ids=device_ids, + ) + + @staticmethod + def _checked_vector(output, expected_size, output_name): + if output is None: + size = 0 + else: + if hasattr(output, "detach"): + output = output.detach().cpu().numpy() + output = np.asarray(output, dtype=np.float32).reshape(-1) + size = output.size + if size != expected_size: + raise RuntimeError( + f"OpenFace 3.0 returned unexpected {output_name} dimensions: {size}" + ) + return output + + @staticmethod + def _aggregate(features, aggregation): + return np.asarray(aggregation.execute(features), dtype=np.float32) + + @staticmethod + def _to_bgr(image): + if hasattr(image, "detach"): + image = image.detach().cpu().numpy() + image = np.asarray(image) + if image.ndim not in (2, 3): + raise ValueError( + f"Expected an image tensor with 2 or 3 dimensions, got {image.shape}" + ) + + if np.issubdtype(image.dtype, np.floating): + finite = image[np.isfinite(image)] + if finite.size and finite.min() >= 0.0 and finite.max() <= 1.0: + image = image * 255.0 + image = np.nan_to_num(image, nan=0.0, posinf=255.0, neginf=0.0) + image = np.clip(image, 0, 255).astype(np.uint8, copy=False) + + if image.ndim == 2: + image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR) + elif image.shape[2] == 1: + image = cv2.cvtColor(image[:, :, 0], cv2.COLOR_GRAY2BGR) + elif image.shape[2] == 3: + image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) + elif image.shape[2] == 4: + image = cv2.cvtColor(image, cv2.COLOR_RGBA2BGR) + else: + raise ValueError( + f"Expected 1, 3, or 4 image channels, got {image.shape[2]}" + ) + return np.ascontiguousarray(image) diff --git a/src/main/python/systemds/scuro/representations/optical_flow.py b/src/main/python/systemds/scuro/representations/optical_flow.py index 943cd2ab499..e4f147dec56 100644 --- a/src/main/python/systemds/scuro/representations/optical_flow.py +++ b/src/main/python/systemds/scuro/representations/optical_flow.py @@ -52,11 +52,14 @@ def transform(self, modality, aggregation=None): ) for video_id, instance in enumerate(modality.data): + if instance.dtype == np.float16: + instance = instance.astype(np.float32) + transformed_modality.data.append([]) - previous_gray = cv2.cvtColor(instance[0], cv2.COLOR_BGR2GRAY) + previous_gray = cv2.cvtColor(instance[0], cv2.COLOR_RGB2GRAY) for frame_id in range(1, len(instance)): - gray = cv2.cvtColor(instance[frame_id], cv2.COLOR_BGR2GRAY) + gray = cv2.cvtColor(instance[frame_id], cv2.COLOR_RGB2GRAY) flow = cv2.calcOpticalFlowFarneback( previous_gray, @@ -72,5 +75,6 @@ def transform(self, modality, aggregation=None): ) transformed_modality.data[video_id].append(flow) + previous_gray = gray transformed_modality.update_metadata() return transformed_modality diff --git a/src/main/python/systemds/scuro/representations/resnet.py b/src/main/python/systemds/scuro/representations/resnet.py index 26a3350e9c8..ccefd4e11b4 100644 --- a/src/main/python/systemds/scuro/representations/resnet.py +++ b/src/main/python/systemds/scuro/representations/resnet.py @@ -21,6 +21,14 @@ from systemds.scuro.dataloader.image_loader import ImageStats from systemds.scuro.dataloader.video_loader import VideoStats from systemds.scuro.representations.representation import RepresentationStats +from systemds.scuro.representations.utils import ( + OwnerAccumulator, + flatten_owned_sequences, + get_sequence_lengths, + inference_context, + move_batch_to_device, + pin_memory_for, +) from systemds.scuro.utils.torch_dataset import CustomDataset from systemds.scuro.modality.transformed import TransformedModality from systemds.scuro.representations.unimodal import UnimodalRepresentation @@ -41,6 +49,9 @@ def forward(self, input_: torch.Tensor) -> torch.Tensor: @register_representation([ModalityType.IMAGE, ModalityType.VIDEO]) class ResNet(UnimodalRepresentation): + supports_aggregation_pushdown = True + cache_in_worker = True + def __init__( self, model_name="ResNet18", @@ -51,6 +62,8 @@ def __init__( ): self.data_type = torch.float32 self.model = None + self._activation_hook = None + self.activation = None self.gpu_id = None self.device = get_device() if params is not None: @@ -63,6 +76,7 @@ def __init__( self.model_name = model_name parameters = self._get_parameters() super().__init__("ResNet", ModalityType.EMBEDDING, parameters) + self.params = params self.output_file = output_file self.model.eval() @@ -116,16 +130,15 @@ def model_name(self, model_name): raise NotImplementedError def estimate_output_memory_bytes(self, input_stats: ImageStats) -> int: - if isinstance(input_stats, VideoStats): - return ( - input_stats.num_instances - * input_stats.max_length - * 512 - * self.data_type.itemsize - ) - return input_stats.num_instances * 512 * self.data_type.itemsize + shape = self.get_output_stats(input_stats).output_shape + return int(input_stats.num_instances * np.prod(shape) * self.data_type.itemsize) def get_output_stats(self, input_stats) -> RepresentationStats: + if self.params and "_pushdown_aggregation" in self.params: + return RepresentationStats( + input_stats.num_instances, (512,), aggregate_dim=None + ) + if isinstance(input_stats, VideoStats): return RepresentationStats( input_stats.num_instances, @@ -178,7 +191,11 @@ def estimate_peak_memory_bytes(self, input_stats: ImageStats) -> dict: return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": gpu_peak} def _get_parameters(self, high_level=True): - parameters = {"model_name": [], "layer_name": []} + parameters = { + "batch_size": [1, 2, 4, 8, 16, 32, 64, 128], + "model_name": [], + "layer_name": [], + } for m in ["ResNet18", "ResNet34", "ResNet50", "ResNet101", "ResNet152"]: parameters["model_name"].append(m) @@ -199,76 +216,68 @@ def _get_parameters(self, high_level=True): def transform(self, modality, aggregation=None): if next(self.model.parameters()).dtype != self.data_type: self.model = self.model.to(self.data_type) - - embeddings = {} - dataset = CustomDataset(modality.data, self.data_type, self.device) - res5c_output = None + self.model = self.model.to(self.device) + self.model.eval() + self.activation = None def get_features(name_): def hook( _module: torch.nn.Module, input_: Tuple[torch.Tensor], output: Any ): - nonlocal res5c_output - res5c_output = output + self.activation = output return hook - if self.layer_name: + if self.layer_name and self._activation_hook is None: for name, layer in self.model.named_modules(): if name == self.layer_name: - layer.register_forward_hook(get_features(name)) + self._activation_hook = layer.register_forward_hook( + get_features(name) + ) break - if modality.modality_type == ModalityType.IMAGE: - embeddings = [] - for batch in torch.utils.data.DataLoader( - dataset, batch_size=self.batch_size - ): - image_batch = batch["data"] - _ = self.model(image_batch) - output = res5c_output - embeddings.extend( - output.squeeze().detach().cpu().numpy().astype(modality.data_type) - ) - torch.cuda.empty_cache() + is_image = modality.modality_type == ModalityType.IMAGE + if is_image: + samples = modality.data + owner_ids = list(range(len(samples))) else: - for instance in torch.utils.data.DataLoader(dataset): - video_id = instance["id"][0] - frames = instance["data"][0] - embeddings[video_id] = [] - batch_size = 64 - - if modality.modality_type == ModalityType.IMAGE: - frames = frames.unsqueeze(0) - - for start_index in range(0, len(frames), batch_size): - end_index = min(start_index + batch_size, len(frames)) - frame_ids_range = range(start_index, end_index) - frame_batch = frames[frame_ids_range] - - _ = self.model(frame_batch) - output = res5c_output - if len(output.shape) > 2: - output = torch.nn.functional.adaptive_avg_pool2d(output, (1, 1)) - # TODO: check if the dimensions are correct here - embeddings[video_id].extend( - torch.flatten(output, 1) - .detach() - .cpu() - .float() - .numpy() - .astype(np.float32) - ) + lengths = get_sequence_lengths(modality.data, modality.metadata) + samples, owner_ids = flatten_owned_sequences(modality.data, lengths) + + dataset = CustomDataset(samples, self.data_type, "cpu") + dataloader = torch.utils.data.DataLoader( + dataset, + batch_size=self.batch_size, + shuffle=False, + pin_memory=pin_memory_for(self.device), + ) + owner_by_chunk = torch.tensor(owner_ids, dtype=torch.long) + accumulator = OwnerAccumulator(len(modality.data), len(dataset), aggregation) + + with inference_context(self.device): + for batch in dataloader: + chunk_ids = batch["id"].long() + batch = move_batch_to_device(batch, self.device) + _ = self.model(batch["data"]) + output = self.activation + if output.ndim > 2: + output = torch.nn.functional.adaptive_avg_pool2d(output, (1, 1)) + accumulator.update( + torch.flatten(output, 1), + owner_by_chunk.index_select(0, chunk_ids), + chunk_ids, + ) - embeddings[video_id] = np.array(embeddings[video_id]) + embeddings = accumulator.finalize(grouped=not is_image and aggregation is None) + if is_image and aggregation is None: + embeddings = list(embeddings) transformed_modality = TransformedModality( modality, self, self.output_modality_type ) - - if isinstance(embeddings, dict): - transformed_modality.data = list(embeddings.values()) - else: - transformed_modality.data = embeddings - + transformed_modality.data_type = np.float32 + transformed_modality.aggregate_dim = ( + None if aggregation is not None or is_image else (0,) + ) + transformed_modality.data = embeddings return transformed_modality diff --git a/src/main/python/systemds/scuro/representations/swin_video_transformer.py b/src/main/python/systemds/scuro/representations/swin_video_transformer.py index 7bb40e278f3..f57ac18f037 100644 --- a/src/main/python/systemds/scuro/representations/swin_video_transformer.py +++ b/src/main/python/systemds/scuro/representations/swin_video_transformer.py @@ -31,10 +31,17 @@ from systemds.scuro.modality.type import ModalityType from systemds.scuro.drsearch.operator_registry import register_representation from systemds.scuro.dataloader.video_loader import VideoStats +from systemds.scuro.representations.utils import ( + LengthBucketBatchSampler, + OwnerAccumulator, + get_sequence_lengths, + move_batch_to_device, + pin_memory_for, + transformer_inference_context, +) from systemds.scuro.utils.torch_dataset import CustomDataset from systemds.scuro.utils.static_variables import ( - compute_batch_size, get_device, get_device_for_model, ) @@ -43,8 +50,9 @@ @register_representation([ModalityType.VIDEO]) class SwinVideoTransformer(UnimodalRepresentation): _EMBED_DIM = 768 + cache_in_worker = True - def __init__(self, layer_name="avgpool", params=None): + def __init__(self, layer_name="avgpool", batch_size=8, params=None): parameters = { "layer_name": [ "features", @@ -56,19 +64,33 @@ def __init__(self, layer_name="avgpool", params=None): "features.6", "avgpool", ], + "batch_size": [1, 2, 4, 8, 16, 32], } self.data_type = torch.float32 super().__init__("SwinVideoTransformer", ModalityType.EMBEDDING, parameters) if params is not None: layer_name = params.get("layer_name", layer_name) + batch_size = int(params.get("batch_size", batch_size)) self.layer_name = layer_name + self.batch_size = batch_size self.model = swin3d_t(weights=models.video.Swin3D_T_Weights.KINETICS400_V1) self.device = get_device_for_model(self.model, memory_factor=1.5) + self._gpu_id = self.device.index + self._activation_hook = None self.model = self.model.to(self.device) self.model.eval() for param in self.model.parameters(): param.requires_grad = False + @property + def gpu_id(self): + return self._gpu_id + + @gpu_id.setter + def gpu_id(self, gpu_id): + self._gpu_id = gpu_id + self.device = get_device(gpu_id) + def get_output_stats(self, input_stats) -> RepresentationStats: num_instances = getattr(input_stats, "num_instances", 0) return RepresentationStats(num_instances, (self._EMBED_DIM,)) @@ -116,61 +138,57 @@ def estimate_peak_memory_bytes(self, input_stats: VideoStats) -> dict: return {"cpu_peak_bytes": int(cpu_peak), "gpu_peak_bytes": int(gpu_peak)} def transform(self, modality, aggregation=None): - embeddings = {} - swin_output = None + self.model = self.model.to(self.device) + self.model.eval() + self.swin_output = None def get_features(name_): def hook( _module: torch.nn.Module, input_: Tuple[torch.Tensor], output: Any ): - nonlocal swin_output - swin_output = output + self.swin_output = output return hook - sample = modality.data[0] if modality.data else "" - self.batch_size = compute_batch_size( - model=self.model, - device=self.device, - sample_data=sample, - tokenizer=None, - max_seq_length=None, - max_batch_size=128, - ) - - if self.layer_name: + if self.layer_name and self._activation_hook is None: for name, layer in self.model.named_modules(): if name == self.layer_name: - layer.register_forward_hook(get_features(name)) + self._activation_hook = layer.register_forward_hook( + get_features(name) + ) break - dataset = CustomDataset(modality.data, self.data_type, self.device) - for instance in torch.utils.data.DataLoader(dataset): - video_id = instance["id"][0] - frames = instance["data"][0] - embeddings[video_id] = [] - - frames = frames.unsqueeze(0).permute(0, 2, 1, 3, 4) - - _ = self.model(frames) - values = swin_output - pooled = torch.nn.functional.adaptive_avg_pool2d(values, (1, 1)) - - embeddings[video_id].extend( - torch.flatten(pooled, 1) - .detach() - .cpu() - .numpy() - .flatten() - .astype(modality.data_type) - ) - - embeddings[video_id] = np.array(embeddings[video_id]) + dataset = CustomDataset(modality.data, self.data_type, "cpu") + lengths = get_sequence_lengths(modality.data, modality.metadata) + dataloader = torch.utils.data.DataLoader( + dataset, + batch_sampler=LengthBucketBatchSampler( + lengths, self.batch_size, exact=True + ), + pin_memory=pin_memory_for(self.device), + ) + accumulator = OwnerAccumulator(len(dataset), len(dataset), aggregation) + + with transformer_inference_context(self.device): + for batch in dataloader: + batch = move_batch_to_device(batch, self.device) + video_ids = batch["id"].long() + frames = batch["data"].permute(0, 2, 1, 3, 4) + _ = self.model(frames) + values = self.swin_output + if isinstance(values, tuple): + values = values[0] + if values.ndim == 2: + pooled = values + elif self.layer_name.startswith("features"): + pooled = values.mean(dim=tuple(range(1, values.ndim - 1))) + else: + pooled = values.mean(dim=tuple(range(2, values.ndim))) + accumulator.update(pooled, video_ids, video_ids) transformed_modality = TransformedModality( modality, self, self.output_modality_type ) - - transformed_modality.data = list(embeddings.values()) - + transformed_modality.data = accumulator.finalize() + transformed_modality.data_type = np.float32 return transformed_modality diff --git a/src/main/python/systemds/scuro/representations/tfidf.py b/src/main/python/systemds/scuro/representations/tfidf.py index bea18a56024..a68fe50e407 100644 --- a/src/main/python/systemds/scuro/representations/tfidf.py +++ b/src/main/python/systemds/scuro/representations/tfidf.py @@ -41,6 +41,7 @@ def __init__(self, min_df=2, output_file=None, params=None): self.min_df = int(min_df) self.output_file = output_file self.data_type = np.float32 + self.requires_dimensionality_reduction = True def get_output_stats(self, input_stats: TextStats) -> RepresentationStats: vocab_estimate = min( diff --git a/src/main/python/systemds/scuro/representations/timeseries_representations.py b/src/main/python/systemds/scuro/representations/timeseries_representations.py index 2c7fbbe7404..e6aa999e8f2 100644 --- a/src/main/python/systemds/scuro/representations/timeseries_representations.py +++ b/src/main/python/systemds/scuro/representations/timeseries_representations.py @@ -249,10 +249,8 @@ def get_output_stats(self, input_stats): @register_context_representation_operator(ModalityType.PHYSIOLOGICAL) @register_context_representation_operator(ModalityType.AUDIO) class Kurtosis(TimeSeriesRepresentation): - min_input_length = 4 # the fourth moment is undefined below four samples - def __init__(self, params=None): - super().__init__("Kurtosis") + super().__init__("Kurtosis", min_input_length=4) def compute_feature(self, signal, axis=-1): return np.array(stats.kurtosis(signal, fisher=True, bias=True, axis=axis)) diff --git a/src/main/python/systemds/scuro/representations/utils.py b/src/main/python/systemds/scuro/representations/utils.py index 5041e18770c..04e6829d7ab 100644 --- a/src/main/python/systemds/scuro/representations/utils.py +++ b/src/main/python/systemds/scuro/representations/utils.py @@ -20,8 +20,297 @@ # ------------------------------------------------------------- import os import pickle +from bisect import bisect_right +from collections.abc import Sequence +from contextlib import contextmanager import numpy as np +import torch + + +def pool_transformer_output(hidden_state, attention_mask, use_cls=False): + """Pool transformer tokens without including padding tokens.""" + if hidden_state.ndim == 2: + return hidden_state + if hidden_state.ndim != 3: + raise ValueError( + f"Unexpected transformer output shape: {tuple(hidden_state.shape)}" + ) + if use_cls: + return hidden_state[:, 0, :] + + mask = attention_mask.unsqueeze(-1).to( + device=hidden_state.device, dtype=hidden_state.dtype + ) + token_count = mask.sum(dim=1).clamp_min(1) + return (hidden_state * mask).sum(dim=1) / token_count + + +def aggregate_chunk_embeddings(embeddings, aggregation): + """Aggregate all chunks of one instance while they are still on the GPU.""" + name = aggregation.aggregation_function + if name == "mean": + return embeddings.mean(dim=0) + if name == "max": + return embeddings.max(dim=0).values + if name == "min": + return embeddings.min(dim=0).values + if name == "sum": + return embeddings.sum(dim=0) + if name == "median": + return torch.quantile(embeddings, 0.5, dim=0) + if name == "mode": + return embeddings.mode(dim=0).values + raise ValueError(f"Unsupported aggregation function: {name}") + + +class LengthBucketBatchSampler(torch.utils.data.Sampler): + """Build deterministic batches from samples with similar sequence lengths.""" + + def __init__(self, lengths, batch_size, exact=False): + self.lengths = [int(length) for length in lengths] + self.batch_size = max(1, int(batch_size)) + self.exact = exact + self._batches = self._build_batches() + + def _build_batches(self): + indices = sorted(range(len(self.lengths)), key=lambda i: (self.lengths[i], i)) + if not self.exact: + return [ + indices[start : start + self.batch_size] + for start in range(0, len(indices), self.batch_size) + ] + + batches = [] + start = 0 + while start < len(indices): + length = self.lengths[indices[start]] + end = start + while end < len(indices) and self.lengths[indices[end]] == length: + end += 1 + batches.extend( + indices[offset : min(offset + self.batch_size, end)] + for offset in range(start, end, self.batch_size) + ) + start = end + return batches + + def __iter__(self): + return iter(self._batches) + + def __len__(self): + return len(self._batches) + + +class OwnedSequenceDataset(torch.utils.data.Dataset): + """Sequence samples with stable chunk and owner identifiers.""" + + def __init__(self, samples, owner_ids=None): + self.samples = list(samples) + if owner_ids is None: + owner_ids = range(len(self.samples)) + self.owner_ids = [int(owner_id) for owner_id in owner_ids] + if len(self.samples) != len(self.owner_ids): + raise ValueError("Each sequence must have exactly one owner_id") + + def __getitem__(self, chunk_id): + return self.samples[chunk_id], self.owner_ids[chunk_id], chunk_id + + def __len__(self): + return len(self.samples) + + +class FlattenedSequence(Sequence): + """Lazy flattened view over per-owner sequences.""" + + def __init__(self, sequences, lengths): + self.sequences = sequences + self.lengths = tuple(int(length) for length in lengths) + if len(self.sequences) != len(self.lengths): + raise ValueError("Each sequence must have exactly one length") + + self.offsets = [0] + for length in self.lengths: + self.offsets.append(self.offsets[-1] + length) + self._cached_owner = None + self._cached_sequence = None + + @property + def owner_ids(self): + return [ + owner_id + for owner_id, length in enumerate(self.lengths) + for _ in range(length) + ] + + def __getitem__(self, index): + if isinstance(index, slice): + return [self[i] for i in range(*index.indices(len(self)))] + if index < 0: + index += len(self) + if index < 0 or index >= len(self): + raise IndexError(index) + + owner_id = bisect_right(self.offsets, index) - 1 + if owner_id != self._cached_owner: + self._cached_sequence = self.sequences[owner_id] + self._cached_owner = owner_id + return self._cached_sequence[index - self.offsets[owner_id]] + + def __len__(self): + return self.offsets[-1] + + +def get_sequence_lengths(sequences, metadata): + if len(metadata) == len(sequences) and all("length" in md for md in metadata): + return [int(md["length"]) for md in metadata] + return [len(sequence) for sequence in sequences] + + +def flatten_owned_sequences(sequences, lengths=None): + """Flatten per-owner sequences while retaining their owner identifiers.""" + if lengths is not None: + samples = FlattenedSequence(sequences, lengths) + return samples, samples.owner_ids + + samples = [] + owner_ids = [] + for owner_id, owner_samples in enumerate(sequences): + samples.extend(owner_samples) + owner_ids.extend([owner_id] * len(owner_samples)) + return samples, owner_ids + + +def pin_memory_for(device): + device = torch.device(device) + return device.type == "cuda" and torch.cuda.is_available() + + +def move_batch_to_device(batch, device): + non_blocking = pin_memory_for(device) + return { + key: ( + value.to(device, non_blocking=non_blocking) + if isinstance(value, torch.Tensor) + else value + ) + for key, value in batch.items() + } + + +@contextmanager +def inference_context(device): + """Enable inference-only execution and mixed precision on CUDA.""" + device = torch.device(device) + with torch.inference_mode(): + if device.type != "cuda": + yield + return + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + with torch.autocast(device_type="cuda", dtype=dtype): + yield + + +transformer_inference_context = inference_context + + +class OwnerAccumulator: + """Collect chunk vectors on-device and restore or aggregate their owners.""" + + def __init__(self, num_owners, num_chunks, aggregation=None): + self.num_owners = int(num_owners) + self.num_chunks = int(num_chunks) + self.aggregation = aggregation + self._values = None + self._counts = None + self._owner_ids = None + + @property + def aggregation_name(self): + if self.aggregation is None: + return None + return self.aggregation.aggregation_function + + def _initialize(self, vectors): + hidden_dim = vectors.shape[1] + name = self.aggregation_name + if name is None or name in ("median", "mode"): + self._values = torch.empty( + (self.num_chunks, hidden_dim), + device=vectors.device, + dtype=torch.float32, + ) + self._owner_ids = torch.empty( + self.num_chunks, device=vectors.device, dtype=torch.long + ) + else: + fill = 0.0 + if name == "max": + fill = -torch.inf + elif name == "min": + fill = torch.inf + self._values = torch.full( + (self.num_owners, hidden_dim), + fill, + device=vectors.device, + dtype=torch.float32, + ) + self._counts = torch.zeros( + self.num_owners, device=vectors.device, dtype=torch.long + ) + + def update(self, vectors, owner_ids, chunk_ids): + vectors = vectors.detach().float() + owner_ids = owner_ids.to(device=vectors.device, dtype=torch.long) + chunk_ids = chunk_ids.to(device=vectors.device, dtype=torch.long) + if self._values is None: + self._initialize(vectors) + + name = self.aggregation_name + if name is None or name in ("median", "mode"): + self._values.index_copy_(0, chunk_ids, vectors) + self._owner_ids.index_copy_(0, chunk_ids, owner_ids) + return + + self._counts.index_add_( + 0, owner_ids, torch.ones_like(owner_ids, dtype=torch.long) + ) + if name in ("mean", "sum"): + self._values.index_add_(0, owner_ids, vectors) + elif name in ("max", "min"): + indices = owner_ids[:, None].expand_as(vectors) + self._values.scatter_reduce_( + 0, indices, vectors, reduce=f"a{name}", include_self=True + ) + else: + raise ValueError(f"Unsupported aggregation function: {name}") + + def finalize(self, grouped=False): + if self._values is None: + return np.empty((self.num_owners, 0), dtype=np.float32) + + name = self.aggregation_name + if name is not None: + if name == "mean": + values = self._values / self._counts.clamp_min(1).unsqueeze(1) + elif name in ("sum", "max", "min"): + values = self._values + else: + values = torch.stack( + [ + aggregate_chunk_embeddings( + self._values[self._owner_ids == owner], self.aggregation + ) + for owner in range(self.num_owners) + ] + ) + return values.cpu().numpy().astype(np.float32, copy=False) + + values = self._values.cpu().numpy().astype(np.float32, copy=False) + if not grouped: + return values + owner_ids = self._owner_ids.cpu().numpy() + return [values[owner_ids == owner] for owner in range(self.num_owners)] def dense_instance_batch(data): diff --git a/src/main/python/systemds/scuro/representations/vgg.py b/src/main/python/systemds/scuro/representations/vgg.py index c2b56e8d6bd..d73753b2288 100644 --- a/src/main/python/systemds/scuro/representations/vgg.py +++ b/src/main/python/systemds/scuro/representations/vgg.py @@ -27,7 +27,6 @@ from systemds.scuro.drsearch.operator_registry import register_representation import torch.utils.data import torch -import re import torchvision.models as models import numpy as np from systemds.scuro.modality.type import ModalityType @@ -36,6 +35,14 @@ ) from systemds.scuro.dataloader.image_loader import ImageStats from systemds.scuro.representations.representation import RepresentationStats +from systemds.scuro.representations.utils import ( + OwnerAccumulator, + flatten_owned_sequences, + get_sequence_lengths, + inference_context, + move_batch_to_device, + pin_memory_for, +) class Identity(torch.nn.Module): @@ -45,18 +52,25 @@ def forward(self, input_: torch.Tensor) -> torch.Tensor: @register_representation([ModalityType.IMAGE, ModalityType.VIDEO]) class VGG19(UnimodalRepresentation): + supports_aggregation_pushdown = True + cache_in_worker = True + def __init__( self, layer="classifier.0", output_file=None, params=None, batch_size=32 ): self.data_type = torch.bfloat16 self.model = None + self._activation_hook = None + self.activation = None self.gpu_id = None self.device = get_device() self.model = models.vgg19(weights=models.VGG19_Weights.DEFAULT) self.model = self.model.to(self.device) parameters = self._get_parameters() super().__init__("VGG19", ModalityType.EMBEDDING, parameters) + self.params = params if params is not None: + batch_size = int(params.get("batch_size", batch_size)) layer = params.get("layer_name", layer) self.output_file = output_file self.layer_name = layer @@ -81,27 +95,28 @@ def gpu_id(self, gpu_id): def _get_parameters(self): parameters = { + "batch_size": [1, 2, 4, 8, 16, 32, 64, 128], "layer_name": [ "features.35", "classifier.0", "classifier.3", "classifier.6", - ] + ], } - return parameters def estimate_output_memory_bytes(self, input_stats: ImageStats) -> int: - if isinstance(input_stats, VideoStats): - return ( - input_stats.num_instances - * input_stats.max_length - * 4096 - * np.dtype(np.float32).itemsize - ) - return input_stats.num_instances * 4096 * np.dtype(np.float32).itemsize + shape = self.get_output_stats(input_stats).output_shape + return int( + input_stats.num_instances * np.prod(shape) * np.dtype(np.float32).itemsize + ) def get_output_stats(self, input_stats) -> RepresentationStats: + if self.params and "_pushdown_aggregation" in self.params: + return RepresentationStats( + input_stats.num_instances, (4096,), aggregate_dim=None + ) + if isinstance(input_stats, VideoStats): return RepresentationStats( input_stats.num_instances, @@ -121,10 +136,12 @@ def estimate_peak_memory_bytes(self, input_stats: ImageStats) -> dict: * input_stats.max_channels * self.data_type.itemsize ) - model = models.vgg19(weights=models.VGG19_Weights.DEFAULT) - param_bytes = sum(p.numel() for p in model.parameters()) - buffer_bytes = sum(b.numel() for b in model.buffers()) - model_size_bytes = param_bytes * 4 + buffer_bytes * 4 + model_size_bytes = sum( + p.nelement() * p.element_size() for p in self.model.parameters() + ) + model_size_bytes += sum( + b.nelement() * b.element_size() for b in self.model.buffers() + ) return { "cpu_peak_bytes": ( @@ -150,84 +167,68 @@ def transform(self, modality, aggregation=None): self.data_type = torch.float32 if next(self.model.parameters()).dtype != self.data_type: self.model = self.model.to(self.data_type) - - self.activations = {} + self.model = self.model.to(self.device) + self.model.eval() + self.activation = None def get_activation(name_): def hook( _module: torch.nn.Module, input_: Tuple[torch.Tensor], output: Any ): - self.activations[name_] = output + self.activation = output return hook - digit = re.findall(r"\d+", self.layer_name)[0] - if "feature" in self.layer_name: - self.model.features[int(digit)].register_forward_hook( - get_activation(self.layer_name) - ) + if self._activation_hook is None: + for name, layer in self.model.named_modules(): + if name == self.layer_name: + self._activation_hook = layer.register_forward_hook( + get_activation(name) + ) + break + + is_image = modality.modality_type == ModalityType.IMAGE + if is_image: + samples = modality.data + owner_ids = list(range(len(samples))) else: - self.model.classifier[int(digit)].register_forward_hook( - get_activation(self.layer_name) - ) - is_image = len(modality.data[0].shape) == 3 - embeddings = ( - self._transform_image_modality(modality) - if is_image - else self._transform_video_modality(modality) + lengths = get_sequence_lengths(modality.data, modality.metadata) + samples, owner_ids = flatten_owned_sequences(modality.data, lengths) + + dataset = CustomDataset(samples, self.data_type, "cpu") + dataloader = torch.utils.data.DataLoader( + dataset, + batch_size=self.batch_size, + shuffle=False, + pin_memory=pin_memory_for(self.device), ) + owner_by_chunk = torch.tensor(owner_ids, dtype=torch.long) + accumulator = OwnerAccumulator(len(modality.data), len(dataset), aggregation) + + with inference_context(self.device): + for batch in dataloader: + chunk_ids = batch["id"].long() + batch = move_batch_to_device(batch, self.device) + _ = self.model(batch["data"]) + output = self.activation + if output.ndim > 2: + output = torch.nn.functional.adaptive_avg_pool2d(output, (1, 1)) + accumulator.update( + torch.flatten(output, 1), + owner_by_chunk.index_select(0, chunk_ids), + chunk_ids, + ) + + embeddings = accumulator.finalize(grouped=not is_image and aggregation is None) + if is_image and aggregation is None: + embeddings = np.asarray(embeddings) transformed_modality = TransformedModality( modality, self, self.output_modality_type ) - + transformed_modality.data_type = np.float32 + transformed_modality.aggregate_dim = ( + None if aggregation is not None or is_image else (0,) + ) transformed_modality.data = embeddings - return transformed_modality - - def _transform_image_modality(self, modality): - dataset = CustomDataset(modality.data, self.data_type, self.device) - embeddings = [] - for instance in torch.utils.data.DataLoader(dataset, self.batch_size): - frames = instance["data"] - - _ = self.model(frames) - output = self.activations[self.layer_name] - - if len(output.shape) == 4: - output = torch.nn.functional.adaptive_avg_pool2d(output, (1, 1)) - - embeddings.extend(output.detach().cpu().float().numpy().astype(np.float32)) - - return np.array(embeddings) - - def _transform_video_modality(self, modality): - dataset = CustomDataset(modality.data, self.data_type, self.device) - embeddings = {} - for instance in torch.utils.data.DataLoader(dataset): - video_id = instance["id"][0] - frames = instance["data"][0] - embeddings[video_id] = [] - - for start_index in range(0, frames.shape[0], self.batch_size): - end_index = min(start_index + self.batch_size, frames.shape[0]) - frame_batch = frames[start_index:end_index] - - _ = self.model(frame_batch) - output = self.activations[self.layer_name] - - if len(output.shape) == 4: - output = torch.nn.functional.adaptive_avg_pool2d(output, (1, 1)) - - embeddings[video_id].extend( - torch.flatten(output, 1) - .detach() - .cpu() - .float() - .numpy() - .astype(np.float32) - ) - - embeddings[video_id] = np.array(embeddings[video_id]) - - return list(embeddings.values()) diff --git a/src/main/python/systemds/scuro/representations/wav2vec.py b/src/main/python/systemds/scuro/representations/wav2vec.py index c9f4025579a..d50465ca5c0 100644 --- a/src/main/python/systemds/scuro/representations/wav2vec.py +++ b/src/main/python/systemds/scuro/representations/wav2vec.py @@ -29,6 +29,16 @@ from systemds.scuro.representations.unimodal import UnimodalRepresentation from systemds.scuro.drsearch.operator_registry import register_representation from systemds.scuro.utils.memory_utility import get_device +from systemds.scuro.representations.utils import ( + LengthBucketBatchSampler, + OwnerAccumulator, + OwnedSequenceDataset, + move_batch_to_device, + pin_memory_for, + pool_transformer_output, + transformer_inference_context, +) +from torch.utils.data import DataLoader from transformers.utils import logging as transformers_logging @@ -38,14 +48,18 @@ @register_representation(ModalityType.AUDIO) class Wav2Vec(UnimodalRepresentation): cache_in_worker = True - instance_parallel = True + instance_parallel = False MODEL_NAME = "facebook/wav2vec2-base-960h" - def __init__(self, params=None): - super().__init__("Wav2Vec", ModalityType.TIMESERIES, {}) + def __init__(self, batch_size=8, params=None): + parameters = {"batch_size": [1, 2, 4, 8, 16, 32, 64]} + super().__init__("Wav2Vec", ModalityType.TIMESERIES, parameters) + self.batch_size = int((params or {}).get("batch_size", batch_size)) self._processor = None self._model = None + self.gpu_id = None + self.device = get_device() @staticmethod def _from_pretrained(loader_cls, name): @@ -66,29 +80,80 @@ def model(self): self._model = self._from_pretrained(Wav2Vec2Model, self.MODEL_NAME).float() return self._model + @property + def gpu_id(self): + return self._gpu_id + + @gpu_id.setter + def gpu_id(self, gpu_id): + self._gpu_id = gpu_id + self.device = get_device(gpu_id) + def transform(self, modality, aggregation=None): transformed_modality = TransformedModality( modality, self, self.output_modality_type ) - - result = [] - for i, sample in enumerate(modality.data): - sr = modality.metadata[i]["frequency"] - audio_resampled = librosa.resample( - np.array(sample), orig_sr=sr, target_sr=16000 + samples = [ + librosa.resample( + np.asarray(sample), + orig_sr=modality.metadata[owner_id]["frequency"], + target_sr=16000, ) - input = self.processor( - audio_resampled, sampling_rate=16000, return_tensors="pt", padding=True + for owner_id, sample in enumerate(modality.data) + ] + dataset = OwnedSequenceDataset(samples) + lengths = [len(sample) for sample in samples] + + def collate(batch): + audio, owner_ids, chunk_ids = zip(*batch) + inputs = self.processor( + list(audio), + sampling_rate=16000, + return_tensors="pt", + padding=True, + return_attention_mask=True, + ) + inputs = dict(inputs) + inputs["input_values"] = inputs["input_values"].float() + return ( + inputs, + torch.tensor(owner_ids, dtype=torch.long), + torch.tensor(chunk_ids, dtype=torch.long), ) - input.input_values = input.input_values.float() - input.data["input_values"] = input.data["input_values"].float() - with torch.no_grad(): - outputs = self.model(**input) - features = outputs.extract_features - # TODO: check how to get intermediate representations - result.append(torch.flatten(features.mean(dim=1)).detach().cpu().numpy()) - transformed_modality.data = np.array(result) + dataloader = DataLoader( + dataset, + batch_sampler=LengthBucketBatchSampler(lengths, self.batch_size), + collate_fn=collate, + pin_memory=pin_memory_for(self.device), + ) + model = self.model.to(self.device) + model.eval() + accumulator = OwnerAccumulator(len(dataset), len(dataset), aggregation) + + with transformer_inference_context(self.device): + for inputs, owner_ids, chunk_ids in dataloader: + inputs = move_batch_to_device(inputs, self.device) + outputs = model(**inputs) + features = outputs.extract_features + attention_mask = inputs.get("attention_mask") + if attention_mask is not None and hasattr( + model, "_get_feature_vector_attention_mask" + ): + attention_mask = model._get_feature_vector_attention_mask( + features.shape[1], attention_mask + ) + elif attention_mask is None: + attention_mask = torch.ones( + features.shape[:2], + dtype=torch.long, + device=features.device, + ) + pooled = pool_transformer_output(features, attention_mask) + accumulator.update(pooled, owner_ids, chunk_ids) + + transformed_modality.data = accumulator.finalize() + transformed_modality.data_type = np.float32 return transformed_modality def get_output_stats(self, input_stats) -> RepresentationStats: diff --git a/src/main/python/systemds/scuro/representations/window_aggregation.py b/src/main/python/systemds/scuro/representations/window_aggregation.py index 713a398312a..e8189e94312 100644 --- a/src/main/python/systemds/scuro/representations/window_aggregation.py +++ b/src/main/python/systemds/scuro/representations/window_aggregation.py @@ -20,8 +20,11 @@ # ------------------------------------------------------------- import inspect -import numpy as np import math +import os +from collections import defaultdict + +import numpy as np from systemds.scuro.modality.type import DataLayout, ModalityType @@ -50,6 +53,38 @@ def _accepts_axis(compute_feature): return accepts +_WINDOW_FEATURE_BATCH_SIZE = max( + 1, int(os.environ.get("SCURO_WINDOW_FEATURE_BATCH_SIZE", "64")) +) + + +def _compute_window_features(aggregation_function, windows): + """Compute independent windows in bounded vectorized batches when supported.""" + windows = [np.asarray(window) for window in windows] + compute_batched = getattr(aggregation_function, "compute_features_batched", None) + if not callable(compute_batched): + return [aggregation_function.compute_feature(window) for window in windows] + + results = [None] * len(windows) + grouped = defaultdict(list) + for index, window in enumerate(windows): + grouped[(window.shape, window.dtype.str)].append((index, window)) + + for group in grouped.values(): + for start in range(0, len(group), _WINDOW_FEATURE_BATCH_SIZE): + chunk = group[start : start + _WINDOW_FEATURE_BATCH_SIZE] + batch = np.stack([window for _, window in chunk]) + batch_result = np.asarray(compute_batched(batch)) + if batch_result.ndim == 0 or batch_result.shape[0] != len(chunk): + raise ValueError( + f"{aggregation_function.name}.compute_features_batched() must " + "preserve the leading batch dimension" + ) + for row, (index, _) in enumerate(chunk): + results[index] = batch_result[row] + return results + + def nested_aggregation_param_names(agg_cls): if not inspect.isclass(agg_cls): return set() @@ -497,15 +532,13 @@ def window_aggregate_single_level(self, instance, new_length): tail_result = self.aggregation_function.compute_feature(tail) full_result = _append_tail_row(full_result, tail_result) else: - full_result = np.stack( - [ - self.aggregation_function.compute_feature(full_batches[i]) - for i in range(full_batches.shape[0]) - ] - ) + windows = [full_batches[i] for i in range(full_batches.shape[0])] if tail.size: - tail_result = self.aggregation_function.compute_feature(tail) - full_result = _append_tail_row(full_result, tail_result) + windows.append(tail) + features = _compute_window_features(self.aggregation_function, windows) + full_result = np.stack(features[: full_batches.shape[0]]) + if tail.size: + full_result = _append_tail_row(full_result, features[-1]) return full_result @@ -585,8 +618,9 @@ def estimate_peak_memory_bytes(self, input_stats: RepresentationStats) -> dict: return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} def execute(self, modality): - windowed_data = [] + instance_windows = [] for instance in modality.data: + instance = np.asarray(instance) window_size = int(np.ceil(len(instance) / self.num_windows)) padding_size = int(window_size * self.num_windows - len(instance)) pad_width = [(0, 0)] * instance.ndim @@ -594,23 +628,28 @@ def execute(self, modality): instance = np.pad( instance, pad_width=pad_width, mode="constant", constant_values=0 ) - full_batches = instance.reshape( - self.num_windows, window_size, *instance.shape[1:] + instance_windows.append( + instance.reshape(self.num_windows, window_size, *instance.shape[1:]) ) - if _accepts_axis(self.aggregation_function.compute_feature): - f = self.aggregation_function.compute_feature(full_batches, axis=1) - else: - f = np.stack( - [ - self.aggregation_function.compute_feature(full_batches[i]) - for i in range(full_batches.shape[0]) - ] - ) + if _accepts_axis(self.aggregation_function.compute_feature): + windowed_data = [ + self.aggregation_function.compute_feature(windows, axis=1) + for windows in instance_windows + ] + else: + flat_windows = [ + window for windows in instance_windows for window in windows + ] + flat_features = _compute_window_features( + self.aggregation_function, flat_windows + ) + windowed_data = [ + np.stack(flat_features[start : start + self.num_windows]) + for start in range(0, len(flat_features), self.num_windows) + ] - windowed_data.append(f) - windowed_data = _pad_stack(windowed_data) - return windowed_data + return _pad_stack(windowed_data) @register_context_operator( @@ -694,19 +733,29 @@ def estimate_peak_memory_bytes(self, input_stats: RepresentationStats) -> dict: return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} def execute(self, modality): - windowed_data = [] + all_windows = [] + windows_per_instance = [] for instance in modality.data: + instance = np.asarray(instance) indices = np.cumsum(self._window_sizes(len(instance))) - output = [] start = 0 + count = 0 for end in indices: window = instance[start:end] window.setflags(write=False) - output.append(self.aggregation_function.compute_feature(window)) + all_windows.append(window) start = end + count += 1 + windows_per_instance.append(count) + + all_features = _compute_window_features(self.aggregation_function, all_windows) + windowed_data = [] + start = 0 + for count in windows_per_instance: + windowed_data.append(_pad_stack(all_features[start : start + count])) + start += count - windowed_data.append(_pad_stack(output)) windowed_data = _pad_stack(windowed_data) self.assert_output_stats(windowed_data) return windowed_data diff --git a/src/main/python/systemds/scuro/representations/word2vec.py b/src/main/python/systemds/scuro/representations/word2vec.py index fd1e148e117..0954a17e290 100644 --- a/src/main/python/systemds/scuro/representations/word2vec.py +++ b/src/main/python/systemds/scuro/representations/word2vec.py @@ -43,9 +43,9 @@ def get_embedding(sentence, model): @register_representation(ModalityType.TEXT) class W2V(UnimodalRepresentation): - def __init__(self, vector_size=150, min_count=1, output_file=None, params=None): + def __init__(self, vector_size=128, min_count=1, output_file=None, params=None): parameters = { - "vector_size": [50, 100, 150, 200], + "vector_size": [64, 128, 256, 512, 1024], "min_count": [1, 2, 4, 8], } super().__init__("Word2Vec", ModalityType.EMBEDDING, parameters) diff --git a/src/main/python/systemds/scuro/representations/x3d.py b/src/main/python/systemds/scuro/representations/x3d.py index bba22434fc4..fae60d1f1cd 100644 --- a/src/main/python/systemds/scuro/representations/x3d.py +++ b/src/main/python/systemds/scuro/representations/x3d.py @@ -18,25 +18,34 @@ # under the License. # # ------------------------------------------------------------- +import math +from typing import Any, Tuple + +import numpy as np +import torch +import torch.utils.data +import torchvision.models as models +from torchvision.models.video import r3d_18, s3d + +from systemds.scuro.dataloader.video_loader import VideoStats +from systemds.scuro.drsearch.operator_registry import register_representation +from systemds.scuro.modality.transformed import TransformedModality +from systemds.scuro.modality.type import ModalityType +from systemds.scuro.representations.representation import RepresentationStats +from systemds.scuro.representations.unimodal import UnimodalRepresentation +from systemds.scuro.representations.utils import ( + LengthBucketBatchSampler, + get_sequence_lengths, + inference_context, + move_batch_to_device, + pin_memory_for, + save_embeddings, +) from systemds.scuro.utils.static_variables import ( - compute_batch_size, get_device, get_device_for_model, ) from systemds.scuro.utils.torch_dataset import CustomDataset -from systemds.scuro.modality.transformed import TransformedModality -from systemds.scuro.representations.unimodal import UnimodalRepresentation -from systemds.scuro.representations.representation import RepresentationStats -from typing import Tuple, Any, Union -import torch.utils.data -import torch -from torchvision.models.video import r3d_18, s3d -import torchvision.models as models -import numpy as np -from systemds.scuro.modality.type import ModalityType -from systemds.scuro.drsearch.operator_registry import register_representation -from systemds.scuro.dataloader.video_loader import VideoStats -import math class Identity(torch.nn.Module): @@ -46,28 +55,53 @@ def forward(self, input_: torch.Tensor) -> torch.Tensor: @register_representation([ModalityType.VIDEO]) class X3D(UnimodalRepresentation): + cache_in_worker = True + def __init__( - self, layer="classifier.1", model_name="s3d", output_file=None, params=None + self, + layer="classifier.1", + model_name="s3d", + output_file=None, + batch_size=8, + params=None, ): self.data_type = torch.float32 if params is not None: model_name = params.get("model_name", model_name) layer = params.get("layer_name", layer) + batch_size = int(params.get("batch_size", batch_size)) self.model_name = model_name parameters = self._get_parameters() super().__init__("X3D", ModalityType.EMBEDDING, parameters) self.output_file = output_file self.layer_name = layer + self.batch_size = batch_size + self._gpu_id = self.device.index + self._activation_hook = None + self.activation = None self.model.eval() for param in self.model.parameters(): param.requires_grad = False self.model.fc = Identity() + @property + def gpu_id(self): + return self._gpu_id + + @gpu_id.setter + def gpu_id(self, gpu_id): + self._gpu_id = gpu_id + self.device = get_device(gpu_id) + if self.model is not None: + self.model = self.model.to(self.device) + def get_output_stats(self, input_stats) -> RepresentationStats: embedding_dim = 400 * math.floor((max(input_stats.max_length, 14) - 5) / 8) - return RepresentationStats(input_stats.num_instances, (embedding_dim,)) + return RepresentationStats( + input_stats.num_instances, (embedding_dim,), dtype=self.data_type + ) def estimate_output_memory_bytes(self, input_stats: VideoStats) -> int: embedding_dim = 400 * math.floor((max(input_stats.max_length, 14) - 5) / 8) @@ -76,7 +110,8 @@ def estimate_output_memory_bytes(self, input_stats: VideoStats) -> int: def estimate_peak_memory_bytes(self, input_stats: VideoStats) -> dict: temporal = max(input_stats.max_length, 14) input_bytes = ( - self.data_type.itemsize + self.batch_size + * self.data_type.itemsize * input_stats.max_channels * temporal * input_stats.max_height @@ -84,9 +119,11 @@ def estimate_peak_memory_bytes(self, input_stats: VideoStats) -> dict: ) output_bytes = self.estimate_output_memory_bytes(input_stats) n = max(input_stats.num_instances, 1) - output_bytes_batch = output_bytes / n + output_bytes_batch = output_bytes / n * self.batch_size - batch_peak_bytes = (input_bytes + 512 * self.data_type.itemsize) * 2 + batch_peak_bytes = ( + input_bytes + self.batch_size * 512 * self.data_type.itemsize + ) * 2 safety_margin_bytes = 100 * 1024 * 1024 @@ -129,7 +166,11 @@ def model_name(self, model_name): raise NotImplementedError def _get_parameters(self, high_level=True): - parameters = {"model_name": [], "layer_name": []} + parameters = { + "batch_size": [1, 2, 4, 8, 16, 32], + "model_name": [], + "layer_name": [], + } for m in ["r3d", "s3d"]: parameters["model_name"].append(m) @@ -160,65 +201,99 @@ def _get_parameters(self, high_level=True): parameters["layer_name"].append(name) return parameters - def transform(self, modality, aggregation=None): - sample = modality.data[0] if modality.data else "" - self.batch_size = compute_batch_size( - model=self.model, - device=self.device, - sample_data=sample, - tokenizer=None, - max_seq_length=None, - max_batch_size=128, - ) - dataset = CustomDataset(modality.data, self.data_type, self.device) + @staticmethod + def _collate_videos(samples): + video_ids = torch.tensor([sample["id"] for sample in samples]) + target_length = max(14, max(sample["data"].shape[0] for sample in samples)) + videos = [] + for sample in samples: + frames = sample["data"] + if frames.shape[0] < target_length: + pad = torch.zeros( + (target_length - frames.shape[0], *frames.shape[1:]), + dtype=frames.dtype, + ) + frames = torch.cat((frames, pad), dim=0) + videos.append(frames) + return {"id": video_ids, "data": torch.stack(videos)} - embeddings = {} - - activation = None + def transform(self, modality, aggregation=None): + self.model = self.model.to(self.device) + self.model.eval() + self.activation = None def get_features(name_): def hook( _module: torch.nn.Module, input_: Tuple[torch.Tensor], output: Any ): - nonlocal activation - activation = output + self.activation = output return hook - if self.layer_name: + if self.layer_name and self._activation_hook is None: for name, layer in self.model.named_modules(): if name == self.layer_name: - layer.register_forward_hook(get_features(name)) + self._activation_hook = layer.register_forward_hook( + get_features(name) + ) break - for instance in dataset: - video_id = instance["id"] - frames = instance["data"].to(self.device) - embeddings[video_id] = [] - - frames = frames.unsqueeze(0).permute(0, 2, 1, 3, 4) - if frames.shape[2] < 14: - pad_width = (0, 0, 0, 0, 0, 14 - frames.shape[2], 0, 0, 0, 0) - frames = torch.nn.functional.pad(frames, pad_width, mode="constant") - _ = self.model(frames) - values = activation - pooled = torch.nn.functional.adaptive_avg_pool2d(values, (1, 1)) - - embeddings[video_id] = ( - torch.flatten(pooled, 1).detach().cpu().numpy().flatten() - ) + dataset = CustomDataset(modality.data, self.data_type, "cpu") + lengths = [ + max(length, 14) + for length in get_sequence_lengths(modality.data, modality.metadata) + ] + dataloader = torch.utils.data.DataLoader( + dataset, + batch_sampler=LengthBucketBatchSampler( + lengths, self.batch_size, exact=True + ), + collate_fn=self._collate_videos, + pin_memory=pin_memory_for(self.device), + ) + embeddings = [None] * len(dataset) + + with inference_context(self.device): + for batch in dataloader: + batch = move_batch_to_device(batch, self.device) + video_ids = batch["id"].long() + frames = batch["data"].permute(0, 2, 1, 3, 4) + _ = self.model(frames) + values = self.activation + if isinstance(values, tuple): + values = values[0] + if values.ndim > 2: + values = torch.nn.functional.adaptive_avg_pool2d(values, (1, 1)) + vectors = torch.flatten(values, 1).detach().float().cpu().numpy() + for video_id, vector in zip(video_ids.cpu().tolist(), vectors): + embeddings[video_id] = vector + + if self.output_file is not None: + save_embeddings(embeddings, self.output_file) transformed_modality = TransformedModality( modality, self, self.output_modality_type ) - - transformed_modality.data = list(embeddings.values()) - + transformed_modality.data = embeddings + transformed_modality.data_type = np.float32 return transformed_modality class I3D(UnimodalRepresentation): - def __init__(self, layer="blocks.6", model_name="i3d", output_file=None): + _EMBEDDING_DIM = 400 + cache_in_worker = True + + def __init__( + self, + layer="blocks.6", + model_name="i3d", + output_file=None, + batch_size=8, + params=None, + ): + if params is not None: + layer = params.get("layer_name", layer) + batch_size = int(params.get("batch_size", batch_size)) self.model_name = model_name self.model = torch.hub.load( "facebookresearch/pytorchvideo", "i3d_r50", pretrained=True @@ -230,12 +305,42 @@ def __init__(self, layer="blocks.6", model_name="i3d", output_file=None): self.output_file = output_file self.layer_name = layer + self.batch_size = batch_size + self.data_type = torch.float32 + self._gpu_id = self.device.index + self._activation_hook = None + self.features = None self.model.eval() for param in self.model.parameters(): param.requires_grad = False + @property + def gpu_id(self): + return self._gpu_id + + @gpu_id.setter + def gpu_id(self, gpu_id): + self._gpu_id = gpu_id + self.device = get_device(gpu_id) + if self.model is not None: + self.model = self.model.to(self.device) + + def get_output_stats(self, input_stats) -> RepresentationStats: + return RepresentationStats( + input_stats.num_instances, + (self._EMBEDDING_DIM,), + output_shape_is_known=self.layer_name == "blocks.6", + dtype=self.data_type, + ) + + def estimate_output_memory_bytes(self, input_stats: VideoStats) -> int: + return input_stats.num_instances * self._EMBEDDING_DIM * self.data_type.itemsize + def _get_parameters(self, high_level=True): - parameters = {"layer_name": []} + parameters = { + "batch_size": [1, 2, 4, 8, 16, 32], + "layer_name": [], + } if high_level: parameters["layer_name"] = [ @@ -252,52 +357,58 @@ def _get_parameters(self, high_level=True): parameters["layer_name"].append(name) return parameters - def transform(self, modality): - sample = modality.data[0] if modality.data else "" - self.batch_size = compute_batch_size( - model=self.model, - device=self.device, - sample_data=sample, - tokenizer=None, - max_seq_length=None, - max_batch_size=128, - ) - dataset = CustomDataset(modality.data, torch.float32, self.device) - embeddings = {} - - features = None + def transform(self, modality, aggregation=None): + self.model = self.model.to(self.device) + self.model.eval() + self.features = None def get_features(name_): def hook( _module: torch.nn.Module, input_: Tuple[torch.Tensor], output: Any ): - # pooled = torch.nn.functional.adaptive_avg_pool3d(output, 1).squeeze() - nonlocal features - features = output.detach().cpu().numpy() + self.features = output return hook - if self.layer_name: + if self.layer_name and self._activation_hook is None: for name, layer in self.model.named_modules(): if name == self.layer_name: - layer.register_forward_hook(get_features(name)) + self._activation_hook = layer.register_forward_hook( + get_features(name) + ) break - for instance in dataset: - video_id = instance["id"] - frames = instance["data"].to(self.device) - embeddings[video_id] = [] - - batch = torch.transpose(frames, 1, 0) - batch = batch.unsqueeze(0) - _ = self.model(batch) - - embeddings[video_id] = features.flatten() + dataset = CustomDataset(modality.data, self.data_type, "cpu") + dataloader = torch.utils.data.DataLoader( + dataset, + batch_sampler=LengthBucketBatchSampler( + get_sequence_lengths(modality.data, modality.metadata), + self.batch_size, + exact=True, + ), + pin_memory=pin_memory_for(self.device), + ) + embeddings = [None] * len(dataset) + + with inference_context(self.device): + for batch in dataloader: + batch = move_batch_to_device(batch, self.device) + video_ids = batch["id"].long() + frames = batch["data"].permute(0, 2, 1, 3, 4) + _ = self.model(frames) + values = self.features + if isinstance(values, tuple): + values = values[0] + vectors = torch.flatten(values, 1).detach().float().cpu().numpy() + for video_id, vector in zip(video_ids.cpu().tolist(), vectors): + embeddings[video_id] = vector + + if self.output_file is not None: + save_embeddings(embeddings, self.output_file) transformed_modality = TransformedModality( modality, self, self.output_modality_type ) - - transformed_modality.data = list(embeddings.values()) - + transformed_modality.data = embeddings + transformed_modality.data_type = np.float32 return transformed_modality diff --git a/src/main/python/systemds/scuro/utils/checkpointing.py b/src/main/python/systemds/scuro/utils/checkpointing.py index e821798534c..2928dd0963a 100644 --- a/src/main/python/systemds/scuro/utils/checkpointing.py +++ b/src/main/python/systemds/scuro/utils/checkpointing.py @@ -123,7 +123,7 @@ def save(self, results: Any, meta: Dict[str, Any]) -> str: def save_checkpoint(self, results: Any, extra_meta: Dict[str, Any] = {}): meta = {"eval_count": self.eval_count} - # meta.update(extra_meta or {}) + meta.update(extra_meta or {}) self.save(results, meta) def checkpoint_if_due(self, results: Any, extra_meta: Dict[str, Any] = None): diff --git a/src/main/python/systemds/scuro/utils/memory_utility.py b/src/main/python/systemds/scuro/utils/memory_utility.py index 88698fa53cc..ece738d050e 100644 --- a/src/main/python/systemds/scuro/utils/memory_utility.py +++ b/src/main/python/systemds/scuro/utils/memory_utility.py @@ -167,8 +167,28 @@ def get_gpu_memory_mb(device): def gpu_memory_info(): - infos = [] num_gpus = torch.cuda.device_count() + if num_gpus == 0: + return [] + try: + import pynvml + + pynvml.nvmlInit() + try: + infos = [] + for i in range(num_gpus): + handle = pynvml.nvmlDeviceGetHandleByIndex(i) + mem = pynvml.nvmlDeviceGetMemoryInfo(handle) + infos.append( + dict(index=i, free_b=int(mem.free), total_b=int(mem.total)) + ) + return infos + finally: + pynvml.nvmlShutdown() + except Exception: + pass + + infos = [] for i in range(num_gpus): torch.cuda.set_device(i) free_b, total_b = torch.cuda.mem_get_info() diff --git a/src/main/python/tests/scuro/data_generator.py b/src/main/python/tests/scuro/data_generator.py index b30946fb7df..d5965d219b1 100644 --- a/src/main/python/tests/scuro/data_generator.py +++ b/src/main/python/tests/scuro/data_generator.py @@ -76,13 +76,13 @@ def __init__(self, indices, chunk_size, modality_type, data, data_type, metadata 30, max(d.shape[0] for d in data), sum(d.shape[0] for d in data) / len(data), - max(d.shape[1] for d in data), max(d.shape[2] for d in data), + max(d.shape[1] for d in data), max(d.shape[3] for d in data), chunk_size if chunk_size is not None else len(data), len(data), ) - elif modality_type == ModalityType.TIMESERIES: + elif modality_type in (ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL): self.stats = TimeseriesStats( max(len(d) for d in data), len(data), @@ -94,8 +94,8 @@ def __init__(self, indices, chunk_size, modality_type, data, data_type, metadata ) elif modality_type == ModalityType.IMAGE: self.stats = ImageStats( - max(d.shape[0] for d in data), max(d.shape[1] for d in data), + max(d.shape[0] for d in data), max(d.shape[2] for d in data), len(data), ( @@ -103,8 +103,8 @@ def __init__(self, indices, chunk_size, modality_type, data, data_type, metadata max(d.shape[1] for d in data), max(d.shape[2] for d in data), ), - average_width=sum(d.shape[0] for d in data) / len(data), - average_height=sum(d.shape[1] for d in data) / len(data), + average_width=sum(d.shape[1] for d in data) / len(data), + average_height=sum(d.shape[0] for d in data) / len(data), average_channels=sum(d.shape[2] for d in data) / len(data), ) @@ -255,6 +255,43 @@ def create_timeseries_data(self, num_instances, sequence_length, num_features=1) ] return data, self.metadata + def create_physiological_data( + self, num_instances, sequence_length, kind="ecg", fs=500.0 + ): + self.modality_type = ModalityType.PHYSIOLOGICAL + rng = np.random.default_rng(7) + data = [] + + for _ in range(num_instances): + t = np.arange(sequence_length) / fs + samples = np.arange(sequence_length) + if kind == "ecg": + signal = rng.normal(0.0, 0.01, sequence_length) + width = max(1.0, 0.02 * fs) + position = int(0.2 * fs) + while position < sequence_length: + signal += np.exp(-(((samples - position) / width) ** 2)) + position += int(rng.uniform(0.7, 0.9) * fs) + elif kind == "eda": + signal = 0.5 + 0.01 * t + rng.normal(0.0, 0.005, sequence_length) + width = max(1.0, 1.5 * fs) + for peak_time in np.arange(5.0, max(t[-1], 5.0), 10.0): + centre = peak_time * fs + signal += np.exp(-(((samples - centre) / width) ** 2)) + elif kind == "resp": + signal = np.sin(2 * np.pi * 0.25 * t) + rng.normal( + 0.0, 0.02, sequence_length + ) + else: + raise ValueError(f"Unsupported physiological signal kind: {kind}") + data.append(signal.astype(self.data_type)) + + self.metadata = [ + self.modality_type.create_metadata(["signal"], data[i]) + for i in range(num_instances) + ] + return data, self.metadata + def create_text_data(self, num_instances, num_sentences_per_instance=1): self.modality_type = ModalityType.TEXT subjects = [ diff --git a/src/main/python/tests/scuro/test_chunked_leaf_execution.py b/src/main/python/tests/scuro/test_chunked_leaf_execution.py new file mode 100644 index 00000000000..91bf0a2f2bf --- /dev/null +++ b/src/main/python/tests/scuro/test_chunked_leaf_execution.py @@ -0,0 +1,352 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +import os +from types import SimpleNamespace +import unittest + +import numpy as np + + +def _skip_if_session_uses_fork(test_case): + if test_case._session_uses_fork: + test_case.skipTest( + "session is running under SCURO_MP_CONTEXT=fork; creating a " + "worker pool here deadlocks the CUDA-using tests later in the " + "session" + ) + + +from systemds.scuro.drsearch.modality_shared_memory import unlink_shm +from systemds.scuro.drsearch.node_executor import ( + NodeExecutor, + _execute_leaf_batch_worker, +) +from systemds.scuro.drsearch.representation_dag import ( + CSEAwareDAGBuilder, + RepresentationDag, + RepresentationNode, +) +from systemds.scuro.drsearch.task import PerformanceMeasure +from systemds.scuro.modality.transformed import TransformedModality +from systemds.scuro.modality.type import ModalityType +from systemds.scuro.modality.unimodal_modality import UnimodalModality +from systemds.scuro.representations.representation import RepresentationStats +from systemds.scuro.representations.unimodal import UnimodalRepresentation +from tests.scuro.data_generator import TestDataLoader + +NUM_INSTANCES = 12 +CHUNK_SIZE = 4 + + +def _make_modality(chunk_size): + rng = np.random.default_rng(0) + data = [rng.random(160, dtype=np.float32) for _ in range(NUM_INSTANCES)] + metadata = [ + ModalityType.AUDIO.create_metadata(16000, data[i]) for i in range(NUM_INSTANCES) + ] + loader = TestDataLoader( + indices=np.arange(NUM_INSTANCES), + chunk_size=chunk_size, + modality_type=ModalityType.AUDIO, + data=data, + data_type=np.float32, + metadata=metadata, + ) + return UnimodalModality(data_loader=loader) + + +class CountingOperation(UnimodalRepresentation): + """Emits one 4-vector per instance it is given.""" + + def __init__(self, params=None): + super().__init__("CountingOperation", ModalityType.EMBEDDING) + + def get_output_stats(self, input_stats): + return RepresentationStats(input_stats.num_instances, (4,)) + + def estimate_memory_bytes(self, input_stats): + return 1024 + + def estimate_peak_memory_bytes(self, input_stats): + return {"cpu_peak_bytes": 1024, "gpu_peak_bytes": 0} + + def transform(self, modality, aggregation=None): + transformed = TransformedModality( + modality, self, self.output_modality_type, set_data=False + ) + n = len(modality.data) + transformed._data = [np.full(4, float(n), dtype=np.float32) for _ in range(n)] + return transformed + + +class FrameOperation(UnimodalRepresentation): + """A same-named frame encoder that honors pushed-down aggregation.""" + + def __init__(self, params=None): + super().__init__("FrameOperation", ModalityType.EMBEDDING) + + def transform(self, modality, aggregation=None): + transformed = TransformedModality( + modality, self, self.output_modality_type, set_data=False + ) + frame_embedding = np.arange(12, dtype=np.float32).reshape(3, 4) + transformed._data = [frame_embedding.copy() for _ in modality.data] + if aggregation is not None: + return aggregation.transform(transformed) + return transformed + + def get_output_stats(self, input_stats): + return RepresentationStats(input_stats.num_instances, (3, 4)) + + def estimate_memory_bytes(self, input_stats): + return 1024 + + def estimate_peak_memory_bytes(self, input_stats): + return {"cpu_peak_bytes": 1024, "gpu_peak_bytes": 0} + + +class RaggedFrameOperation(FrameOperation): + """Emits variable-length frame sequences without aggregation.""" + + def __init__(self, params=None): + super().__init__(params=params) + self.name = "RaggedFrameOperation" + + def transform(self, modality, aggregation=None): + transformed = TransformedModality( + modality, self, self.output_modality_type, set_data=False + ) + transformed._data = [ + np.full((index % 3 + 1, 4), index, dtype=np.float32) + for index in range(len(modality.data)) + ] + if aggregation is not None: + return aggregation.transform(transformed) + return transformed + + +class InstanceCountingTask: + """Reports how many instances actually reached the task.""" + + def estimate_peak_memory_bytes(self, input_stats): + return {"cpu_peak_bytes": 1024, "gpu_peak_bytes": 0} + + def get_output_stats(self, input_stats): + return RepresentationStats(input_stats.num_instances, (1,)) + + def run(self, data): + count = float(len(data)) + scores = [] + for split in ("train", "val", "test"): + measure = PerformanceMeasure(split, "accuracy") + measure.scores["accuracy"] = [count] + scores.append(measure.compute_averages()) + return scores + + +def _build_dag(modality): + builder = CSEAwareDAGBuilder() + leaf_id = builder.create_leaf_node(modality_id=modality.modality_id) + op_id = builder.create_operation_node(CountingOperation, [leaf_id], {}) + dag = builder.build(op_id) + + task_root_id = f"task_{dag.root_node_id}_0" + task_node = RepresentationNode( + node_id=task_root_id, + operation=None, + inputs=[dag.root_node_id], + parameters={ + "_node_kind": "task", + "_task_idx": 0, + "_dag_root_id": dag.root_node_id, + }, + ) + return [RepresentationDag(nodes=[*dag.nodes, task_node], root_node_id=task_root_id)] + + +class TestChunkedLeafExecution(unittest.TestCase): + def setUp(self): + self._session_uses_fork = os.environ.get("SCURO_MP_CONTEXT") == "fork" + previous = os.environ.get("SCURO_MP_CONTEXT") + os.environ["SCURO_MP_CONTEXT"] = "spawn" + if previous is None: + self.addCleanup(os.environ.pop, "SCURO_MP_CONTEXT", None) + else: + self.addCleanup(os.environ.__setitem__, "SCURO_MP_CONTEXT", previous) + + def _executor(self, modality): + """A NodeExecutor whose pool is torn down even if the test fails.""" + _skip_if_session_uses_fork(self) + executor = NodeExecutor( + dags=_build_dag(modality), + modalities=[modality], + tasks=[InstanceCountingTask()], + max_num_workers=2, + enable_checkpointing=False, + ) + self.addCleanup(executor._pool.shutdown) + return executor + + def test_chunked_leaf_is_not_preloaded(self): + """A streaming modality must not be materialised before scheduling. + + `has_data()` staying False is the observable consequence: the executor + left the leaf alone, and the chunk loop inside `apply_representations` + is what reads the data. + """ + modality = _make_modality(chunk_size=CHUNK_SIZE) + executor = self._executor(modality) + self.assertTrue(executor._loads_in_chunks(modality)) + + executor._load_leaf_modalities() + self.assertFalse( + modality.has_data(), + "chunked leaf was preloaded; BaseLoader.load() would have " + "returned only the first chunk", + ) + + def test_chunked_subset_remains_lazy_and_uses_full_dataset_indices(self): + """Test-only subsets must not index into whichever chunk is resident.""" + modality = _make_modality(chunk_size=CHUNK_SIZE) + modality.extract_raw_data() + self.assertEqual(len(modality.data), CHUNK_SIZE) + + subset_indices = [1, 5, 10] + subset = modality.subset(subset_indices) + + self.assertFalse(subset.has_data()) + self.assertEqual( + subset.data_loader.indices, + [modality.data_loader.indices[i] for i in subset_indices], + ) + transformed = subset.apply_representations([CountingOperation()]) + self.assertEqual(len(transformed["CountingOperation"].data), 3) + + def test_unchunked_leaf_is_still_preloaded(self): + """The skip must be narrow: a non-streaming leaf still loads up front.""" + modality = _make_modality(chunk_size=None) + executor = self._executor(modality) + self.assertFalse(executor._loads_in_chunks(modality)) + + executor._load_leaf_modalities() + self.assertTrue(modality.has_data()) + self.assertEqual(len(modality.data), NUM_INSTANCES) + executor._cleanup_leaf_shared_memory() + + def test_unchunked_ragged_representation_is_padded(self): + modality = _make_modality(chunk_size=None) + transformed = modality.apply_representation(RaggedFrameOperation()) + + self.assertEqual(len(transformed.data), NUM_INSTANCES) + self.assertTrue(all(value.shape == (3, 4) for value in transformed.data)) + np.testing.assert_array_equal(transformed.data[0][1:], np.zeros((2, 4))) + masks = [metadata["attention_masks"] for metadata in transformed.metadata] + np.testing.assert_array_equal(masks[0], np.array([1.0, 0.0, 0.0])) + np.testing.assert_array_equal(masks[1], np.array([1.0, 1.0, 0.0])) + np.testing.assert_array_equal(masks[2], np.array([1.0, 1.0, 1.0])) + + def test_chunked_run_sees_every_instance(self): + """End to end, the search must score the whole dataset. + + This one holds with or without the preload -- `iter_raw_data_chunks` + resets the loader and re-reads everything, so the preload wasted time + and memory rather than truncating results. It is here to pin that + skipping the preload did not cost coverage of the whole dataset. + """ + modality = _make_modality(chunk_size=CHUNK_SIZE) + executor = self._executor(modality) + result = executor.run() + + self.assertEqual(len(result["task_results"]), 1) + entry = result["task_results"][0] + self.assertIsNotNone(entry.val_score, "candidate produced no score at all") + self.assertEqual( + entry.val_score["accuracy"], + float(NUM_INSTANCES), + "the task saw a partial dataset", + ) + + def test_chunked_run_produces_one_metadata_entry_per_instance(self): + """Metadata must not be double-counted. + + TransformedModality seeds its metadata from the source modality's and + the chunk loop appends one entry per instance on top, so a leaf that + arrived carrying preloaded metadata produced len(chunk) extra entries. + """ + modality = _make_modality(chunk_size=CHUNK_SIZE) + # Exactly the state the old preload left the leaf in: carrying the + # first chunk's data and metadata. Without this the modality starts + # empty and the doubling cannot occur, so the test would pass either + # way and prove nothing. + modality.extract_raw_data() + self.assertEqual(len(modality.metadata), CHUNK_SIZE) + + transformed = modality.apply_representations([CountingOperation()]) + out = transformed["CountingOperation"] + + self.assertEqual(len(out.data), NUM_INSTANCES) + self.assertEqual( + len(out.metadata), + NUM_INSTANCES, + "metadata was seeded from the leaf and then appended to per " + "instance, so the preloaded chunk got counted twice", + ) + + def test_leaf_batch_keeps_same_named_nodes_and_pushes_down_aggregation(self): + """Batched frame encoders must yield one 2-D result per DAG node.""" + modality = _make_modality(chunk_size=CHUNK_SIZE) + aggregation = { + "aggregation": "mean", + "target_dimensions": 1, + "aggregate_leading": True, + } + nodes = [ + SimpleNamespace( + node_id=f"frame_node_{index}", + operation=FrameOperation, + parameters={"_pushdown_aggregation": aggregation}, + ) + for index in range(2) + ] + + value = _execute_leaf_batch_worker(nodes, modality, gpu_id=None) + self.addCleanup( + lambda: [ + unlink_shm(info["shm_name"]) + for info in value["shm_info"].values() + if info.get("shm_name") is not None + ] + ) + + self.assertEqual(set(value["results"]), {node.node_id for node in nodes}) + self.assertEqual(value["failed_nodes"], {}) + for transformed in value["results"].values(): + self.assertEqual(len(transformed.data), NUM_INSTANCES) + self.assertEqual( + np.asarray(transformed.data).shape, + (NUM_INSTANCES, 4), + "pushed-down frame aggregation was lost, leaving a 3-D result", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/main/python/tests/scuro/test_lazy_visual_loading.py b/src/main/python/tests/scuro/test_lazy_visual_loading.py new file mode 100644 index 00000000000..e2da64394fb --- /dev/null +++ b/src/main/python/tests/scuro/test_lazy_visual_loading.py @@ -0,0 +1,177 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +import shutil +import unittest +from unittest.mock import patch + +import numpy as np +import torch + +from systemds.scuro.dataloader.base_loader import LazyFileSequence +from systemds.scuro.dataloader.image_loader import ImageLoader +from systemds.scuro.dataloader.video_loader import VideoLoader +from systemds.scuro.modality.type import ModalityType +from systemds.scuro.modality.unimodal_modality import UnimodalModality +from systemds.scuro.representations.color_histogram import ColorHistogram +from systemds.scuro.representations.optical_flow import OpticalFlow +from systemds.scuro.representations.utils import flatten_owned_sequences +from systemds.scuro.utils.torch_dataset import CustomDataset +from tests.scuro.data_generator import setup_data + + +class TestLazyVisualLoading(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.test_file_path = "test_lazy_visual_data" + cls.data_generator = setup_data( + [ModalityType.IMAGE, ModalityType.VIDEO], + 2, + cls.test_file_path, + ) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.test_file_path, ignore_errors=True) + + def _loader(self, loader_type, indices=None, **kwargs): + modality_type = ( + ModalityType.IMAGE if loader_type is ImageLoader else ModalityType.VIDEO + ) + return loader_type( + self.data_generator.get_modality_path(modality_type), + indices or self.data_generator.indices, + **kwargs, + ) + + def test_unchunked_load_keeps_only_file_references(self): + for loader_type in (ImageLoader, VideoLoader): + with self.subTest(loader=loader_type.__name__): + loader = self._loader(loader_type) + with patch.object( + loader, "_decode_file", wraps=loader._decode_file + ) as decode: + data, metadata = loader.load() + + self.assertIsInstance(data, LazyFileSequence) + self.assertEqual(len(data), 2) + self.assertEqual(len(metadata), 2) + decode.assert_not_called() + + self.assertIsInstance(data[0], np.ndarray) + decode.assert_called_once() + + def test_custom_dataset_decodes_only_the_requested_batch(self): + loader = self._loader(ImageLoader) + with patch.object(loader, "_decode_file", wraps=loader._decode_file) as decode: + data, _ = loader.load() + dataset = CustomDataset(data, torch.float32, "cpu") + dataloader = torch.utils.data.DataLoader(dataset, batch_size=1) + + first_batch = next(iter(dataloader)) + + self.assertEqual(first_batch["data"].shape[0], 1) + decode.assert_called_once() + + def test_modality_subset_preserves_lazy_file_references(self): + loader = self._loader(ImageLoader) + modality = UnimodalModality(loader) + with patch.object(loader, "_decode_file", wraps=loader._decode_file) as decode: + modality.extract_raw_data() + subset = modality.subset([1]) + + self.assertIsInstance(subset.data, LazyFileSequence) + self.assertEqual(len(subset.data), 1) + decode.assert_not_called() + self.assertIsInstance(subset.data[0], np.ndarray) + decode.assert_called_once() + + def test_unchunked_peak_memory_is_not_the_full_corpus(self): + for loader_type in (ImageLoader, VideoLoader): + loader = self._loader(loader_type) + modality = UnimodalModality(loader) + peak = modality.estimate_peak_memory_bytes()["cpu_peak_bytes"] + total = modality.estimate_memory_bytes() + self.assertLess(peak, total) + + def test_chunked_loading_still_returns_decoded_chunks(self): + loader = self._loader(ImageLoader, chunk_size=1) + + first_data, first_metadata = loader.load() + second_data, second_metadata = loader.load() + + self.assertIsInstance(first_data, list) + self.assertNotIsInstance(first_data, LazyFileSequence) + self.assertEqual(len(first_data), 1) + self.assertEqual(len(first_metadata), 1) + self.assertEqual(len(second_data), 1) + self.assertEqual(len(second_metadata), 1) + + def test_histogram_streams_lazy_images_and_videos(self): + for loader_type in (ImageLoader, VideoLoader): + with self.subTest(loader=loader_type.__name__): + loader = self._loader(loader_type) + modality = UnimodalModality(loader) + with patch.object( + loader, "_decode_file", wraps=loader._decode_file + ) as decode: + transformed = modality.apply_representation( + ColorHistogram(bins=4, normalize=True) + ) + + self.assertEqual(len(transformed.data), 2) + self.assertEqual(decode.call_count, 2) + + def test_optical_flow_streams_one_lazy_video_at_a_time(self): + loader = self._loader(VideoLoader, indices=self.data_generator.indices[:1]) + modality = UnimodalModality(loader) + with patch.object(loader, "_decode_file", wraps=loader._decode_file) as decode: + transformed = modality.apply_representation(OpticalFlow()) + + self.assertEqual(len(transformed.data), 1) + self.assertEqual(len(transformed.data[0]), loader.stats.max_length - 1) + decode.assert_called_once() + + def test_flattened_frame_view_caches_only_the_current_owner(self): + class CountingSequences: + def __init__(self): + self.values = [ + np.arange(2).reshape(2, 1), + np.arange(3).reshape(3, 1), + ] + self.reads = [] + + def __getitem__(self, index): + self.reads.append(index) + return self.values[index] + + def __len__(self): + return len(self.values) + + sequences = CountingSequences() + frames, owner_ids = flatten_owned_sequences(sequences, [2, 3]) + + np.testing.assert_array_equal(frames[0], [0]) + np.testing.assert_array_equal(frames[1], [1]) + self.assertEqual(sequences.reads, [0]) + np.testing.assert_array_equal(frames[2], [0]) + self.assertEqual(sequences.reads, [0, 1]) + self.assertEqual(owner_ids, [0, 0, 1, 1, 1]) diff --git a/src/main/python/tests/scuro/test_modality_pad.py b/src/main/python/tests/scuro/test_modality_pad.py new file mode 100644 index 00000000000..b6c0001250c --- /dev/null +++ b/src/main/python/tests/scuro/test_modality_pad.py @@ -0,0 +1,105 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +import unittest + +import numpy as np + +from systemds.scuro.modality.modality import Modality +from systemds.scuro.modality.type import ModalityType +from systemds.scuro.representations.sum import Sum + + +def _embedding_metadata(embedding_dim): + return [ + { + "data_layout": { + "shape": (embedding_dim,), + "type": np.float32, + "representation": "embedding", + } + } + ] + + +class TestModalityPad(unittest.TestCase): + def test_pad_single_instance_1d_embedding(self): + modality = Modality( + ModalityType.EMBEDDING, + modality_id=1, + metadata=_embedding_metadata(4), + data_type=np.float32, + ) + modality._data = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32) + + modality.pad(max_len=6) + + self.assertEqual(modality.data.shape, (1, 6)) + np.testing.assert_array_equal( + modality.data[0, :4], np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32) + ) + np.testing.assert_array_equal(modality.data[0, 4:], np.zeros(2)) + + def test_pad_2d_embedding_columns(self): + modality = Modality( + ModalityType.EMBEDDING, + modality_id=1, + metadata=_embedding_metadata(3) * 2, + data_type=np.float32, + ) + modality._data = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], dtype=np.float32) + + modality.pad(max_len=5) + + self.assertEqual(modality.data.shape, (2, 5)) + np.testing.assert_array_equal(modality.data[0, :3], np.array([1.0, 2.0, 3.0])) + np.testing.assert_array_equal(modality.data[1, :3], np.array([4.0, 5.0, 6.0])) + + def test_fusion_sum_aligns_mismatched_embedding_sizes(self): + metadata_a = _embedding_metadata(4) + metadata_b = _embedding_metadata(6) + + modality_a = Modality( + ModalityType.EMBEDDING, + modality_id=1, + metadata=metadata_a, + data_type=np.float32, + ) + modality_a._data = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32) + + modality_b = Modality( + ModalityType.EMBEDDING, + modality_id=2, + metadata=metadata_b, + data_type=np.float32, + ) + modality_b._data = np.array([[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]], dtype=np.float32) + + fused = Sum().transform([modality_a, modality_b]) + + self.assertEqual(fused.shape, (1, 6)) + np.testing.assert_array_equal( + fused[0], np.array([2.0, 4.0, 6.0, 8.0, 5.0, 6.0], dtype=np.float32) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/main/python/tests/scuro/test_neural_encoder_batching.py b/src/main/python/tests/scuro/test_neural_encoder_batching.py new file mode 100644 index 00000000000..3094e15f1a2 --- /dev/null +++ b/src/main/python/tests/scuro/test_neural_encoder_batching.py @@ -0,0 +1,156 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- +import unittest +from types import SimpleNamespace + +import numpy as np +import torch + +from systemds.scuro.modality.type import ModalityType +from systemds.scuro.representations.aggregated_representation import ( + AggregatedRepresentation, +) +from systemds.scuro.representations.clip import CLIPVisual +from systemds.scuro.representations.resnet import ResNet +from systemds.scuro.representations.vgg import VGG19 + + +class _ResNetModel(torch.nn.Module): + def __init__(self): + super().__init__() + self.scale = torch.nn.Parameter(torch.tensor(1.0)) + self.avgpool = torch.nn.Identity() + + def forward(self, images): + values = images.flatten(2).mean(dim=2)[:, :2] * self.scale + return self.avgpool(values[:, :, None, None]) + + +class _VGGModel(torch.nn.Module): + def __init__(self): + super().__init__() + self.scale = torch.nn.Parameter(torch.tensor(1.0)) + self.classifier = torch.nn.Sequential(torch.nn.Identity()) + + def forward(self, images): + values = images.flatten(2).mean(dim=2)[:, :2] * self.scale + return self.classifier[0](values) + + +def _video_modality(videos): + height, width, channels = videos[0][0].shape + return SimpleNamespace( + modality_type=ModalityType.VIDEO, + modality_id=0, + metadata=[ + ModalityType.VIDEO.create_metadata(30, len(video), width, height, channels) + for video in videos + ], + data_type=np.float32, + transform_time=0, + data=videos, + ) + + +def _representation(representation_class, batch_size): + representation = object.__new__(representation_class) + representation.data_type = torch.float32 + representation.device = torch.device("cpu") + representation.batch_size = batch_size + representation._activation_hook = None + representation.activation = None + representation.output_modality_type = ModalityType.EMBEDDING + if representation_class is ResNet: + representation.model = _ResNetModel() + representation.layer_name = "avgpool" + else: + representation.model = _VGGModel() + representation.layer_name = "classifier.0" + return representation + + +class TestNeuralEncoderBatching(unittest.TestCase): + def setUp(self): + self.videos = [ + [ + np.full((8, 8, 3), 32, dtype=np.uint8), + np.full((8, 8, 3), 64, dtype=np.uint8), + ], + [ + np.full((8, 8, 3), 96, dtype=np.uint8), + np.full((8, 8, 3), 128, dtype=np.uint8), + np.full((8, 8, 3), 160, dtype=np.uint8), + ], + ] + self.aggregation = AggregatedRepresentation("mean") + + def test_global_frame_batching_is_invariant_to_batch_size(self): + for representation_class in (ResNet, VGG19): + with self.subTest(representation=representation_class.__name__): + results = [ + _representation(representation_class, batch_size) + .transform(_video_modality(self.videos), self.aggregation) + .data + for batch_size in (1, 4) + ] + self.assertEqual(results[0].shape, (len(self.videos), 2)) + np.testing.assert_allclose(results[1], results[0]) + + def test_global_frame_batching_matches_patient_by_patient(self): + for representation_class in (ResNet, VGG19): + with self.subTest(representation=representation_class.__name__): + global_result = ( + _representation(representation_class, 3) + .transform(_video_modality(self.videos), self.aggregation) + .data + ) + per_patient = np.concatenate( + [ + _representation(representation_class, 3) + .transform(_video_modality([video]), self.aggregation) + .data + for video in self.videos + ], + axis=0, + ) + np.testing.assert_allclose(global_result, per_patient) + + def test_frame_encoders_support_aggregation_pushdown(self): + input_stats = SimpleNamespace(num_instances=3) + for representation_class, hidden_dim in ( + (ResNet, 512), + (VGG19, 4096), + (CLIPVisual, 512), + ): + with self.subTest(representation=representation_class.__name__): + representation = object.__new__(representation_class) + representation.params = {"_pushdown_aggregation": {}} + representation.data_type = torch.float32 + + self.assertTrue(representation.supports_aggregation_pushdown) + stats = representation.get_output_stats(input_stats) + self.assertEqual(stats.num_instances, input_stats.num_instances) + self.assertEqual(stats.output_shape, (hidden_dim,)) + self.assertIsNone(stats.aggregate_dim) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/main/python/tests/scuro/test_operator_registry.py b/src/main/python/tests/scuro/test_operator_registry.py index 93afba342b0..99bfeb2f2a4 100644 --- a/src/main/python/tests/scuro/test_operator_registry.py +++ b/src/main/python/tests/scuro/test_operator_registry.py @@ -75,7 +75,10 @@ from systemds.scuro.representations.mel_spectrogram import MelSpectrogram from systemds.scuro.representations.spectrogram import Spectrogram from systemds.scuro.representations.hadamard import Hadamard +from systemds.scuro.representations.image_bind import ImageBind from systemds.scuro.representations.resnet import ResNet +from systemds.scuro.representations.openface import OpenFace +from systemds.scuro.representations.color_histogram import ColorHistogram from systemds.scuro.representations.multimodal_attention_fusion import AttentionFusion from systemds.scuro.representations.physiological_window import ( AdaptiveWindow, @@ -89,6 +92,7 @@ def test_audio_representations_in_registry(self): assert registry.get_representations(ModalityType.AUDIO) == [ MelSpectrogram, MFCC, + ImageBind, Spectrogram, Wav2Vec, Spectral, @@ -101,8 +105,11 @@ def test_video_representations_in_registry(self): registry = Registry() assert registry.get_representations(ModalityType.VIDEO) == [ ResNet, + OpenFace, + ImageBind, SwinVideoTransformer, X3D, + ColorHistogram, VGG19, CLIPVisual, ] diff --git a/src/main/python/tests/scuro/test_transformer_text_aggregation.py b/src/main/python/tests/scuro/test_transformer_text_aggregation.py new file mode 100644 index 00000000000..ded26859917 --- /dev/null +++ b/src/main/python/tests/scuro/test_transformer_text_aggregation.py @@ -0,0 +1,353 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- +import copy +import unittest +from types import SimpleNamespace + +import numpy as np +import torch + +from systemds.scuro.drsearch.representation_dag import ( + CSEAwareDAGBuilder, + pushdown_aggregation, +) +from systemds.scuro.representations.aggregated_representation import ( + AggregatedRepresentation, +) +from systemds.scuro.representations.bert import Bert +from systemds.scuro.representations.clip import CLIPText +from systemds.scuro.representations.representation import RepresentationStats +from systemds.scuro.representations.utils import pool_transformer_output + + +class _BatchEncoding(dict): + @property + def data(self): + return self + + def to(self, device): + for key, value in self.items(): + self[key] = value.to(device) + return self + + +class _Tokenizer: + def __call__(self, batch, **kwargs): + ids = torch.tensor([int(text) for text in batch]) + input_ids = ids.unsqueeze(1).repeat(1, 3) + attention_mask = torch.tensor([[1, 1, 0]]).repeat(len(batch), 1) + return _BatchEncoding( + input_ids=input_ids, + attention_mask=attention_mask, + offset_mapping=torch.zeros((len(batch), 3, 2), dtype=torch.long), + ) + + +class _BertModel: + def __call__(self, input_ids, attention_mask): + ids = input_ids[:, 0].float() + hidden = torch.stack( + ( + torch.stack((ids, ids + 10), dim=1), + torch.stack((ids + 100, ids + 200), dim=1), + torch.full((len(ids), 2), 1000.0, device=ids.device), + ), + dim=1, + ) + return SimpleNamespace(last_hidden_state=hidden) + + +class _DynamicTokenizer: + def __call__(self, batch, **kwargs): + tokens = [[int(token) for token in text.split()] for text in batch] + max_length = kwargs.get("max_length") + if max_length is not None: + tokens = [values[:max_length] for values in tokens] + + if kwargs.get("return_tensors") != "pt": + return _BatchEncoding( + input_ids=tokens, + attention_mask=[[1] * len(values) for values in tokens], + ) + + padded_length = max(len(values) for values in tokens) + input_ids = torch.zeros((len(tokens), padded_length), dtype=torch.long) + attention_mask = torch.zeros_like(input_ids) + for row, values in enumerate(tokens): + input_ids[row, : len(values)] = torch.tensor(values) + attention_mask[row, : len(values)] = 1 + return _BatchEncoding( + input_ids=input_ids, + attention_mask=attention_mask, + offset_mapping=torch.zeros( + (len(tokens), padded_length, 2), dtype=torch.long + ), + ) + + +class _IntermediateBertModel: + def __init__(self, representation): + self.representation = representation + + def __call__(self, input_ids, attention_mask): + values = input_ids.float() + hidden = torch.stack((values, values + 10), dim=-1) + hidden = torch.where( + attention_mask.unsqueeze(-1).bool(), + hidden, + torch.full_like(hidden, 1000), + ) + self.representation.bert_output = hidden + return SimpleNamespace(last_hidden_state=hidden) + + +class _CLIPProcessor: + def __call__(self, text, **kwargs): + ids = torch.tensor([int(value) for value in text]) + return _BatchEncoding( + input_ids=ids.unsqueeze(1), + attention_mask=torch.tensor([[1, 1, 0]]).repeat(len(text), 1), + ) + + +class _CLIPTextModel: + def __init__(self, representation): + self.representation = representation + + def __call__(self, input_ids, attention_mask): + ids = input_ids[:, 0].float() + self.representation.clip_output = torch.stack( + ( + torch.stack((ids, ids + 2), dim=1), + torch.stack((ids + 2, ids + 4), dim=1), + torch.full((len(ids), 2), 1000.0, device=ids.device), + ), + dim=1, + ) + + +class _CLIPModel: + def __init__(self, representation): + self.text_model = _CLIPTextModel(representation) + + +class TestTransformerTextAggregation(unittest.TestCase): + def test_token_pooling_selects_cls_or_masked_mean(self): + hidden = torch.tensor( + [ + [[1.0, 2.0], [3.0, 4.0], [1000.0, 1000.0]], + [[5.0, 6.0], [1000.0, 1000.0], [1000.0, 1000.0]], + ] + ) + mask = torch.tensor([[1, 1, 0], [1, 0, 0]]) + + np.testing.assert_allclose( + pool_transformer_output(hidden, mask, use_cls=True).numpy(), + [[1.0, 2.0], [5.0, 6.0]], + ) + np.testing.assert_allclose( + pool_transformer_output(hidden, mask).numpy(), + [[2.0, 3.0], [5.0, 6.0]], + ) + + def test_output_stats_describe_pooled_chunk_vectors(self): + raw_stats = SimpleNamespace(num_instances=3) + context_stats = RepresentationStats(3, (5, 77)) + + bert_plain = Bert().get_output_stats(raw_stats) + bert_context = Bert().get_output_stats(context_stats) + clip_plain = CLIPText().get_output_stats(raw_stats) + clip_context = CLIPText().get_output_stats(context_stats) + + self.assertEqual(bert_plain.output_shape, (768,)) + self.assertEqual(bert_context.output_shape, (5, 768)) + self.assertEqual(bert_context.aggregate_dim, (0,)) + self.assertEqual(clip_plain.output_shape, (512,)) + self.assertEqual(clip_context.output_shape, (5, 512)) + self.assertEqual(clip_context.aggregate_dim, (0,)) + + def test_bert_cls_aggregation_is_independent_of_batch_boundaries(self): + representation = Bert(batch_size=2, max_seq_length=3) + result = representation.create_embeddings( + ["0", "1", "2", "3", "4"], + _BertModel(), + _Tokenizer(), + AggregatedRepresentation("mean"), + ) + + np.testing.assert_allclose(result, [2.0, 12.0]) + self.assertEqual(result.shape, (2,)) + + def test_global_batching_returns_one_vector_per_patient(self): + chunks = ["1", "3 5", "7 9 11", "2", "4 6"] + owner_ids = [0, 0, 1, 2, 2] + representation = Bert(layer="intermediate", batch_size=3, max_seq_length=8) + result = representation.create_embeddings( + chunks, + _IntermediateBertModel(representation), + _DynamicTokenizer(), + AggregatedRepresentation("mean"), + owner_ids=owner_ids, + num_owners=3, + ) + + self.assertEqual(result.shape, (3, 2)) + + def test_global_batching_is_invariant_to_batch_size(self): + chunks = ["1", "3 5", "7 9 11", "2", "4 6"] + owner_ids = [0, 0, 1, 2, 2] + results = [] + for batch_size in (1, 2, 4): + representation = Bert( + layer="intermediate", + batch_size=batch_size, + max_seq_length=8, + ) + results.append( + representation.create_embeddings( + chunks, + _IntermediateBertModel(representation), + _DynamicTokenizer(), + AggregatedRepresentation("mean"), + owner_ids=owner_ids, + num_owners=3, + ) + ) + + for result in results[1:]: + np.testing.assert_allclose(result, results[0]) + + def test_dynamic_padding_does_not_change_embeddings(self): + short = "2 4" + representation = Bert(layer="intermediate", batch_size=2, max_seq_length=8) + model = _IntermediateBertModel(representation) + tokenizer = _DynamicTokenizer() + + alone = representation.create_embeddings([short], model, tokenizer)[0] + mixed = representation.create_embeddings( + [short, "10 20 30 40 50"], model, tokenizer + )[0] + + np.testing.assert_allclose(mixed, alone) + + def test_global_batching_matches_patient_by_patient_batching(self): + patient_chunks = [["1", "3 5"], ["7 9 11"], ["2", "4 6"]] + chunks = [chunk for patient in patient_chunks for chunk in patient] + owner_ids = [ + owner_id for owner_id, patient in enumerate(patient_chunks) for _ in patient + ] + aggregation = AggregatedRepresentation("mean") + representation = Bert(layer="intermediate", batch_size=3, max_seq_length=8) + model = _IntermediateBertModel(representation) + tokenizer = _DynamicTokenizer() + + global_result = representation.create_embeddings( + chunks, + model, + tokenizer, + aggregation, + owner_ids=owner_ids, + num_owners=len(patient_chunks), + ) + per_patient_result = np.stack( + [ + representation.create_embeddings(patient, model, tokenizer, aggregation) + for patient in patient_chunks + ] + ) + + np.testing.assert_allclose(global_result, per_patient_result) + + def test_clip_intermediate_layer_uses_masked_mean_before_chunk_aggregation(self): + representation = CLIPText(batch_size=2, layer_name="intermediate") + representation.processor = _CLIPProcessor() + result = representation.create_text_embeddings( + ["0", "2", "4"], + _CLIPModel(representation), + AggregatedRepresentation("mean"), + ) + + np.testing.assert_allclose(result, [3.0, 5.0]) + self.assertEqual(result.shape, (2,)) + + def test_pushdown_keeps_shared_plain_and_aggregated_paths_distinct(self): + builder = CSEAwareDAGBuilder() + leaf_id = builder.create_leaf_node("transformer_pushdown") + bert = Bert() + bert_id = builder.create_operation_node( + Bert, [leaf_id], bert.get_current_parameters() + ) + aggregation = AggregatedRepresentation( + "mean", target_dimensions=1, aggregate_leading=True + ) + aggregation_id = builder.create_operation_node( + AggregatedRepresentation, + [bert_id], + aggregation.get_current_parameters(), + ) + + plain_dag = builder.build(bert_id) + aggregated_dag = builder.build(aggregation_id) + aggregation_params = copy.deepcopy( + aggregated_dag.get_node_by_id(aggregation_id).parameters + ) + + pushdown_aggregation([plain_dag, aggregated_dag]) + + plain_node = plain_dag.get_node_by_id(bert_id) + self.assertIs(plain_node.operation, Bert) + self.assertNotIn("_pushdown_aggregation", plain_node.parameters) + + pushed_node = aggregated_dag.get_node_by_id(aggregation_id) + self.assertIs(pushed_node.operation, Bert) + self.assertEqual(pushed_node.inputs, [leaf_id]) + self.assertEqual( + pushed_node.parameters["_pushdown_aggregation"], aggregation_params + ) + self.assertIsNone(aggregated_dag.get_node_by_id(bert_id)) + + def test_clip_text_supports_aggregation_pushdown(self): + builder = CSEAwareDAGBuilder() + leaf_id = builder.create_leaf_node("clip_pushdown") + clip_id = builder.create_operation_node( + CLIPText, [leaf_id], CLIPText().get_current_parameters() + ) + aggregation = AggregatedRepresentation("max", target_dimensions=1) + aggregation_id = builder.create_operation_node( + AggregatedRepresentation, + [clip_id], + aggregation.get_current_parameters(), + ) + dag = builder.build(aggregation_id) + + pushdown_aggregation([dag]) + + pushed_node = dag.get_node_by_id(aggregation_id) + self.assertIs(pushed_node.operation, CLIPText) + pushed_aggregation = AggregatedRepresentation( + params=pushed_node.parameters["_pushdown_aggregation"] + ) + self.assertEqual(pushed_aggregation.aggregation_function, "max") + self.assertIsNone(dag.get_node_by_id(clip_id)) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/main/python/tests/scuro/test_unimodal_optimizer.py b/src/main/python/tests/scuro/test_unimodal_optimizer.py index 11c3aa29ea6..41717b4e3a8 100644 --- a/src/main/python/tests/scuro/test_unimodal_optimizer.py +++ b/src/main/python/tests/scuro/test_unimodal_optimizer.py @@ -28,11 +28,24 @@ from systemds.scuro.drsearch.unimodal_optimizer import UnimodalOptimizer from systemds.scuro.representations.covarep_audio_features import ZeroCrossing +from systemds.scuro.representations.covarep_audio_features import ( + Spectral, + RMSE, + Pitch, +) from systemds.scuro.representations.resnet import ResNet from systemds.scuro.representations.mel_spectrogram import MelSpectrogram +from systemds.scuro.representations.mfcc import MFCC +from systemds.scuro.representations.mlp_averaging import MLPAveraging +from systemds.scuro.representations.spectrogram import Spectrogram from systemds.scuro.representations.tfidf import TfIdf from systemds.scuro.representations.bow import BoW from systemds.scuro.representations.bert import Bert +from systemds.scuro.representations.word2vec import W2V +from tests.scuro.test_unimodal_representations import ( + PHYSIOLOGICAL_REPRESENTATIONS, + TIMESERIES_REPRESENTATIONS, +) from systemds.scuro.modality.unimodal_modality import UnimodalModality from tests.scuro.data_generator import ( ModalityRandomDataGenerator, @@ -62,6 +75,37 @@ ModalityType.EMBEDDING: [], } +#: Every registered representation that runs without downloading a pretrained +#: model. The transformer- and CNN-based ones (Bert, RoBERTa, CLIP, GloVe, X3D, +#: VGG19, Swin, Wav2Vec) are deliberately absent: they pull hundreds of MB over +#: the network, which is the same reason the video representation test in +#: test_unimodal_representations.py is commented out. +FULL_TEXT_REPRESENTATIONS = [BoW, TfIdf, W2V] +FULL_AUDIO_REPRESENTATIONS = [ + MFCC, + MelSpectrogram, + Spectrogram, + Spectral, + RMSE, + Pitch, + ZeroCrossing, +] +FULL_IMAGE_REPRESENTATIONS = [ColorHistogram] +FULL_TIMESERIES_REPRESENTATIONS = TIMESERIES_REPRESENTATIONS +FULL_PHYSIOLOGICAL_REPRESENTATIONS = PHYSIOLOGICAL_REPRESENTATIONS + + +def registry_for(modality_type, representations): + """A registry holding `representations` for one modality and nothing else. + + Every modality type has to be present: the optimizer looks its modality up + directly, and a partial dict would raise a KeyError rather than search an + empty space. + """ + registry = {m_type: [] for m_type in ModalityType} + registry[modality_type] = representations + return registry + class TestUnimodalRepresentationOptimizer(unittest.TestCase): data_generator = None @@ -89,6 +133,38 @@ def test_unimodal_optimizer_for_text_modality(self): ) self.optimize_unimodal_representation_for_modality([text]) + def test_bow_and_tfidf_require_dimensionality_reduction_before_task(self): + text_data, text_md = ModalityRandomDataGenerator().create_text_data( + self.num_instances, 10 + ) + text = UnimodalModality( + TestDataLoader( + self.indices, None, ModalityType.TEXT, text_data, str, text_md + ) + ) + + dimensionality_reduction_operators = {ModalityType.EMBEDDING: [MLPAveraging]} + for representation in (BoW, TfIdf): + with self.subTest(representation=representation.__name__), patch.object( + Registry, + "_representations", + registry_for(ModalityType.TEXT, [representation]), + ), patch.object( + Registry, + "_dimensionality_reduction_operators", + dimensionality_reduction_operators, + ): + optimizer = UnimodalOptimizer( + [text], self.tasks, False, enable_checkpointing=False + ) + _, _, task_dags = optimizer._build_execution_dags_for_modality(text) + + self.assertGreater(len(task_dags), 0) + for dag in task_dags: + task_node = dag.get_node_by_id(dag.root_node_id) + task_input = dag.get_node_by_id(task_node.inputs[0]) + self.assertIs(task_input.operation, MLPAveraging) + def test_unimodal_optimizer_for_image_modality(self): image_data, image_md = ModalityRandomDataGenerator().create_visual_modality( self.num_instances, 1, 10, 10 @@ -133,7 +209,7 @@ def test_unimodal_optimizer_for_audio_modality(self): def test_unimodal_optimizer_for_video_modality(self): video_data, video_md = ModalityRandomDataGenerator().create_visual_modality( - self.num_instances, 10, 10 + self.num_instances, 10, 10, 10 ) video = UnimodalModality( TestDataLoader( @@ -142,6 +218,122 @@ def test_unimodal_optimizer_for_video_modality(self): ) self.optimize_unimodal_representation_for_modality([video]) + # ------------------------------------------------------------------ + # Every registered representation, run through the optimizer + # ------------------------------------------------------------------ + # + # The tests above check that the optimizer runs at all. These check that no + # individual representation breaks it: a rep whose get_output_stats, + # preconditions or transform disagree with what the executor expects takes + # the whole search down, and with a two-representation registry that would + # never surface. + + def _optimize_with_registry(self, modality, registry): + with patch.object(Registry, "_representations", registry): + Registry() + unimodal_optimizer = UnimodalOptimizer( + [modality], + self.tasks, + False, + k=1, + max_num_workers=1, + enable_checkpointing=False, + ) + unimodal_optimizer.optimize() + + self.assertIn( + modality.modality_id, + unimodal_optimizer.operator_performance.modality_ids, + ) + result, _ = unimodal_optimizer.operator_performance.get_k_best_results( + modality, self.tasks[0], "accuracy" + ) + self.assertEqual(len(result), 1) + return unimodal_optimizer + + def test_unimodal_optimizer_with_all_text_representations(self): + text_data, text_md = ModalityRandomDataGenerator().create_text_data( + self.num_instances, 10 + ) + text = UnimodalModality( + TestDataLoader( + self.indices, None, ModalityType.TEXT, text_data, str, text_md + ) + ) + self._optimize_with_registry( + text, registry_for(ModalityType.TEXT, FULL_TEXT_REPRESENTATIONS) + ) + + def test_unimodal_optimizer_with_all_audio_representations(self): + audio_data, audio_md = ModalityRandomDataGenerator().create_audio_data( + self.num_instances, 4000 + ) + audio = UnimodalModality( + TestDataLoader( + self.indices, None, ModalityType.AUDIO, audio_data, np.float32, audio_md + ) + ) + self._optimize_with_registry( + audio, registry_for(ModalityType.AUDIO, FULL_AUDIO_REPRESENTATIONS) + ) + + def test_unimodal_optimizer_with_all_image_representations(self): + image_data, image_md = ModalityRandomDataGenerator().create_visual_modality( + self.num_instances, 1, 10, 10 + ) + image = UnimodalModality( + TestDataLoader( + self.indices, None, ModalityType.IMAGE, image_data, np.float32, image_md + ) + ) + self._optimize_with_registry( + image, registry_for(ModalityType.IMAGE, FULL_IMAGE_REPRESENTATIONS) + ) + + def test_unimodal_optimizer_with_all_timeseries_representations(self): + ts_data, ts_md = ModalityRandomDataGenerator().create_timeseries_data( + self.num_instances, 256 + ) + timeseries = UnimodalModality( + TestDataLoader( + self.indices, + None, + ModalityType.TIMESERIES, + ts_data, + np.float32, + ts_md, + ) + ) + optimizer = self._optimize_with_registry( + timeseries, + registry_for(ModalityType.TIMESERIES, FULL_TIMESERIES_REPRESENTATIONS), + ) + # A search over windowed timeseries always proposes some configurations + # the input cannot express (a lag longer than the window, a moment on a + # two-sample window). Those must be pruned up front, not executed. + self.assertGreater(len(optimizer.pruned), 0) + + def test_unimodal_optimizer_with_all_physiological_representations(self): + data, md = ModalityRandomDataGenerator().create_physiological_data( + self.num_instances, 2000, kind="ecg", fs=500.0 + ) + physiological = UnimodalModality( + TestDataLoader( + self.indices, + None, + ModalityType.PHYSIOLOGICAL, + data, + np.float32, + md, + ) + ) + self._optimize_with_registry( + physiological, + registry_for( + ModalityType.PHYSIOLOGICAL, FULL_PHYSIOLOGICAL_REPRESENTATIONS + ), + ) + def test_aggregation_pushdown_preserves_dag_id_and_bert_node_parameters(self): builder = CSEAwareDAGBuilder() modality_id = "test_modality_agg_pushdown" @@ -184,12 +376,13 @@ def test_aggregation_pushdown_preserves_dag_id_and_bert_node_parameters(self): pushdown_aggregation([dag]) self.assertEqual(dag.dag_id, expected_dag_id) - self.assertEqual(dag.root_node_id, bert_id) + self.assertEqual(dag.root_node_id, agg_id) self.assertEqual(len(dag.nodes), 2) - self.assertIsNone(dag.get_node_by_id(agg_id)) + self.assertIsNone(dag.get_node_by_id(bert_id)) - bert_after = dag.get_node_by_id(bert_id) + bert_after = dag.get_node_by_id(agg_id) self.assertIsNotNone(bert_after) + self.assertIs(bert_after.operation, Bert) self.assertEqual(bert_after.inputs, [leaf_id]) self.assertIn("_pushdown_aggregation", bert_after.parameters) self.assertEqual( diff --git a/src/main/python/tests/scuro/test_unimodal_representations.py b/src/main/python/tests/scuro/test_unimodal_representations.py index 59bef40ef64..27e09d48711 100644 --- a/src/main/python/tests/scuro/test_unimodal_representations.py +++ b/src/main/python/tests/scuro/test_unimodal_representations.py @@ -42,10 +42,15 @@ ModalityRandomDataGenerator, ) from systemds.scuro.modality.type import ModalityType +from systemds.scuro.modality.transformed import TransformedModality +from systemds.scuro.representations.representation import RepresentationStats +from systemds.scuro.representations.tabular_features import TabularFeatures +from systemds.scuro.representations.word2vec import W2V from systemds.scuro.representations.timeseries_representations import ( Mean, Max, Min, + Sum, Kurtosis, Skew, Std, @@ -56,6 +61,80 @@ Quantile, ZeroCrossingRate, BandpowerFFT, + LastValue, + TransitionCount, + ObservationDensity, +) +from systemds.scuro.representations.physiological_representations import ( + SDNN, + RMSSD, + pNN, + RRPerMinute, + HRVBandPower, + HRVVLF, + HRVLF, + HRVHF, + HRVLFHF, + PoincareSD1, + PoincareSD2, + SCLSlope, + SCLDynamicRange, + SCRPeaksPerMinute, + SCRAverageAmplitude, + SCRAverageDuration, + BreathingRate, + BreathIntervalRMSSD, + BreathAmplitude, +) + +TIMESERIES_REPRESENTATIONS = [ + Mean, + Min, + Max, + Sum, + Std, + Skew, + Quantile, + Kurtosis, + RMS, + ZeroCrossingRate, + LastValue, + TransitionCount, + ObservationDensity, + ACF, + FrequencyMagnitude, + SpectralCentroid, + BandpowerFFT, +] + + +ECG_REPRESENTATIONS = [ + SDNN, + RMSSD, + pNN, + RRPerMinute, + HRVBandPower, + HRVVLF, + HRVLF, + HRVHF, + HRVLFHF, + PoincareSD1, + PoincareSD2, +] +EDA_REPRESENTATIONS = [ + SCLSlope, + SCLDynamicRange, + SCRPeaksPerMinute, + SCRAverageAmplitude, + SCRAverageDuration, +] +RESPIRATION_REPRESENTATIONS = [ + BreathingRate, + BreathIntervalRMSSD, + BreathAmplitude, +] +PHYSIOLOGICAL_REPRESENTATIONS = ( + ECG_REPRESENTATIONS + EDA_REPRESENTATIONS + RESPIRATION_REPRESENTATIONS ) @@ -147,21 +226,7 @@ def test_audio_representations(self): assert (audio.data[i] == original_data[i]).all() def test_timeseries_representations(self): - ts_representations = [ - Mean(), - Max(), - Min(), - Kurtosis(), - Skew(), - Std(), - RMS(), - ACF(), - FrequencyMagnitude(), - SpectralCentroid(), - Quantile(), - ZeroCrossingRate(), - BandpowerFFT(), - ] + ts_representations = [cls() for cls in TIMESERIES_REPRESENTATIONS] ts_data, ts_md = ModalityRandomDataGenerator().create_timeseries_data( self.num_instances, 100 ) @@ -182,6 +247,417 @@ def test_timeseries_representations(self): for i in range(self.num_instances): assert (ts.data[i] == original_data[i]).all() + def _create_timeseries_modality(self, data, metadata): + modality = UnimodalModality( + TestDataLoader( + np.array(range(len(data))), + None, + ModalityType.TIMESERIES, + data, + np.float32, + metadata, + ) + ) + modality.extract_raw_data() + return modality + + def test_timeseries_output_stats_match_transformed_data(self): + sequence_length = 64 + ts_data, ts_md = ModalityRandomDataGenerator().create_timeseries_data( + self.num_instances, sequence_length + ) + ts = self._create_timeseries_modality(ts_data, ts_md) + input_stats = RepresentationStats(self.num_instances, (sequence_length,)) + + for representation_class in TIMESERIES_REPRESENTATIONS: + with self.subTest(representation=representation_class.__name__): + representation = representation_class() + transformed = ts.apply_representation(representation) + stats = representation.get_output_stats(input_stats) + + self.assertEqual(stats.num_instances, self.num_instances) + self.assertEqual( + np.array(transformed.data).shape[0], self.num_instances + ) + self.assertEqual( + np.array(transformed.data).shape[1:], tuple(stats.output_shape) + ) + + def test_timeseries_batched_path_matches_per_instance_path(self): + sequence_length = 48 + ts_data, _ = ModalityRandomDataGenerator().create_timeseries_data( + self.num_instances, sequence_length + ) + batch = np.stack(ts_data) + + for representation_class in TIMESERIES_REPRESENTATIONS: + with self.subTest(representation=representation_class.__name__): + representation = representation_class() + batched = np.asarray(representation.compute_features_batched(batch)) + if batched.ndim == 1: + batched = batched[:, None] + per_instance = np.stack( + [ + np.atleast_1d(representation.compute_feature(instance)) + for instance in ts_data + ] + ) + np.testing.assert_allclose(batched, per_instance, rtol=1e-5, atol=1e-5) + + def test_timeseries_representations_on_variable_length_instances(self): + lengths = [40 + 7 * i for i in range(self.num_instances)] + ts_data = [np.random.rand(length).astype(np.float32) for length in lengths] + ts_md = [ + ModalityType.TIMESERIES.create_metadata(["signal"], instance) + for instance in ts_data + ] + ts = self._create_timeseries_modality(ts_data, ts_md) + + for representation_class in TIMESERIES_REPRESENTATIONS: + with self.subTest(representation=representation_class.__name__): + transformed = ts.apply_representation(representation_class()) + self.assertEqual( + np.array(transformed.data).shape[0], self.num_instances + ) + self.assertTrue(np.isfinite(np.array(transformed.data)).all()) + + spectrum = ts.apply_representation(FrequencyMagnitude()) + self.assertEqual(np.array(spectrum.data).shape[1], max(lengths) // 2 + 1) + + def test_timeseries_representations_on_degenerate_signals(self): + for name, instance in [ + ("constant", np.full(32, 3.5, dtype=np.float32)), + ("zeros", np.zeros(32, dtype=np.float32)), + ]: + ts_data = [instance.copy() for _ in range(self.num_instances)] + ts_md = [ + ModalityType.TIMESERIES.create_metadata(["signal"], d) for d in ts_data + ] + ts = self._create_timeseries_modality(ts_data, ts_md) + for representation_class in TIMESERIES_REPRESENTATIONS: + if representation_class in (Skew, Kurtosis): + continue + with self.subTest( + signal=name, representation=representation_class.__name__ + ): + transformed = ts.apply_representation(representation_class()) + self.assertTrue(np.isfinite(transformed.data).all()) + + def test_timeseries_minimum_input_length_preconditions(self): + for representation_class in TIMESERIES_REPRESENTATIONS: + representation = representation_class() + minimum = representation.min_input_length + with self.subTest(representation=representation_class.__name__): + self.assertIsNone( + representation.check_preconditions( + RepresentationStats(self.num_instances, (minimum,)) + ) + ) + if minimum > 1: + rejection = representation.check_preconditions( + RepresentationStats(self.num_instances, (minimum - 1,)) + ) + self.assertIsNotNone(rejection) + self.assertIn(str(minimum), rejection) + + def test_acf_rejects_and_narrows_lags_the_input_cannot_express(self): + acf = ACF(k=10) + self.assertIsNotNone( + acf.check_preconditions(RepresentationStats(self.num_instances, (5,))) + ) + self.assertIsNone( + acf.check_preconditions(RepresentationStats(self.num_instances, (50,))) + ) + + candidates = [1, 2, 5, 10, 20] + self.assertEqual( + acf.filter_parameter_domain( + "k", candidates, RepresentationStats(self.num_instances, (5,)) + ), + [1, 2], + ) + # Never empty: a node with no candidates left would stop being tunable. + self.assertEqual( + acf.filter_parameter_domain( + "k", [5, 10], RepresentationStats(self.num_instances, (2,)) + ), + [1], + ) + # Unrelated parameters pass through untouched. + self.assertEqual( + acf.filter_parameter_domain( + "unrelated", candidates, RepresentationStats(self.num_instances, (5,)) + ), + candidates, + ) + + def test_spectral_operators_bind_sampling_rate_to_the_input(self): + for representation in [SpectralCentroid(), BandpowerFFT()]: + with self.subTest(representation=representation.name): + representation.configure_for_input( + RepresentationStats(self.num_instances, (10,), sampling_rate=250.0) + ) + self.assertEqual(representation.fs, 250.0) + + # An input that does not know its rate must not reset the bound one. + representation.configure_for_input( + RepresentationStats(self.num_instances, (10,), sampling_rate=None) + ) + self.assertEqual(representation.fs, 250.0) + self.assertEqual(representation.get_current_parameters()["fs"], 250.0) + + def test_bandpower_band_is_clamped_to_nyquist(self): + self.assertEqual(BandpowerFFT(band_low=0.5, band_width=1.0).band_high, 1.0) + self.assertEqual(BandpowerFFT(band_low=0.0, band_width=0.25).band_high, 0.25) + + def test_quantile_returns_one_column_per_requested_quantile(self): + quantiles = [0.25, 0.5, 0.75] + sequence_length = 64 + ts_data, ts_md = ModalityRandomDataGenerator().create_timeseries_data( + self.num_instances, sequence_length + ) + ts = self._create_timeseries_modality(ts_data, ts_md) + + quantile = Quantile(quantile=quantiles) + transformed = ts.apply_representation(quantile) + stats = quantile.get_output_stats( + RepresentationStats(self.num_instances, (sequence_length,)) + ) + self.assertEqual(tuple(stats.output_shape), (len(quantiles),)) + self.assertEqual( + np.array(transformed.data).shape, (self.num_instances, len(quantiles)) + ) + # np.quantile prepends its own axis; the columns must still come back in + # the requested order rather than transposed. + np.testing.assert_allclose( + transformed.data[0], + np.quantile(ts_data[0], quantiles), + rtol=1e-5, + atol=1e-5, + ) + + def _create_physiological_modality(self, data, metadata): + modality = UnimodalModality( + TestDataLoader( + np.array(range(len(data))), + None, + ModalityType.PHYSIOLOGICAL, + data, + np.float32, + metadata, + ) + ) + modality.extract_raw_data() + return modality + + def test_physiological_representations_output_shapes(self): + data, md = ModalityRandomDataGenerator().create_physiological_data( + self.num_instances, 2000, kind="ecg", fs=500.0 + ) + physiological = self._create_physiological_modality(data, md) + + for representation_class in PHYSIOLOGICAL_REPRESENTATIONS: + with self.subTest(representation=representation_class.__name__): + transformed = physiological.apply_representation(representation_class()) + transformed_data = np.asarray(transformed.data) + self.assertEqual(transformed_data.shape, (self.num_instances, 1)) + self.assertTrue(np.isfinite(transformed_data).all()) + + def test_ecg_features_recover_the_generated_heart_rate(self): + """The generator lays down R peaks at 0.7-0.9 s intervals, i.e. 66-86 + bpm. Anything outside that means the detector is locking onto the noise + floor rather than the beats.""" + fs = 500.0 + data, md = ModalityRandomDataGenerator().create_physiological_data( + self.num_instances, int(fs * 20), kind="ecg", fs=fs + ) + physiological = self._create_physiological_modality(data, md) + transformed_data = np.asarray(physiological.data) + heart_rate = physiological.apply_representation(RRPerMinute(fs=fs)) + self.assertTrue( + ( + (np.array(heart_rate.data) > 60.0) & (np.array(heart_rate.data) < 95.0) + ).all() + ) + + # Jittered intervals mean real, non-zero variability. + for representation_class in [SDNN, RMSSD, PoincareSD1, PoincareSD2]: + with self.subTest(representation=representation_class.__name__): + transformed = physiological.apply_representation( + representation_class(fs=fs) + ) + self.assertTrue((np.array(transformed.data) > 0.0).all()) + + def test_eda_features_recover_the_generated_scr_peaks(self): + """One SCR bump every 10 s is 6 per minute.""" + fs = 4.0 + data, md = ModalityRandomDataGenerator().create_physiological_data( + self.num_instances, 400, kind="eda", fs=fs + ) + physiological = self._create_physiological_modality(data, md) + + peaks_per_minute = physiological.apply_representation(SCRPeaksPerMinute(fs=fs)) + np.testing.assert_allclose(peaks_per_minute.data, 6.0) + + for representation_class in [SCRAverageAmplitude, SCRAverageDuration]: + with self.subTest(representation=representation_class.__name__): + transformed = physiological.apply_representation( + representation_class(fs=fs) + ) + self.assertTrue((np.array(transformed.data) > 0.0).all()) + + # A rising tonic level has a positive slope and a non-degenerate range. + self.assertTrue( + (np.array(physiological.apply_representation(SCLSlope()).data) > 0.0).all() + ) + self.assertTrue( + ( + np.array(physiological.apply_representation(SCLDynamicRange()).data) + > 0.0 + ).all() + ) + + def test_respiration_features_recover_the_generated_breathing_rate(self): + """0.25 Hz is 15 breaths per minute.""" + fs = 500.0 + data, md = ModalityRandomDataGenerator().create_physiological_data( + self.num_instances, int(fs * 20), kind="resp", fs=fs + ) + physiological = self._create_physiological_modality(data, md) + + breathing_rate = physiological.apply_representation(BreathingRate(fs=fs)) + np.testing.assert_allclose(breathing_rate.data, 15.0, rtol=0.1) + + amplitude = physiological.apply_representation(BreathAmplitude(fs=fs)) + np.testing.assert_allclose(amplitude.data, 2.0, rtol=0.2) + + def test_scl_slope_recovers_a_known_linear_trend(self): + ramp = (3.0 * np.arange(100) + 2.0).astype(np.float32) + np.testing.assert_allclose(SCLSlope().compute_feature(ramp), 3.0, rtol=1e-4) + np.testing.assert_allclose( + SCLDynamicRange().compute_feature(np.array([1.0, 5.0, -2.0])), 7.0 + ) + + def test_physiological_representations_on_degenerate_signals(self): + """A flat signal has no beats, no SCRs and no breaths. Every detector + has to fall back to 0.0 instead of dividing by an empty interval list.""" + for name, instance in [ + ("constant", np.ones(400, dtype=np.float32)), + ("zeros", np.zeros(400, dtype=np.float32)), + ("single_sample", np.array([0.5], dtype=np.float32)), + ]: + data = [instance.copy() for _ in range(self.num_instances)] + md = [ + ModalityType.PHYSIOLOGICAL.create_metadata(["signal"], d) for d in data + ] + physiological = self._create_physiological_modality(data, md) + for representation_class in PHYSIOLOGICAL_REPRESENTATIONS: + with self.subTest( + signal=name, representation=representation_class.__name__ + ): + transformed = physiological.apply_representation( + representation_class() + ) + self.assertTrue(np.isfinite(transformed.data).all()) + np.testing.assert_allclose(transformed.data, 0.0, atol=1e-6) + + def test_tabular_features(self): + data_generator = ModalityRandomDataGenerator() + data_generator.modality_type = ModalityType.EMBEDDING + rows = [[1.0, 2.0, 3.0] for _ in range(self.num_instances)] + + modality = TransformedModality(data_generator, "test_transformation") + modality.data = rows + modality.metadata = [ + ModalityType.EMBEDDING.create_metadata(np.asarray(row)) for row in rows + ] + + tabular_features = TabularFeatures() + transformed = tabular_features.transform(modality) + self.assertEqual(transformed.data.shape, (self.num_instances, 3)) + self.assertEqual(transformed.data.dtype, np.float32) + np.testing.assert_allclose(transformed.data, np.asarray(rows)) + + def test_word2vec_representation(self): + vector_size = 20 + text_data, text_md = ModalityRandomDataGenerator().create_text_data( + self.num_instances, 3 + ) + text = UnimodalModality( + TestDataLoader( + self.indices, None, ModalityType.TEXT, text_data, str, text_md + ) + ) + transformed = text.apply_representation(W2V(vector_size=vector_size)) + transformed_data = np.asarray(transformed.data) + self.assertEqual(transformed_data.shape, (self.num_instances, vector_size)) + self.assertTrue(np.isfinite(transformed_data).all()) + + def test_audio_representations_on_a_signal_shorter_than_one_frame(self): + """librosa's default frame is 2048 samples. A shorter instance must + still come back as a single frame rather than an empty array a + downstream aggregation would then reduce over nothing.""" + audio = self._create_audio_modality(signal_length=8) + + for representation in [ + MFCC(), + MelSpectrogram(), + Spectrogram(), + Spectral(), + ZeroCrossing(), + RMSE(), + Pitch(), + ]: + with self.subTest(representation=representation.name): + transformed = representation.transform(audio) + self.assertEqual(len(transformed.data), self.num_instances) + for instance in transformed.data: + self.assertEqual(instance.shape[0], 1) + self.assertTrue(np.isfinite(instance).all()) + + def test_color_histogram_color_spaces_and_normalization(self): + image_data, image_md = ModalityRandomDataGenerator().create_visual_modality( + self.num_instances, 1, height=8, width=8 + ) + image = UnimodalModality( + TestDataLoader( + self.indices, None, ModalityType.IMAGE, image_data, np.float32, image_md + ) + ) + image.extract_raw_data() + + for color_space in ["RGB", "HSV", "GRAY"]: + with self.subTest(color_space=color_space): + representation = ColorHistogram( + color_space=color_space, bins=4, normalize=True + ) + transformed = np.asarray(representation.transform(image).data) + self.assertEqual( + transformed.shape, + (self.num_instances, representation.calculate_hist_dim()), + ) + np.testing.assert_allclose(transformed.sum(axis=1), 1.0, rtol=1e-5) + + # A single-colour image puts every pixel in one bin -- the degenerate + # case a downstream model can learn nothing from. + uniform = [ + np.full((8, 8, 3), 7, dtype=np.uint8) for _ in range(self.num_instances) + ] + uniform_md = [ + ModalityType.IMAGE.create_metadata(8, 8, 3) + for _ in range(self.num_instances) + ] + uniform_image = UnimodalModality( + TestDataLoader( + self.indices, None, ModalityType.IMAGE, uniform, np.float32, uniform_md + ) + ) + uniform_image.extract_raw_data() + histogram = np.asarray( + ColorHistogram(bins=4, normalize=True).transform(uniform_image).data + ) + np.testing.assert_array_equal((histogram > 0).sum(axis=1), 1) + def test_image_representations(self): image_data, image_md = ModalityRandomDataGenerator().create_visual_modality( self.num_instances, 1, height=8, width=8 @@ -248,6 +724,10 @@ def test_chunked_video_representations(self): assert len(r.metadata) == self.num_instances -# TODO: add unit tests for the other representations +# TODO: the representations still untested here are the ones that download a +# pretrained model at construction time -- Bert, RoBERTa, CLIPText/CLIPVisual, +# GloVe, W2V's larger variants, Wav2Vec, VGG19, X3D and SwinVideoTransformer. +# They need either a cached-model fixture or a network-marked test suite, which +# is also why test_video_representations above is commented out. if __name__ == "__main__": unittest.main() diff --git a/src/main/python/tests/scuro/test_window_operations.py b/src/main/python/tests/scuro/test_window_operations.py index a8c86374801..c6a258fb465 100644 --- a/src/main/python/tests/scuro/test_window_operations.py +++ b/src/main/python/tests/scuro/test_window_operations.py @@ -29,13 +29,45 @@ from tests.scuro.data_generator import ModalityRandomDataGenerator, TestDataLoader from systemds.scuro.modality.type import ModalityType from systemds.scuro.modality.unimodal_modality import UnimodalModality +from systemds.scuro.representations.aggregate import Aggregation +from systemds.scuro.representations.timeseries_representations import ( + FrequencyMagnitude, + Mean, + Quantile, + Std, +) +from systemds.scuro.representations.physiological_window import ( + AdaptiveWindow, + PhysiologicalEventWindow, +) from systemds.scuro.representations.window_aggregation import ( StaticWindow, DynamicWindow, WindowAggregation, + resolve_aggregation_function, ) +class _FakeModality: + """The smallest surface a context operator's execute() actually touches. + + Lets a test hand a window operator hand-built instances -- an empty one, a + flat one -- without going through a loader that would reject them first. + """ + + def __init__(self, data, metadata=None): + self.data = data + self.metadata = metadata or [ + ModalityType.TIMESERIES.create_metadata(["signal"], np.asarray(instance)) + for instance in data + ] + + def get_data_layout(self): + from systemds.scuro.modality.type import DataLayout + + return DataLayout.SINGLE_LEVEL + + class TestWindowOperations(unittest.TestCase): @classmethod def setUpClass(cls): @@ -141,6 +173,372 @@ def test_window_aggregation_on_2d_modality(self): windowed_modality = embedding_modality.context(window_operator) + def _timeseries_modality(self, signal_length=100): + return self.data_generator.create1DModality( + self.num_instances, signal_length, ModalityType.TIMESERIES + ) + + # ------------------------------------------------------------------ + # WindowAggregation: window size against signal length + # ------------------------------------------------------------------ + + def test_window_size_of_one_leaves_the_signal_unchanged(self): + signal_length = 60 + modality = self._timeseries_modality(signal_length) + windowed = np.asarray( + WindowAggregation("mean", window_size=1).execute(modality) + ) + self.assertEqual(windowed.shape, (self.num_instances, signal_length)) + np.testing.assert_allclose(windowed, modality.data, rtol=1e-5, atol=1e-5) + + def test_window_larger_than_the_signal_collapses_to_one_padded_window(self): + """The single short window is zero-padded up to window_size, so a mean + over it divides by the nominal size and not by the sample count. That + dilution is the behaviour a caller has to be able to rely on.""" + signal_length = 100 + window_size = 250 + modality = self._timeseries_modality(signal_length) + + windowed = np.asarray( + WindowAggregation("mean", window_size=window_size).execute(modality) + ) + self.assertEqual(windowed.shape, (self.num_instances, 1)) + np.testing.assert_allclose( + windowed[:, 0], + modality.data.sum(axis=1) / window_size, + rtol=1e-5, + atol=1e-5, + ) + + def test_window_size_that_does_not_divide_the_signal_keeps_a_tail_window(self): + signal_length = 100 + window_size = 7 + modality = self._timeseries_modality(signal_length) + + windowed = np.asarray( + WindowAggregation("mean", window_size=window_size).execute(modality) + ) + self.assertEqual( + windowed.shape, + (self.num_instances, math.ceil(signal_length / window_size)), + ) + # The tail window covers only the samples that are actually there. + tail_start = (windowed.shape[1] - 1) * window_size + np.testing.assert_allclose( + windowed[:, -1], + modality.data[:, tail_start:].mean(axis=1), + rtol=1e-5, + atol=1e-5, + ) + + def test_batched_and_per_instance_paths_agree(self): + """Equal-length numeric instances take a vectorized path; anything else + falls back to the per-instance loop. A window's value must not depend on + which of the two ran.""" + modality = self._timeseries_modality(300) + for window_size in [1, 7, 10, 300, 400]: + with self.subTest(window_size=window_size): + batched = np.asarray( + WindowAggregation("mean", window_size=window_size).execute(modality) + ) + per_instance_operator = WindowAggregation( + "mean", window_size=window_size + ) + per_instance = np.stack( + [ + per_instance_operator.window_aggregate_single_level( + np.asarray(instance), + math.ceil(len(instance) / window_size), + ) + for instance in modality.data + ] + ) + np.testing.assert_allclose(batched, per_instance, rtol=1e-5, atol=1e-5) + + def test_window_aggregation_without_padding_returns_one_array_per_instance(self): + modality = self._timeseries_modality(100) + windowed = WindowAggregation("mean", window_size=10, pad=False).execute( + modality + ) + self.assertIsInstance(windowed, list) + self.assertEqual(len(windowed), self.num_instances) + for instance in windowed: + self.assertEqual(np.asarray(instance).shape, (10,)) + + # ------------------------------------------------------------------ + # Preconditions + # ------------------------------------------------------------------ + + def test_empty_instance_is_rejected(self): + """An empty instance has no window to reduce. Failing loudly beats + returning an empty feature the model would only choke on later.""" + empty = _FakeModality( + [np.array([], dtype=np.float32)], + [ + ModalityType.TIMESERIES.create_metadata( + ["signal"], np.zeros(1, dtype=np.float32) + ) + ], + ) + with self.assertRaises(ValueError): + WindowAggregation("mean", window_size=10).execute(empty) + + def test_invalid_aggregation_function_is_rejected(self): + for operator in [WindowAggregation, StaticWindow, DynamicWindow]: + with self.subTest(operator=operator.__name__): + with self.assertRaises(ValueError): + operator(aggregation_function=object()) + # Aggregation itself only knows a fixed set of names. + with self.assertRaises(ValueError): + Aggregation("not_an_aggregation") + + # ------------------------------------------------------------------ + # StaticWindow / DynamicWindow: window count against signal length + # ------------------------------------------------------------------ + + def test_single_window_reduces_the_whole_signal(self): + modality = self._timeseries_modality(100) + for operator_class in [StaticWindow, DynamicWindow]: + with self.subTest(operator=operator_class.__name__): + windowed = np.asarray( + operator_class("mean", num_windows=1).execute(modality) + ) + self.assertEqual(windowed.shape, (self.num_instances, 1)) + np.testing.assert_allclose( + windowed[:, 0], modality.data.mean(axis=1), rtol=1e-5, atol=1e-5 + ) + + def test_static_window_pads_beyond_the_signal_length(self): + """StaticWindow honours num_windows literally: asking for more windows + than there are samples zero-pads rather than clamping.""" + signal_length = 100 + num_windows = 250 + modality = self._timeseries_modality(signal_length) + + operator = StaticWindow("mean", num_windows=num_windows) + windowed = np.asarray(operator.execute(modality)) + self.assertEqual(windowed.shape, (self.num_instances, num_windows)) + self.assertEqual( + tuple( + operator.get_output_stats( + RepresentationStats(self.num_instances, (signal_length,)) + ).output_shape + ), + (num_windows,), + ) + + def test_dynamic_window_clamps_num_windows_to_the_signal_length(self): + """DynamicWindow splits the signal into geometrically growing windows, + which cannot be shorter than one sample -- so the count is capped.""" + signal_length = 100 + modality = self._timeseries_modality(signal_length) + + operator = DynamicWindow("mean", num_windows=250) + windowed = np.asarray(operator.execute(modality)) + self.assertEqual(windowed.shape, (self.num_instances, signal_length)) + self.assertEqual( + tuple( + operator.get_output_stats( + RepresentationStats(self.num_instances, (signal_length,)) + ).output_shape + ), + (signal_length,), + ) + + def test_window_operators_on_variable_length_instances(self): + """A window *count* is length-independent, so ragged instances still + stack into one rectangular block.""" + num_windows = 5 + instances = [ + np.random.rand(100 + 37 * i).astype(np.float32) + for i in range(self.num_instances) + ] + modality = _FakeModality(instances) + + for operator_class in [StaticWindow, DynamicWindow]: + with self.subTest(operator=operator_class.__name__): + windowed = np.asarray( + operator_class("mean", num_windows=num_windows).execute(modality) + ) + self.assertEqual(windowed.shape, (self.num_instances, num_windows)) + self.assertTrue(np.isfinite(windowed).all()) + + # ------------------------------------------------------------------ + # Aggregation functions + # ------------------------------------------------------------------ + + def test_window_operators_accept_a_representation_as_aggregation(self): + """A window may reduce with a full representation, not just a named + aggregation -- and then the per-window feature can be multi-valued, so + get_output_stats has to carry that extra shape.""" + signal_length = 100 + modality = self._timeseries_modality(signal_length) + input_stats = RepresentationStats(self.num_instances, (signal_length,)) + + window_size = 10 + for aggregation, expected_feature_shape in [ + (Mean(), ()), + (Std(), ()), + (Quantile(), ()), + (FrequencyMagnitude(), (window_size // 2 + 1,)), + ]: + with self.subTest(aggregation=aggregation.name): + operator = WindowAggregation(aggregation, window_size=window_size) + windowed = np.asarray(operator.execute(modality)) + stats = operator.get_output_stats(input_stats) + + expected = ( + signal_length // window_size, + *expected_feature_shape, + ) + self.assertEqual(windowed.shape, (self.num_instances, *expected)) + self.assertEqual(tuple(stats.output_shape), expected) + + for operator_class in [StaticWindow, DynamicWindow]: + with self.subTest(operator=operator_class.__name__): + operator = operator_class(Std(), num_windows=5) + windowed = np.asarray(operator.execute(modality)) + self.assertEqual(windowed.shape, (self.num_instances, 5)) + + def test_window_operator_current_parameters_expose_the_nested_aggregation(self): + """The tuner reads its search space from get_current_parameters, so a + representation used as an aggregation has to surface its own parameters + under a prefixed name rather than disappearing behind the class.""" + window = WindowAggregation("mean", window_size=16) + parameters = window.get_current_parameters() + self.assertEqual(parameters["window_size"], 16) + self.assertIs(parameters["aggregation_function"], Aggregation) + self.assertEqual( + parameters["aggregation_function_aggregation_function"], "mean" + ) + + nested = WindowAggregation(Quantile(quantile=0.5), window_size=16) + nested_parameters = nested.get_current_parameters() + self.assertIs(nested_parameters["aggregation_function"], Quantile) + self.assertEqual(nested_parameters["aggregation_function_quantile"], 0.5) + + static = StaticWindow("max", num_windows=7) + self.assertEqual(static.get_current_parameters()["num_windows"], 7) + + def test_resolve_aggregation_function(self): + self.assertEqual(resolve_aggregation_function("mean", None), "mean") + self.assertEqual( + resolve_aggregation_function("mean", {"aggregation_function": "max"}), "max" + ) + # A class is instantiated ... + self.assertIsInstance( + resolve_aggregation_function("mean", {"aggregation_function": Mean}), Mean + ) + # ... and its prefixed parameters are threaded into the instance. + resolved = resolve_aggregation_function( + "mean", + { + "aggregation_function": Quantile, + "aggregation_function_quantile": 0.25, + }, + ) + self.assertIsInstance(resolved, Quantile) + self.assertEqual(resolved.quantile, 0.25) + + # ------------------------------------------------------------------ + # Data-dependent windows + # ------------------------------------------------------------------ + + def test_adaptive_window_output_shape_is_only_an_estimate(self): + """The window count depends on the signal's local variance, so it is + not knowable from statistics. The operator must say so rather than + report a shape the executor would then assert against.""" + signal_length = 300 + modality = self._timeseries_modality(signal_length) + operator = AdaptiveWindow( + "mean", base_window_size=64, overlap=0.5, min_window_size=16 + ) + + stats = operator.get_output_stats( + RepresentationStats(self.num_instances, (signal_length,)) + ) + self.assertFalse(stats.output_shape_is_known) + self.assertEqual(stats.num_instances, self.num_instances) + + windowed = np.asarray(operator.execute(modality)) + self.assertEqual(windowed.shape[0], self.num_instances) + self.assertGreater(windowed.shape[1], 0) + self.assertTrue(np.isfinite(windowed).all()) + + def test_adaptive_window_clamps_a_floor_above_the_nominal_size(self): + """A minimum larger than the nominal window is contradictory; it + collapses onto the nominal size instead of silently inverting.""" + operator = AdaptiveWindow( + "mean", base_window_size=8, overlap=0.5, min_window_size=64 + ) + self.assertEqual(operator.base_window_size, 8) + self.assertEqual(operator.min_window_size, 8) + + def test_adaptive_window_always_advances(self): + """With a small window and a high overlap the stride truncates to zero + samples, which would never move the cursor. The floor of one sample is + what keeps execute() from spinning forever.""" + modality = _FakeModality( + [np.ones(200, dtype=np.float32) for _ in range(self.num_instances)] + ) + for base_window_size, overlap in [(8, 1.0), (1, 0.9), (4, 0.99)]: + with self.subTest(base_window_size=base_window_size, overlap=overlap): + windowed = np.asarray( + AdaptiveWindow( + "mean", + base_window_size=base_window_size, + overlap=overlap, + min_window_size=1, + ).execute(modality) + ) + self.assertEqual(windowed.shape[0], self.num_instances) + self.assertGreater(windowed.shape[1], 0) + + def test_physiological_event_window_splits_on_detected_events(self): + signal_length = 300 + modality = self._timeseries_modality(signal_length) + operator = PhysiologicalEventWindow( + "mean", event_threshold=0.5, min_distance=32 + ) + + stats = operator.get_output_stats( + RepresentationStats(self.num_instances, (signal_length,)) + ) + self.assertFalse(stats.output_shape_is_known) + + windowed = np.asarray(operator.execute(modality)) + self.assertEqual(windowed.shape[0], self.num_instances) + self.assertGreater(windowed.shape[1], 0) + self.assertTrue(np.isfinite(windowed).all()) + + def test_physiological_event_window_falls_back_when_no_event_is_found(self): + """A flat signal has no peaks to split on, so the operator falls back to + an even split instead of producing zero windows.""" + min_distance = 32 + for name, instance in [ + ("constant", np.ones(200, dtype=np.float32)), + ("zeros", np.zeros(200, dtype=np.float32)), + ]: + with self.subTest(signal=name): + modality = _FakeModality( + [instance.copy() for _ in range(self.num_instances)] + ) + windowed = np.asarray( + PhysiologicalEventWindow( + "mean", event_threshold=0.5, min_distance=min_distance + ).execute(modality) + ) + self.assertEqual( + windowed.shape, + (self.num_instances, len(instance) // min_distance), + ) + self.assertTrue(np.isfinite(windowed).all()) + + def test_physiological_event_window_floors_min_distance(self): + self.assertEqual( + PhysiologicalEventWindow("mean", min_distance=0).min_distance, 1 + ) + def verify_window_operation( self, aggregation, modality, windowed_modality, window_size ): diff --git a/src/main/python/tests/scuro/test_window_representation_batching.py b/src/main/python/tests/scuro/test_window_representation_batching.py new file mode 100644 index 00000000000..e561830d081 --- /dev/null +++ b/src/main/python/tests/scuro/test_window_representation_batching.py @@ -0,0 +1,141 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- +import math +import unittest + +import numpy as np + +from systemds.scuro.modality.type import DataLayout, ModalityType +from systemds.scuro.representations.mel_spectrogram import MelSpectrogram +from systemds.scuro.representations.mfcc import MFCC +from systemds.scuro.representations.representation import RepresentationStats +from systemds.scuro.representations.unimodal import UnimodalRepresentation +from systemds.scuro.representations.window_aggregation import ( + DynamicWindow, + StaticWindow, + WindowAggregation, +) + + +class _FakeModality: + def __init__(self, data): + self.data = np.asarray(data, dtype=np.float32) + self.metadata = [ + ModalityType.TIMESERIES.create_metadata(["signal"], instance) + for instance in self.data + ] + + def get_data_layout(self): + return DataLayout.SINGLE_LEVEL + + +class _BatchedMean(UnimodalRepresentation): + def __init__(self): + super().__init__("BatchedMean", ModalityType.EMBEDDING) + self.scalar_calls = 0 + self.batch_calls = 0 + + def compute_feature(self, signal): + self.scalar_calls += 1 + return np.asarray(signal).mean() + + def compute_features_batched(self, data): + self.batch_calls += 1 + data = np.asarray(data) + return data.mean(axis=tuple(range(1, data.ndim))) + + def transform(self, modality, aggregation=None): + raise NotImplementedError + + def get_output_stats(self, input_stats): + return RepresentationStats(input_stats.num_instances, (1,)) + + +class TestWindowRepresentationBatching(unittest.TestCase): + def setUp(self): + rng = np.random.default_rng(7) + self.data = rng.normal(size=(4, 100)).astype(np.float32) + self.modality = _FakeModality(self.data) + + def test_static_window_batches_across_instances_and_windows(self): + representation = _BatchedMean() + result = StaticWindow(representation, num_windows=20).execute(self.modality) + expected = self.data.reshape(4, 20, 5).mean(axis=2) + + np.testing.assert_allclose(result, expected, rtol=1e-6, atol=1e-6) + self.assertEqual(representation.scalar_calls, 0) + self.assertGreater(representation.batch_calls, 0) + self.assertLess(representation.batch_calls, self.data.shape[0] * 20) + + def test_dynamic_window_batches_equal_shape_windows(self): + representation = _BatchedMean() + operator = DynamicWindow(representation, num_windows=16) + result = operator.execute(self.modality) + + expected = [] + for instance in self.data: + ends = np.cumsum(operator._window_sizes(len(instance))) + starts = np.concatenate(([0], ends[:-1])) + expected.append( + [instance[start:end].mean() for start, end in zip(starts, ends)] + ) + + np.testing.assert_allclose(result, np.asarray(expected), rtol=1e-6, atol=1e-6) + self.assertEqual(representation.scalar_calls, 0) + self.assertGreater(representation.batch_calls, 0) + self.assertLess(representation.batch_calls, self.data.shape[0] * 16) + + def test_window_aggregation_batches_full_windows_and_preserves_tail(self): + representation = _BatchedMean() + operator = WindowAggregation(representation, window_size=7) + result = operator.execute(self.modality) + expected = np.stack( + [ + [ + instance[start : min(start + 7, len(instance))].mean() + for start in range(0, len(instance), 7) + ] + for instance in self.data + ] + ) + + self.assertEqual(result.shape[1], math.ceil(self.data.shape[1] / 7)) + np.testing.assert_allclose(result, expected, rtol=1e-6, atol=1e-6) + self.assertEqual(representation.scalar_calls, 0) + + def test_audio_representations_preserve_results_when_batched(self): + rng = np.random.default_rng(11) + windows = rng.normal(size=(3, 64)).astype(np.float32) + representations = ( + MFCC(n_mfcc=4, n_mels=8, hop_length=8, n_fft=32), + MelSpectrogram(n_mels=8, hop_length=8, n_fft=32), + ) + for representation in representations: + with self.subTest(representation=representation.name): + batched = representation.compute_features_batched(windows) + per_window = np.stack( + [representation.compute_feature(window) for window in windows] + ) + np.testing.assert_allclose(batched, per_window, rtol=1e-5, atol=1e-5) + + +if __name__ == "__main__": + unittest.main() From 3254d94a3e956626faa6e036591e7e7909da3f37 Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Mon, 31 Aug 2026 19:01:59 +0200 Subject: [PATCH 126/132] [SYSTEMDS-3967] Add a multimodal GA optimizer approach In this patch a new multimodal GA optimizer is added that implements various mutation operations. It reuses the worker pool and shared memory modules of the node executor. The GA optimizer uses the deap library. Assisted-by: AI --- .github/workflows/python.yml | 1 + .../scuro/drsearch/multimodal_ga_optimizer.py | 1103 +++++++++++++++ .../scuro/drsearch/operator_registry.py | 10 +- .../scuro/test_multimodal_ga_optimizer.py | 1220 +++++++++++++++++ 4 files changed, 2329 insertions(+), 5 deletions(-) create mode 100644 src/main/python/systemds/scuro/drsearch/multimodal_ga_optimizer.py create mode 100644 src/main/python/tests/scuro/test_multimodal_ga_optimizer.py diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index d679aa74a92..d6e66813708 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -180,6 +180,7 @@ jobs: scikit-optimize \ flair \ optuna \ + deap \ openface-test \ imagebind \ "pytorchvideo @ git+https://github.com/facebookresearch/pytorchvideo.git@eb04d1b" diff --git a/src/main/python/systemds/scuro/drsearch/multimodal_ga_optimizer.py b/src/main/python/systemds/scuro/drsearch/multimodal_ga_optimizer.py new file mode 100644 index 00000000000..44858d96a0e --- /dev/null +++ b/src/main/python/systemds/scuro/drsearch/multimodal_ga_optimizer.py @@ -0,0 +1,1103 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +from __future__ import annotations + +import copy +import os +import pickle +import random +import tempfile +import threading +import time +import traceback +from dataclasses import dataclass, field +from itertools import chain +from typing import Any, Dict, List, Optional, Tuple, Union + +from deap import base, tools + +from systemds.scuro.drsearch.modality_shared_memory import ( + add_shared_memory_candidate, + unlink_shm, +) + +from systemds.scuro.drsearch.operator_registry import Registry +from systemds.scuro.drsearch.representation_dag import ( + RepresentationDAGBuilder, + RepresentationDag, +) +from systemds.scuro.drsearch.task import Task +from systemds.scuro.drsearch.worker_pool import PersistentWorkerPool, create_mp_context +from systemds.scuro.representations.aggregated_representation import ( + AggregatedRepresentation, +) +from systemds.scuro.utils.schema_helpers import get_shape + +Tree = Union[int, Tuple["Tree", "Tree"]] + + +def _collect_internal_paths(tree: Tree, path: str = "") -> List[str]: + if isinstance(tree, int): + return [] + left, right = tree + return ( + [path] + + _collect_internal_paths(left, path + "L") + + _collect_internal_paths(right, path + "R") + ) + + +def _get_subtree(tree: Tree, path: str) -> Tree: + if not path: + return copy.deepcopy(tree) + left, right = tree + branch = left if path[0] == "L" else right + return _get_subtree(branch, path[1:]) + + +def _replace_subtree(tree: Tree, path: str, replacement: Tree) -> Tree: + if not path: + return copy.deepcopy(replacement) + left, right = tree + if path[0] == "L": + return _replace_subtree(left, path[1:], replacement), copy.deepcopy(right) + return copy.deepcopy(left), _replace_subtree(right, path[1:], replacement) + + +def _collect_leaf_indices(tree: Tree) -> List[int]: + if isinstance(tree, int): + return [tree] + return _collect_leaf_indices(tree[0]) + _collect_leaf_indices(tree[1]) + + +def _remove_leaf_from_tree(tree: Tree, leaf: int) -> Optional[Tree]: + if isinstance(tree, int): + return None if tree == leaf else tree + left = _remove_leaf_from_tree(tree[0], leaf) + right = _remove_leaf_from_tree(tree[1], leaf) + if left is None: + return right + if right is None: + return left + return left, right + + +def _reindex_tree(tree: Tree, index_map: Dict[int, int]) -> Tree: + if isinstance(tree, int): + return index_map[tree] + return _reindex_tree(tree[0], index_map), _reindex_tree(tree[1], index_map) + + +def _rebuild_fusion_ops( + tree: Tree, + existing: Dict[str, type], + rng: random.Random, + operators: List[type], + randomized_prefixes: Optional[List[str]] = None, +) -> Dict[str, type]: + prefixes = randomized_prefixes or [] + rebuilt = {} + for path in _collect_internal_paths(tree): + randomized = any(not prefix or path.startswith(prefix) for prefix in prefixes) + rebuilt[path] = ( + rng.choice(operators) + if randomized or path not in existing + else existing[path] + ) + return rebuilt + + +@dataclass +class FusionSearchResult: + dag: RepresentationDag + train_score: dict + val_score: dict + test_score: dict + runtime: float = 0.0 + task_time: float = 0.0 + representation_time: float = 0.0 + task_name: str = "" + + val_fold_scores: dict = field(default_factory=dict) + train_fold_scores: dict = field(default_factory=dict) + test_fold_scores: dict = field(default_factory=dict) + + task_timing: dict = field(default_factory=dict) + + generation: int = -1 + eval_index: int = -1 + t_since_search_start_s: float = 0.0 + t_eval_end_unix: float = 0.0 + + +@dataclass +class DagGenome: + leaves: List[Tuple[str, int]] + tree: Tree + fusion_ops: Dict[str, type] + + +_TIMING_OBJECTIVES = {"runtime", "task_time", "representation_time"} + +ObjectiveSpec = Tuple[str, str] # (name, "max" | "min") + + +def _objective_value( + name: str, val_score: Dict[str, float], timing: Dict[str, float] +) -> float: + if name in _TIMING_OBJECTIVES: + return timing[name] + return val_score[name] + + +def _failure_fitness(objective_specs: List[ObjectiveSpec]) -> Tuple[float, ...]: + return tuple( + float("-inf") if direction == "max" else float("inf") + for _, direction in objective_specs + ) + + +def _fold_scores(performance_measure) -> Dict[str, List[float]]: + return { + metric: [float(value) for value in values] + for metric, values in performance_measure.scores.items() + } + + +def _evaluate_genome_body( + dag: RepresentationDag, + task: Task, + modalities: List[Any], + objective_specs: List[ObjectiveSpec], +) -> Tuple[Optional[Tuple[float, ...]], Optional[Dict[str, Any]]]: + start = time.time() + fused = dag.execute(modalities, task, enable_cache=False) + if fused is None: + return None, None + + if isinstance(fused, dict): + fused = fused[list(fused.keys())[-1]] + + if task.expected_dim == 1 and get_shape(fused.metadata) > 1: + fused = AggregatedRepresentation().transform(fused) + + t0 = time.time() + scores = task.run(fused.data) + task_time = time.time() - t0 + total = time.time() - start + + val_score = scores[1].average_scores + timing = { + "runtime": total, + "task_time": task_time, + "representation_time": total - task_time, + } + fitness = tuple( + _objective_value(name, val_score, timing) for name, _ in objective_specs + ) + payload = { + "train_score": scores[0].average_scores, + "val_score": val_score, + "test_score": scores[2].average_scores, + "train_fold_scores": _fold_scores(scores[0]), + "val_fold_scores": _fold_scores(scores[1]), + "test_fold_scores": _fold_scores(scores[2]), + "task_timing": getattr(task, "last_run_timing", {}), + **timing, + } + return fitness, payload + + +def _dispatch_genome_evaluation(payload, _gpu_id): + dag, task, modalities, objective_specs = payload + fitness, result = _evaluate_genome_body(dag, task, modalities, objective_specs) + if fitness is None: + fitness = _failure_fitness(objective_specs) + return fitness, result + + +_WORKER_DISPATCH = {"genome": _dispatch_genome_evaluation} + + +class _FusionIndividual(list): + def __init__(self, values, fitness_type): + super().__init__(values) + self.fitness = fitness_type() + + +class MultimodalDeapOptimizer: + def __init__( + self, + modalities: List[Any], + unimodal_optimization_results: Any, + tasks: List[Task], + debug: bool = True, + min_modalities: int = 2, + max_modalities: int = None, + metric: str = "accuracy", + objectives: Optional[List[ObjectiveSpec]] = None, + population_size: int = 32, + generations: int = 20, + crossover_probability: float = 0.7, + mutation_probability: float = 0.4, + random_seed: int = 42, + maximize_metric: bool = True, + elite_size: int = 2, + max_workers: int = 1, + batch_size: Optional[int] = None, + early_stopping_patience: Optional[int] = 5, + early_stopping_min_delta: float = 1e-6, + novelty_breeding: bool = True, + hall_of_fame_size: int = 5, + allow_repeated_modalities: bool = False, + threads_per_worker: Optional[int] = None, + ): + self.modalities = modalities + self.tasks = tasks + self.debug = debug + self.allow_repeated_modalities = allow_repeated_modalities + + self.min_modalities = max(1, min_modalities) + requested_max = max_modalities or len(modalities) + self.max_modalities = ( + requested_max + if allow_repeated_modalities + else min(requested_max, len(modalities)) + ) + if self.max_modalities < self.min_modalities: + raise ValueError( + f"max_modalities ({self.max_modalities}) is below min_modalities " + f"({self.min_modalities})" + ) + self.metric_name = metric + self.maximize_metric = maximize_metric + + if objectives is not None: + if len(objectives) < 1: + raise ValueError( + "objectives must contain at least one (name, direction) pair" + ) + for name, direction in objectives: + if direction not in ("max", "min"): + raise ValueError( + f"objective direction must be 'max' or 'min', got " + f"{direction!r} for objective {name!r}" + ) + self.objective_specs: List[ObjectiveSpec] = list(objectives) + self.metric_name = self.objective_specs[0][0] + self.maximize_metric = self.objective_specs[0][1] == "max" + else: + self.objective_specs = [ + (self.metric_name, "max" if self.maximize_metric else "min") + ] + self.is_multi_objective = len(self.objective_specs) > 1 + + if len(self.modalities) < self.min_modalities: + raise ValueError( + f"MultimodalDeapOptimizer requires at least {self.min_modalities} " + f"modalities, got {len(self.modalities)}." + ) + + self.operator_registry = Registry() + self.fusion_operators = self.operator_registry.get_fusion_operators() + if not self.fusion_operators: + raise ValueError( + "MultimodalDeapOptimizer requires at least one registered " + "fusion operator." + ) + self.k_best_representations = self._extract_k_best( + unimodal_optimization_results + ) + + self.optimization_results: Dict[str, List[FusionSearchResult]] = {} + self.evaluation_errors: Dict[str, int] = {} + + self.hall_of_fame_size = max(1, hall_of_fame_size) + self.hall_of_fame: Dict[str, List[FusionSearchResult]] = {} + self._hof_fitness: Dict[str, List[Tuple[float, ...]]] = {} + self.rng = random.Random(random_seed) + self._eval_counter = 0 + self._current_generation = -1 + self._search_start = time.perf_counter() + self.population_size = max(1, population_size) + self.generations = max(1, generations) + self.crossover_probability = crossover_probability + self.mutation_probability = mutation_probability + self.random_seed = random_seed + self._fitness_cache: Dict[str, Dict[Tuple, Tuple[float, ...]]] = {} + + self.elite_size = max(0, min(elite_size, self.population_size - 1)) + self.max_workers = max(1, max_workers) + self.batch_size = max(1, batch_size or self.max_workers) + cpu_count = os.cpu_count() or 1 + self.threads_per_worker = max( + 1, + ( + threads_per_worker + if threads_per_worker is not None + else cpu_count // self.max_workers + ), + ) + self.early_stopping_patience = early_stopping_patience + self.early_stopping_min_delta = early_stopping_min_delta + self.novelty_breeding = novelty_breeding + self._current_task_name = None + self._optimize_lock = threading.Lock() + self._worker_pool: Optional[PersistentWorkerPool] = None + self._parallel_task_name: Optional[str] = None + self._parallel_modalities: Optional[List[Any]] = None + self._parallel_shm_names: List[str] = [] + + desired_weights = tuple( + 1.0 if direction == "max" else -1.0 for _, direction in self.objective_specs + ) + self._objective_weights = desired_weights + self._fitness_type = type( + f"FusionFitness_{id(self)}", (base.Fitness,), {"weights": desired_weights} + ) + + def optimize( + self, + ) -> Dict[str, List[FusionSearchResult]]: + with self._optimize_lock: + try: + return self._optimize() + finally: + self._shutdown_parallel_runtime() + + def _optimize( + self, + ) -> Dict[str, List[FusionSearchResult]]: + for task in self.tasks: + task_name = task.model.name + self._current_task_name = task_name + self.optimization_results.setdefault(task_name, []) + self.evaluation_errors.setdefault(task_name, 0) + + self._eval_counter = 0 + self._search_start = time.perf_counter() + + if self.max_workers > 1: + self._start_parallel_runtime(task_name) + + population = self._build_initial_population(task_name) + best_ever = None + no_improve = 0 + + for gen in range(self.generations): + self._current_generation = gen + self._evaluate_population(population, task) + + if self.is_multi_objective: + front = tools.sortNondominated( + population, len(population), first_front_only=True + )[0] + front_signature = frozenset( + self._genome_signature(ind[0]) for ind in front + ) + if best_ever is None or front_signature != best_ever: + best_ever = front_signature + no_improve = 0 + else: + no_improve += 1 + debug_msg = f"front_size={len(front)}" + else: + gen_best = max(population, key=lambda ind: ind.fitness.values[0]) + if ( + best_ever is None + or gen_best.fitness.values[0] + > best_ever.fitness.values[0] + self.early_stopping_min_delta + ): + best_ever = self._clone_individual(gen_best) + no_improve = 0 + else: + no_improve += 1 + debug_msg = f"best={gen_best.fitness.values[0]:.4f}" + + if self.debug: + print( + f"[GA] task={task_name} gen={gen} {debug_msg} " + f"no_improve={no_improve} " + f"errors={self.evaluation_errors.get(task_name, 0)}" + ) + + stagnated = ( + self.early_stopping_patience is not None + and no_improve >= self.early_stopping_patience + ) + if stagnated or gen == self.generations - 1: + if self.debug and stagnated: + print( + f"[GA] task={task_name} early stopping after " + f"{no_improve} generations without improvement" + ) + break + + population = self._next_generation(population, task_name, task) + + return self.optimization_results + + def _make_individual(self, genome: DagGenome): + return _FusionIndividual([genome], self._fitness_type) + + def _clone_individual(self, ind): + clone = self._make_individual(copy.deepcopy(ind[0])) + if ind.fitness.valid: + clone.fitness.values = ind.fitness.values + return clone + + def _append_if_unique( + self, + population: List[Any], + genome: DagGenome, + seen_signatures: set, + ) -> bool: + if len(population) >= self.population_size: + return False + sig = self._genome_signature(genome) + if sig in seen_signatures: + return False + seen_signatures.add(sig) + population.append(self._make_individual(genome)) + return True + + def _build_initial_population(self, task_name: str) -> List[Any]: + population: List[Any] = [] + seen: set = set() + retry_budget = max(20, self.population_size * 10) + retries = 0 + while len(population) < self.population_size and retries < retry_budget: + genome = self._random_genome(task_name) + if self._append_if_unique(population, genome, seen): + retries = 0 + else: + retries += 1 + while len(population) < self.population_size: + population.append(self._make_individual(self._random_genome(task_name))) + return population + + def _next_generation( + self, population: List[Any], task_name: str, task: Task + ) -> List[Any]: + if self.is_multi_objective: + offspring = self._breed_offspring( + population, task_name, seen=self._novelty_archive(task_name) + ) + self._evaluate_population(offspring, task) + combined = list(population) + list(offspring) + return list(tools.selNSGA2(combined, self.population_size)) + + ranked = sorted(population, key=lambda ind: ind.fitness.values[0], reverse=True) + elite = [self._clone_individual(ind) for ind in ranked[: self.elite_size]] + seen = {self._genome_signature(ind[0]) for ind in elite} + seen |= self._novelty_archive(task_name) + return self._breed_offspring(population, task_name, initial=elite, seen=seen) + + def _novelty_archive(self, task_name: str) -> set: + if not self.novelty_breeding: + return set() + return set(self._fitness_cache.get(task_name, {})) + + def _breed_offspring( + self, + population: List[Any], + task_name: str, + initial: Optional[List[Any]] = None, + seen: Optional[set] = None, + ) -> List[Any]: + next_population = list(initial) if initial else [] + seen = set(seen) if seen else set() + + retry_budget = max(20, self.population_size * 10) + retries = 0 + tournsize = max(1, min(3, len(population))) + while len(next_population) < self.population_size and retries < retry_budget: + p1, p2 = tools.selTournament(population, 2, tournsize=tournsize) + + if self.rng.random() < self.crossover_probability: + g1, g2 = self._crossover_genomes(p1[0], p2[0]) + else: + g1, g2 = copy.deepcopy(p1[0]), copy.deepcopy(p2[0]) + + if self.rng.random() < self.mutation_probability: + g1 = self._mutate_genome(g1, task_name) + if self.rng.random() < self.mutation_probability: + g2 = self._mutate_genome(g2, task_name) + + added1 = self._append_if_unique(next_population, g1, seen) + added2 = self._append_if_unique(next_population, g2, seen) + retries = 0 if (added1 or added2) else retries + 1 + + while len(next_population) < self.population_size: + genome = self._random_genome(task_name) + if not self._append_if_unique(next_population, genome, seen): + next_population.append(self._make_individual(genome)) + + return next_population + + def _evaluate_population(self, population: List[Any], task: Task) -> None: + to_evaluate = [ind for ind in population if not ind.fitness.valid] + if not to_evaluate: + return + if self.max_workers > 1 and len(to_evaluate) > 1: + self._evaluate_individuals_parallel(to_evaluate, task) + else: + for ind in to_evaluate: + fitness = self._evaluate_genome(ind[0], task) + ind.fitness.values = fitness + + def _start_parallel_runtime(self, task_name: str) -> None: + if self._worker_pool is not None and self._parallel_task_name == task_name: + return + self._shutdown_parallel_runtime() + modalities = list( + chain.from_iterable(self.k_best_representations[task_name].values()) + ) + shared_modalities = [] + shm_names = [] + try: + for modality in modalities: + shared_modality = copy.copy(modality) + resident_bytes = 0 + try: + resident_bytes = modality.calculate_memory_usage() + except Exception: + pass + wrapped, shm_name, _, _ = add_shared_memory_candidate( + modality.data, resident_bytes + ) + if wrapped is not None: + shared_modality._data = wrapped + shm_names.append(shm_name) + shared_modalities.append(shared_modality) + worker_pool = PersistentWorkerPool( + self.max_workers, + _WORKER_DISPATCH, + ctx=create_mp_context(), + threads_per_worker=self.threads_per_worker, + ) + except Exception: + for shm_name in shm_names: + unlink_shm(shm_name) + raise + self._worker_pool = worker_pool + self._parallel_task_name = task_name + self._parallel_modalities = shared_modalities + self._parallel_shm_names = shm_names + + def _shutdown_parallel_runtime(self) -> None: + if self._worker_pool is not None: + self._worker_pool.shutdown() + self._worker_pool = None + self._parallel_task_name = None + self._parallel_modalities = None + for shm_name in self._parallel_shm_names: + unlink_shm(shm_name) + self._parallel_shm_names = [] + + def _evaluate_individuals_parallel( + self, individuals: List[Any], task: Task + ) -> None: + task_name = task.model.name + manage_runtime = self._worker_pool is None + if manage_runtime: + self._start_parallel_runtime(task_name) + cache = self._fitness_cache.setdefault(task_name, {}) + pending_followers: Dict[Tuple, List[Any]] = {} + pending_work = [] + jobs: Dict[int, Tuple[Any, RepresentationDag, Tuple]] = {} + + for ind in individuals: + genome = ind[0] + sig = self._genome_signature(genome) + cached = cache.get(sig) + if cached is not None: + ind.fitness.values = cached + continue + if sig in pending_followers: + pending_followers[sig].append(ind) + continue + dag = self._genome_to_dag(genome) + pending_followers[sig] = [] + pending_work.append((ind, dag, sig)) + + try: + while pending_work or jobs: + while ( + pending_work + and self._worker_pool.has_idle_worker + and len(jobs) < self.batch_size + ): + ind, dag, sig = pending_work.pop(0) + job_id = self._worker_pool.submit( + "genome", + (dag, task, self._parallel_modalities, self.objective_specs), + ) + jobs[job_id] = (ind, dag, sig) + + jr = self._worker_pool.wait() + ind, dag, sig = jobs.pop(jr.job_id) + if jr.ok: + fitness, payload = jr.value + error = None + else: + fitness = _failure_fitness(self.objective_specs) + payload = None + error = jr.error + ind.fitness.values = fitness + self._record_evaluation(task_name, dag, payload, error) + cache[sig] = fitness + for follower in pending_followers.pop(sig, []): + follower.fitness.values = fitness + finally: + if manage_runtime: + self._shutdown_parallel_runtime() + + def _record_evaluation( + self, + task_name: str, + dag: RepresentationDag, + payload: Optional[Dict[str, Any]], + error: Optional[str], + ) -> None: + self.optimization_results.setdefault(task_name, []) + if error is not None or payload is None: + self.evaluation_errors[task_name] = ( + self.evaluation_errors.get(task_name, 0) + 1 + ) + if self.debug and error is not None: + last_line = error.strip().splitlines()[-1] if error.strip() else error + print( + f"[GA] genome evaluation failed for task={task_name}: {last_line}" + ) + return + + result = FusionSearchResult( + dag=dag, + train_score=payload["train_score"], + val_score=payload["val_score"], + test_score=payload["test_score"], + train_fold_scores=payload.get("train_fold_scores", {}), + val_fold_scores=payload.get("val_fold_scores", {}), + test_fold_scores=payload.get("test_fold_scores", {}), + task_timing=payload.get("task_timing", {}), + runtime=payload["runtime"], + task_time=payload["task_time"], + representation_time=payload["representation_time"], + task_name=task_name, + generation=self._current_generation, + eval_index=self._eval_counter, + t_since_search_start_s=time.perf_counter() - self._search_start, + t_eval_end_unix=time.time(), + ) + self.optimization_results.setdefault(task_name, []).append(result) + self._update_hall_of_fame(task_name, result) + self._eval_counter += 1 + + def _dominates(self, a: Tuple[float, ...], b: Tuple[float, ...]) -> bool: + """True if objective tuple `a` Pareto-dominates `b`, direction-aware.""" + wa = [w * v for w, v in zip(self._objective_weights, a)] + wb = [w * v for w, v in zip(self._objective_weights, b)] + return all(x >= y for x, y in zip(wa, wb)) and any( + x > y for x, y in zip(wa, wb) + ) + + def _update_hall_of_fame(self, task_name: str, result: FusionSearchResult) -> None: + timing = { + "runtime": result.runtime, + "task_time": result.task_time, + "representation_time": result.representation_time, + } + try: + fitness = tuple( + _objective_value(name, result.val_score, timing) + for name, _ in self.objective_specs + ) + except KeyError: + return + + hof = self.hall_of_fame.setdefault(task_name, []) + fits = self._hof_fitness.setdefault(task_name, []) + + if self.is_multi_objective: + if any(self._dominates(f, fitness) or f == fitness for f in fits): + return + keep = [i for i, f in enumerate(fits) if not self._dominates(fitness, f)] + self.hall_of_fame[task_name] = [hof[i] for i in keep] + [result] + self._hof_fitness[task_name] = [fits[i] for i in keep] + [fitness] + return + + weight = self._objective_weights[0] + hof.append(result) + fits.append(fitness) + order = sorted( + range(len(fits)), key=lambda i: weight * fits[i][0], reverse=True + ) + order = order[: self.hall_of_fame_size] + self.hall_of_fame[task_name] = [hof[i] for i in order] + self._hof_fitness[task_name] = [fits[i] for i in order] + + def get_hall_of_fame(self, task_name: str) -> List[FusionSearchResult]: + return list(self.hall_of_fame.get(task_name, [])) + + def _extract_k_best(self, unimodal_results) -> Dict[str, Dict[str, List[Any]]]: + k_best = {} + for task in self.tasks: + name = task.model.name + k_best[name] = {} + for modality in self.modalities: + _, cached_data = unimodal_results.get_k_best_results( + modality, task, self.metric_name + ) + k_best[name][modality.modality_id] = cached_data + return k_best + + def _available_modality_ids(self, task_name: str) -> List[Any]: + reps = self.k_best_representations[task_name] + return [ + m.modality_id + for m in self.modalities + if len(reps.get(m.modality_id, [])) > 0 + ] + + def _leaf_capacity(self, task_name: str) -> int: + reps = self.k_best_representations[task_name] + ids = self._available_modality_ids(task_name) + if not self.allow_repeated_modalities: + return len(ids) + return sum(len(reps[mid]) for mid in ids) + + def _random_genome(self, task_name: str) -> DagGenome: + reps = self.k_best_representations[task_name] + available_modality_ids = self._available_modality_ids(task_name) + capacity = self._leaf_capacity(task_name) + if capacity < self.min_modalities: + raise ValueError( + f"Need at least {self.min_modalities} distinct leaves for task " + f"'{task_name}', but only {capacity} are available across " + f"{len(available_modality_ids)} modalities." + ) + + upper = min(self.max_modalities, capacity) + lower = min(self.min_modalities, upper) + r = self.rng.randint(lower, upper) + + if self.allow_repeated_modalities: + pool = [ + (mid, idx) + for mid in available_modality_ids + for idx in range(len(reps[mid])) + ] + leaves = self.rng.sample(pool, r) + else: + chosen = self.rng.sample(available_modality_ids, r) + leaves = [(mid, self.rng.randrange(len(reps[mid]))) for mid in chosen] + + tree = self._random_binary_tree(len(leaves)) + fusion_ops = {} + self._assign_fusion_ops(tree, fusion_ops, "") + return DagGenome(leaves=leaves, tree=tree, fusion_ops=fusion_ops) + + def _internal_paths(self, tree): + return _collect_internal_paths(tree) + + def _random_binary_tree(self, n: int) -> Tree: + nodes: List[Tree] = list(range(n)) + while len(nodes) > 1: + i, j = self.rng.sample(range(len(nodes)), 2) + a, b = nodes.pop(max(i, j)), nodes.pop(min(i, j)) + nodes.append((a, b)) + return nodes[0] + + def _assign_fusion_ops(self, subtree: Tree, ops: Dict[str, Any], path: str) -> None: + if isinstance(subtree, int): + return + ops[path] = self.rng.choice(self.fusion_operators) + left, right = subtree + self._assign_fusion_ops(left, ops, path + "L") + self._assign_fusion_ops(right, ops, path + "R") + + def _genome_to_dag(self, genome: DagGenome) -> RepresentationDag: + builder = RepresentationDAGBuilder() + leaf_ids = [ + builder.create_leaf_node(mod_id, repr_idx) + for mod_id, repr_idx in genome.leaves + ] + + def build(subtree: Tree, path: str) -> str: + if isinstance(subtree, int): + return leaf_ids[subtree] + left, right = subtree + left_id = build(left, path + "L") + right_id = build(right, path + "R") + op_cls = genome.fusion_ops[path] + op = op_cls() + return builder.create_operation_node( + op.__class__, [left_id, right_id], op.get_current_parameters() + ) + + return builder.build(build(genome.tree, "")) + + def _genome_signature(self, g: DagGenome) -> Tuple: + def norm(t: Tree): + return t if isinstance(t, int) else (norm(t[0]), norm(t[1])) + + return ( + tuple(g.leaves), + norm(g.tree), + tuple(sorted((p, c.__name__) for p, c in g.fusion_ops.items())), + ) + + def _evaluate_genome(self, genome: DagGenome, task: Task) -> Tuple[float, ...]: + task_name = task.model.name + sig = self._genome_signature(genome) + cache = self._fitness_cache.setdefault(task_name, {}) + if sig in cache: + return cache[sig] + + dag = self._genome_to_dag(genome) + modalities = list( + chain.from_iterable(self.k_best_representations[task_name].values()) + ) + + try: + fitness, payload = _evaluate_genome_body( + dag, task, modalities, self.objective_specs + ) + error = None + if fitness is None: + fitness = _failure_fitness(self.objective_specs) + except Exception: + fitness = _failure_fitness(self.objective_specs) + payload, error = None, traceback.format_exc() + + self._record_evaluation(task_name, dag, payload, error) + cache[sig] = fitness + return fitness + + def _crossover_genomes( + self, g1: DagGenome, g2: DagGenome + ) -> Tuple[DagGenome, DagGenome]: + c1, c2 = copy.deepcopy(g1), copy.deepcopy(g2) + + if c1.leaves == c2.leaves: + paths1 = self._internal_paths(c1.tree) + paths2 = self._internal_paths(c2.tree) + if paths1 and paths2: + path1 = self.rng.choice(paths1) + path2 = self.rng.choice(paths2) + subtree1 = _get_subtree(c1.tree, path1) + subtree2 = _get_subtree(c2.tree, path2) + c1.tree = _replace_subtree(c1.tree, path1, subtree2) + c2.tree = _replace_subtree(c2.tree, path2, subtree1) + c1.fusion_ops = _rebuild_fusion_ops( + c1.tree, + {**c1.fusion_ops, **c2.fusion_ops}, + self.rng, + self.fusion_operators, + randomized_prefixes=[path1], + ) + c2.fusion_ops = _rebuild_fusion_ops( + c2.tree, + {**c2.fusion_ops, **c1.fusion_ops}, + self.rng, + self.fusion_operators, + randomized_prefixes=[path2], + ) + return c1, c2 + + shared_paths = set(c1.fusion_ops) & set(c2.fusion_ops) + for path in shared_paths: + if self.rng.random() < 0.5: + c1.fusion_ops[path], c2.fusion_ops[path] = ( + c2.fusion_ops[path], + c1.fusion_ops[path], + ) + return c1, c2 + + def _mutate_genome(self, g: DagGenome, task_name: str) -> DagGenome: + op = self.rng.choice( + [ + self._mutate_change_fusion, + lambda gg: self._mutate_swap_leaf_repr(gg, task_name), + lambda gg: self.mutate_add_leaf(gg, task_name), + self.mutate_remove_leaf, + self.mutate_replace_subtree, + ] + ) + return op(g) + + def _mutate_change_fusion(self, g: DagGenome) -> DagGenome: + g = copy.deepcopy(g) + paths = [p for p in g.fusion_ops] + if not paths: + return g + path = self.rng.choice(paths) + choices = [op for op in self.fusion_operators if op != g.fusion_ops[path]] + if choices: + g.fusion_ops[path] = self.rng.choice(choices) + return g + + def _mutate_swap_leaf_repr(self, g: DagGenome, task_name: str) -> DagGenome: + g = copy.deepcopy(g) + i = self.rng.randrange(len(g.leaves)) + mod_id, current = g.leaves[i] + k = len(self.k_best_representations[task_name][mod_id]) + if k <= 1: + return g + taken = { + idx for j, (mid, idx) in enumerate(g.leaves) if mid == mod_id and j != i + } + choices = [idx for idx in range(k) if idx != current and idx not in taken] + if not choices: + return g + g.leaves[i] = (mod_id, self.rng.choice(choices)) + return g + + def mutate_add_leaf(self, g: DagGenome, task_name: str) -> DagGenome: + if len(g.leaves) >= self.max_modalities: + return g + g = copy.deepcopy(g) + reps = self.k_best_representations[task_name] + if self.allow_repeated_modalities: + existing_leaves = set(g.leaves) + candidates = [ + (mid, idx) + for mid in self._available_modality_ids(task_name) + for idx in range(len(reps[mid])) + if (mid, idx) not in existing_leaves + ] + if not candidates: + return g + new_leaf = self.rng.choice(candidates) + else: + existing = {l[0] for l in g.leaves} + available = [ + m.modality_id + for m in self.modalities + if m.modality_id not in existing + and len(reps.get(m.modality_id, [])) > 0 + ] + if not available: + return g + mod_id = self.rng.choice(available) + new_leaf = (mod_id, self.rng.randrange(len(reps[mod_id]))) + new_idx = len(g.leaves) + g.leaves.append(new_leaf) + if isinstance(g.tree, int): + g.tree = (g.tree, new_idx) + g.fusion_ops = {"": self.rng.choice(self.fusion_operators)} + else: + paths = self._internal_paths(g.tree) + path = self.rng.choice(paths) + sub = _get_subtree(g.tree, path) + g.tree = _replace_subtree(g.tree, path, (sub, new_idx)) + g.fusion_ops = _rebuild_fusion_ops( + g.tree, + g.fusion_ops, + self.rng, + self.fusion_operators, + randomized_prefixes=[path], + ) + return g + + def mutate_remove_leaf(self, g: DagGenome) -> DagGenome: + if len(g.leaves) <= self.min_modalities: + return g + g = copy.deepcopy(g) + drop = self.rng.randrange(len(g.leaves)) + new_tree = _remove_leaf_from_tree(g.tree, drop) + if new_tree is None: + return g + keep = [i for i in range(len(g.leaves)) if i != drop] + index_map = {old: new for new, old in enumerate(keep)} + g.leaves = [g.leaves[i] for i in keep] + g.tree = _reindex_tree(new_tree, index_map) + g.fusion_ops = _rebuild_fusion_ops( + g.tree, g.fusion_ops, self.rng, self.fusion_operators + ) + return g + + def mutate_replace_subtree(self, g: DagGenome) -> DagGenome: + g = copy.deepcopy(g) + paths = self._internal_paths(g.tree) + if not paths: + return g + path = self.rng.choice(paths) + sub = _get_subtree(g.tree, path) + if isinstance(sub, int): + return g + + if self.rng.random() < 0.5: + child = sub[0] if self.rng.random() < 0.5 else sub[1] + candidate = _replace_subtree(g.tree, path, child) + kept = sorted(set(_collect_leaf_indices(candidate))) + if len(kept) >= self.min_modalities: + index_map = {old: new for new, old in enumerate(kept)} + g.leaves = [g.leaves[i] for i in kept] + g.tree = _reindex_tree(candidate, index_map) + g.fusion_ops = _rebuild_fusion_ops( + g.tree, {}, self.rng, self.fusion_operators + ) + return g + + leaf_idxs = _collect_leaf_indices(sub) + new_sub = self._random_binary_tree(len(leaf_idxs)) + local_map = {i: leaf_idxs[i] for i in range(len(leaf_idxs))} + new_sub = _reindex_tree(new_sub, local_map) + g.tree = _replace_subtree(g.tree, path, new_sub) + g.fusion_ops = _rebuild_fusion_ops( + g.tree, + g.fusion_ops, + self.rng, + self.fusion_operators, + randomized_prefixes=[path], + ) + return g + + def store_results(self, file_name: str = None, overwrite: bool = False) -> str: + if file_name is None: + timestr = time.strftime("%Y%m%d-%H%M%S") + file_name = f"multimodal_optimizer_{timestr}.pkl" + + directory = os.path.dirname(file_name) or "." + os.makedirs(directory, exist_ok=True) + + if os.path.exists(file_name) and not overwrite: + raise FileExistsError( + f"Refusing to overwrite existing results file '{file_name}'. " + "Pass overwrite=True if this is intentional, or choose a " + "different file_name." + ) + + fd, tmp_path = tempfile.mkstemp( + dir=directory, prefix=".tmp_multimodal_results_", suffix=".pkl" + ) + try: + with os.fdopen(fd, "wb") as f: + pickle.dump(self.optimization_results, f) + os.replace(tmp_path, file_name) + except Exception: + if os.path.exists(tmp_path): + os.remove(tmp_path) + raise + return file_name diff --git a/src/main/python/systemds/scuro/drsearch/operator_registry.py b/src/main/python/systemds/scuro/drsearch/operator_registry.py index 7a80aafa913..d9ae798f867 100644 --- a/src/main/python/systemds/scuro/drsearch/operator_registry.py +++ b/src/main/python/systemds/scuro/drsearch/operator_registry.py @@ -47,9 +47,9 @@ def __new__(cls): def set_fusion_operators(self, fusion_operators): if isinstance(fusion_operators, list): - self._fusion_operators = fusion_operators + type(self)._fusion_operators = fusion_operators else: - self._fusion_operators = [fusion_operators] + type(self)._fusion_operators = [fusion_operators] def set_representations(self, modality_type, representations): if isinstance(representations, list): @@ -89,7 +89,7 @@ def add_context_operator(self, context_operator, modality_type): self._context_operators[m_type].append(context_operator) def add_fusion_operator(self, fusion_operator): - self._fusion_operators.append(fusion_operator) + type(self)._fusion_operators.append(fusion_operator) def add_dimensionality_reduction_operator( self, dimensionality_reduction_operator, modality_type @@ -134,10 +134,10 @@ def get_dimensionality_reduction_operators(self, modality_type): return self._dimensionality_reduction_operators.get(modality_type, []) def get_fusion_operators(self): - return self._fusion_operators + return type(self)._fusion_operators def get_fusion_operator_by_name(self, fusion_name): - for fusion in self._fusion_operators: + for fusion in type(self)._fusion_operators: if fusion.__name__ == fusion_name: return fusion return None diff --git a/src/main/python/tests/scuro/test_multimodal_ga_optimizer.py b/src/main/python/tests/scuro/test_multimodal_ga_optimizer.py new file mode 100644 index 00000000000..f79157f0247 --- /dev/null +++ b/src/main/python/tests/scuro/test_multimodal_ga_optimizer.py @@ -0,0 +1,1220 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +import copy +import os +import pickle +import random +import tempfile +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from deap import tools + +from systemds.scuro.drsearch.multimodal_ga_optimizer import ( + DagGenome, + MultimodalDeapOptimizer, + _collect_leaf_indices, + _failure_fitness, + _objective_value, +) +from systemds.scuro.drsearch.operator_registry import Registry, register_fusion_operator +from systemds.scuro.modality.type import ModalityType +from systemds.scuro.representations.average import Average +from systemds.scuro.representations.sum import Sum +from systemds.scuro.representations.concatenation import Concatenation +from systemds.scuro.representations.fusion import Fusion +from tests.scuro.data_generator import ModalityRandomDataGenerator, TestTask + +MODULE = "systemds.scuro.drsearch.multimodal_ga_optimizer" + + +@register_fusion_operator() +class _AlwaysFailingFusion(Fusion): + """A fusion operator that always raises - used to prove that one bad + genome can no longer crash the rest of a population's evaluation.""" + + def __init__(self, params=None): + super().__init__("AlwaysFailingFusion") + + def execute(self, modalities): + raise RuntimeError("intentional failure for testing") + + +class _FakeModality: + def __init__(self, modality_id): + self.modality_id = modality_id + + +class _FakeUnimodalResults: + """Stand-in for UnimodalOptimizer.operator_performance: hands back a + fixed list of representations per modality without running any real + unimodal search.""" + + def __init__(self, reps_per_modality): + self.reps_per_modality = reps_per_modality + + def get_k_best_results(self, modality, task, performance_metric_name): + reps = self.reps_per_modality.get(modality.modality_id, []) + return list(range(len(reps))), reps + + +def _make_task(name="task0"): + return SimpleNamespace(model=SimpleNamespace(name=name)) + + +def _make_optimizer(n_modalities=3, reps_per_modality=2, task=None, **kwargs): + modalities = [_FakeModality(f"m{i}") for i in range(n_modalities)] + reps = { + f"m{i}": [object() for _ in range(reps_per_modality)] + for i in range(n_modalities) + } + task = task or _make_task() + kwargs.setdefault("debug", False) + optimizer = MultimodalDeapOptimizer( + modalities, _FakeUnimodalResults(reps), [task], **kwargs + ) + return optimizer, modalities, task + + +def _fake_success_body(_dag, _task, _modalities, _objective_specs, value=0.5): + return (value,), { + "train_score": {}, + "val_score": {"accuracy": value}, + "test_score": {}, + "runtime": 0.0, + "task_time": 0.0, + "representation_time": 0.0, + } + + +class _SynchronousPool: + instances = [] + + def __init__(self, n_workers, dispatch, ctx=None, threads_per_worker=1): + self.n_workers = n_workers + self.dispatch = dispatch + self.threads_per_worker = threads_per_worker + self.pending = None + self.next_job_id = 0 + self.shutdown_called = False + self.instances.append(self) + + @property + def has_idle_worker(self): + return self.pending is None + + def submit(self, kind, payload, gpu_id=None): + job_id = self.next_job_id + self.next_job_id += 1 + try: + value = self.dispatch[kind](payload, gpu_id) + self.pending = SimpleNamespace(job_id=job_id, ok=True, value=value) + except Exception as exc: + self.pending = SimpleNamespace( + job_id=job_id, ok=False, value=None, error=str(exc) + ) + return job_id + + def wait(self): + result = self.pending + self.pending = None + return result + + def shutdown(self): + self.shutdown_called = True + + +def _make_real_representation(modality_id, num_instances, num_features): + gen = ModalityRandomDataGenerator() + rep = gen.create1DModality(num_instances, num_features, ModalityType.TIMESERIES) + rep.modality_id = modality_id + return rep + + +def _build_real_optimizer(num_instances=10, fusion_ops=None, **kwargs): + """Builds an optimizer wired to real (but tiny/synthetic) modalities, + a real cheap task/model, and real fusion operators, so it can execute + genuine RepresentationDag.execute() calls - including across process + boundaries, which rules out unittest.mock patches (a spawned worker + re-imports the module fresh and never sees main-process patches).""" + modality_ids = [0, 1] + modalities = [_FakeModality(mid) for mid in modality_ids] + reps = { + mid: [_make_real_representation(mid, num_instances, 4)] for mid in modality_ids + } + task = TestTask("mm_ga_test_task", "mm_ga_test_model", num_instances) + kwargs.setdefault("min_modalities", 2) + kwargs.setdefault("max_modalities", 2) + kwargs.setdefault("population_size", 4) + kwargs.setdefault("generations", 3) + kwargs.setdefault("elite_size", 1) + kwargs.setdefault("debug", False) + optimizer = MultimodalDeapOptimizer( + modalities, _FakeUnimodalResults(reps), [task], **kwargs + ) + optimizer.fusion_operators = fusion_ops or [Concatenation] + return optimizer, task + + +class TestConstructorValidation(unittest.TestCase): + def test_rejects_too_few_modalities(self): + modalities = [_FakeModality("m0")] + task = _make_task() + with self.assertRaises(ValueError): + MultimodalDeapOptimizer( + modalities, + _FakeUnimodalResults({"m0": [object()]}), + [task], + min_modalities=2, + debug=False, + ) + + def test_rejects_no_registered_fusion_operators(self): + modalities = [_FakeModality("m0"), _FakeModality("m1")] + reps = {"m0": [object()], "m1": [object()]} + task = _make_task() + with patch.object(Registry, "_fusion_operators", []): + with self.assertRaises(ValueError): + MultimodalDeapOptimizer( + modalities, _FakeUnimodalResults(reps), [task], debug=False + ) + + def test_elite_size_clamped_below_population_size(self): + optimizer, _, _ = _make_optimizer(population_size=3, elite_size=10) + self.assertLessEqual(optimizer.elite_size, 2) + + def test_min_modalities_clamped_to_available_when_too_high(self): + # 2 modalities available but caller asks for min_modalities=5: + # construction itself should not explode, and genome sampling + # must clamp instead of calling randint(5, 2). + optimizer, _, task = _make_optimizer( + n_modalities=2, reps_per_modality=2, min_modalities=2, max_modalities=2 + ) + optimizer.fusion_operators = [Concatenation] + genome = optimizer._random_genome(task.model.name) + self.assertEqual(len(genome.leaves), 2) + + def test_batch_size_defaults_to_max_workers(self): + optimizer, _, _ = _make_optimizer(max_workers=4) + self.assertEqual(optimizer.batch_size, 4) + + def test_batch_size_respects_explicit_value(self): + optimizer, _, _ = _make_optimizer(max_workers=8, batch_size=2) + self.assertEqual(optimizer.batch_size, 2) + self.assertEqual(optimizer.max_workers, 8) + + +class TestGenomeGeneration(unittest.TestCase): + def test_random_genome_respects_min_max_modalities(self): + optimizer, _, task = _make_optimizer( + n_modalities=4, min_modalities=2, max_modalities=3 + ) + optimizer.fusion_operators = [Concatenation, Average] + for _ in range(50): + genome = optimizer._random_genome(task.model.name) + self.assertGreaterEqual(len(genome.leaves), 2) + self.assertLessEqual(len(genome.leaves), 3) + self.assertEqual(len(genome.fusion_ops), len(genome.leaves) - 1) + self.assertEqual( + set(genome.fusion_ops.keys()), + set(optimizer._internal_paths(genome.tree)), + ) + + def test_random_genome_skips_modalities_without_representations(self): + modalities = [_FakeModality("m0"), _FakeModality("m1"), _FakeModality("m2")] + reps = {"m0": [object(), object()], "m1": [], "m2": [object()]} + task = _make_task() + optimizer = MultimodalDeapOptimizer( + modalities, + _FakeUnimodalResults(reps), + [task], + debug=False, + min_modalities=2, + max_modalities=3, + ) + optimizer.fusion_operators = [Concatenation] + for _ in range(50): + genome = optimizer._random_genome(task.model.name) + used = {mod_id for mod_id, _ in genome.leaves} + self.assertNotIn("m1", used) + + def test_random_genome_raises_when_not_enough_modalities_have_reps(self): + modalities = [_FakeModality("m0"), _FakeModality("m1")] + reps = {"m0": [object()], "m1": []} + task = _make_task() + optimizer = MultimodalDeapOptimizer( + modalities, + _FakeUnimodalResults(reps), + [task], + debug=False, + min_modalities=2, + max_modalities=2, + ) + optimizer.fusion_operators = [Concatenation] + with self.assertRaises(ValueError): + optimizer._random_genome(task.model.name) + + +class TestMutations(unittest.TestCase): + def test_add_leaf_noop_beyond_max_modalities(self): + optimizer, _, task = _make_optimizer( + n_modalities=2, min_modalities=2, max_modalities=2 + ) + optimizer.fusion_operators = [Concatenation] + genome = optimizer._random_genome(task.model.name) + mutated = optimizer.mutate_add_leaf(genome, task.model.name) + self.assertEqual(mutated.leaves, genome.leaves) + + def test_add_leaf_skips_modalities_without_representations(self): + modalities = [_FakeModality("m0"), _FakeModality("m1"), _FakeModality("m2")] + reps = {"m0": [object()], "m1": [], "m2": [object()]} + task = _make_task() + optimizer = MultimodalDeapOptimizer( + modalities, + _FakeUnimodalResults(reps), + [task], + debug=False, + min_modalities=2, + max_modalities=3, + ) + optimizer.fusion_operators = [Concatenation] + genome = DagGenome(leaves=[("m0", 0)], tree=0, fusion_ops={}) + for _ in range(20): + mutated = optimizer.mutate_add_leaf(genome, task.model.name) + used = {mod_id for mod_id, _ in mutated.leaves} + self.assertNotIn("m1", used) + + def test_remove_leaf_noop_at_min_modalities(self): + optimizer, _, task = _make_optimizer( + n_modalities=3, min_modalities=2, max_modalities=2 + ) + optimizer.fusion_operators = [Concatenation] + genome = optimizer._random_genome(task.model.name) + mutated = optimizer.mutate_remove_leaf(genome) + self.assertEqual(mutated.leaves, genome.leaves) + + def test_remove_leaf_reduces_and_stays_consistent(self): + optimizer, _, task = _make_optimizer( + n_modalities=4, reps_per_modality=2, min_modalities=2, max_modalities=4 + ) + optimizer.fusion_operators = [Concatenation, Average] + genome = None + for _ in range(50): + candidate = optimizer._random_genome(task.model.name) + if len(candidate.leaves) > 2: + genome = candidate + break + self.assertIsNotNone(genome) + mutated = optimizer.mutate_remove_leaf(genome) + self.assertEqual(len(mutated.leaves), len(genome.leaves) - 1) + self.assertEqual( + set(mutated.fusion_ops.keys()), + set(optimizer._internal_paths(mutated.tree)), + ) + + def test_replace_subtree_never_drops_below_min_modalities(self): + """Regression test: the collapse branch used to replace a two-leaf + root by one of its children, leaving a bare leaf index as the tree + while genome.leaves kept both entries -- so a min_modalities=2 search + evaluated unimodal pipelines.""" + optimizer, _, task = _make_optimizer( + n_modalities=3, reps_per_modality=2, min_modalities=2, max_modalities=2 + ) + optimizer.fusion_operators = [Concatenation, Average] + for _ in range(200): + genome = optimizer._random_genome(task.model.name) + mutated = optimizer.mutate_replace_subtree(genome) + self.assertGreaterEqual(len(mutated.leaves), optimizer.min_modalities) + self.assertNotIsInstance(mutated.tree, int) + self.assertEqual( + sorted(set(_collect_leaf_indices(mutated.tree))), + list(range(len(mutated.leaves))), + ) + + def test_replace_subtree_collapse_prunes_and_reindexes_leaves(self): + """When the collapse is allowed (enough leaves survive), the dropped + leaves must leave genome.leaves too, and the tree must be reindexed + onto the surviving ones.""" + optimizer, _, task = _make_optimizer( + n_modalities=4, reps_per_modality=2, min_modalities=2, max_modalities=4 + ) + optimizer.fusion_operators = [Concatenation, Average] + saw_collapse = False + for _ in range(400): + genome = optimizer._random_genome(task.model.name) + if len(genome.leaves) < 3: + continue + mutated = optimizer.mutate_replace_subtree(genome) + leaf_idxs = _collect_leaf_indices(mutated.tree) + self.assertEqual(sorted(set(leaf_idxs)), list(range(len(mutated.leaves)))) + self.assertEqual( + set(mutated.fusion_ops.keys()), + set(optimizer._internal_paths(mutated.tree)), + ) + if len(mutated.leaves) < len(genome.leaves): + saw_collapse = True + # every surviving leaf still names a leaf of the parent genome + for leaf in mutated.leaves: + self.assertIn(leaf, genome.leaves) + self.assertTrue(saw_collapse, "collapse branch never taken") + + +class TestRepeatedModalities(unittest.TestCase): + """allow_repeated_modalities lets one modality contribute several leaves, + each a different representation of it (intra-modal fusion).""" + + def test_off_by_default_one_leaf_per_modality(self): + optimizer, _, task = _make_optimizer( + n_modalities=3, reps_per_modality=4, min_modalities=2, max_modalities=3 + ) + for _ in range(100): + g = optimizer._random_genome(task.model.name) + mods = [mid for mid, _ in g.leaves] + self.assertEqual(len(mods), len(set(mods))) + + def test_max_modalities_clamped_to_modality_count_when_off(self): + optimizer, _, _ = _make_optimizer( + n_modalities=3, reps_per_modality=5, max_modalities=12 + ) + self.assertEqual(optimizer.max_modalities, 3) + + def test_max_modalities_can_exceed_modality_count_when_on(self): + optimizer, _, _ = _make_optimizer( + n_modalities=3, + reps_per_modality=5, + max_modalities=12, + allow_repeated_modalities=True, + ) + self.assertEqual(optimizer.max_modalities, 12) + + def test_random_genome_may_repeat_a_modality_and_leaves_stay_distinct(self): + optimizer, _, task = _make_optimizer( + n_modalities=2, + reps_per_modality=4, + min_modalities=2, + max_modalities=6, + allow_repeated_modalities=True, + ) + saw_repeat = False + for _ in range(200): + g = optimizer._random_genome(task.model.name) + self.assertEqual(len(set(g.leaves)), len(g.leaves)) + self.assertLessEqual(len(g.leaves), 8) # 2 modalities x 4 reps + if len({mid for mid, _ in g.leaves}) < len(g.leaves): + saw_repeat = True + self.assertTrue(saw_repeat, "no genome ever repeated a modality") + + def test_leaf_capacity_bounds_genome_size(self): + optimizer, _, task = _make_optimizer( + n_modalities=2, + reps_per_modality=3, + min_modalities=2, + max_modalities=99, + allow_repeated_modalities=True, + ) + self.assertEqual(optimizer._leaf_capacity(task.model.name), 6) + for _ in range(100): + g = optimizer._random_genome(task.model.name) + self.assertLessEqual(len(g.leaves), 6) + + def test_add_leaf_can_repeat_a_modality_without_duplicating_a_leaf(self): + optimizer, _, task = _make_optimizer( + n_modalities=2, + reps_per_modality=3, + min_modalities=2, + max_modalities=6, + allow_repeated_modalities=True, + ) + name = task.model.name + g = optimizer._random_genome(name) + for _ in range(20): + g = optimizer.mutate_add_leaf(g, name) + self.assertEqual(len(set(g.leaves)), len(g.leaves)) + self.assertEqual(len(g.leaves), 6) # saturates at capacity, no duplicates + + def test_swap_leaf_repr_never_creates_a_duplicate_leaf(self): + optimizer, _, task = _make_optimizer( + n_modalities=2, + reps_per_modality=2, + min_modalities=2, + max_modalities=4, + allow_repeated_modalities=True, + ) + name = task.model.name + for _ in range(200): + g = optimizer._random_genome(name) + mutated = optimizer._mutate_swap_leaf_repr(g, name) + self.assertEqual(len(set(mutated.leaves)), len(mutated.leaves)) + + def test_min_modalities_one_admits_unimodal_genomes(self): + optimizer, _, task = _make_optimizer( + n_modalities=3, reps_per_modality=2, min_modalities=1, max_modalities=3 + ) + self.assertEqual(optimizer.min_modalities, 1) + sizes = { + len(optimizer._random_genome(task.model.name).leaves) for _ in range(200) + } + self.assertIn(1, sizes) + + def test_rejects_max_below_min(self): + with self.assertRaises(ValueError): + _make_optimizer(n_modalities=4, min_modalities=3, max_modalities=2) + + +class TestHallOfFame(unittest.TestCase): + def test_keeps_best_single_objective_results_in_order(self): + optimizer, _, task = _make_optimizer(hall_of_fame_size=2) + optimizer.fusion_operators = [Concatenation] + name = task.model.name + + for value in (0.3, 0.9, 0.5, 0.7): + genome = optimizer._random_genome(name) + with patch( + f"{MODULE}._evaluate_genome_body", + side_effect=lambda *a, v=value, **kw: _fake_success_body(*a, value=v), + ): + optimizer._evaluate_genome(genome, task) + + hof = optimizer.get_hall_of_fame(name) + self.assertEqual([r.val_score["accuracy"] for r in hof], [0.9, 0.7]) + # the full result list is untouched by the hall of fame + self.assertEqual(len(optimizer.optimization_results[name]), 4) + + def test_direction_aware_for_a_minimised_objective(self): + optimizer, _, task = _make_optimizer( + objectives=[("accuracy", "min")], hall_of_fame_size=1 + ) + optimizer.fusion_operators = [Concatenation] + name = task.model.name + for value in (0.8, 0.2, 0.6): + genome = optimizer._random_genome(name) + with patch( + f"{MODULE}._evaluate_genome_body", + side_effect=lambda *a, v=value, **kw: _fake_success_body(*a, value=v), + ): + optimizer._evaluate_genome(genome, task) + hof = optimizer.get_hall_of_fame(name) + self.assertEqual([r.val_score["accuracy"] for r in hof], [0.2]) + + def test_multi_objective_keeps_non_dominated_front_only(self): + optimizer, _, task = _make_optimizer( + objectives=[("accuracy", "max"), ("runtime", "min")] + ) + optimizer.fusion_operators = [Concatenation] + name = task.model.name + + # (accuracy, runtime): B dominates C; A and B are mutually non-dominated. + points = [(0.9, 10.0), (0.5, 1.0), (0.4, 2.0)] + for accuracy, runtime in points: + genome = optimizer._random_genome(name) + + def body(*_a, acc=accuracy, rt=runtime, **_kw): + return (acc, rt), { + "train_score": {}, + "val_score": {"accuracy": acc}, + "test_score": {}, + "runtime": rt, + "task_time": 0.0, + "representation_time": rt, + } + + with patch(f"{MODULE}._evaluate_genome_body", side_effect=body): + optimizer._evaluate_genome(genome, task) + + front = { + (r.val_score["accuracy"], r.runtime) + for r in optimizer.get_hall_of_fame(name) + } + self.assertEqual(front, {(0.9, 10.0), (0.5, 1.0)}) + + def test_failed_evaluations_never_enter_the_hall_of_fame(self): + optimizer, _, task = _make_optimizer() + optimizer.fusion_operators = [Concatenation] + genome = optimizer._random_genome(task.model.name) + with patch(f"{MODULE}._evaluate_genome_body", side_effect=RuntimeError("boom")): + optimizer._evaluate_genome(genome, task) + self.assertEqual(optimizer.get_hall_of_fame(task.model.name), []) + + +class TestCrossover(unittest.TestCase): + def test_same_leaves_produces_structurally_valid_children(self): + optimizer, _, task = _make_optimizer(n_modalities=3, reps_per_modality=2) + optimizer.fusion_operators = [Concatenation, Average] + g1 = optimizer._random_genome(task.model.name) + g2 = copy.deepcopy(g1) + g2.tree = optimizer._random_binary_tree(len(g2.leaves)) + g2.fusion_ops = {} + optimizer._assign_fusion_ops(g2.tree, g2.fusion_ops, "") + + c1, c2 = optimizer._crossover_genomes(g1, g2) + for child in (c1, c2): + self.assertEqual(sorted(child.leaves), sorted(g1.leaves)) + self.assertEqual( + set(child.fusion_ops.keys()), set(optimizer._internal_paths(child.tree)) + ) + + def test_mismatched_leaves_still_recombines(self): + """Regression test: the original crossover returned both parents + completely unchanged whenever they didn't select the exact same + modality subset/order/index - which, since leaves are sampled + independently per genome, made crossover a near total no-op even + though it fired with 70% probability every generation.""" + optimizer, _, task = _make_optimizer( + n_modalities=4, reps_per_modality=2, min_modalities=2, max_modalities=4 + ) + optimizer.fusion_operators = [Concatenation, Average, Sum] + optimizer.rng = random.Random(0) + + g1 = optimizer._random_genome(task.model.name) + g2 = optimizer._random_genome(task.model.name) + for _ in range(50): + if g1.leaves != g2.leaves: + break + g2 = optimizer._random_genome(task.model.name) + self.assertNotEqual(g1.leaves, g2.leaves, "test setup needs mismatched parents") + + changed = False + for _ in range(100): + c1, c2 = optimizer._crossover_genomes(g1, g2) + if c1.fusion_ops != g1.fusion_ops or c2.fusion_ops != g2.fusion_ops: + changed = True + break + self.assertTrue( + changed, "crossover with mismatched leaf sets never recombined anything" + ) + # leaves/tree topology are untouched by the op-only fallback + self.assertEqual(c1.leaves, g1.leaves) + self.assertEqual(c2.leaves, g2.leaves) + + +class TestGenomeSignature(unittest.TestCase): + def test_signature_ignores_fusion_ops_dict_insertion_order(self): + optimizer, _, task = _make_optimizer(n_modalities=3, reps_per_modality=2) + optimizer.fusion_operators = [Concatenation, Average] + genome = optimizer._random_genome(task.model.name) + reordered = DagGenome( + leaves=list(genome.leaves), + tree=genome.tree, + fusion_ops=dict(reversed(list(genome.fusion_ops.items()))), + ) + self.assertEqual( + optimizer._genome_signature(genome), optimizer._genome_signature(reordered) + ) + + def test_signature_differs_for_different_fusion_op(self): + optimizer, _, task = _make_optimizer(n_modalities=2, reps_per_modality=1) + optimizer.fusion_operators = [Concatenation, Average] + genome = DagGenome( + leaves=[("m0", 0), ("m1", 0)], tree=(0, 1), fusion_ops={"": Concatenation} + ) + other = DagGenome( + leaves=[("m0", 0), ("m1", 0)], tree=(0, 1), fusion_ops={"": Average} + ) + self.assertNotEqual( + optimizer._genome_signature(genome), optimizer._genome_signature(other) + ) + + +class TestEvaluateGenome(unittest.TestCase): + def test_records_result_on_success(self): + optimizer, _, task = _make_optimizer() + optimizer.fusion_operators = [Concatenation] + genome = optimizer._random_genome(task.model.name) + + with patch(f"{MODULE}._evaluate_genome_body", side_effect=_fake_success_body): + fitness = optimizer._evaluate_genome(genome, task) + + self.assertEqual(fitness, (0.5,)) + self.assertEqual(len(optimizer.optimization_results[task.model.name]), 1) + self.assertEqual(optimizer.evaluation_errors.get(task.model.name, 0), 0) + + def test_survives_exception_without_crashing(self): + """Regression test for the crash bug: a failure used to raise + NameError (missing `traceback` import in the except-block) and then + TypeError (unpacking the commented-out None return), instead of + just scoring the genome -inf and moving on.""" + optimizer, _, task = _make_optimizer() + optimizer.fusion_operators = [Concatenation] + genome = optimizer._random_genome(task.model.name) + + with patch(f"{MODULE}._evaluate_genome_body", side_effect=RuntimeError("boom")): + fitness = optimizer._evaluate_genome(genome, task) + + self.assertEqual(fitness, (float("-inf"),)) + self.assertEqual(len(optimizer.optimization_results[task.model.name]), 0) + self.assertEqual(optimizer.evaluation_errors[task.model.name], 1) + + def test_uses_cache_for_repeated_signature(self): + optimizer, _, task = _make_optimizer() + optimizer.fusion_operators = [Concatenation] + genome = optimizer._random_genome(task.model.name) + + call_count = {"n": 0} + + def counting_body(*args, **kwargs): + call_count["n"] += 1 + return _fake_success_body(*args, **kwargs) + + with patch(f"{MODULE}._evaluate_genome_body", side_effect=counting_body): + optimizer._evaluate_genome(genome, task) + optimizer._evaluate_genome(copy.deepcopy(genome), task) + + self.assertEqual(call_count["n"], 1) + + +class TestNextGenerationAndPopulation(unittest.TestCase): + def test_next_generation_carries_over_elite_individuals(self): + optimizer, _, task = _make_optimizer( + n_modalities=3, reps_per_modality=3, population_size=5, elite_size=2 + ) + optimizer.fusion_operators = [Concatenation, Average] + population = optimizer._build_initial_population(task.model.name) + for i, ind in enumerate(population): + ind.fitness.values = (float(i),) + + ranked = sorted(population, key=lambda ind: ind.fitness.values[0], reverse=True) + top_signatures = { + optimizer._genome_signature(ind[0]) + for ind in ranked[: optimizer.elite_size] + } + + next_population = optimizer._next_generation(population, task.model.name, task) + next_signatures = { + optimizer._genome_signature(ind[0]) for ind in next_population + } + self.assertTrue(top_signatures.issubset(next_signatures)) + + elites_in_next = [ + ind + for ind in next_population + if optimizer._genome_signature(ind[0]) in top_signatures + ] + for ind in elites_in_next: + self.assertTrue(ind.fitness.valid) + + def test_novelty_breeding_rejects_genomes_from_earlier_generations(self): + # The dedup set must include the whole fitness cache, not just the + # current elite. Without that, offspring identical to something + # scored in an earlier generation are accepted, served from the + # cache, and occupy a population slot that explores nothing. + optimizer, _, task = _make_optimizer( + n_modalities=3, reps_per_modality=4, population_size=6, elite_size=1 + ) + optimizer.fusion_operators = [Concatenation, Average] + name = task.model.name + population = optimizer._build_initial_population(name) + for i, ind in enumerate(population): + ind.fitness.values = (float(i),) + + # Pretend a previous generation already scored these genomes. + cache = optimizer._fitness_cache.setdefault(name, {}) + for ind in population: + cache[optimizer._genome_signature(ind[0])] = ind.fitness.values + stale = set(cache) + + nxt = optimizer._next_generation(population, name, task) + elite = { + optimizer._genome_signature(ind[0]) + for ind in sorted( + population, key=lambda i: i.fitness.values[0], reverse=True + )[: optimizer.elite_size] + } + # Elites are exempt; every other slot must be a genome never scored. + non_elite = [ + optimizer._genome_signature(ind[0]) + for ind in nxt + if optimizer._genome_signature(ind[0]) not in elite + ] + self.assertTrue(non_elite) + self.assertEqual([g for g in non_elite if g in stale], []) + + def test_novelty_breeding_can_be_disabled(self): + optimizer, _, task = _make_optimizer( + n_modalities=2, + reps_per_modality=1, + population_size=4, + min_modalities=2, + max_modalities=2, + novelty_breeding=False, + ) + optimizer.fusion_operators = [Concatenation] + name = task.model.name + self.assertEqual(optimizer._novelty_archive(name), set()) + + def test_novelty_breeding_still_terminates_when_space_is_exhausted(self): + # Archive covers the only genome that exists: breeding must fall back + # to duplicates rather than spinning on the retry budget forever. + optimizer, _, task = _make_optimizer( + n_modalities=2, + reps_per_modality=1, + population_size=5, + min_modalities=2, + max_modalities=2, + ) + optimizer.fusion_operators = [Concatenation] + name = task.model.name + population = optimizer._build_initial_population(name) + for ind in population: + ind.fitness.values = (0.5,) + cache = optimizer._fitness_cache.setdefault(name, {}) + for ind in population: + cache[optimizer._genome_signature(ind[0])] = ind.fitness.values + + nxt = optimizer._next_generation(population, name, task) + self.assertEqual(len(nxt), 5) + + def test_next_generation_terminates_with_tiny_search_space(self): + # 2 modalities x 1 representation x 1 fusion op => exactly one + # distinct genome is possible; requesting a bigger population must + # not hang. + optimizer, _, task = _make_optimizer( + n_modalities=2, + reps_per_modality=1, + population_size=8, + min_modalities=2, + max_modalities=2, + ) + optimizer.fusion_operators = [Concatenation] + population = optimizer._build_initial_population(task.model.name) + self.assertEqual(len(population), 8) + for ind in population: + ind.fitness.values = (0.5,) + + next_population = optimizer._next_generation(population, task.model.name, task) + self.assertEqual(len(next_population), 8) + + +class TestOptimizeLoop(unittest.TestCase): + def test_end_to_end_with_stubbed_fitness(self): + optimizer, _, task = _make_optimizer( + n_modalities=3, + reps_per_modality=3, + population_size=6, + generations=8, + elite_size=1, + early_stopping_patience=3, + ) + optimizer.fusion_operators = [Concatenation, Average] + + def fake_body(dag, _task, _modalities, _metric): + h = hash(str(dag.nodes)) % 1000 / 1000.0 + return _fake_success_body(dag, _task, _modalities, _metric, value=h) + + with patch(f"{MODULE}._evaluate_genome_body", side_effect=fake_body): + results = optimizer.optimize() + + self.assertIn(task.model.name, results) + self.assertGreater(len(results[task.model.name]), 0) + self.assertEqual(optimizer.evaluation_errors.get(task.model.name, 0), 0) + + def test_survives_partial_evaluation_failures(self): + optimizer, _, task = _make_optimizer( + population_size=6, generations=4, elite_size=1, early_stopping_patience=None + ) + optimizer.fusion_operators = [Concatenation, Average] + + call_counter = {"n": 0} + + def flaky_body(dag, _task, _modalities, _metric): + call_counter["n"] += 1 + if call_counter["n"] % 3 == 0: + raise RuntimeError("simulated fusion failure") + return _fake_success_body(dag, _task, _modalities, _metric, value=0.6) + + with patch(f"{MODULE}._evaluate_genome_body", side_effect=flaky_body): + results = optimizer.optimize() + + self.assertGreater(optimizer.evaluation_errors.get(task.model.name, 0), 0) + self.assertGreater(len(results[task.model.name]), 0) + + def test_early_stopping_triggers_before_generation_budget(self): + optimizer, _, task = _make_optimizer( + population_size=6, + generations=50, + elite_size=1, + early_stopping_patience=2, + early_stopping_min_delta=1e-6, + ) + optimizer.fusion_operators = [Concatenation] + + original_next_gen = optimizer._next_generation + gens_run = {"n": 0} + + def counting_next_gen(pop, name, task_): + gens_run["n"] += 1 + return original_next_gen(pop, name, task_) + + optimizer._next_generation = counting_next_gen + + with patch(f"{MODULE}._evaluate_genome_body", side_effect=_fake_success_body): + optimizer.optimize() + + self.assertLess(gens_run["n"], 10) + + +class TestStoreResults(unittest.TestCase): + def test_refuses_overwrite_by_default(self): + optimizer, _, task = _make_optimizer() + optimizer.optimization_results[task.model.name] = ["dummy"] + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "results.pkl") + optimizer.store_results(path) + with open(path, "rb") as f: + original = f.read() + + optimizer.optimization_results[task.model.name] = ["different"] + with self.assertRaises(FileExistsError): + optimizer.store_results(path) + + with open(path, "rb") as f: + self.assertEqual(f.read(), original) + + def test_overwrite_true_replaces_file(self): + optimizer, _, task = _make_optimizer() + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "results.pkl") + optimizer.optimization_results[task.model.name] = ["v1"] + optimizer.store_results(path) + + optimizer.optimization_results[task.model.name] = ["v2"] + optimizer.store_results(path, overwrite=True) + + with open(path, "rb") as f: + loaded = pickle.load(f) + self.assertEqual(loaded[task.model.name], ["v2"]) + + def test_write_is_atomic_no_partial_file_on_failure(self): + optimizer, _, _ = _make_optimizer() + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "results.pkl") + with patch("pickle.dump", side_effect=RuntimeError("disk full")): + with self.assertRaises(RuntimeError): + optimizer.store_results(path) + self.assertFalse(os.path.exists(path)) + self.assertEqual(os.listdir(d), []) + + +class TestRealFusionIntegration(unittest.TestCase): + """Exercises the actual RepresentationDag.execute() path end-to-end, + including across process boundaries for the parallel case (a spawned + worker re-imports this module fresh, so unittest.mock patches from the + parent process cannot reach it - these tests need real, picklable + fusion ops/tasks/modalities).""" + + def test_optimize_end_to_end_real_serial(self): + optimizer, task = _build_real_optimizer(max_workers=1) + results = optimizer.optimize() + self.assertGreater(len(results[task.model.name]), 0) + self.assertEqual(optimizer.evaluation_errors.get(task.model.name, 0), 0) + + def test_optimize_end_to_end_real_parallel(self): + optimizer, task = _build_real_optimizer(max_workers=2, batch_size=2) + results = optimizer.optimize() + self.assertGreater(len(results[task.model.name]), 0) + self.assertEqual(optimizer.evaluation_errors.get(task.model.name, 0), 0) + + def test_optimize_reuses_one_pool_across_generations(self): + optimizer, task = _build_real_optimizer( + max_workers=2, batch_size=2, threads_per_worker=3 + ) + _SynchronousPool.instances = [] + with patch(f"{MODULE}.PersistentWorkerPool", _SynchronousPool), patch( + f"{MODULE}.create_mp_context", return_value=None + ): + results = optimizer.optimize() + + self.assertGreater(len(results[task.model.name]), 0) + self.assertEqual(len(_SynchronousPool.instances), 1) + pool = _SynchronousPool.instances[0] + self.assertEqual(pool.threads_per_worker, 3) + self.assertTrue(pool.shutdown_called) + self.assertIsNone(optimizer._worker_pool) + + def test_parallel_runtime_uses_shared_copies_and_unlinks_them(self): + optimizer, task = _build_real_optimizer(max_workers=2, batch_size=2) + task_name = task.model.name + source_modalities = list( + modality + for reps in optimizer.k_best_representations[task_name].values() + for modality in reps + ) + original_data = [modality.data for modality in source_modalities] + wrappers = [object(), object()] + shared_results = [ + (wrappers[0], "shm-0", 1024, 0), + (wrappers[1], "shm-1", 1024, 0), + ] + _SynchronousPool.instances = [] + + with patch( + f"{MODULE}.add_shared_memory_candidate", side_effect=shared_results + ), patch(f"{MODULE}.unlink_shm") as unlink, patch( + f"{MODULE}.PersistentWorkerPool", _SynchronousPool + ), patch( + f"{MODULE}.create_mp_context", return_value=None + ): + optimizer._start_parallel_runtime(task_name) + self.assertEqual( + [modality.data for modality in optimizer._parallel_modalities], + wrappers, + ) + self.assertEqual([m.data for m in source_modalities], original_data) + optimizer._shutdown_parallel_runtime() + + self.assertEqual( + [call.args[0] for call in unlink.call_args_list], ["shm-0", "shm-1"] + ) + + def test_parallel_evaluation_dedupes_identical_genomes_in_same_batch(self): + optimizer, task = _build_real_optimizer(max_workers=2, batch_size=2) + genome = optimizer._random_genome(task.model.name) + ind1 = optimizer._make_individual(copy.deepcopy(genome)) + ind2 = optimizer._make_individual(copy.deepcopy(genome)) + + optimizer._evaluate_individuals_parallel([ind1, ind2], task) + + self.assertTrue(ind1.fitness.valid) + self.assertTrue(ind2.fitness.valid) + self.assertEqual(ind1.fitness.values, ind2.fitness.values) + self.assertEqual(len(optimizer.optimization_results[task.model.name]), 1) + + def test_parallel_evaluation_survives_a_failing_genome(self): + """A genome whose fusion operator always raises must not take down + the rest of the (parallel) batch.""" + optimizer, task = _build_real_optimizer( + max_workers=2, + batch_size=2, + fusion_ops=[Concatenation, _AlwaysFailingFusion], + ) + good_genome = DagGenome( + leaves=[(0, 0), (1, 0)], tree=(0, 1), fusion_ops={"": Concatenation} + ) + bad_genome = DagGenome( + leaves=[(0, 0), (1, 0)], + tree=(0, 1), + fusion_ops={"": _AlwaysFailingFusion}, + ) + ind_good = optimizer._make_individual(good_genome) + ind_bad = optimizer._make_individual(bad_genome) + + optimizer._evaluate_individuals_parallel([ind_good, ind_bad], task) + + self.assertTrue(ind_good.fitness.valid) + self.assertTrue(ind_bad.fitness.valid) + self.assertEqual(ind_bad.fitness.values[0], float("-inf")) + self.assertGreater(ind_good.fitness.values[0], float("-inf")) + self.assertEqual(optimizer.evaluation_errors.get(task.model.name, 0), 1) + self.assertEqual(len(optimizer.optimization_results[task.model.name]), 1) + + +class TestMultiObjective(unittest.TestCase): + def test_objective_value_reads_timing_vs_val_score(self): + val_score = {"accuracy": 0.8, "f1": 0.7} + timing = {"runtime": 1.5, "task_time": 1.0, "representation_time": 0.5} + self.assertEqual(_objective_value("accuracy", val_score, timing), 0.8) + self.assertEqual(_objective_value("f1", val_score, timing), 0.7) + self.assertEqual(_objective_value("runtime", val_score, timing), 1.5) + self.assertEqual(_objective_value("task_time", val_score, timing), 1.0) + + def test_failure_fitness_is_direction_aware(self): + """A failed evaluation must always be the worst possible candidate, + regardless of whether an objective is maximized or minimized - + using -inf for a 'min' objective (e.g. runtime) would make a failure + look infinitely fast and win every tournament/dominance check.""" + specs = [("accuracy", "max"), ("runtime", "min")] + worst = _failure_fitness(specs) + self.assertEqual(worst, (float("-inf"), float("inf"))) + + def test_constructor_rejects_invalid_direction(self): + with self.assertRaises(ValueError): + _make_optimizer(objectives=[("accuracy", "sideways")]) + + def test_constructor_rejects_empty_objectives(self): + with self.assertRaises(ValueError): + _make_optimizer(objectives=[]) + + def test_is_multi_objective_flag_and_weights(self): + multi, _, _ = _make_optimizer( + objectives=[("accuracy", "max"), ("runtime", "min")] + ) + single, _, _ = _make_optimizer() + self.assertTrue(multi.is_multi_objective) + self.assertEqual( + multi.objective_specs, [("accuracy", "max"), ("runtime", "min")] + ) + multi_ind = multi._make_individual(DagGenome([("m0", 0)], 0, {})) + single_ind = single._make_individual(DagGenome([("m0", 0)], 0, {})) + self.assertEqual(multi_ind.fitness.weights, (1.0, -1.0)) + self.assertEqual(single_ind.fitness.weights, (1.0,)) + + def test_single_objective_by_default(self): + optimizer, _, _ = _make_optimizer() + self.assertFalse(optimizer.is_multi_objective) + self.assertEqual(optimizer.objective_specs, [("accuracy", "max")]) + + def test_evaluate_genome_returns_tuple_per_objective(self): + optimizer, _, task = _make_optimizer( + objectives=[("accuracy", "max"), ("runtime", "min")] + ) + optimizer.fusion_operators = [Concatenation] + genome = optimizer._random_genome(task.model.name) + + def fake_body(_dag, _task, _modalities, objective_specs): + self.assertEqual(objective_specs, optimizer.objective_specs) + return (0.9, 1.2), { + "train_score": {}, + "val_score": {"accuracy": 0.9}, + "test_score": {}, + "runtime": 1.2, + "task_time": 1.0, + "representation_time": 0.2, + } + + with patch(f"{MODULE}._evaluate_genome_body", side_effect=fake_body): + fitness = optimizer._evaluate_genome(genome, task) + + self.assertEqual(fitness, (0.9, 1.2)) + self.assertEqual(len(optimizer.optimization_results[task.model.name]), 1) + + def test_evaluate_genome_failure_uses_direction_aware_sentinel(self): + optimizer, _, task = _make_optimizer( + objectives=[("accuracy", "max"), ("runtime", "min")] + ) + optimizer.fusion_operators = [Concatenation] + genome = optimizer._random_genome(task.model.name) + + with patch(f"{MODULE}._evaluate_genome_body", side_effect=RuntimeError("boom")): + fitness = optimizer._evaluate_genome(genome, task) + + self.assertEqual(fitness, (float("-inf"), float("inf"))) + + def test_next_generation_multi_objective_returns_population_sized_front(self): + optimizer, _, task = _make_optimizer( + n_modalities=3, + reps_per_modality=3, + population_size=6, + objectives=[("accuracy", "max"), ("runtime", "min")], + ) + optimizer.fusion_operators = [Concatenation, Average] + population = optimizer._build_initial_population(task.model.name) + for i, ind in enumerate(population): + ind.fitness.values = ( + float(i) / len(population), + float(len(population) - i), + ) + + def fake_body(dag, _task, _modalities, _objective_specs): + h = hash(str(dag.nodes)) % 1000 / 1000.0 + return (h, 1.0 - h), { + "train_score": {}, + "val_score": {"accuracy": h}, + "test_score": {}, + "runtime": 1.0 - h, + "task_time": 0.0, + "representation_time": 0.0, + } + + with patch(f"{MODULE}._evaluate_genome_body", side_effect=fake_body): + next_population = optimizer._next_generation( + population, task.model.name, task + ) + + self.assertEqual(len(next_population), optimizer.population_size) + for ind in next_population: + self.assertTrue(ind.fitness.valid) + self.assertEqual(len(ind.fitness.values), 2) + + def test_optimize_end_to_end_multi_objective_stubbed(self): + optimizer, _, task = _make_optimizer( + n_modalities=3, + reps_per_modality=3, + population_size=6, + generations=6, + objectives=[("accuracy", "max"), ("runtime", "min")], + early_stopping_patience=3, + ) + optimizer.fusion_operators = [Concatenation, Average] + + def fake_body(dag, _task, _modalities, _objective_specs): + h = hash(str(dag.nodes)) % 1000 / 1000.0 + return (h, 1.0 - h), { + "train_score": {}, + "val_score": {"accuracy": h}, + "test_score": {}, + "runtime": 1.0 - h, + "task_time": 0.0, + "representation_time": 0.0, + } + + with patch(f"{MODULE}._evaluate_genome_body", side_effect=fake_body): + results = optimizer.optimize() + + self.assertIn(task.model.name, results) + self.assertGreater(len(results[task.model.name]), 0) + self.assertEqual(optimizer.evaluation_errors.get(task.model.name, 0), 0) + + def test_optimize_multi_objective_survives_partial_failures(self): + optimizer, _, task = _make_optimizer( + population_size=6, + generations=4, + objectives=[("accuracy", "max"), ("runtime", "min")], + early_stopping_patience=None, + ) + optimizer.fusion_operators = [Concatenation, Average] + + call_counter = {"n": 0} + + def flaky_body(dag, _task, _modalities, _objective_specs): + call_counter["n"] += 1 + if call_counter["n"] % 3 == 0: + raise RuntimeError("simulated fusion failure") + return (0.7, 0.3), { + "train_score": {}, + "val_score": {"accuracy": 0.7}, + "test_score": {}, + "runtime": 0.3, + "task_time": 0.0, + "representation_time": 0.0, + } + + with patch(f"{MODULE}._evaluate_genome_body", side_effect=flaky_body): + results = optimizer.optimize() + + self.assertGreater(optimizer.evaluation_errors.get(task.model.name, 0), 0) + self.assertGreater(len(results[task.model.name]), 0) + + def test_real_fusion_multi_objective_end_to_end(self): + """Exercises the real dag.execute() path (not stubbed) with two + objectives to make sure runtime is actually threaded through from + real evaluation timing, not just from stubbed payloads.""" + optimizer, task = _build_real_optimizer( + objectives=[("accuracy", "max"), ("runtime", "min")] + ) + results = optimizer.optimize() + self.assertGreater(len(results[task.model.name]), 0) + for result in results[task.model.name]: + self.assertGreaterEqual(result.runtime, 0.0) + + +if __name__ == "__main__": + unittest.main() From b76bbd68dc77bbf84e6fd3303e2baddc0589c549 Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Tue, 1 Sep 2026 11:10:56 +0200 Subject: [PATCH 127/132] [SYSTEMDS-3968] Add test set evaluator - #2602 This patch adds functionality to apply representations on the test set only and perform inference on it. It measures the performance metrics as well as runtime for each step. Assisted-by: AI --- .../scuro/dataloader/timeseries_loader.py | 2 + .../python/systemds/scuro/drsearch/ranking.py | 136 +++++++++++++++++- .../python/systemds/scuro/drsearch/task.py | 110 ++++++++++++++ .../scuro/drsearch/test_set_evaluation.py | 121 ++++++++++++++++ .../scuro/drsearch/unimodal_optimizer.py | 23 ++- .../timeseries_representations.py | 8 +- .../tests/scuro/test_unimodal_optimizer.py | 31 +++- 7 files changed, 422 insertions(+), 9 deletions(-) create mode 100644 src/main/python/systemds/scuro/drsearch/test_set_evaluation.py diff --git a/src/main/python/systemds/scuro/dataloader/timeseries_loader.py b/src/main/python/systemds/scuro/dataloader/timeseries_loader.py index 8e6c11316b0..a2f515f7d26 100644 --- a/src/main/python/systemds/scuro/dataloader/timeseries_loader.py +++ b/src/main/python/systemds/scuro/dataloader/timeseries_loader.py @@ -52,11 +52,13 @@ def __init__( normalize: bool = True, file_format: str = "npy", modality_type: Optional[ModalityType] = ModalityType.TIMESERIES, + channel_index: Optional[int] = None, ): super().__init__(source_path, indices, data_type, chunk_size, modality_type) self.signal_names = signal_names self.sampling_rate = sampling_rate self.normalize = normalize + self.channel_index = channel_index self.file_format = file_format.lower() self.stats = self.get_stats(source_path, sampling_rate) if self.file_format not in ["npy", "mat", "hdf5", "txt"]: diff --git a/src/main/python/systemds/scuro/drsearch/ranking.py b/src/main/python/systemds/scuro/drsearch/ranking.py index 381a6d1fb9d..935bc565356 100644 --- a/src/main/python/systemds/scuro/drsearch/ranking.py +++ b/src/main/python/systemds/scuro/drsearch/ranking.py @@ -21,11 +21,127 @@ from typing import Callable, Iterable, Optional +import numpy as np + + +def _operator_signature(entry) -> frozenset: + """Set of operator class names in an entry's DAG, ignoring hyperparameters.""" + try: + return frozenset( + node.operation.__name__ + for node in entry.dag.nodes + if getattr(node, "operation", None) is not None + ) + except Exception: + return frozenset() + + +def _dag_size(entry) -> int: + try: + return sum( + 1 + for node in entry.dag.nodes + if getattr(node, "operation", None) is not None + ) + except Exception: + return 0 + + +def rank_by_robustness( + entries: Iterable, + *, + performance_metric_name: str = "accuracy", + neighbourhood_weight: float = 0.5, + sharpness: int = 4, + one_se_parsimony: bool = True, + cache_scores: bool = True, + score_attr: str = "robustness_score", +): + entries = list(entries) + if not entries: + return [], [] + + def perf_of(entry): + if entry is None: + return None + try: + score = float(entry.val_score[performance_metric_name]) + except (KeyError, TypeError, ValueError): + return None + return score if np.isfinite(score) else None + + indexed_entries = [ + (index, entry, score) + for index, entry in enumerate(entries) + if (score := perf_of(entry)) is not None + ] + if not indexed_entries: + return [], [] + + original_indices, entries, performance = zip(*indexed_entries) + entries = list(entries) + perf = np.array(performance, dtype=float) + sizes = np.array([_dag_size(e) if e is not None else 0 for e in entries], float) + + smoothed = perf + if neighbourhood_weight > 0.0 and len(entries) > 1: + signatures = [ + _operator_signature(e) if e is not None else frozenset() for e in entries + ] + vocabulary = sorted({op for sig in signatures for op in sig}) + if vocabulary: + position = {op: i for i, op in enumerate(vocabulary)} + membership = np.zeros((len(entries), len(vocabulary)), dtype=np.float32) + for row, sig in enumerate(signatures): + for op in sig: + membership[row, position[op]] = 1.0 + intersection = membership @ membership.T + counts = membership.sum(1) + union = counts[:, None] + counts[None, :] - intersection + jaccard = np.divide( + intersection, + union, + out=np.zeros_like(intersection), + where=union > 0, + ) + weights = jaccard**sharpness + denominator = weights.sum(1) + neighbourhood = np.divide( + (weights * perf[None, :].astype(np.float32)).sum(1), + denominator, + out=perf.astype(np.float32).copy(), + where=denominator > 0, + ) + smoothed = ( + 1.0 - neighbourhood_weight + ) * perf + neighbourhood_weight * neighbourhood + + if cache_scores: + for entry, score in zip(entries, smoothed): + if entry is not None: + setattr(entry, score_attr, float(score)) + + if one_se_parsimony and len(entries) > 1: + standard_error = float(smoothed.std()) / np.sqrt(len(smoothed)) + threshold = float(smoothed.max()) - standard_error + keys = [ + (True, -sz, float(s)) if s >= threshold else (False, 0.0, float(s)) + for s, sz in zip(smoothed, sizes) + ] + else: + keys = [(True, 0.0, float(s)) for s in smoothed] + + local_indices = sorted(range(len(entries)), key=lambda i: keys[i], reverse=True) + sorted_entries = [entries[i] for i in local_indices] + sorted_indices = [original_indices[i] for i in local_indices] + + return sorted_entries, sorted_indices + def rank_by_tradeoff( entries: Iterable, *, - weights=(0.7, 0.3), + weights=(1.0, 0.0), performance_metric_name: str = "accuracy", runtime_accessor: Optional[Callable[[object], float]] = None, cache_scores: bool = True, @@ -48,8 +164,10 @@ def runtime_accessor(entry): task = getattr(entry, "task_time", 0.0) return rep + task - performance = [float(performance_score_accessor(e)) for e in entries] - runtimes = [float(runtime_accessor(e)) for e in entries] + performance = [ + float(performance_score_accessor(e)) if e is not None else 0.0 for e in entries + ] + runtimes = [float(runtime_accessor(e)) if e is not None else 0.0 for e in entries] perf_min, perf_max = min(performance), max(performance) run_min, run_max = min(runtimes), max(runtimes) @@ -76,17 +194,25 @@ def safe_normalize(values, vmin, vmax): if cache_scores: for entry, score in zip(entries, scores): + if entry is None: + continue if hasattr(entry, score_attr): setattr(entry, score_attr, score) else: setattr(entry, score_attr, score) - sorted_entries = sorted(entries, key=lambda e: e.tradeoff_score, reverse=True) + sorted_entries = sorted( + entries, + key=lambda e: e.tradeoff_score if hasattr(e, "tradeoff_score") else 0.0, + reverse=True, + ) sorted_indices = [ i for i, _ in sorted( - enumerate(entries), key=lambda pair: pair[1].tradeoff_score, reverse=True + enumerate(entries), + key=lambda pair: pair[1].tradeoff_score if pair is not None else None, + reverse=True, ) ] diff --git a/src/main/python/systemds/scuro/drsearch/task.py b/src/main/python/systemds/scuro/drsearch/task.py index 7977c628c12..7ddeaede76c 100644 --- a/src/main/python/systemds/scuro/drsearch/task.py +++ b/src/main/python/systemds/scuro/drsearch/task.py @@ -62,6 +62,18 @@ def compute_averages(self): self.average_scores[self.metrics] = np.mean(self.scores[self.metrics]) return self + def fold_scores(self): + return { + metric: [float(v) for v in values] for metric, values in self.scores.items() + } + + def score_stds(self): + """Per-metric standard deviation across folds (0.0 for a single fold).""" + return { + metric: (float(np.std(values, ddof=1)) if len(values) > 1 else 0.0) + for metric, values in self.scores.items() + } + class Task: def __init__( @@ -95,6 +107,7 @@ def __init__( self.measure_performance = measure_performance self.inference_time = [] self.training_time = [] + self.last_run_timing = {} self.expected_dim = 1 self.performance_measures = performance_measures self.train_scores = PerformanceMeasure("train", performance_measures) @@ -286,12 +299,109 @@ def run(self, data): if hasattr(model, "clean_up"): model.clean_up() del model + + self.last_run_timing = { + "train_time_per_fold_s": list(self.training_time), + "test_inference_time_per_fold_s": list(self.inference_time), + "train_time_mean_s": ( + float(np.mean(self.training_time)) if self.training_time else 0.0 + ), + "test_inference_time_mean_s": ( + float(np.mean(self.inference_time)) if self.inference_time else 0.0 + ), + "n_test_instances": len(self.test_indices) if self.test_indices else 0, + } return [ self.train_scores.compute_averages(), self.val_scores.compute_averages(), self.test_scores.compute_averages(), ] + def fit_once_and_time_inference( + self, data, latency_repeats: int = 200, latency_warmup: int = 20 + ): + model = self.create_model() + + train_X = self._gather_by_indices(data, self.train_indices) + train_y = self._gather_by_indices(self.labels, self.train_indices) + test_X = self._gather_by_indices(data, self.test_indices) + test_y = self._gather_by_indices(self.labels, self.test_indices) + + t0 = time.perf_counter() + model.fit(train_X, train_y, test_X, test_y) + fit_time = time.perf_counter() - t0 + + t0 = time.perf_counter() + test_score = model.test(np.asarray(test_X), test_y) + batch_inference_time = time.perf_counter() - t0 + + latency = self._measure_single_sample_latency( + model, test_X, test_y, latency_repeats, latency_warmup + ) + + scores = PerformanceMeasure("test_single_fit", self.performance_measures) + scores.add_scores(test_score[0]) + scores.compute_averages() + + if hasattr(model, "clean_up"): + model.clean_up() + + n_test = len(test_X) + return { + "single_fit_train_time_s": fit_time, + "test_batch_inference_time_s": batch_inference_time, + "test_inference_time_per_instance_ms": ( + batch_inference_time / n_test * 1000.0 if n_test else 0.0 + ), + "n_test_instances": n_test, + "single_fit_test_scores": scores.average_scores, + **latency, + } + + @staticmethod + def _prediction_callable(model): + if hasattr(model, "predict"): + return model.predict, "model.predict" + clf = getattr(model, "clf", None) + if clf is not None and hasattr(clf, "predict"): + return clf.predict, "clf.predict" + return None, None + + def _measure_single_sample_latency( + self, model, test_X, test_y, repeats: int, warmup: int + ): + empty = { + "inference_latency_ms_median": None, + "inference_latency_ms_p95": None, + "inference_latency_samples": 0, + "inference_latency_source": None, + } + if repeats <= 0 or not len(test_X): + return empty + + predict, source = self._prediction_callable(model) + if predict is None: + return empty + + timings = [] + for i in range(warmup + repeats): + sample_X = np.asarray([test_X[i % len(test_X)]]) + t0 = time.perf_counter() + try: + predict(sample_X) + except Exception: + return empty + elapsed = (time.perf_counter() - t0) * 1000.0 + if i >= warmup: + timings.append(elapsed) + + return { + "inference_latency_ms_median": float(np.median(timings)), + "inference_latency_ms_p95": float(np.percentile(timings, 95)), + "inference_latency_samples": len(timings), + "inference_latency_source": source, + } + def _reset_params(self): self.inference_time = [] self.training_time = [] diff --git a/src/main/python/systemds/scuro/drsearch/test_set_evaluation.py b/src/main/python/systemds/scuro/drsearch/test_set_evaluation.py new file mode 100644 index 00000000000..f160cd4ccd0 --- /dev/null +++ b/src/main/python/systemds/scuro/drsearch/test_set_evaluation.py @@ -0,0 +1,121 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- +from __future__ import annotations + +import time +from typing import Any, Dict, List + +from systemds.scuro.modality.modality import Modality +from systemds.scuro.drsearch.representation_dag import RepresentationDag +from systemds.scuro.representations.aggregated_representation import ( + AggregatedRepresentation, +) +from systemds.scuro.utils.schema_helpers import get_shape + + +def _unwrap(result): + if isinstance(result, dict): + if not result: + return None + return result[list(result.keys())[-1]] + return result + + +def _match_expected_dim(modality, task): + if modality is None or task is None: + return modality + if getattr(task, "expected_dim", 1) == 1 and get_shape(modality.metadata) > 1: + return AggregatedRepresentation().transform(modality) + return modality + + +def measure_representation_time_on_test_set( + dag: RepresentationDag, + modalities: List[Modality], + test_indices: List[int], + task=None, + repeats: int = 1, +) -> Dict[str, Any]: + subsets = [modality.subset(test_indices) for modality in modalities] + + timings = [] + output = None + for _ in range(max(1, repeats)): + t0 = time.perf_counter() + output = _match_expected_dim( + _unwrap(dag.execute(subsets, task, enable_cache=False)), task + ) + timings.append(time.perf_counter() - t0) + + timings.sort() + median = timings[len(timings) // 2] + n_test = len(test_indices) + + output_shape = None + if output is not None and getattr(output, "data", None) is not None: + try: + output_shape = tuple(getattr(output.data[0], "shape", ())) + except (IndexError, TypeError): + output_shape = None + + return { + "test_only_representation_time_s": median, + "test_only_representation_time_all_runs_s": timings, + "test_only_representation_time_per_instance_ms": ( + median / n_test * 1000.0 if n_test else 0.0 + ), + "test_only_n_instances": n_test, + "test_only_output_shape": output_shape, + "test_only_features_are_valid_for_scoring": False, + } + + +def measure_test_set_application( + dag: RepresentationDag, + modalities: List[Modality], + task, + full_data=None, + repeats: int = 1, + latency_repeats: int = 200, + latency_warmup: int = 20, +) -> Dict[str, Any]: + record = measure_representation_time_on_test_set( + dag, modalities, task.test_indices, task=task, repeats=repeats + ) + + if full_data is None: + t0 = time.perf_counter() + output = _match_expected_dim( + _unwrap(dag.execute(modalities, task, enable_cache=False)), task + ) + record["full_representation_time_s"] = time.perf_counter() - t0 + full_data = None if output is None else output.data + + if full_data is not None: + record.update( + task.fit_once_and_time_inference( + full_data, + latency_repeats=latency_repeats, + latency_warmup=latency_warmup, + ) + ) + + return record diff --git a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py index b185bd45dfb..5f130c369a3 100644 --- a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py +++ b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py @@ -30,7 +30,7 @@ from systemds.scuro.modality.type import ModalityType from systemds.scuro.drsearch.node_executor import NodeExecutor, ResultEntry from systemds.scuro.representations.representation import RepresentationStats -from systemds.scuro.drsearch.ranking import rank_by_tradeoff +from systemds.scuro.drsearch.ranking import rank_by_tradeoff, rank_by_robustness from systemds.scuro.drsearch.task import PerformanceMeasure from systemds.scuro.representations.concatenation import Concatenation from systemds.scuro.representations.hadamard import Hadamard @@ -994,6 +994,27 @@ def get_k_best_results( return results, cache + def get_k_most_robust_results( + self, + modality, + task, + performance_metric_name, + k=None, + neighbourhood_weight=0.5, + one_se_parsimony=True, + ): + task_results = self.results[modality.modality_id][task.model.name] + + results, sorted_indices = rank_by_robustness( + task_results, + performance_metric_name=performance_metric_name, + neighbourhood_weight=neighbourhood_weight, + one_se_parsimony=one_se_parsimony, + ) + + limit = self.k if k is None else k + return results[:limit], sorted_indices[:limit] + def add_worker_stat(self, worker_stats, modality_id): self.worker_stats[modality_id] = worker_stats diff --git a/src/main/python/systemds/scuro/representations/timeseries_representations.py b/src/main/python/systemds/scuro/representations/timeseries_representations.py index e6aa999e8f2..c67fde8692e 100644 --- a/src/main/python/systemds/scuro/representations/timeseries_representations.py +++ b/src/main/python/systemds/scuro/representations/timeseries_representations.py @@ -210,7 +210,9 @@ def __init__(self, params=None): super().__init__("Skew", min_input_length=3) def compute_feature(self, signal, axis=-1): - return np.array(stats.skew(signal, axis=axis)) + result = np.asarray(stats.skew(signal, axis=axis)) + zero_variance = np.std(signal, axis=axis) <= np.finfo(float).eps + return np.where(zero_variance, 0.0, result) @register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) @@ -253,7 +255,9 @@ def __init__(self, params=None): super().__init__("Kurtosis", min_input_length=4) def compute_feature(self, signal, axis=-1): - return np.array(stats.kurtosis(signal, fisher=True, bias=True, axis=axis)) + result = np.asarray(stats.kurtosis(signal, fisher=True, bias=True, axis=axis)) + zero_variance = np.std(signal, axis=axis) <= np.finfo(float).eps + return np.where(zero_variance, 0.0, result) @register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL]) diff --git a/src/main/python/tests/scuro/test_unimodal_optimizer.py b/src/main/python/tests/scuro/test_unimodal_optimizer.py index 41717b4e3a8..f27c721aa25 100644 --- a/src/main/python/tests/scuro/test_unimodal_optimizer.py +++ b/src/main/python/tests/scuro/test_unimodal_optimizer.py @@ -21,11 +21,16 @@ import unittest +from types import SimpleNamespace import numpy as np from systemds.scuro.representations.color_histogram import ColorHistogram from systemds.scuro.drsearch.operator_registry import Registry -from systemds.scuro.drsearch.unimodal_optimizer import UnimodalOptimizer +from systemds.scuro.drsearch.node_executor import ResultEntry +from systemds.scuro.drsearch.unimodal_optimizer import ( + UnimodalOptimizer, + UnimodalResults, +) from systemds.scuro.representations.covarep_audio_features import ZeroCrossing from systemds.scuro.representations.covarep_audio_features import ( @@ -133,6 +138,30 @@ def test_unimodal_optimizer_for_text_modality(self): ) self.optimize_unimodal_representation_for_modality([text]) + def test_robust_results_ignore_non_finite_scores(self): + modality = SimpleNamespace(modality_id="modality") + task = SimpleNamespace(model=SimpleNamespace(name="task")) + results = UnimodalResults([modality], [task], k=2) + scores = [0.8, np.nan, np.inf, -np.inf, None, 0.6] + entries = [ + ResultEntry( + val_score=None if score is None else {"accuracy": score}, + dag=SimpleNamespace( + nodes=[SimpleNamespace(operation=UnimodalOptimizer)] + ), + ) + for score in scores + ] + results.results[modality.modality_id][task.model.name] = entries + + robust, indices = results.get_k_most_robust_results( + modality, task, "accuracy", one_se_parsimony=False + ) + + self.assertEqual(robust, [entries[0], entries[5]]) + self.assertEqual(indices, [0, 5]) + self.assertTrue(all(np.isfinite(entry.robustness_score) for entry in robust)) + def test_bow_and_tfidf_require_dimensionality_reduction_before_task(self): text_data, text_md = ModalityRandomDataGenerator().create_text_data( self.num_instances, 10 From 55e6bf6bf64f3d495f8f5d2ba3c246c5fe0fb2ed Mon Sep 17 00:00:00 2001 From: bruno Date: Tue, 1 Sep 2026 14:28:00 +0200 Subject: [PATCH 128/132] dev/format-changed.sh --- .../java/org/apache/sysds/api/DMLScript.java | 10 +- .../org/apache/sysds/common/Builtins.java | 376 +++++------------- .../java/org/apache/sysds/hops/BinaryOp.java | 4 +- src/main/java/org/apache/sysds/hops/Hop.java | 14 +- .../java/org/apache/sysds/hops/UnaryOp.java | 9 +- .../sysds/hops/estim/EstimationUtils.java | 12 +- .../sysds/hops/rewrite/ProgramRewriter.java | 6 +- ...riteMatrixMultChainOptimizationSparse.java | 19 +- .../parser/BuiltinFunctionExpression.java | 24 -- .../apache/sysds/parser/DMLTranslator.java | 87 +--- .../apache/sysds/parser/DataExpression.java | 206 +++------- .../compress/CompressedMatrixBlock.java | 2 +- .../runtime/compress/colgroup/AColGroup.java | 17 +- .../compress/colgroup/AColGroupValue.java | 1 - .../runtime/compress/colgroup/ASDCZero.java | 6 +- .../compress/colgroup/ColGroupDDC.java | 4 +- .../compress/colgroup/ColGroupEmpty.java | 7 +- .../colgroup/ColGroupLinearFunctional.java | 2 +- .../compress/colgroup/ColGroupOLE.java | 2 +- .../compress/colgroup/ColGroupRLE.java | 4 +- .../compress/colgroup/ColGroupSDCFOR.java | 4 +- .../colgroup/ColGroupUncompressed.java | 13 +- .../colgroup/ColGroupUncompressedArray.java | 2 +- .../dictionary/AIdentityDictionary.java | 4 +- .../colgroup/dictionary/DeltaDictionary.java | 4 +- .../colgroup/dictionary/IDictionary.java | 6 +- .../dictionary/IdentityDictionary.java | 4 +- .../dictionary/IdentityDictionarySlice.java | 4 +- .../compress/colgroup/mapping/AMapToData.java | 2 +- .../compress/colgroup/offset/AOffset.java | 10 +- .../compress/colgroup/offset/OffsetEmpty.java | 1 + .../compress/lib/CLALibBinaryCellOp.java | 6 +- .../runtime/compress/lib/CLALibMMChain.java | 2 +- .../compress/lib/CLALibRemoveEmpty.java | 15 +- .../runtime/compress/lib/CLALibSort.java | 8 +- .../controlprogram/caching/FrameObject.java | 4 +- .../controlprogram/caching/MatrixObject.java | 2 +- .../context/SparkExecutionContext.java | 34 +- .../federated/FederatedWorker.java | 3 +- .../frame/data/columns/ArrayFactory.java | 22 +- .../frame/data/lib/MatrixBlockFromFrame.java | 4 +- .../runtime/functionobjects/Builtin.java | 139 +++---- .../instructions/cp/BinaryCPInstruction.java | 2 +- .../cp/BinaryFrameScalarCPInstruction.java | 10 +- .../cp/BinaryMatrixMatrixCPInstruction.java | 4 +- .../cp/ParameterizedBuiltinCPInstruction.java | 7 +- .../instructions/ooc/ReorgOOCInstruction.java | 8 +- .../ooc/ReshapeOOCInstruction.java | 69 ++-- .../spark/QuantilePickSPInstruction.java | 3 +- .../spark/data/IndexedMatrixValue.java | 7 +- .../sysds/runtime/io/DeltaKernelUtils.java | 257 ++++++------ .../apache/sysds/runtime/io/ReaderDelta.java | 107 ++--- .../sysds/runtime/io/ReaderDeltaParallel.java | 96 ++--- .../apache/sysds/runtime/io/WriterDelta.java | 74 ++-- .../runtime/matrix/data/LibMatrixReorg.java | 34 +- .../runtime/matrix/data/MatrixBlock.java | 34 +- .../sysds/runtime/ooc/cache/OOCFuture.java | 9 +- .../runtime/ooc/cache/io/CloseableQueue.java | 38 +- .../cache/io/OOCBufferedDataInputStream.java | 12 +- .../cache/io/OOCBufferedDataOutputStream.java | 20 +- .../runtime/ooc/cache/io/OOCIOHandler.java | 25 +- .../ooc/cache/io/OOCMatrixIOHandler.java | 161 ++++---- .../runtime/ooc/cache/io/SpillableObject.java | 6 +- .../ooc/cache/legacy/OOCCacheScheduler.java | 44 +- .../cache/legacy/OOCLRUCacheScheduler.java | 207 +++++----- .../runtime/transform/decode/Decoder.java | 10 +- .../runtime/transform/decode/DecoderBin.java | 4 +- .../transform/decode/DecoderDummycode.java | 2 +- .../transform/decode/DecoderFactory.java | 53 ++- .../transform/decode/DecoderRecode.java | 18 +- .../sysds/runtime/util/CommonThreadPool.java | 8 +- .../sysds/runtime/util/DataConverter.java | 9 +- .../org/apache/sysds/utils/DoubleParser.java | 2 +- .../apache/sysds/utils/SettingsChecker.java | 13 +- .../org/apache/sysds/performance/Main.java | 7 +- .../apache/sysds/test/AutomatedTestBase.java | 23 +- .../java/org/apache/sysds/test/TestUtils.java | 9 +- .../component/compile/CompilerTestBase.java | 21 +- .../SparkTransitiveExecTypeCompileTest.java | 50 +-- .../compress/CompressedSortTest.java | 6 +- .../compress/lib/CLALibMMChainTest.java | 4 +- ...CompressedBinaryMatrixMatrixSolveTest.java | 15 +- .../OffsetClassInitConcurrencyTest.java | 4 +- .../SparkContextReferenceCountTest.java | 32 +- .../component/federated/FedWorkerBase.java | 19 +- .../federated/FedWorkerMatrixCompress.java | 8 +- .../component/frame/FrameToStringTest.java | 14 +- .../frame/MatrixFromFrameSafeCastTest.java | 12 +- .../frame/transform/DecoderCompositeTest.java | 8 +- .../GetCategoricalMaskInstructionTest.java | 21 +- .../TransformDecodeRoundTripTest.java | 39 +- .../frame/transform/TransformDecodeTest.java | 4 +- .../component/io/DeltaMatrixCoverageTest.java | 97 +++-- .../io/DeltaMatrixReadWriteTest.java | 333 ++++++++++------ .../io/DeltaMatrixSparkInteropTest.java | 105 +++-- .../component/matrix/QuantilePickTest.java | 13 +- .../component/tensor/TensorToStringTest.java | 14 +- .../functions/binary/matrix/QuantileTest.java | 18 +- .../builtin/part2/BuiltinSTEPGlmTest.java | 3 +- .../FederatedBackendPerformanceTest.java | 6 +- .../part4/FederatedLogicalTest.java | 5 +- .../functions/indexing/LeftIndexingTest.java | 48 +-- .../sysds/test/functions/io/ScalarIOTest.java | 12 +- .../io/delta/DeltaReadWriteTest.java | 40 +- .../io/parquet/FrameParquetSchemaTest.java | 3 +- .../functions/jmlc/JMLConnectionTest.java | 42 +- .../functions/lineage/FedFullReuseTest.java | 17 +- .../functions/lineage/FedUDFReuseTest.java | 5 +- .../test/functions/misc/ToStringTest.java | 33 +- .../sysds/test/functions/ooc/ReshapeTest.java | 14 +- .../functions/reorg/MatrixReshapeTest.java | 6 +- .../functions/reorg/VectorReshapeTest.java | 6 +- .../rewrite/RewriteMatrixChainDPTest.java | 4 +- .../RewriteMatrixMultChainOptSparseTest.java | 7 +- ...writeQuantizationFusedCompressionTest.java | 2 +- .../transform/GetCategoricalMaskTest.java | 20 +- .../TransformFrameEncodeBagOfWords.java | 3 +- .../vect/LeftIndexingChainUpdateTest.java | 2 +- 118 files changed, 1654 insertions(+), 1958 deletions(-) diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java b/src/main/java/org/apache/sysds/api/DMLScript.java index a7a175bb7b6..0bb1e9b462d 100644 --- a/src/main/java/org/apache/sysds/api/DMLScript.java +++ b/src/main/java/org/apache/sysds/api/DMLScript.java @@ -508,9 +508,9 @@ private static void execute(String dmlScriptStr, String fnameOptConfig, Map inHops1 = new ArrayList<>(); - inHops1.add(expr); - inHops1.add(expr2); - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), inHops1); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - case AVG_POOL: - case MAX_POOL: { - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForPoolingForwardIM2COL(expr, source, 1, hops)); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - case AVG_POOL_BACKWARD: - case MAX_POOL_BACKWARD: { - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForConvOpPoolingCOL2IM(expr, source, 1, hops)); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - case CONV2D: - case CONV2D_BACKWARD_FILTER: - case CONV2D_BACKWARD_DATA: { - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForConvOp(expr, source, 1, hops)); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - - case ROW_COUNT_DISTINCT: - currBuiltinOp = new AggUnaryOp(target.getName(), - DataType.MATRIX, target.getValueType(), AggOp.COUNT_DISTINCT, Direction.Row, expr); - break; - - case COL_COUNT_DISTINCT: - currBuiltinOp = new AggUnaryOp(target.getName(), - DataType.MATRIX, target.getValueType(), AggOp.COUNT_DISTINCT, Direction.Col, expr); - break; - - case GET_CATEGORICAL_MASK: - currBuiltinOp = new BinaryOp(target.getName(), DataType.MATRIX, ValueType.FP64, OpOp2.GET_CATEGORICAL_MASK, expr, expr2); - break; - default: - throw new ParseException("Unsupported builtin function type: "+source.getOpCode()); - } - - boolean isConvolution = source.getOpCode() == Builtins.CONV2D || source.getOpCode() == Builtins.CONV2D_BACKWARD_DATA || - source.getOpCode() == Builtins.CONV2D_BACKWARD_FILTER || - source.getOpCode() == Builtins.MAX_POOL || source.getOpCode() == Builtins.MAX_POOL_BACKWARD || - source.getOpCode() == Builtins.AVG_POOL || source.getOpCode() == Builtins.AVG_POOL_BACKWARD; - if( !isConvolution) { + boolean isConvolution = source.getOpCode() == Builtins.CONV2D || + source.getOpCode() == Builtins.CONV2D_BACKWARD_DATA || + source.getOpCode() == Builtins.CONV2D_BACKWARD_FILTER || source.getOpCode() == Builtins.MAX_POOL || + source.getOpCode() == Builtins.MAX_POOL_BACKWARD || source.getOpCode() == Builtins.AVG_POOL || + source.getOpCode() == Builtins.AVG_POOL_BACKWARD; + if(!isConvolution) { // Since the dimension of output doesnot match that of input variable for these operations setIdentifierParams(currBuiltinOp, source.getOutput()); } diff --git a/src/main/java/org/apache/sysds/parser/DataExpression.java b/src/main/java/org/apache/sysds/parser/DataExpression.java index 68a3d1b7ffe..3d3a90b4f6f 100644 --- a/src/main/java/org/apache/sysds/parser/DataExpression.java +++ b/src/main/java/org/apache/sysds/parser/DataExpression.java @@ -1176,52 +1176,72 @@ else if( getVarParam(READNNZPARAM) != null ) { boolean isHDF5 = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.HDF5.toString())); - boolean isCOG = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.COG.toString())); + // handle all csv default parameters + handleCSVDefaultParam(DELIM_DELIMITER, ValueType.STRING, conditional); + handleCSVDefaultParam(DELIM_FILL_VALUE, ValueType.FP64, conditional); + handleCSVDefaultParam(DELIM_HAS_HEADER_ROW, ValueType.BOOLEAN, conditional); + handleCSVDefaultParam(DELIM_FILL, ValueType.BOOLEAN, conditional); + handleCSVDefaultParam(DELIM_NA_STRINGS, ValueType.STRING, conditional); + } - // Delta tables are self-describing (schema + dimensions discovered from the - // transaction log at read time), so dimensions are optional like CSV. - boolean isDelta = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.DELTA.toString())); + boolean isLIBSVM = false; + isLIBSVM = (formatTypeString != null && + formatTypeString.equalsIgnoreCase(FileFormat.LIBSVM.toString())); + if(isLIBSVM) { + // Handle libsvm file format + shouldReadMTD = true; + + // only allow IO_FILENAME, READROWPARAM, READCOLPARAM + // as valid parameters + if(!inferredFormatType) { + for(String key : _varParams.keySet()) { + if(!(key.equals(IO_FILENAME) || key.equals(FORMAT_TYPE) || key.equals(READROWPARAM) || + key.equals(READCOLPARAM) || key.equals(READNNZPARAM) || key.equals(DATATYPEPARAM) || + key.equals(VALUETYPEPARAM) || key.equals(DELIM_DELIMITER) || + key.equals(LIBSVM_INDEX_DELIM))) { + String msg = "Only parameters allowed are: " + IO_FILENAME + "," + READROWPARAM + "," + + READCOLPARAM + DELIM_DELIMITER + "," + LIBSVM_INDEX_DELIM; + + raiseValidateError( + "Invalid parameter " + key + " in read statement: " + toString() + ". " + msg, + conditional, LanguageErrorCodes.INVALID_PARAMETERS); + } + } + } + // handle all default parameters + handleCSVDefaultParam(DELIM_DELIMITER, ValueType.STRING, conditional); + handleCSVDefaultParam(LIBSVM_INDEX_DELIM, ValueType.STRING, conditional); + } - dataTypeString = (getVarParam(DATATYPEPARAM) == null) ? null : getVarParam(DATATYPEPARAM).toString(); - - if ( dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE) - || dataTypeString.equalsIgnoreCase(Statement.FRAME_DATA_TYPE)) { - - boolean isMatrix = false; - if ( dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE)) + boolean isHDF5 = (formatTypeString != null && + formatTypeString.equalsIgnoreCase(FileFormat.HDF5.toString())); + + boolean isCOG = (formatTypeString != null && + formatTypeString.equalsIgnoreCase(FileFormat.COG.toString())); + + // Delta tables are self-describing (schema + dimensions discovered from the + // transaction log at read time), so dimensions are optional like CSV. + boolean isDelta = (formatTypeString != null && + formatTypeString.equalsIgnoreCase(FileFormat.DELTA.toString())); + + dataTypeString = (getVarParam(DATATYPEPARAM) == null) ? null : getVarParam(DATATYPEPARAM).toString(); + + if(dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE) || + dataTypeString.equalsIgnoreCase(Statement.FRAME_DATA_TYPE)) { + + boolean isMatrix = false; + if(dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE)) isMatrix = true; - - // set data type - getOutput().setDataType(isMatrix ? DataType.MATRIX : DataType.FRAME); - - // set number non-zeros - Expression ennz = getVarParam("nnz"); - long nnz = -1; - if( ennz != null ) { - nnz = Long.valueOf(ennz.toString()); - getOutput().setNnz(nnz); - } - // Following dimension checks must be done when data type = MATRIX_DATA_TYPE - // initialize size of target data identifier to UNKNOWN - getOutput().setDimensions(-1, -1); - - if (!isCSV && !isLIBSVM && !isHDF5 && !isCOG && !isDelta && ConfigurationManager.getCompilerConfig() - .getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) //skip check for csv/libsvm/delta format / jmlc api - && (getVarParam(READROWPARAM) == null || getVarParam(READCOLPARAM) == null) ) { - raiseValidateError("Missing or incomplete dimension information in read statement: " - + mtdFileName, conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - - if (getVarParam(READROWPARAM) instanceof ConstIdentifier - && getVarParam(READCOLPARAM) instanceof ConstIdentifier) - { - // these are strings that are long values - Long dim1 = (getVarParam(READROWPARAM) == null) ? null : Long.valueOf( getVarParam(READROWPARAM).toString()); - Long dim2 = (getVarParam(READCOLPARAM) == null) ? null : Long.valueOf( getVarParam(READCOLPARAM).toString()); - if ( !isCSV && !isDelta && (dim1 < 0 || dim2 < 0) && ConfigurationManager - .getCompilerConfig().getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) ) { - raiseValidateError("Invalid dimension information in read statement", conditional, LanguageErrorCodes.INVALID_PARAMETERS); + // set data type + getOutput().setDataType(isMatrix ? DataType.MATRIX : DataType.FRAME); + + // set number non-zeros + Expression ennz = getVarParam("nnz"); + long nnz = -1; + if(ennz != null) { + nnz = Long.valueOf(ennz.toString()); + getOutput().setNnz(nnz); } // set dim1 and dim2 values @@ -1252,104 +1272,10 @@ && getVarParam(READCOLPARAM) instanceof ConstIdentifier) catch(Exception ex) { raiseValidateError("Invalid format '" + fmt+ "' in statement: " + toString(), conditional); } - - if (getVarParam(ROWBLOCKCOUNTPARAM) instanceof ConstIdentifier && getVarParam(COLUMNBLOCKCOUNTPARAM) instanceof ConstIdentifier) { - Integer rowBlockCount = (getVarParam(ROWBLOCKCOUNTPARAM) == null) ? - null : Integer.valueOf(getVarParam(ROWBLOCKCOUNTPARAM).toString()); - getOutput().setBlocksize(rowBlockCount != null ? rowBlockCount : -1); - } - - // block dimensions must be -1x-1 when format="text" - // NOTE MB: disabled validate of default blocksize for inputs w/ format="binary" - // because we automatically introduce reblocks if blocksizes don't match - if ( (getOutput().getFileFormat().isTextFormat() || !isMatrix) && getOutput().getBlocksize() != -1 ){ - raiseValidateError("Invalid block dimensions (" + getOutput().getBlocksize() + ") when format=" + getVarParam(FORMAT_TYPE) + " in \"" + this.toString() + "\".", conditional); - } - - } - else if ( dataTypeString.equalsIgnoreCase(Statement.SCALAR_DATA_TYPE)) { - getOutput().setDataType(DataType.SCALAR); - getOutput().setNnz(-1L); - } - else if ( dataTypeString.equalsIgnoreCase(DataType.LIST.name())) { - getOutput().setDataType(DataType.LIST); - } - else{ - raiseValidateError("Unknown Data Type " + dataTypeString + ". Valid values: " - + Statement.SCALAR_DATA_TYPE +", " + Statement.MATRIX_DATA_TYPE+", " + Statement.FRAME_DATA_TYPE - +", " + DataType.LIST.name().toLowerCase(), conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - - // handle value type parameter - if (getVarParam(VALUETYPEPARAM) != null && !(getVarParam(VALUETYPEPARAM) instanceof StringIdentifier)){ - raiseValidateError("for read method, parameter " + VALUETYPEPARAM + " can only be a string. " + - "Valid values are: " + Statement.DOUBLE_VALUE_TYPE +", " + Statement.INT_VALUE_TYPE + ", " + Statement.BOOLEAN_VALUE_TYPE + ", " + Statement.STRING_VALUE_TYPE, conditional); - } - // Identify the value type (used only for read method) - String valueTypeString = getVarParam(VALUETYPEPARAM) == null ? null : getVarParam(VALUETYPEPARAM).toString(); - if (valueTypeString != null) { - if (valueTypeString.equalsIgnoreCase(Statement.DOUBLE_VALUE_TYPE)) - getOutput().setValueType(ValueType.FP64); - else if (valueTypeString.equalsIgnoreCase(Statement.STRING_VALUE_TYPE)) - getOutput().setValueType(ValueType.STRING); - else if (valueTypeString.equalsIgnoreCase(Statement.INT_VALUE_TYPE)) - getOutput().setValueType(ValueType.INT64); - else if (valueTypeString.equalsIgnoreCase(Statement.BOOLEAN_VALUE_TYPE)) - getOutput().setValueType(ValueType.BOOLEAN); - else if (valueTypeString.equalsIgnoreCase(ValueType.UNKNOWN.name())) - getOutput().setValueType(ValueType.UNKNOWN); - else { - raiseValidateError("Unknown Value Type " + valueTypeString - + ". Valid values are: " + Statement.DOUBLE_VALUE_TYPE +", " + Statement.INT_VALUE_TYPE + ", " + Statement.BOOLEAN_VALUE_TYPE + ", " + Statement.STRING_VALUE_TYPE, conditional); - } - } else { - getOutput().setValueType(ValueType.FP64); - } - - break; - - case WRITE: - - // for CSV format, if no delimiter specified THEN set default "," - if (getVarParam(FORMAT_TYPE) == null || checkFormatType(FileFormat.CSV) ){ - if (getVarParam(DELIM_DELIMITER) == null) { - addVarParam(DELIM_DELIMITER, new StringIdentifier(DEFAULT_DELIM_DELIMITER, this)); - } - if (getVarParam(DELIM_HAS_HEADER_ROW) == null) { - addVarParam(DELIM_HAS_HEADER_ROW, new BooleanIdentifier(DEFAULT_DELIM_HAS_HEADER_ROW, this)); - } - if (getVarParam(DELIM_SPARSE) == null) { - addVarParam(DELIM_SPARSE, new BooleanIdentifier(DEFAULT_DELIM_SPARSE, this)); - } - } - - // for LIBSVM format, add the default separators if not specified - if (getVarParam(FORMAT_TYPE) == null || checkFormatType(FileFormat.LIBSVM)) { - if(getVarParam(DELIM_DELIMITER) == null) { - addVarParam(DELIM_DELIMITER, new StringIdentifier(DEFAULT_DELIM_DELIMITER, this)); - } - if(getVarParam(LIBSVM_INDEX_DELIM) == null) { - addVarParam(LIBSVM_INDEX_DELIM, new StringIdentifier(DEFAULT_LIBSVM_INDEX_DELIM, this)); - } - if(getVarParam(DELIM_SPARSE) == null) { - addVarParam(DELIM_SPARSE, new BooleanIdentifier(DEFAULT_DELIM_SPARSE, this)); - } - } - - //validate read filename - if (getVarParam(FORMAT_TYPE) == null || FileFormat.isTextFormat(getVarParam(FORMAT_TYPE).toString()) - || checkFormatType(FileFormat.DELTA)) //delta: columnar, no block layout - getOutput().setBlocksize(-1); - else if (checkFormatType(FileFormat.BINARY, FileFormat.COMPRESSED, FileFormat.UNKNOWN)) { - if( getVarParam(ROWBLOCKCOUNTPARAM)!=null ) - getOutput().setBlocksize(Integer.parseInt(getVarParam(ROWBLOCKCOUNTPARAM).toString())); - else - getOutput().setBlocksize(ConfigurationManager.getBlocksize()); - } - else if( getVarParam(FORMAT_TYPE) instanceof StringIdentifier ) //literal format - raiseValidateError("Invalid format " + getVarParam(FORMAT_TYPE) - + " in statement: " + toString(), conditional); - break; + else if(getVarParam(FORMAT_TYPE) instanceof StringIdentifier) // literal format + raiseValidateError("Invalid format " + getVarParam(FORMAT_TYPE) + " in statement: " + toString(), + conditional); + break; case RAND: diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java index d0ba5363939..042e0dc0328 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java @@ -484,7 +484,7 @@ public static CompressedMatrixBlock read(DataInput in) throws IOException { long nonZeros = in.readLong(); boolean overlappingColGroups = in.readBoolean(); List groups = ColGroupIO.readGroups(in, rlen); - CompressedMatrixBlock ret = new CompressedMatrixBlock(rlen, clen, nonZeros, overlappingColGroups, groups); + CompressedMatrixBlock ret = new CompressedMatrixBlock(rlen, clen, nonZeros, overlappingColGroups, groups); LOG.debug("Compressed read serialization time: " + t.stop()); return ret; } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java index 354325e293b..66d4e78cb0f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java @@ -402,7 +402,8 @@ public final AColGroup rightMultByMatrix(MatrixBlock right) { * @param cru The right hand side column upper * @param nRows The number of rows in this column group */ - public void rightDecompressingMult(MatrixBlock right, MatrixBlock ret, int rl, int ru, int nRows, int crl, int cru) { + public void rightDecompressingMult(MatrixBlock right, MatrixBlock ret, int rl, int ru, int nRows, int crl, + int cru) { throw new NotImplementedException( "not supporting right Decompressing Multiply on class: " + this.getClass().getSimpleName()); } @@ -977,9 +978,9 @@ public AColGroup[] splitReshapePushDown(final int multiplier, final int nRow, fi /** * Sort the values of the column group according to double comparison operations and return as another compressed * group. - * + * * This sorting assumes that the column group is sorted independently of everything else. - * + * * @return The sorted group */ public abstract AColGroup sort(); @@ -996,9 +997,9 @@ public String toString() { /** * Return a new column group containing only the selected rows in the given boolean vector. - * + * * Whenever possible only modify the index structure, not the dictionary of the column groups. - * + * * @param selectV The selection vector * @param rOut The number of rows in the output * @return The new column group @@ -1007,9 +1008,9 @@ public String toString() { /** * Return a new column group containing only the selected columns in the given boolean vector. - * + * * Whenever possible only modify the column index, and reduce the dictionaries of the column groups. - * + * * @param selectV The selection vector * @return The new column group, or {@code null} if no column of this group is selected */ @@ -1045,7 +1046,7 @@ public AColGroup removeEmptyCols(boolean[] selectV) { /** * Using the selection of columns, slice out those and return in a new column group with the given column indexes. * Ideally this method should only modify the dictionaries. - * + * * @param newColumnIDs the new column indexes * @param selectedColumns The selected columns of this column group (guaranteed < current number of columns) * @return A new Column group diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java index d825b91f089..d610c1b586c 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java @@ -210,7 +210,6 @@ public void clear() { counts = null; } - @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java index 30de5e120c5..794d90c0d11 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java @@ -212,8 +212,8 @@ public void decompressToSparseBlock(SparseBlock sb, int rl, int ru, int offR, in // TODO make sparse decompression where the iterator is known in argument decompressToSparseBlockSparseDictionary(sb, rl, ru, offR, offC, mb.getSparseBlock()); else - decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, mb.getDenseBlockValues(), - it); + decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, + mb.getDenseBlockValues(), it); } else decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, _dict.getValues(), it); @@ -240,7 +240,7 @@ public void decompressToDenseBlockDenseDictionary(DenseBlock db, int rl, int ru, } public abstract void decompressToSparseBlockDenseDictionaryWithProvidedIterator(SparseBlock db, int rl, int ru, - int offR, int offC, double[] values, AIterator it); + int offR, int offC, double[] values, AIterator it); public abstract void decompressToDenseBlockDenseDictionaryWithProvidedIterator(DenseBlock db, int rl, int ru, int offR, int offC, double[] values, AIterator it); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java index b316e48474a..d643cae440c 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java @@ -674,8 +674,8 @@ private void defaultRightDecompressingMult(MatrixBlock right, MatrixBlock ret, i } } - final void vectMM(double aa, double[] b, double[] c, int endT, int jd, int crl, int cru, int offOut, int k, int vLen, - DoubleVector vVec) { + final void vectMM(double aa, double[] b, double[] c, int endT, int jd, int crl, int cru, int offOut, int k, + int vLen, DoubleVector vVec) { vVec = vVec.broadcast(aa); final int offj = k * jd; final int end = endT + offj; diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java index 64114a054ab..d5ad55772c7 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java @@ -478,14 +478,13 @@ public AColGroup combineWithSameIndex(int nRow, int nCol, List right) return new ColGroupEmpty(combinedIndex); } - @Override - public AColGroup removeEmptyRows(boolean[] selectV, int rOut){ + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { return this; } - @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { return new ColGroupEmpty(newColumnIDs); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java index fa8aa104ffb..e0bea3c3696 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java @@ -747,7 +747,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java index a251d828b5f..b4f0c144a73 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java @@ -738,7 +738,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java index 347cea9c0da..43df7fa3b94 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java @@ -1195,9 +1195,9 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { throw new NotImplementedException("Unimplemented method 'removeEmptyRows'"); } - + @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java index 815ecacf378..4566106a3e2 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java @@ -634,8 +634,8 @@ protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList s for(int i = 0; i < selectedColumns.size(); i++) { ref[i] = _reference[selectedColumns.get(i)]; } - return ColGroupSDCFOR.create(newColumnIDs, _numRows, _dict.sliceColumns(selectedColumns, getNumCols()), _indexes, _data, null, - ref); + return ColGroupSDCFOR.create(newColumnIDs, _numRows, _dict.sliceColumns(selectedColumns, getNumCols()), + _indexes, _data, null, ref); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java index 611add6480f..9797087f8c3 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java @@ -85,7 +85,7 @@ public class ColGroupUncompressed extends AColGroup { /** * Do not use this constructor of column group uncompressed, instead use the create constructor. - * + * * @param mb The contained data. * @param colIndexes Column indexes for this Columngroup */ @@ -96,9 +96,10 @@ protected ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes) { /** * Do not use this constructor of column group quantization-fused uncompressed, instead use the create constructor. - * + * * @param mb The contained data. - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire + * matrix * @param colIndexes Column indexes for this Columngroup */ protected ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes, double[] scaleFactors) { @@ -138,7 +139,8 @@ public static AColGroup create(MatrixBlock mb, IColIndex colIndexes) { * * @param mb The MB / data to contain in the uncompressed column * @param colIndexes The column indexes for the group - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire + * matrix * @return An Uncompressed Column group */ public static AColGroup createQuantized(MatrixBlock mb, IColIndex colIndexes, double[] scaleFactors) { @@ -157,7 +159,8 @@ public static AColGroup createQuantized(MatrixBlock mb, IColIndex colIndexes, do * @param rawBlock The uncompressed block; uncompressed data must be present at the time that the constructor is * called * @param transposed Says if the input matrix raw block have been transposed. - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire + * matrix * @return AColGroup. */ public static AColGroup createQuantized(IColIndex colIndexes, MatrixBlock rawBlock, boolean transposed, diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java index 51e26a3f9d2..de8a740ceb2 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java @@ -290,7 +290,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java index a7e715b59b8..6e66ef6ef9b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java @@ -76,8 +76,8 @@ public double[] productAllRowsToDoubleWithDefault(double[] defaultTuple) { return ret; } - @Override - public int[] sort(){ + @Override + public int[] sort() { throw new NotImplementedException(); } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java index 9a0412145f0..7ebba2f1a76 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java @@ -138,8 +138,8 @@ public IDictionary clone() { throw new NotImplementedException(); } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { throw new NotImplementedException(); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java index c8ddfc4883a..b5e1a99355b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java @@ -1055,7 +1055,7 @@ public IDictionary rightMMPreAggSparse(int numVals, SparseBlock b, IColIndex thi /** * Slice out the selected columns given of this encoded group. - * + * * @param selectedColumns The columns to slice out and return as a new matrix. * @param nCol The number of columns in this dictionary. * @return The returned matrix @@ -1064,9 +1064,9 @@ public IDictionary rightMMPreAggSparse(int numVals, SparseBlock b, IColIndex thi /** * Sort the values of this dictionary via an index of how the values mapped previously. - * + * * In practice this design means we can reuse the previous dictionary for the resulting column group - * + * * @return The sorted index. */ public int[] sort(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java index c2540de959a..4337da7307f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java @@ -541,8 +541,8 @@ public String getString(int colIndexes) { return "IdentityMatrix of size: " + nRowCol + " with empty: " + withEmpty; } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { return getMBDict().sliceColumns(selectedColumns, nCol); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java index c7f642edfd0..47628b43d2a 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java @@ -311,8 +311,8 @@ public String getString(int colIndexes) { return toString(); } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { return getMBDict().sliceColumns(selectedColumns, nCol); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java index 83a74972db7..6d516713689 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java @@ -1064,7 +1064,7 @@ public AMapToData removeEmpty(final boolean[] selectV, final int rOut) { /** * Use the offsets of the select vector to choose which values to keep. - * + * * @param select The row indexes to keep * @return A New MapToData */ diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java index f65876b7f37..bf8ee7f9ee1 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java @@ -56,11 +56,11 @@ public abstract class AOffset implements Serializable { protected static final Log LOG = LogFactory.getLog(AOffset.class.getName()); /** - * Lazy holder for the cached empty slice. The empty slice is built on first use rather than in AOffset's - * static initializer: instantiating the OffsetEmpty subclass from AOffset's {@code } forms a - * superclass/subclass class-initialization cycle that deadlocks when several threads first touch the offset - * classes concurrently (e.g. parallel tests). Deferring it to first use guarantees AOffset is already - * initialized by the time OffsetEmpty is loaded, so no cycle exists. + * Lazy holder for the cached empty slice. The empty slice is built on first use rather than in AOffset's static + * initializer: instantiating the OffsetEmpty subclass from AOffset's {@code } forms a superclass/subclass + * class-initialization cycle that deadlocks when several threads first touch the offset classes concurrently (e.g. + * parallel tests). Deferring it to first use guarantees AOffset is already initialized by the time OffsetEmpty is + * loaded, so no cycle exists. */ private static final class EmptySliceHolder { static final OffsetSliceInfo EMPTY_SLICE = new OffsetSliceInfo(-1, -1, new OffsetEmpty()); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java index 866168ded2f..37ff41cf817 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java @@ -76,6 +76,7 @@ public int getOffsetToLast() { public long getInMemorySize() { return estimateInMemorySize(); } + @Override public boolean equals(AOffset b) { return b instanceof OffsetEmpty; diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java index d981ab87838..7953322350e 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java @@ -139,7 +139,8 @@ private static boolean isDoubleCompressedOpApplicable(CompressedMatrixBlock m1, m1.getColGroups().get(0) instanceof ColGroupDDC && !((CompressedMatrixBlock) that).isOverlapping() && ((CompressedMatrixBlock) that).getColGroups().get(0) instanceof ColGroupDDC && ((IMapToDataGroup) m1.getColGroups().get(0)) - .getMapToData() == ((IMapToDataGroup) ((CompressedMatrixBlock) that).getColGroups().get(0)).getMapToData(); + .getMapToData() == ((IMapToDataGroup) ((CompressedMatrixBlock) that).getColGroups().get(0)) + .getMapToData(); } private static CompressedMatrixBlock doubleCompressedBinaryOp(BinaryOperator op, CompressedMatrixBlock m1, @@ -1062,7 +1063,8 @@ public Long call() { return _ret.recomputeNonZeros(_rl, _ru - 1); } - private final void processBlock(final int rl, final int ru, final List groups, final AIterator[] its) { + private final void processBlock(final int rl, final int ru, final List groups, + final AIterator[] its) { decompressToTmpBlock(rl, ru, tmp.getSparseBlock(), groups, its); // decompressing multiple column groups can leave the temp rows with unsorted column indices, so sort // before reading them in stored order into the (column-sorted) output sparse block. diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java index cc7953f8c5d..a91b75ae73c 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java @@ -96,7 +96,7 @@ public static MatrixBlock mmChain(CompressedMatrixBlock x, MatrixBlock v, Matrix if(x.isEmpty()) return returnEmpty(x, out); - if(ctype == ChainType.XtXv && x.getColGroups().size() < 5 && x.getNumColumns()> 30){ + if(ctype == ChainType.XtXv && x.getColGroups().size() < 5 && x.getNumColumns() > 30) { MatrixBlock tmp = CLALibTSMM.leftMultByTransposeSelf(x, k); return tmp.aggregateBinaryOperations(tmp, v, out, InstructionUtils.getMatMultOperator(k)); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java index 3755e4040e7..802eddffcb8 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java @@ -36,7 +36,7 @@ public class CLALibRemoveEmpty { /** * CP rmempty operation (single input, single output matrix) - * + * * @param in The input matrix * @param ret The output matrix * @param rows If we are removing based on rows, or columns. @@ -66,13 +66,13 @@ private static MatrixBlock rmEmptyCols(CompressedMatrixBlock in, MatrixBlock ret int cOut = (int) select.getNonZeros(); if(cOut == -1) cOut = (int) select.recomputeNonZeros(); - if(cOut == 0){ + if(cOut == 0) { ret.reset(in.getNumRows(), !emptyReturn ? 0 : 1); return ret; } - final boolean[] selectV = DataConverter - .convertToBooleanVector(CompressedMatrixBlock.getUncompressed(select, "decompressing selection in rmempty")); + final boolean[] selectV = DataConverter.convertToBooleanVector( + CompressedMatrixBlock.getUncompressed(select, "decompressing selection in rmempty")); final List inG = in.getColGroups(); final List retG = new ArrayList<>(inG.size()); @@ -102,18 +102,17 @@ private static MatrixBlock rmEmptyRows(CompressedMatrixBlock in, MatrixBlock ret int rOut = (int) select.getNonZeros(); if(rOut == -1) rOut = (int) select.recomputeNonZeros(); - if(rOut == 0){ + if(rOut == 0) { ret.reset(!emptyReturn ? 0 : 1, in.getNumColumns()); return ret; } - // TODO: add optimization to avoid linear scan and make selectV indexes, if selection is small relative to number + // TODO: add optimization to avoid linear scan and make selectV indexes, if selection is small relative to + // number // of rows // TODO: add decompress to boolean vector. final boolean[] selectV = DataConverter.convertToBooleanVector(select); - - final List inG = in.getColGroups(); final List retG = new ArrayList<>(inG.size()); try { diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java index b94f11ae723..5ae7bd5103b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java @@ -40,10 +40,10 @@ private CLALibSort() { /** * Sort (order) a compressed matrix in place of the {@code order} built-in, while keeping the result compressed. * - * The compressed fast-path only supports the case the user can benefit from: a single column held in a single column - * group, sorted ascending and returning the sorted values (not the index permutation). For everything else (multiple - * columns, multiple column groups, descending order, index return, or a column-group encoding without a sort - * implementation) this returns {@code null} so the caller can fall back to a decompressed reorg. + * The compressed fast-path only supports the case the user can benefit from: a single column held in a single + * column group, sorted ascending and returning the sorted values (not the index permutation). For everything else + * (multiple columns, multiple column groups, descending order, index return, or a column-group encoding without a + * sort implementation) this returns {@code null} so the caller can fall back to a decompressed reorg. * * @param mb the compressed matrix to sort * @param fn the sort specification carried by the reorg operator diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java index 87d14dbf87e..9ccaa474f39 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java @@ -208,8 +208,8 @@ protected FrameBlock readBlobFromHDFS(String fname, long[] dims) throws IOExcept if(data == null) throw new IOException("Unable to load frame from file: " + fname); - //Delta and CSV discover dimensions (and Delta also schema) at read time, so - //refresh the cached metadata to reflect the materialized frame block. + // Delta and CSV discover dimensions (and Delta also schema) at read time, so + // refresh the cached metadata to reflect the materialized frame block. if(iimd.getFileFormat() == FileFormat.CSV || iimd.getFileFormat() == FileFormat.DELTA) { _metaData = _metaData instanceof MetaDataFormat ? new MetaDataFormat(data.getDataCharacteristics(), iimd.getFileFormat()) : new MetaData(data.getDataCharacteristics()); diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java index 28fa70f7741..4331da2b426 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java @@ -454,7 +454,7 @@ protected MatrixBlock readBlobFromHDFS(String fname, long[] dims) throws IOExcep rlen, clen, blen, mc.getNonZeros(), getFileFormatProperties()); if(iimd.getFileFormat() == FileFormat.CSV || iimd.getFileFormat() == FileFormat.DELTA) { - //dimensions/nnz are discovered at read time for these self-describing formats + // dimensions/nnz are discovered at read time for these self-describing formats _metaData = _metaData instanceof MetaDataFormat ? new MetaDataFormat(newData.getDataCharacteristics(), iimd.getFileFormat()) : new MetaData(newData.getDataCharacteristics()); } diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java index b52f3777e1f..fbae4925c66 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java @@ -122,9 +122,9 @@ public class SparkExecutionContext extends ExecutionContext //singleton spark context (as there can be only one spark context per JVM) private static JavaSparkContext _spctx = null; - //registered users of the singleton context (guarded by the - //SparkExecutionContext.class monitor); maintained by enterSparkExecution()/ - //exitSparkExecution(), and close() only stops the context once it hits zero + // registered users of the singleton context (guarded by the + // SparkExecutionContext.class monitor); maintained by enterSparkExecution()/ + // exitSparkExecution(), and close() only stops the context once it hits zero private static int _activeExecutions = 0; //registry of parallelized RDDs to enforce that at any time, we spent at most @@ -175,8 +175,8 @@ public synchronized static JavaSparkContext getSparkContextStatic() { initSparkContext(); if(_spctx.sc().isStopped()){ _spctx = null; - //the previous context was stopped; reset the active-execution count so a - //stale registration cannot skip a future legitimate stop of the new one + // the previous context was stopped; reset the active-execution count so a + // stale registration cannot skip a future legitimate stop of the new one _activeExecutions = 0; initSparkContext(); } @@ -196,16 +196,15 @@ public synchronized static boolean isSparkContextCreated() { public static void resetSparkContextStatic() { synchronized(SparkExecutionContext.class) { _spctx = null; - //force-discarding the shared context: drop the active-execution count so - //a stale registration cannot skip a future legitimate stop + // force-discarding the shared context: drop the active-execution count so + // a stale registration cannot skip a future legitimate stop _activeExecutions = 0; } } /** - * Registers an active user of the shared spark context. Must be balanced by a - * later {@link #exitSparkExecution()} so a concurrent execution cannot stop the - * context while this one still has in-flight jobs. + * Registers an active user of the shared spark context. Must be balanced by a later {@link #exitSparkExecution()} + * so a concurrent execution cannot stop the context while this one still has in-flight jobs. */ public static void enterSparkExecution() { synchronized(SparkExecutionContext.class) { @@ -214,9 +213,8 @@ public static void enterSparkExecution() { } /** - * Releases an active user previously registered via {@link #enterSparkExecution()}. - * Only adjusts the count; the actual teardown is left to {@link #close()}, which - * stops the context once no registered execution remains. + * Releases an active user previously registered via {@link #enterSparkExecution()}. Only adjusts the count; the + * actual teardown is left to {@link #close()}, which stops the context once no registered execution remains. */ public static void exitSparkExecution() { synchronized(SparkExecutionContext.class) { @@ -227,13 +225,13 @@ public static void exitSparkExecution() { public void close() { synchronized(SparkExecutionContext.class) { - //keep the shared context alive while a registered execution still uses - //it; close() never changes the count, so an unpaired close() (a caller - //that never entered) cannot stop a context another execution is using + // keep the shared context alive while a registered execution still uses + // it; close() never changes the count, so an unpaired close() (a caller + // that never entered) cannot stop a context another execution is using if(_activeExecutions > 0) { if(LOG.isDebugEnabled()) - LOG.debug("Keeping shared spark context alive; " + _activeExecutions - + " execution(s) still active"); + LOG.debug( + "Keeping shared spark context alive; " + _activeExecutions + " execution(s) still active"); return; } if(_spctx != null) { diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java index 682cc8e3fff..c502817e026 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java @@ -95,8 +95,7 @@ private void run() { int par_conn = ConfigurationManager.getDMLConfig().getIntValue(DMLConfig.FEDERATED_PAR_CONN); final int EVENT_LOOP_THREADS = (par_conn > 0) ? par_conn : InfrastructureAnalyzer.getLocalParallelism(); // Daemon event loops so a leaked in-JVM (test) worker cannot block JVM exit. - NioEventLoopGroup bossGroup = new NioEventLoopGroup(1, - new DefaultThreadFactory("fed-worker-boss", true)); + NioEventLoopGroup bossGroup = new NioEventLoopGroup(1, new DefaultThreadFactory("fed-worker-boss", true)); ThreadPoolExecutor workerTPE = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 10, TimeUnit.SECONDS, new SynchronousQueue(true), new DefaultThreadFactory("fed-worker-pool", true)); NioEventLoopGroup workerGroup = new NioEventLoopGroup(EVENT_LOOP_THREADS, workerTPE); diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java index 80a5d699dfa..ebf05972b87 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java @@ -125,13 +125,15 @@ public static RaggedArray create(T[] col, int m) { /** * Wrap a fully populated raw typed column array into an {@link Array} of the given value type. The runtime type of - * {@code col} must match the primitive backing type of {@code vt} (e.g. {@code double[]} for {@link ValueType#FP64}, - * {@code String[]} for {@link ValueType#STRING}). + * {@code col} must match the primitive backing type of {@code vt} (e.g. {@code double[]} for + * {@link ValueType#FP64}, {@code String[]} for {@link ValueType#STRING}). * - *

For {@link ValueType#BOOLEAN} this mirrors {@link #allocateBoolean(int)}: a {@code boolean[]} longer than - * {@link #bitSetSwitchPoint} is bit-packed into a compact {@link BitSetArray} (so a bulk decoder that fills a - * plain {@code boolean[]} still ends up with the same representation as every other frame allocation path), - * while shorter columns stay a plain {@link BooleanArray}.

+ *

+ * For {@link ValueType#BOOLEAN} this mirrors {@link #allocateBoolean(int)}: a {@code boolean[]} longer than + * {@link #bitSetSwitchPoint} is bit-packed into a compact {@link BitSetArray} (so a bulk decoder that fills a plain + * {@code boolean[]} still ends up with the same representation as every other frame allocation path), while shorter + * columns stay a plain {@link BooleanArray}. + *

* * @param vt the value type of the column * @param col the backing array to wrap @@ -168,10 +170,10 @@ public static Array create(ValueType vt, Object col) { /** * Allocate the raw backing array for a column of the given value type: the inverse of - * {@link #create(ValueType, Object)}. Returns {@code double[]} for {@link ValueType#FP64}, - * {@code int[]} for INT32/UINT/HASH32, {@code long[]} for INT64/HASH64, {@code String[]} for STRING, etc. The - * runtime array type matches what {@link #create(ValueType, Object)} expects, so a bulk decoder can fill this - * primitive array directly and then wrap it via {@code create(vt, backing)}. + * {@link #create(ValueType, Object)}. Returns {@code double[]} for {@link ValueType#FP64}, {@code int[]} for + * INT32/UINT/HASH32, {@code long[]} for INT64/HASH64, {@code String[]} for STRING, etc. The runtime array type + * matches what {@link #create(ValueType, Object)} expects, so a bulk decoder can fill this primitive array directly + * and then wrap it via {@code create(vt, backing)}. * * @param vt the value type of the column * @param nRow the number of rows to allocate diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java b/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java index 9ff58065d97..95be95117e2 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java @@ -41,7 +41,7 @@ public class MatrixBlockFromFrame { public static Boolean WARNED_FOR_FAILED_CAST = false; - private MatrixBlockFromFrame(){ + private MatrixBlockFromFrame() { // private constructor for code coverage. } @@ -115,7 +115,7 @@ private static long convert(FrameBlock frame, MatrixBlock mb, int n, int rl, int return convertStrict(frame, mb, n, rl, ru); } catch(NumberFormatException | DMLRuntimeException e) { - synchronized(WARNED_FOR_FAILED_CAST){ + synchronized(WARNED_FOR_FAILED_CAST) { if(!WARNED_FOR_FAILED_CAST) { LOG.error( "Failed to convert to Matrix because of number format errors, falling back to NaN on incompatible cells", diff --git a/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java b/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java index eed2c58f78c..c12a187cf17 100644 --- a/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java +++ b/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java @@ -30,32 +30,13 @@ import jdk.incubator.vector.VectorSpecies; -/** - * Class with pre-defined set of objects. This class can not be instantiated elsewhere. - * - * Notes on commons.math FastMath: - * * FastMath uses lookup tables and interpolation instead of native calls. - * * The memory overhead for those tables is roughly 48KB in total (acceptable) - * * Micro and application benchmarks showed significantly (30%-3x) performance improvements - * for most operations; without loss of accuracy. - * * atan / sqrt were 20% slower in FastMath and hence, we use Math there - * * round / abs were equivalent in FastMath and hence, we use Math there - * * Finally, there is just one argument against FastMath - The comparison heavily depends - * on the JVM. For example, currently the IBM JDK JIT compiles to HW instructions for sqrt - * which makes this operation very efficient; as soon as other operations like log/exp are - * similarly compiled, we should rerun the micro benchmarks, and switch back if necessary. - * - */ -public class Builtin extends ValueFunction -{ - private static final long serialVersionUID = 3836744687789840574L; - - public enum BuiltinCode { AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, ATAN, LOG, LOG_NZ, MIN, - MAX, ABS, SIGN, SQRT, EXP, PLOGP, PRINT, PRINTF, NROW, NCOL, LENGTH, LINEAGE, ROUND, MAXINDEX, MININDEX, - STOP, CEIL, FLOOR, CUMSUM, ROWCUMSUM, CUMPROD, CUMMIN, CUMMAX, CUMSUMPROD, INVERSE, SPROP, SIGMOID, EVAL, LIST, - TYPEOF, APPLY_SCHEMA, DETECTSCHEMA, ISNA, ISNAN, ISINF, DROP_INVALID_TYPE, - DROP_INVALID_LENGTH, VALUE_SWAP, FRAME_ROW_REPLICATE, GET_CATEGORICAL_MASK, - MAP, COUNT_DISTINCT, COUNT_DISTINCT_APPROX, UNIQUE} + public enum BuiltinCode { + AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, ATAN, LOG, LOG_NZ, MIN, MAX, ABS, SIGN, SQRT, EXP, PLOGP, + PRINT, PRINTF, NROW, NCOL, LENGTH, LINEAGE, ROUND, MAXINDEX, MININDEX, STOP, CEIL, FLOOR, CUMSUM, ROWCUMSUM, + CUMPROD, CUMMIN, CUMMAX, CUMSUMPROD, INVERSE, SPROP, SIGMOID, EVAL, LIST, TYPEOF, APPLY_SCHEMA, DETECTSCHEMA, + ISNA, ISNAN, ISINF, DROP_INVALID_TYPE, DROP_INVALID_LENGTH, VALUE_SWAP, FRAME_ROW_REPLICATE, + GET_CATEGORICAL_MASK, MAP, COUNT_DISTINCT, COUNT_DISTINCT_APPROX, UNIQUE + } private static final VectorSpecies SPECIES = DoubleVector.SPECIES_PREFERRED; private static final int vLen = SPECIES.length(); @@ -68,59 +49,59 @@ public enum BuiltinCode { AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, static public HashMap String2BuiltinCode; static { String2BuiltinCode = new HashMap<>(); - String2BuiltinCode.put( "autoDiff" , BuiltinCode.AUTODIFF); - String2BuiltinCode.put( "sin" , BuiltinCode.SIN); - String2BuiltinCode.put( "cos" , BuiltinCode.COS); - String2BuiltinCode.put( "tan" , BuiltinCode.TAN); - String2BuiltinCode.put( "sinh" , BuiltinCode.SINH); - String2BuiltinCode.put( "cosh" , BuiltinCode.COSH); - String2BuiltinCode.put( "tanh" , BuiltinCode.TANH); - String2BuiltinCode.put( "asin" , BuiltinCode.ASIN); - String2BuiltinCode.put( "acos" , BuiltinCode.ACOS); - String2BuiltinCode.put( "atan" , BuiltinCode.ATAN); - String2BuiltinCode.put( "log" , BuiltinCode.LOG); - String2BuiltinCode.put( "log_nz" , BuiltinCode.LOG_NZ); - String2BuiltinCode.put( "min" , BuiltinCode.MIN); - String2BuiltinCode.put( "max" , BuiltinCode.MAX); - String2BuiltinCode.put( "maxindex", BuiltinCode.MAXINDEX); - String2BuiltinCode.put( "minindex", BuiltinCode.MININDEX); - String2BuiltinCode.put( "abs" , BuiltinCode.ABS); - String2BuiltinCode.put( "sign" , BuiltinCode.SIGN); - String2BuiltinCode.put( "sqrt" , BuiltinCode.SQRT); - String2BuiltinCode.put( "exp" , BuiltinCode.EXP); - String2BuiltinCode.put( "plogp" , BuiltinCode.PLOGP); - String2BuiltinCode.put( "print" , BuiltinCode.PRINT); - String2BuiltinCode.put( "printf" , BuiltinCode.PRINTF); - String2BuiltinCode.put( "eval" , BuiltinCode.EVAL); - String2BuiltinCode.put( "list" , BuiltinCode.LIST); - String2BuiltinCode.put( "nrow" , BuiltinCode.NROW); - String2BuiltinCode.put( "ncol" , BuiltinCode.NCOL); - String2BuiltinCode.put( "length" , BuiltinCode.LENGTH); - String2BuiltinCode.put( "round" , BuiltinCode.ROUND); - String2BuiltinCode.put( "stop" , BuiltinCode.STOP); - String2BuiltinCode.put( "ceil" , BuiltinCode.CEIL); - String2BuiltinCode.put( "floor" , BuiltinCode.FLOOR); - String2BuiltinCode.put( "ucumk+" , BuiltinCode.CUMSUM); - String2BuiltinCode.put( "urowcumk+" , BuiltinCode.ROWCUMSUM); - String2BuiltinCode.put( "ucum*" , BuiltinCode.CUMPROD); - String2BuiltinCode.put( "ucumk+*", BuiltinCode.CUMSUMPROD); - String2BuiltinCode.put( "ucummin", BuiltinCode.CUMMIN); - String2BuiltinCode.put( "ucummax", BuiltinCode.CUMMAX); - String2BuiltinCode.put( "inverse", BuiltinCode.INVERSE); - String2BuiltinCode.put( "sprop", BuiltinCode.SPROP); - String2BuiltinCode.put( "sigmoid", BuiltinCode.SIGMOID); - String2BuiltinCode.put( "typeOf", BuiltinCode.TYPEOF); - String2BuiltinCode.put( "detectSchema", BuiltinCode.DETECTSCHEMA); - String2BuiltinCode.put( "isna", BuiltinCode.ISNA); - String2BuiltinCode.put( "isnan", BuiltinCode.ISNAN); - String2BuiltinCode.put( "isinf", BuiltinCode.ISINF); - String2BuiltinCode.put( "dropInvalidType", BuiltinCode.DROP_INVALID_TYPE); - String2BuiltinCode.put( "freplicate", BuiltinCode.FRAME_ROW_REPLICATE); - String2BuiltinCode.put( "dropInvalidLength", BuiltinCode.DROP_INVALID_LENGTH); - String2BuiltinCode.put( "_map", BuiltinCode.MAP); - String2BuiltinCode.put( "valueSwap", BuiltinCode.VALUE_SWAP); - String2BuiltinCode.put( "applySchema", BuiltinCode.APPLY_SCHEMA); - String2BuiltinCode.put( "get_categorical_mask", BuiltinCode.GET_CATEGORICAL_MASK); + String2BuiltinCode.put("autoDiff", BuiltinCode.AUTODIFF); + String2BuiltinCode.put("sin", BuiltinCode.SIN); + String2BuiltinCode.put("cos", BuiltinCode.COS); + String2BuiltinCode.put("tan", BuiltinCode.TAN); + String2BuiltinCode.put("sinh", BuiltinCode.SINH); + String2BuiltinCode.put("cosh", BuiltinCode.COSH); + String2BuiltinCode.put("tanh", BuiltinCode.TANH); + String2BuiltinCode.put("asin", BuiltinCode.ASIN); + String2BuiltinCode.put("acos", BuiltinCode.ACOS); + String2BuiltinCode.put("atan", BuiltinCode.ATAN); + String2BuiltinCode.put("log", BuiltinCode.LOG); + String2BuiltinCode.put("log_nz", BuiltinCode.LOG_NZ); + String2BuiltinCode.put("min", BuiltinCode.MIN); + String2BuiltinCode.put("max", BuiltinCode.MAX); + String2BuiltinCode.put("maxindex", BuiltinCode.MAXINDEX); + String2BuiltinCode.put("minindex", BuiltinCode.MININDEX); + String2BuiltinCode.put("abs", BuiltinCode.ABS); + String2BuiltinCode.put("sign", BuiltinCode.SIGN); + String2BuiltinCode.put("sqrt", BuiltinCode.SQRT); + String2BuiltinCode.put("exp", BuiltinCode.EXP); + String2BuiltinCode.put("plogp", BuiltinCode.PLOGP); + String2BuiltinCode.put("print", BuiltinCode.PRINT); + String2BuiltinCode.put("printf", BuiltinCode.PRINTF); + String2BuiltinCode.put("eval", BuiltinCode.EVAL); + String2BuiltinCode.put("list", BuiltinCode.LIST); + String2BuiltinCode.put("nrow", BuiltinCode.NROW); + String2BuiltinCode.put("ncol", BuiltinCode.NCOL); + String2BuiltinCode.put("length", BuiltinCode.LENGTH); + String2BuiltinCode.put("round", BuiltinCode.ROUND); + String2BuiltinCode.put("stop", BuiltinCode.STOP); + String2BuiltinCode.put("ceil", BuiltinCode.CEIL); + String2BuiltinCode.put("floor", BuiltinCode.FLOOR); + String2BuiltinCode.put("ucumk+", BuiltinCode.CUMSUM); + String2BuiltinCode.put("urowcumk+", BuiltinCode.ROWCUMSUM); + String2BuiltinCode.put("ucum*", BuiltinCode.CUMPROD); + String2BuiltinCode.put("ucumk+*", BuiltinCode.CUMSUMPROD); + String2BuiltinCode.put("ucummin", BuiltinCode.CUMMIN); + String2BuiltinCode.put("ucummax", BuiltinCode.CUMMAX); + String2BuiltinCode.put("inverse", BuiltinCode.INVERSE); + String2BuiltinCode.put("sprop", BuiltinCode.SPROP); + String2BuiltinCode.put("sigmoid", BuiltinCode.SIGMOID); + String2BuiltinCode.put("typeOf", BuiltinCode.TYPEOF); + String2BuiltinCode.put("detectSchema", BuiltinCode.DETECTSCHEMA); + String2BuiltinCode.put("isna", BuiltinCode.ISNA); + String2BuiltinCode.put("isnan", BuiltinCode.ISNAN); + String2BuiltinCode.put("isinf", BuiltinCode.ISINF); + String2BuiltinCode.put("dropInvalidType", BuiltinCode.DROP_INVALID_TYPE); + String2BuiltinCode.put("freplicate", BuiltinCode.FRAME_ROW_REPLICATE); + String2BuiltinCode.put("dropInvalidLength", BuiltinCode.DROP_INVALID_LENGTH); + String2BuiltinCode.put("_map", BuiltinCode.MAP); + String2BuiltinCode.put("valueSwap", BuiltinCode.VALUE_SWAP); + String2BuiltinCode.put("applySchema", BuiltinCode.APPLY_SCHEMA); + String2BuiltinCode.put("get_categorical_mask", BuiltinCode.GET_CATEGORICAL_MASK); } protected Builtin(BuiltinCode bf) { diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java index 86184f47be6..08d28512d5c 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java @@ -59,7 +59,7 @@ else if (in1.getDataType() == DataType.TENSOR && in2.getDataType() == DataType.T return new BinaryTensorTensorCPInstruction(operator, in1, in2, out, opcode, str); else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.FRAME) return new BinaryFrameFrameCPInstruction(operator, in1, in2, out, opcode, str); - else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.SCALAR) + else if(in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.SCALAR) return new BinaryFrameScalarCPInstruction(operator, in1, in2, out, opcode, str); else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.MATRIX) return new BinaryFrameMatrixCPInstruction(operator, in1, in2, out, opcode, str); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java index 193894fd9bc..de76fca18b8 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java @@ -37,8 +37,8 @@ public class BinaryFrameScalarCPInstruction extends BinaryCPInstruction { // private static final Log LOG = LogFactory.getLog(BinaryFrameFrameCPInstruction.class.getName()); - private static final TfMethod[] UNSUPPORTED_MASK_METHODS = new TfMethod[] {TfMethod.BIN, - TfMethod.WORD_EMBEDDING, TfMethod.BAG_OF_WORDS, TfMethod.UDF}; + private static final TfMethod[] UNSUPPORTED_MASK_METHODS = new TfMethod[] {TfMethod.BIN, TfMethod.WORD_EMBEDDING, + TfMethod.BAG_OF_WORDS, TfMethod.UDF}; protected BinaryFrameScalarCPInstruction(MultiThreadedOperator op, CPOperand in1, CPOperand in2, CPOperand out, String opcode, String istr) { @@ -96,9 +96,9 @@ public void processGetCategorical(ExecutionContext ec, FrameBlock f, ScalarObjec } /** - * Accumulates, per input column, how many output columns it expands to (lengths) and whether those - * output columns are categorical (categorical). The arrays are allocated lazily: a column that no - * method touches keeps the implicit default of a single, non-categorical output column. + * Accumulates, per input column, how many output columns it expands to (lengths) and whether those output columns + * are categorical (categorical). The arrays are allocated lazily: a column that no method touches keeps the + * implicit default of a single, non-categorical output column. */ private static final class CategoricalMask { private final FrameBlock f; diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java index d76dbe0d45e..2c8093c3717 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java @@ -80,10 +80,10 @@ public void processInstruction(ExecutionContext ec) { retBlock = inBlock1; } else { - if(LibCommonsMath.isSupportedMatrixMatrixOperation(getOpcode()) ){ + if(LibCommonsMath.isSupportedMatrixMatrixOperation(getOpcode())) { if(compressedLeft) inBlock1 = CompressedMatrixBlock.getUncompressed(inBlock1, getOpcode()); - + if(compressedRight) inBlock2 = CompressedMatrixBlock.getUncompressed(inBlock2, getOpcode()); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java index e53958ac4b8..97ae151ecc0 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java @@ -350,9 +350,10 @@ else if(opcode.equalsIgnoreCase(Opcodes.TRANSFORMDECODE.toString())) { String[] colnames = meta.getColumnNames(); // compute transformdecode - Decoder decoder = DecoderFactory - .createDecoder(getParameterMap().get("spec"), colnames, null, meta, data.getNumColumns()); - FrameBlock fbout = decoder.decode(data, new FrameBlock(decoder.getSchema()), InfrastructureAnalyzer.getLocalParallelism()); + Decoder decoder = DecoderFactory.createDecoder(getParameterMap().get("spec"), colnames, null, meta, + data.getNumColumns()); + FrameBlock fbout = decoder.decode(data, new FrameBlock(decoder.getSchema()), + InfrastructureAnalyzer.getLocalParallelism()); fbout.setColumnNames(Arrays.copyOfRange(colnames, 0, fbout.getNumColumns())); // release locks diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java index 40a677e5d71..04353806ca3 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java @@ -45,8 +45,8 @@ protected ReorgOOCInstruction(ReorgOperator op, CPOperand in1, CPOperand out, St this(op, in1, out, null, null, null, opcode, istr); } - private ReorgOOCInstruction(Operator op, CPOperand in, CPOperand out, CPOperand col, CPOperand desc, CPOperand ixret, - String opcode, String istr) { + private ReorgOOCInstruction(Operator op, CPOperand in, CPOperand out, CPOperand col, CPOperand desc, + CPOperand ixret, String opcode, String istr) { super(OOCType.Reorg, op, in, out, opcode, istr); _col = col; _desc = desc; @@ -76,8 +76,8 @@ else if(opcode.equalsIgnoreCase(Opcodes.SORT.toString())) { CPOperand desc = new CPOperand(parts[3]); CPOperand ixret = new CPOperand(parts[4]); int k = Integer.parseInt(parts[6]); - return new ReorgOOCInstruction(new ReorgOperator(new SortIndex(1, false, false), k), - in, out, col, desc, ixret, opcode, str); + return new ReorgOOCInstruction(new ReorgOperator(new SortIndex(1, false, false), k), in, out, col, desc, + ixret, opcode, str); } else throw new NotImplementedException(); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java index 7590438b949..091c6785b3c 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java @@ -128,17 +128,20 @@ public void processInstruction(ExecutionContext ec) { long r = tmp.getIndexes().getRowIndex(); long c = tmp.getIndexes().getColumnIndex(); // adapt index to new position in row - return new IndexedMatrixValue(new MatrixIndexes(1, (r - 1) * numBlocksPerRowIn + c), tmp.getValue()); + return new IndexedMatrixValue(new MatrixIndexes(1, (r - 1) * numBlocksPerRowIn + c), + tmp.getValue()); }); } else { f.join(); - reshapeFullColBlocks(rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, singleRowBlocks, qOut); + reshapeFullColBlocks(rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, + singleRowBlocks, qOut); } } else { f.join(); - reshapePartialColBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, singleRowBlocks, qOut); + reshapePartialColBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, + numBlocksPerColOut, singleRowBlocks, qOut); } } else { @@ -166,17 +169,20 @@ public void processInstruction(ExecutionContext ec) { long r = tmp.getIndexes().getRowIndex(); long c = tmp.getIndexes().getColumnIndex(); // adapt index to new position in col - return new IndexedMatrixValue(new MatrixIndexes((c - 1) * numBlocksPerColIn + r, 1), tmp.getValue()); + return new IndexedMatrixValue(new MatrixIndexes((c - 1) * numBlocksPerColIn + r, 1), + tmp.getValue()); }); } else { f.join(); - reshapeFullRowBlocks(rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, singleColBlocks, qOut); + reshapeFullRowBlocks(rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, + singleColBlocks, qOut); } } else { f.join(); - reshapePartialRowBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, singleColBlocks, qOut); + reshapePartialRowBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, + numBlocksPerColOut, singleColBlocks, qOut); } } } @@ -226,7 +232,8 @@ private void reshapeFullColBlocks(long rows, long cols, int blen, int numBlocksP } private void reshapePartialColBlocks(long rlen, long clen, long rows, long cols, int blen, int numBlocksPerRowIn, - int numBlocksPerRowOut, int numBlocksPerColOut, OOCStream singleRowBlocks, OOCStream qOut) { + int numBlocksPerRowOut, int numBlocksPerColOut, OOCStream singleRowBlocks, + OOCStream qOut) { // use cache for accessing input rows by index CachingStream singleRowBlockCache = new CachingStream(singleRowBlocks); singleRowBlockCache.incrSubscriberCount(1); @@ -237,14 +244,16 @@ private void reshapePartialColBlocks(long rlen, long clen, long rows, long cols, int r = 0; // allocate row of output blocks - MatrixBlock[] outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut,true); + MatrixBlock[] outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, + true); int offsetOut = 0; int localColsOut = (cols > blen) ? blen : (int) cols; // iterate through input rows and add to row of output blocks for(int i = 1; i <= rlen; i++) { for(int j = 1; j <= numBlocksPerRowIn; j++) { - try(OOCStream.QueueCallback qcb = singleRowBlockCache.findCached(new MatrixIndexes(i, j))) { + try(OOCStream.QueueCallback qcb = singleRowBlockCache + .findCached(new MatrixIndexes(i, j))) { MatrixBlock blk = (MatrixBlock) qcb.get().getValue(); int offsetIn = 0; @@ -278,10 +287,12 @@ else if(remIn == remOut) { if(r == outputBlockRow[0].getNumRows()) { // enqueue filled output blocks and allocate new ones for(int b = 0; b < outputBlockRow.length; b++) - qOut.enqueue(new IndexedMatrixValue(new MatrixIndexes(br + 1, b + 1), outputBlockRow[b])); + qOut.enqueue( + new IndexedMatrixValue(new MatrixIndexes(br + 1, b + 1), outputBlockRow[b])); br++; // allocate new block row - outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, true); + outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, + numBlocksPerColOut, true); r = 0; } bc = 0; @@ -341,7 +352,8 @@ private void reshapeFullRowBlocks(long rows, long cols, int blen, int numBlocksP } private void reshapePartialRowBlocks(long rlen, long clen, long rows, long cols, int blen, int numBlocksPerRowOut, - int numBlocksPerColIn, int numBlocksPerColOut, OOCStream singleColBlocks, OOCStream qOut) { + int numBlocksPerColIn, int numBlocksPerColOut, OOCStream singleColBlocks, + OOCStream qOut) { // use cache for accessing input cols by index CachingStream singleRowBlockCache = new CachingStream(singleColBlocks); singleRowBlockCache.incrSubscriberCount(1); @@ -352,14 +364,16 @@ private void reshapePartialRowBlocks(long rlen, long clen, long rows, long cols, int c = 0; // allocate col of output blocks - MatrixBlock[] outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, false); + MatrixBlock[] outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, + false); int offsetOut = 0; int localRowsOut = (rows > blen) ? blen : (int) rows; // iterate through input cols and add to col of output blocks for(int j = 1; j <= clen; j++) { for(int i = 1; i <= numBlocksPerColIn; i++) { - try(OOCStream.QueueCallback qcb = singleRowBlockCache.findCached(new MatrixIndexes(i, j))) { + try(OOCStream.QueueCallback qcb = singleRowBlockCache + .findCached(new MatrixIndexes(i, j))) { MatrixBlock blk = (MatrixBlock) qcb.get().getValue(); int offsetIn = 0; @@ -394,11 +408,13 @@ else if(remIn == remOut) { // enqueue filled output blocks and allocate new ones for(int b = 0; b < outputBlockCol.length; b++) { outputBlockCol[b].recomputeNonZeros(); - qOut.enqueue(new IndexedMatrixValue(new MatrixIndexes(b + 1, bc + 1), outputBlockCol[b])); + qOut.enqueue( + new IndexedMatrixValue(new MatrixIndexes(b + 1, bc + 1), outputBlockCol[b])); } bc++; // allocate new block col - outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, false); + outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, + numBlocksPerColOut, false); c = 0; } br = 0; @@ -411,16 +427,20 @@ else if(remIn == remOut) { qOut.closeInput(); } - private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int blen, int numBlocksPerRow, int numBlocksPerCol, boolean isBlockRowSlice) { + private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int blen, int numBlocksPerRow, + int numBlocksPerCol, boolean isBlockRowSlice) { int num = isBlockRowSlice ? numBlocksPerRow : numBlocksPerCol; MatrixBlock[] res = new MatrixBlock[num]; // full inner blocks, adjust for outer blocks - int localRows = ((!isBlockRowSlice || idx == numBlocksPerCol - 1) && rows % blen != 0) ? (int) rows % blen : blen; - int localCols = ((isBlockRowSlice || idx == numBlocksPerRow - 1) && cols % blen != 0) ? (int) cols % blen : blen; + int localRows = ((!isBlockRowSlice || idx == numBlocksPerCol - 1) && rows % blen != 0) ? (int) rows % + blen : blen; + int localCols = ((isBlockRowSlice || idx == numBlocksPerRow - 1) && cols % blen != 0) ? (int) cols % + blen : blen; for(int k = 0; k < num - 1; k++) { - res[k] = isBlockRowSlice ? new MatrixBlock(localRows, blen, false) : new MatrixBlock(blen, localCols, false); + res[k] = isBlockRowSlice ? new MatrixBlock(localRows, blen, false) : new MatrixBlock(blen, localCols, + false); res[k].allocateDenseBlock(); } res[num - 1] = new MatrixBlock(localRows, localCols, false); @@ -428,10 +448,13 @@ private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int ble return res; } - private void setOutputEntries(MatrixBlock src, MatrixBlock dest, int idx, int srcOffset, int destOffset, int length, boolean rowWise) { + private void setOutputEntries(MatrixBlock src, MatrixBlock dest, int idx, int srcOffset, int destOffset, int length, + boolean rowWise) { if(rowWise) - ((DenseBlockFP64) dest.getDenseBlock()).setPartialRow(src.getDenseBlock(), idx, srcOffset, destOffset, length); + ((DenseBlockFP64) dest.getDenseBlock()).setPartialRow(src.getDenseBlock(), idx, srcOffset, destOffset, + length); else - ((DenseBlockFP64) dest.getDenseBlock()).setPartialCol(src.getDenseBlock(), idx, srcOffset, destOffset, length); + ((DenseBlockFP64) dest.getDenseBlock()).setPartialCol(src.getDenseBlock(), idx, srcOffset, destOffset, + length); } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java index 75f84882478..e25219b80ba 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java @@ -114,8 +114,7 @@ public void processInstruction(ExecutionContext ec) { case VALUEPICK: { if( input2.isScalar() ) { ScalarObject quantile = ec.getScalarInput(input2); - double[] wt = getWeightedQuantileSummary(in, mc, - new double[] {quantile.getDoubleValue()}, true); + double[] wt = getWeightedQuantileSummary(in, mc, new double[] {quantile.getDoubleValue()}, true); ec.setScalarOutput(output.getName(), new DoubleObject(wt[3])); } else { diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java index 2f83caa5526..f007558ebdc 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java @@ -30,8 +30,7 @@ import org.apache.sysds.runtime.matrix.data.MatrixValue; import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; -public class IndexedMatrixValue implements SpillableObject, Serializable -{ +public class IndexedMatrixValue implements SpillableObject, Serializable { private static final long serialVersionUID = 6723389820806752110L; private MatrixIndexes _indexes = null; @@ -110,8 +109,8 @@ public void discard() { @Override public void read(DataInput dataInput) throws IOException { - _indexes = new MatrixIndexes(); - _value = new MatrixBlock(); + _indexes = new MatrixIndexes(); + _value = new MatrixBlock(); _indexes.readFields(dataInput); _value.readFields(dataInput); } diff --git a/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java index c3b9351d3d3..cc8491d4515 100644 --- a/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java +++ b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java @@ -87,11 +87,10 @@ import io.delta.kernel.utils.FileStatus; /** - * Shared helpers for the native (Spark-free) Delta Lake read/write paths used - * by both the matrix and frame readers/writers. Centralizes engine creation, - * path qualification, the scan loop (snapshot -> data files -> logical - * columnar batches, honoring deletion vectors), and the write transaction - * (logical data -> parquet -> commit). + * Shared helpers for the native (Spark-free) Delta Lake read/write paths used by both the matrix and frame + * readers/writers. Centralizes engine creation, path qualification, the scan loop (snapshot -> data files -> + * logical columnar batches, honoring deletion vectors), and the write transaction (logical data -> parquet -> + * commit). */ public class DeltaKernelUtils { @@ -102,23 +101,29 @@ public class DeltaKernelUtils { /** Reused thread-safe JSON reader for the per-file Delta stats (numRecords). */ private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); - /** Delta Kernel config key: number of rows per parquet read batch, overridable via - * {@link org.apache.sysds.conf.DMLConfig#DELTA_READER_BATCH_SIZE}. */ + /** + * Delta Kernel config key: number of rows per parquet read batch, overridable via + * {@link org.apache.sysds.conf.DMLConfig#DELTA_READER_BATCH_SIZE}. + */ private static final String CONF_READER_BATCH_SIZE = "delta.kernel.default.parquet.reader.batch-size"; - /** Delta Kernel config key: target size (bytes) at which the writer rolls a new data file, overridable via - * {@link org.apache.sysds.conf.DMLConfig#DELTA_WRITER_TARGET_FILE_SIZE}. */ + /** + * Delta Kernel config key: target size (bytes) at which the writer rolls a new data file, overridable via + * {@link org.apache.sysds.conf.DMLConfig#DELTA_WRITER_TARGET_FILE_SIZE}. + */ private static final String CONF_WRITER_TARGET_FILE_SIZE = "delta.kernel.default.parquet.writer.targetMaxFileSize"; - /** Internal Delta column type codes shared by the matrix and frame readers to - * dispatch boxing-free primitive column access. */ - public static final int T_DOUBLE = 0; - public static final int T_FLOAT = 1; - public static final int T_LONG = 2; - public static final int T_INT = 3; - public static final int T_SHORT = 4; - public static final int T_BYTE = 5; + /** + * Internal Delta column type codes shared by the matrix and frame readers to dispatch boxing-free primitive column + * access. + */ + public static final int T_DOUBLE = 0; + public static final int T_FLOAT = 1; + public static final int T_LONG = 2; + public static final int T_INT = 3; + public static final int T_SHORT = 4; + public static final int T_BYTE = 5; public static final int T_BOOLEAN = 6; - public static final int T_STRING = 7; + public static final int T_STRING = 7; /** * Parquet physical type each {@code T_*} column is stored as, indexed by type code (delta int/short/byte columns @@ -128,22 +133,22 @@ public class DeltaKernelUtils { PrimitiveTypeName.INT64, PrimitiveTypeName.INT32, PrimitiveTypeName.INT32, PrimitiveTypeName.INT32, PrimitiveTypeName.BOOLEAN, PrimitiveTypeName.BINARY}; - //derived configuration cached to avoid copying the (large) base conf on every - //engine creation (createEngine is called once per data file in parallel reads); - //rebuilt whenever the base conf or the relevant SystemDS settings change. + // derived configuration cached to avoid copying the (large) base conf on every + // engine creation (createEngine is called once per data file in parallel reads); + // rebuilt whenever the base conf or the relevant SystemDS settings change. private static Configuration cachedConf; private static Configuration cachedConfBase; private static int cachedBatchSize; private static long cachedTargetFileSize; - private DeltaKernelUtils() {} + private DeltaKernelUtils() { + } /** - * Consumes a whole columnar batch. {@code selected} is {@code null} when all - * {@code size} rows are live; otherwise {@code selected[r]} indicates whether - * row {@code r} survived the deletion/selection vector. Batch-level consumption - * lets callers extract data column-at-a-time (cache friendly, boxing free) - * instead of paying a per-row callback. + * Consumes a whole columnar batch. {@code selected} is {@code null} when all {@code size} rows are live; otherwise + * {@code selected[r]} indicates whether row {@code r} survived the deletion/selection vector. Batch-level + * consumption lets callers extract data column-at-a-time (cache friendly, boxing free) instead of paying a per-row + * callback. */ @FunctionalInterface public interface BatchConsumer { @@ -151,22 +156,29 @@ public interface BatchConsumer { } /** - * Map a Delta Kernel {@link DataType} to an internal type code (see the - * {@code T_*} constants). Returned once per column so the per-cell read loop - * can switch on a primitive int instead of repeating {@code instanceof} checks. + * Map a Delta Kernel {@link DataType} to an internal type code (see the {@code T_*} constants). Returned once per + * column so the per-cell read loop can switch on a primitive int instead of repeating {@code instanceof} checks. * * @param dt the Delta column data type * @return the matching {@code T_*} code, or {@code -1} if the type is not supported */ public static int typeCode(DataType dt) { - if( dt instanceof DoubleType ) return T_DOUBLE; - if( dt instanceof FloatType ) return T_FLOAT; - if( dt instanceof LongType ) return T_LONG; - if( dt instanceof IntegerType ) return T_INT; - if( dt instanceof ShortType ) return T_SHORT; - if( dt instanceof ByteType ) return T_BYTE; - if( dt instanceof BooleanType ) return T_BOOLEAN; - if( dt instanceof StringType ) return T_STRING; + if(dt instanceof DoubleType) + return T_DOUBLE; + if(dt instanceof FloatType) + return T_FLOAT; + if(dt instanceof LongType) + return T_LONG; + if(dt instanceof IntegerType) + return T_INT; + if(dt instanceof ShortType) + return T_SHORT; + if(dt instanceof ByteType) + return T_BYTE; + if(dt instanceof BooleanType) + return T_BOOLEAN; + if(dt instanceof StringType) + return T_STRING; return -1; } @@ -429,8 +441,10 @@ private static void decodeColumnInto(ColumnReader creader, int maxDef, int nrow, } } - /** Floor on the adaptive writer target file size. Below this the per-file metadata/open - * overhead (and tiny-file proliferation) outweighs the extra read parallelism. */ + /** + * Floor on the adaptive writer target file size. Below this the per-file metadata/open overhead (and tiny-file + * proliferation) outweighs the extra read parallelism. + */ public static final long ADAPTIVE_WRITER_MIN_FILE_SIZE = 4L * 1024 * 1024; private static Configuration buildConf(Configuration base, int batchSize, long targetFileSize) { @@ -444,9 +458,8 @@ private static synchronized Configuration deltaConf() { Configuration base = ConfigurationManager.getCachedJobConf(); int batchSize = ConfigurationManager.getDeltaReaderBatchSize(); long targetFileSize = ConfigurationManager.getDeltaWriterTargetFileSize(); - if(cachedConf == null || cachedConfBase != base - || cachedBatchSize != batchSize || cachedTargetFileSize != targetFileSize) - { + if(cachedConf == null || cachedConfBase != base || cachedBatchSize != batchSize || + cachedTargetFileSize != targetFileSize) { cachedConf = buildConf(base, batchSize, targetFileSize); cachedConfBase = base; cachedBatchSize = batchSize; @@ -460,12 +473,11 @@ public static Engine createEngine() { } /** - * Compute the parquet target data-file size (bytes) for writing a table of the given - * estimated size. With adaptive sizing enabled the writer aims for roughly one data - * file per expected parallel reader (so the native per-file parallel read can use all - * threads): never above the configured target, and never below - * {@code ADAPTIVE_WRITER_MIN_FILE_SIZE} unless the configured target is itself smaller - * than that floor (in which case the configured target wins). + * Compute the parquet target data-file size (bytes) for writing a table of the given estimated size. With adaptive + * sizing enabled the writer aims for roughly one data file per expected parallel reader (so the native per-file + * parallel read can use all threads): never above the configured target, and never below + * {@code ADAPTIVE_WRITER_MIN_FILE_SIZE} unless the configured target is itself smaller than that floor (in which + * case the configured target wins). * * @param estimatedBytes estimate of the table's size (the block in-memory size is a fine proxy) * @return the target max parquet data-file size in bytes @@ -476,7 +488,7 @@ public static long adaptiveWriterTargetFileSize(long estimatedBytes) { return configured; int par = Math.max(1, OptimizerUtils.getParallelBinaryReadParallelism()); long perReader = Math.max(1, estimatedBytes / par); - //never above the configured cap, never below the floor (unless the cap itself is lower) + // never above the configured cap, never below the floor (unless the cap itself is lower) long target = Math.min(configured, Math.max(ADAPTIVE_WRITER_MIN_FILE_SIZE, perReader)); if(LOG.isDebugEnabled()) LOG.debug("Delta adaptive file size: est=" + estimatedBytes + "B par=" + par + " -> target=" + target @@ -485,24 +497,24 @@ public static long adaptiveWriterTargetFileSize(long estimatedBytes) { } /** - * Create an engine for writing a table of the given estimated size, configured with an - * adaptive target data-file size (see {@link #adaptiveWriterTargetFileSize(long)}). A fresh - * (uncached) configuration is built since writes happen once per table, not per data file. + * Create an engine for writing a table of the given estimated size, configured with an adaptive target data-file + * size (see {@link #adaptiveWriterTargetFileSize(long)}). A fresh (uncached) configuration is built since writes + * happen once per table, not per data file. * * @param estimatedBytes estimate of the table's size (the block in-memory size is a fine proxy) * @return a Delta Kernel engine for the write */ public static Engine createWriteEngine(long estimatedBytes) { - //the reader batch size is irrelevant on the write path but is set to keep the - //conf shape identical to deltaConf(); only the target file size matters here. + // the reader batch size is irrelevant on the write path but is set to keep the + // conf shape identical to deltaConf(); only the target file size matters here. Configuration c = buildConf(ConfigurationManager.getCachedJobConf(), ConfigurationManager.getDeltaReaderBatchSize(), adaptiveWriterTargetFileSize(estimatedBytes)); return DefaultEngine.create(c); } /** - * Resolve a (possibly relative) path to a fully-qualified URI so the - * kernel's default engine can locate the table on the right filesystem. + * Resolve a (possibly relative) path to a fully-qualified URI so the kernel's default engine can locate the table + * on the right filesystem. * * @param fname input path * @return fully-qualified table path @@ -519,11 +531,9 @@ public static String qualify(String fname) { } /** - * Opened latest snapshot of a Delta table: the logical schema plus everything - * needed to (re)read its data files, including the list of per-data-file scan - * rows. Delta Kernel scan-file rows are self-contained (the kernel's - * distributed design serializes them to workers), so they can be retained and - * read independently / in parallel. + * Opened latest snapshot of a Delta table: the logical schema plus everything needed to (re)read its data files, + * including the list of per-data-file scan rows. Delta Kernel scan-file rows are self-contained (the kernel's + * distributed design serializes them to workers), so they can be retained and read independently / in parallel. */ public static final class ScanHandle { public final StructType schema; @@ -531,19 +541,18 @@ public static final class ScanHandle { public final StructType physicalReadSchema; public final List scanFiles; /** - * Per-file record counts taken from the Delta {@code numRecords} statistic, - * aligned with {@link #scanFiles}; {@code -1} where the statistic is absent. + * Per-file record counts taken from the Delta {@code numRecords} statistic, aligned with {@link #scanFiles}; + * {@code -1} where the statistic is absent. */ public final long[] numRecords; /** - * Per-file flag indicating a deletion vector is present (so the live row - * count differs from {@link #numRecords}), aligned with {@link #scanFiles}. + * Per-file flag indicating a deletion vector is present (so the live row count differs from + * {@link #numRecords}), aligned with {@link #scanFiles}. */ public final boolean[] hasDeletionVector; - private ScanHandle(StructType schema, Row scanState, StructType physicalReadSchema, - List scanFiles, long[] numRecords, boolean[] hasDeletionVector) - { + private ScanHandle(StructType schema, Row scanState, StructType physicalReadSchema, List scanFiles, + long[] numRecords, boolean[] hasDeletionVector) { this.schema = schema; this.scanState = scanState; this.physicalReadSchema = physicalReadSchema; @@ -553,13 +562,12 @@ private ScanHandle(StructType schema, Row scanState, StructType physicalReadSche } /** - * @return true iff every data file carries a {@code numRecords} statistic - * and none has a deletion vector, i.e. exact per-file row offsets - * can be derived from metadata without reading the data. + * @return true iff every data file carries a {@code numRecords} statistic and none has a deletion vector, i.e. + * exact per-file row offsets can be derived from metadata without reading the data. */ public boolean hasExactRowCounts() { - for( int i=0; i scanFileIter = (scan instanceof ScanImpl) - ? ((ScanImpl) scan).getScanFiles(engine, true) - : scan.getScanFiles(engine); + // request the scan files WITH per-file statistics (numRecords) so callers can + // pre-size output and place rows without reading the data; harmless extra + // column for the data-read path. Fall back to the stats-less iterator if the + // concrete scan does not support it. + CloseableIterator scanFileIter = (scan instanceof ScanImpl) ? ((ScanImpl) scan) + .getScanFiles(engine, true) : scan.getScanFiles(engine); List files = new ArrayList<>(); List recs = new ArrayList<>(); List dvs = new ArrayList<>(); - try( CloseableIterator scanFiles = scanFileIter ) { - while( scanFiles.hasNext() ) { + try(CloseableIterator scanFiles = scanFileIter) { + while(scanFiles.hasNext()) { FilteredColumnarBatch scanFileBatch = scanFiles.next(); - try( CloseableIterator scanFileRows = scanFileBatch.getRows() ) { - while( scanFileRows.hasNext() ) { + try(CloseableIterator scanFileRows = scanFileBatch.getRows()) { + while(scanFileRows.hasNext()) { Row scanFileRow = scanFileRows.next(); files.add(scanFileRow); recs.add(numRecords(scanFileRow)); @@ -609,7 +615,7 @@ public static ScanHandle openScan(Engine engine, String tablePath) throws IOExce } long[] numRecords = new long[recs.size()]; boolean[] hasDv = new boolean[dvs.size()]; - for( int i=0; i physicalData = engine.getParquetHandler() .readParquetFiles(Utils.singletonCloseableIterator(dataFile), physicalReadSchema, Optional.empty()); - try( CloseableIterator logicalData = - Scan.transformPhysicalData(engine, scanState, scanFileRow, physicalData) ) - { - while( logicalData.hasNext() ) + try(CloseableIterator logicalData = Scan.transformPhysicalData(engine, scanState, + scanFileRow, physicalData)) { + while(logicalData.hasNext()) consumeBatch(logicalData.next(), consumer); } } /** - * Scan the latest snapshot of a Delta table sequentially, invoking the batch - * consumer for every data batch. The consumer is created lazily from the table - * schema (so callers can size buffers / derive per-column types up front). + * Scan the latest snapshot of a Delta table sequentially, invoking the batch consumer for every data batch. The + * consumer is created lazily from the table schema (so callers can size buffers / derive per-column types up + * front). * * @param engine delta kernel engine * @param tablePath fully-qualified table path @@ -677,11 +679,10 @@ public static void readScanFile(Engine engine, Row scanState, StructType physica * @throws IOException on read failure */ public static StructType scan(Engine engine, String tablePath, Function consumerFactory) - throws IOException - { + throws IOException { ScanHandle h = openScan(engine, tablePath); BatchConsumer consumer = consumerFactory.apply(h.schema); - for( Row scanFileRow : h.scanFiles ) + for(Row scanFileRow : h.scanFiles) readScanFile(engine, h.scanState, h.physicalReadSchema, scanFileRow, consumer); return h.schema; } @@ -690,28 +691,27 @@ private static void consumeBatch(FilteredColumnarBatch fcb, BatchConsumer consum ColumnarBatch batch = fcb.getData(); int ncol = batch.getSchema().length(); ColumnVector[] cols = new ColumnVector[ncol]; - for( int c=0; c all rows live) + // materialize the deletion/selection mask once (null => all rows live) Optional selVector = fcb.getSelectionVector(); boolean[] selected = null; - if( selVector.isPresent() ) { + if(selVector.isPresent()) { ColumnVector sv = selVector.get(); selected = new boolean[size]; - for( int r=0; r logicalData) throws IOException - { - //replace any existing table at the path (the other SystemDS writers delete - //the output first; the caching layer does not do it on our behalf) + CloseableIterator logicalData) throws IOException { + // replace any existing table at the path (the other SystemDS writers delete + // the output first; the caching layer does not do it on our behalf) HDFSTool.deleteFileIfExistOnHDFS(tablePath); Table table = Table.forPath(engine, tablePath); - TransactionBuilder txnBuilder = table - .createTransactionBuilder(engine, ENGINE_INFO, Operation.CREATE_TABLE) + TransactionBuilder txnBuilder = table.createTransactionBuilder(engine, ENGINE_INFO, Operation.CREATE_TABLE) .withSchema(engine, schema); Transaction txn = txnBuilder.build(engine); Row txnState = txn.getTransactionState(engine); - CloseableIterator physicalData = - Transaction.transformLogicalData(engine, txnState, logicalData, Collections.emptyMap()); - DataWriteContext writeContext = - Transaction.getWriteContext(engine, txnState, Collections.emptyMap()); + CloseableIterator physicalData = Transaction.transformLogicalData(engine, txnState, + logicalData, Collections.emptyMap()); + DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, Collections.emptyMap()); CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles(writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns()); - CloseableIterator appendActions = - Transaction.generateAppendActions(engine, txnState, dataFiles, writeContext); + CloseableIterator appendActions = Transaction.generateAppendActions(engine, txnState, dataFiles, + writeContext); txn.commit(engine, CloseableIterable.inMemoryIterable(appendActions)); } } diff --git a/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java b/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java index 58a98741975..55d8f8f7c2d 100644 --- a/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java +++ b/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java @@ -33,29 +33,27 @@ import io.delta.kernel.types.StructType; /** - * Single-threaded native Delta Lake reader for matrices, built on the - * Spark-free Delta Kernel library. It opens the latest snapshot of a Delta - * table directory, reads its parquet data files through the kernel's default - * engine (honoring deletion vectors), and materializes the numeric columns - * into a dense {@link MatrixBlock}. + * Single-threaded native Delta Lake reader for matrices, built on the Spark-free Delta Kernel library. It opens the + * latest snapshot of a Delta table directory, reads its parquet data files through the kernel's default engine + * (honoring deletion vectors), and materializes the numeric columns into a dense {@link MatrixBlock}. * - *

Only numeric columns (double/float/long/int/short/byte/boolean) are - * supported, matching the all-double nature of a SystemDS matrix. Dimensions - * do not need to be known up front: the row count is discovered while scanning - * and the column count is taken from the table schema.

+ *

+ * Only numeric columns (double/float/long/int/short/byte/boolean) are supported, matching the all-double nature of a + * SystemDS matrix. Dimensions do not need to be known up front: the row count is discovered while scanning and the + * column count is taken from the table schema. + *

*/ public class ReaderDelta extends MatrixReader { @Override public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int blen, long estnnz) - throws IOException, DMLRuntimeException - { + throws IOException, DMLRuntimeException { Engine engine = DeltaKernelUtils.createEngine(); String tablePath = DeltaKernelUtils.qualify(fname); - //Scan column-at-a-time into one row-major buffer per batch (no per-row - //allocation, no boxing, no per-cell set()). Buffers are concatenated into - //the dense output via bulk array copies below. + // Scan column-at-a-time into one row-major buffer per batch (no per-row + // allocation, no boxing, no per-cell set()). Buffers are concatenated into + // the dense output via bulk array copies below. ArrayList batches = new ArrayList<>(); int[] nrowH = new int[1]; StructType schema = DeltaKernelUtils.scan(engine, tablePath, sch -> { @@ -72,7 +70,7 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl long lestnnz = (estnnz >= 0) ? estnnz : (long) nrow * ncol; MatrixBlock ret = createOutputMatrixBlock(nrow, ncol, Math.max(nrow, 1), lestnnz, true, false); - if( nrow > 0 && ncol > 0 ) + if(nrow > 0 && ncol > 0) fillDense(ret, batches); ret.recomputeNonZeros(); ret.examSparsity(); @@ -83,14 +81,14 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl static int[] columnTypes(StructType schema) { int ncol = schema.length(); int[] types = new int[ncol]; - for( int c=0; c batches) { DenseBlock db = ret.getDenseBlock(); - if( db.isContiguous() ) { + if(db.isContiguous()) { double[] dv = db.valuesAt(0); int off = 0; - for( double[] buf : batches ) { + for(double[] buf : batches) { System.arraycopy(buf, 0, dv, off, buf.length); off += buf.length; } } else { - //rare large multi-block fallback: route each row through the block API + // rare large multi-block fallback: route each row through the block API int ncol = ret.getNumColumns(); int r = 0; - for( double[] buf : batches ) { + for(double[] buf : batches) { int rowsInBuf = buf.length / ncol; - for( int i=0; iThe expensive part of a Delta read is the parquet decode, which the kernel - * performs per data file; parallelizing across files is therefore the natural - * way to bridge the gap to the (near-raw) binary reader. A table backed by a - * single data file (the default for tables <= the parquet target file size) - * cannot be split this way, so the reader transparently falls back to the - * sequential {@link ReaderDelta} path in that case.

+ *

+ * The expensive part of a Delta read is the parquet decode, which the kernel performs per data file; parallelizing + * across files is therefore the natural way to bridge the gap to the (near-raw) binary reader. A table backed by a + * single data file (the default for tables <= the parquet target file size) cannot be split this way, so the reader + * transparently falls back to the sequential {@link ReaderDelta} path in that case. + *

*/ public class ReaderDeltaParallel extends ReaderDelta { @@ -57,28 +56,27 @@ public ReaderDeltaParallel() { @Override public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int blen, long estnnz) - throws IOException, DMLRuntimeException - { + throws IOException, DMLRuntimeException { Engine engine = DeltaKernelUtils.createEngine(); String tablePath = DeltaKernelUtils.qualify(fname); DeltaKernelUtils.ScanHandle handle = DeltaKernelUtils.openScan(engine, tablePath); final int nfiles = handle.scanFiles.size(); - //nothing to gain from parallelism for single-file (or empty) tables - if( _numThreads <= 1 || nfiles <= 1 ) + // nothing to gain from parallelism for single-file (or empty) tables + if(_numThreads <= 1 || nfiles <= 1) return super.readMatrixFromHDFS(fname, rlen, clen, blen, estnnz); final int ncol = handle.schema.length(); final int[] types = columnTypes(handle.schema); - //fast path: exact per-file row counts are known from metadata and the dense - //output fits a single contiguous array -> pre-size once and let each thread - //decode directly into its slice (no intermediate buffers, no serial copy). - if( useDirectPath(handle) ) { + // fast path: exact per-file row counts are known from metadata and the dense + // output fits a single contiguous array -> pre-size once and let each thread + // decode directly into its slice (no intermediate buffers, no serial copy). + if(useDirectPath(handle)) { long total = 0; - for( long r : handle.numRecords ) + for(long r : handle.numRecords) total += r; - if( total > 0 && (long) total * ncol <= Integer.MAX_VALUE ) + if(total > 0 && (long) total * ncol <= Integer.MAX_VALUE) return readDirect(fname, handle, ncol, types, (int) total, estnnz); } @@ -86,11 +84,9 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl } /** - * Whether the metadata-driven direct-write fast path can be used for this - * table (exact per-file row counts and no deletion vectors). Visible for - * testing: the buffered fallback is otherwise only reachable for tables - * lacking row statistics or carrying deletion vectors, which the SystemDS - * Delta writer never produces. + * Whether the metadata-driven direct-write fast path can be used for this table (exact per-file row counts and no + * deletion vectors). Visible for testing: the buffered fallback is otherwise only reachable for tables lacking row + * statistics or carrying deletion vectors, which the SystemDS Delta writer never produces. * * @param handle the opened scan handle * @return true if the direct path is applicable @@ -100,38 +96,37 @@ protected boolean useDirectPath(DeltaKernelUtils.ScanHandle handle) { } /** - * Fast path: each thread decodes one data file straight into the final dense - * array at a metadata-derived row offset. Single allocation, fully parallel. + * Fast path: each thread decodes one data file straight into the final dense array at a metadata-derived row + * offset. Single allocation, fully parallel. */ - private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, - int ncol, int[] types, int nrow, long estnnz) throws IOException - { + private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, int ncol, int[] types, int nrow, + long estnnz) throws IOException { final int nfiles = handle.scanFiles.size(); final int[] rowOffset = new int[nfiles]; int acc = 0; - for( int i=0; i> tasks = new ArrayList<>(nfiles); - for( int i=0; i { int[] cur = new int[] {base}; Engine eng = DeltaKernelUtils.createEngine(); DeltaKernelUtils.readScanFile(eng, handle.scanState, handle.physicalReadSchema, scanFileRow, (cols, size, selected) -> { - if( cur[0] + DeltaKernelUtils.countSelected(size, selected) > limit ) + if(cur[0] + DeltaKernelUtils.countSelected(size, selected) > limit) throw new DMLRuntimeException("Delta file produced more rows than its " + "numRecords statistic; refusing parallel direct read of " + fname); cur[0] += extractBatchInto(cols, size, selected, types, ncol, dv, cur[0]); @@ -147,19 +142,17 @@ private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, } /** - * Fallback path: decode each file in parallel into per-file buffers (used when - * row counts are unknown, deletion vectors are present, or the matrix exceeds a - * single contiguous array), then concatenate in file order. + * Fallback path: decode each file in parallel into per-file buffers (used when row counts are unknown, deletion + * vectors are present, or the matrix exceeds a single contiguous array), then concatenate in file order. */ - private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handle, - int ncol, int[] types, long estnnz) throws IOException - { + private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handle, int ncol, int[] types, + long estnnz) throws IOException { final int nfiles = handle.scanFiles.size(); @SuppressWarnings("unchecked") final ArrayList[] fileBufs = new ArrayList[nfiles]; final int[] fileRows = new int[nfiles]; ArrayList> tasks = new ArrayList<>(nfiles); - for( int i=0; i { @@ -179,15 +172,15 @@ private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handl awaitFileTasks(tasks, fname); int nrow = 0; - for( int i=0; i ordered = new ArrayList<>(); - for( int i=0; i= 0) ? estnnz : (long) nrow * ncol; MatrixBlock ret = createOutputMatrixBlock(nrow, ncol, Math.max(nrow, 1), lestnnz, true, false); - if( nrow > 0 && ncol > 0 ) + if(nrow > 0 && ncol > 0) fillDense(ret, ordered); ret.recomputeNonZeros(_numThreads); ret.examSparsity(); @@ -195,16 +188,15 @@ private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handl } /** - * Run one decode task per data file on the shared common thread pool and await - * completion. Full parallelism is requested (the task count, one per data file, - * naturally caps concurrency); this avoids the per-thread pool-size caching in - * {@code CommonThreadPool.get(k)} that could otherwise throttle this reader to a - * smaller pool created earlier on the same thread. + * Run one decode task per data file on the shared common thread pool and await completion. Full parallelism is + * requested (the task count, one per data file, naturally caps concurrency); this avoids the per-thread pool-size + * caching in {@code CommonThreadPool.get(k)} that could otherwise throttle this reader to a smaller pool created + * earlier on the same thread. */ private void awaitFileTasks(List> tasks, String fname) throws IOException { ExecutorService pool = CommonThreadPool.get(_numThreads); try { - for( Future f : pool.invokeAll(tasks) ) + for(Future f : pool.invokeAll(tasks)) f.get(); } catch(Exception ex) { diff --git a/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java b/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java index 55ea8a54297..602b540c407 100644 --- a/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java +++ b/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java @@ -39,60 +39,54 @@ import io.delta.kernel.utils.CloseableIterator; /** - * Single-threaded native Delta Lake writer for matrices, built on the - * Spark-free Delta Kernel library. It creates a Delta table at the target - * directory with an all-double schema {@code c0..c(n-1)} (replacing any existing - * table at that path), streams the {@link MatrixBlock} rows as columnar batches - * into parquet data files via the kernel's default engine, and commits the - * corresponding add-file actions to the transaction log. + * Single-threaded native Delta Lake writer for matrices, built on the Spark-free Delta Kernel library. It creates a + * Delta table at the target directory with an all-double schema {@code c0..c(n-1)} (replacing any existing table at + * that path), streams the {@link MatrixBlock} rows as columnar batches into parquet data files via the kernel's default + * engine, and commits the corresponding add-file actions to the transaction log. */ public class WriterDelta extends MatrixWriter { @Override public void writeMatrixToHDFS(MatrixBlock src, String fname, long rlen, long clen, int blen, long nnz, boolean diag) - throws IOException - { - if( src.getNumRows() != rlen || src.getNumColumns() != clen ) - throw new IOException("Matrix dimensions mismatch with metadata: (" - + src.getNumRows() + "x" + src.getNumColumns() + ") vs (" + rlen + "x" + clen + ")."); + throws IOException { + if(src.getNumRows() != rlen || src.getNumColumns() != clen) + throw new IOException("Matrix dimensions mismatch with metadata: (" + src.getNumRows() + "x" + + src.getNumColumns() + ") vs (" + rlen + "x" + clen + ")."); int ncol = (int) clen; int nrow = (int) rlen; int batchRows = ConfigurationManager.getDeltaWriterBatchSize(); - //fast path: a contiguous dense block lets the column views read straight - //from the backing double[] (avoids per-cell MatrixBlock.get dispatch). - double[] dense = (!src.isInSparseFormat() && src.getDenseBlock() != null - && src.getDenseBlock().isContiguous()) ? src.getDenseBlockValues() : null; - //size data files adaptively (toward one file per parallel reader) for faster parallel reads. - //Delta writes every cell as a double, so size by the dense footprint rather than the (possibly - //sparse) in-memory size, which would understate the on-disk table for sparse inputs. + // fast path: a contiguous dense block lets the column views read straight + // from the backing double[] (avoids per-cell MatrixBlock.get dispatch). + double[] dense = (!src.isInSparseFormat() && src.getDenseBlock() != null && + src.getDenseBlock().isContiguous()) ? src.getDenseBlockValues() : null; + // size data files adaptively (toward one file per parallel reader) for faster parallel reads. + // Delta writes every cell as a double, so size by the dense footprint rather than the (possibly + // sparse) in-memory size, which would understate the on-disk table for sparse inputs. long estimatedBytes = (long) nrow * ncol * 8L; Engine engine = DeltaKernelUtils.createWriteEngine(estimatedBytes); - DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), - buildSchema(ncol), new MatrixBatchIterator(src, dense, nrow, ncol, batchRows)); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), buildSchema(ncol), + new MatrixBatchIterator(src, dense, nrow, ncol, batchRows)); } @Override - public void writeEmptyMatrixToHDFS(String fname, long rlen, long clen, int blen) - throws IOException - { - //empty table: create with schema but no data files + public void writeEmptyMatrixToHDFS(String fname, long rlen, long clen, int blen) throws IOException { + // empty table: create with schema but no data files Engine engine = DeltaKernelUtils.createEngine(); - DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), - buildSchema((int) clen), CloseableIterable.emptyIterable().iterator()); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), buildSchema((int) clen), + CloseableIterable.emptyIterable().iterator()); } private static StructType buildSchema(int ncol) { StructType schema = new StructType(); - for( int c=0; c stream, long rlen, long clen, int blen) - throws IOException - { + public long writeMatrixFromStream(String fname, OOCStream stream, long rlen, long clen, + int blen) throws IOException { throw new UnsupportedOperationException("Out-of-core stream write is not supported for the Delta format."); } @@ -122,18 +116,18 @@ public boolean hasNext() { @Override public FilteredColumnarBatch next() { - if( !hasNext() ) + if(!hasNext()) throw new NoSuchElementException(); int size = Math.min(_batchRows, _nrow - _pos); ColumnarBatch batch = new MatrixColumnarBatch(_mb, _dense, _schema, _pos, size, _ncol); _pos += size; - //no selection vector: all rows in the batch are written + // no selection vector: all rows in the batch are written return new FilteredColumnarBatch(batch, Optional.empty()); } @Override public void close() { - //nothing to release + // nothing to release } } @@ -162,7 +156,7 @@ public StructType getSchema() { @Override public ColumnVector getColumnVector(int ordinal) { - if( ordinal < 0 || ordinal >= _ncol ) + if(ordinal < 0 || ordinal >= _ncol) throw new IndexOutOfBoundsException("column ordinal " + ordinal); return new MatrixColumnVector(_mb, _dense, _rowStart, _size, _ncol, ordinal); } @@ -208,16 +202,14 @@ public boolean isNullAt(int rowId) { @Override public double getDouble(int rowId) { - //dense contiguous single block => index fits in int (getDenseBlockValues - //is only handed over for single-block dense matrices) - return (_dense != null) - ? _dense[(_rowStart + rowId) * _ncol + _col] - : _mb.get(_rowStart + rowId, _col); + // dense contiguous single block => index fits in int (getDenseBlockValues + // is only handed over for single-block dense matrices) + return (_dense != null) ? _dense[(_rowStart + rowId) * _ncol + _col] : _mb.get(_rowStart + rowId, _col); } @Override public void close() { - //nothing to release + // nothing to release } } } diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java index 5f478979104..0bf9f7d87ba 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java @@ -977,7 +977,7 @@ public static MatrixBlock rmempty(MatrixBlock in, MatrixBlock ret, boolean rows, if(ret == null) ret = new MatrixBlock(); MatrixBlock ret2 = rmemptyEarlyAbort(in, ret, rows, emptyReturn, select); - if(ret2 != null ) + if(ret2 != null) return ret2; // core removeEmpty return rmemptyUnsafe(in, ret, rows, emptyReturn, select); @@ -985,7 +985,7 @@ public static MatrixBlock rmempty(MatrixBlock in, MatrixBlock ret, boolean rows, public static MatrixBlock rmemptyUnsafe(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, MatrixBlock select) { - if( rows ) + if(rows) return removeEmptyRows(in, ret, select, emptyReturn); else // cols return removeEmptyColumns(in, ret, select, emptyReturn); @@ -999,14 +999,15 @@ public static MatrixBlock rmemptyUnsafe(MatrixBlock in, MatrixBlock ret, boolean * @param rows If removing based on rows, or columns * @param emptyReturn Return a row/column of zeros for empty input * @param select An optional selection vector - * @return The early-abort result, or {@code null} if no early termination applies and the caller must continue. - * For the select-all case the returned block is the input {@code in} itself (a shallow alias, not a copy). + * @return The early-abort result, or {@code null} if no early termination applies and the caller must continue. For + * the select-all case the returned block is the input {@code in} itself (a shallow alias, not a copy). */ - public static MatrixBlock rmemptyEarlyAbort(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, MatrixBlock select){ - //check for empty inputs - //(the semantics of removeEmpty are that for an empty m-by-n matrix, the output - //is an empty 1-by-n or m-by-1 matrix because we don't allow matrices with dims 0) - if( in.isEmptyBlock(false) && select == null ) { + public static MatrixBlock rmemptyEarlyAbort(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, + MatrixBlock select) { + // check for empty inputs + // (the semantics of removeEmpty are that for an empty m-by-n matrix, the output + // is an empty 1-by-n or m-by-1 matrix because we don't allow matrices with dims 0) + if(in.isEmptyBlock(false) && select == null) { int n = emptyReturn ? 1 : 0; if( rows ) ret.reset(n, in.clen, in.sparse); @@ -3651,7 +3652,7 @@ private static MatrixBlock removeEmptyRows(MatrixBlock in, MatrixBlock ret, Matr /** * Remove selected rows, based on the boolean array given. Note this function is internal use only, and require a * boolean vector to be constructed first. - * + * * @param in Input to remove rows from * @param ret Output to assign the result into * @param emptyReturn If the output is allowed to be empty. @@ -3672,9 +3673,9 @@ public static MatrixBlock removeEmptyRows(MatrixBlock in, MatrixBlock ret, boole ret.reset(rlen2, n, sp); if( in.isEmptyBlock(false) ) return ret; - - if( SHALLOW_COPY_REORG && m == rlen2 && selectNull ) { - // the condition m==rlen2 is not enough with non-empty 1-row input but empty + + if(SHALLOW_COPY_REORG && m == rlen2 && selectNull) { + // the condition m==rlen2 is not enough with non-empty 1-row input but empty // 1-row select vector because if emptyReturn should output a single empty row ret.sparse = in.sparse; if( ret.sparse ) @@ -3714,10 +3715,9 @@ else if( !in.sparse && !ret.sparse ) //DENSE <- DENSE ci++; } } - - //check sparsity - ret.nonZeros = (selectNull) ? - in.nonZeros : ret.recomputeNonZeros(); + + // check sparsity + ret.nonZeros = (selectNull) ? in.nonZeros : ret.recomputeNonZeros(); ret.examSparsity(); return ret; diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java index 7525dab2f7f..c450ecbe1f5 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java @@ -4756,13 +4756,13 @@ public static double computeIQMCorrection(double sum, double sum_wt, } /** - * Pick the quantiles out of this matrix. If this matrix contains two columns it is weighted quantile picking. - * If a single column it is unweighted. - * + * Pick the quantiles out of this matrix. If this matrix contains two columns it is weighted quantile picking. If a + * single column it is unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantiles The quantiles to pick - * @param ret The result matrix + * @param ret The result matrix * @return The result matrix */ public final MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret) { @@ -4792,11 +4792,11 @@ public MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret, boolean av } /** - * Pick the median value from this matrix. If this matrix has two columns it is weighted picking using the - * weight column, otherwise it is unweighted over the single column. - * + * Pick the median value from this matrix. If this matrix has two columns it is weighted picking using the weight + * column, otherwise it is unweighted over the single column. + * * Note the values are assumed to be sorted. - * + * * @return The median value */ public double median() { @@ -4807,10 +4807,11 @@ public double median() { } /** - * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is unweighted. - * + * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is + * unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantile The quantile to pick * @return The quantile */ @@ -4819,12 +4820,13 @@ public final double pickValue(double quantile){ } /** - * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is unweighted. - * + * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is + * unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantile The quantile to pick - * @param average If the quantile is averaged. + * @param average If the quantile is averaged. * @return The quantile */ public final double pickValue(double quantile, boolean average) { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java index 9e3494a7d48..9ea39bd1e2f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java @@ -34,8 +34,8 @@ import java.util.function.Function; /** - * Small future implementation for OOC hot paths. It supports multiple synchronous subscribers without - * the completion-stage support of {@link java.util.concurrent.CompletableFuture}. + * Small future implementation for OOC hot paths. It supports multiple synchronous subscribers without the + * completion-stage support of {@link java.util.concurrent.CompletableFuture}. */ public class OOCFuture { private Subscriber _subscribers; @@ -240,7 +240,7 @@ private static void accept(Function mapper, Consu if(resultError == null) { try { @SuppressWarnings("unchecked") - R mapped = mapper == null ? (R)value : mapper.apply(value); + R mapped = mapper == null ? (R) value : mapper.apply(value); result = mapped; } catch(Throwable t) { @@ -345,8 +345,7 @@ public T get() throws InterruptedException, ExecutionException { } @Override - public T get(long timeout, TimeUnit unit) - throws InterruptedException, ExecutionException, TimeoutException { + public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { try { return mapper.apply(source.get(timeout, unit)); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java index 94411242df1..df981f40223 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java @@ -25,20 +25,22 @@ public class CloseableQueue { private final BlockingQueue queue = new LinkedBlockingQueue<>(); - private final Object POISON = new Object(); // sentinel + private final Object POISON = new Object(); // sentinel private volatile boolean closed = false; - public CloseableQueue() { } + public CloseableQueue() { + } /** * Enqueue if the queue is not closed. + * * @return false if already closed */ public boolean enqueueIfOpen(T task) throws InterruptedException { - if (task == null) + if(task == null) throw new IllegalArgumentException("null tasks not allowed"); - synchronized (this) { - if (closed) + synchronized(this) { + if(closed) return false; queue.put(task); } @@ -47,12 +49,12 @@ public boolean enqueueIfOpen(T task) throws InterruptedException { @SuppressWarnings("unchecked") public T take() throws InterruptedException { - if (closed && queue.isEmpty()) + if(closed && queue.isEmpty()) return null; Object x = queue.take(); - if (x == POISON) + if(x == POISON) return null; return (T) x; @@ -60,33 +62,31 @@ public T take() throws InterruptedException { /** * Poll with max timeout. - * @return item, or null if: - * - timeout, or - * - queue has been closed and this consumer reached its poison pill + * + * @return item, or null if: - timeout, or - queue has been closed and this consumer reached its poison pill */ @SuppressWarnings("unchecked") public T poll(long timeout, TimeUnit unit) throws InterruptedException { - if (closed && queue.isEmpty()) + if(closed && queue.isEmpty()) return null; Object x = queue.poll(timeout, unit); - if (x == null) - return null; // timeout + if(x == null) + return null; // timeout - if (x == POISON) + if(x == POISON) return null; return (T) x; } /** - * Close queue for N consumers. - * Each consumer will receive exactly one poison pill and then should stop. + * Close queue for N consumers. Each consumer will receive exactly one poison pill and then should stop. */ public boolean close() throws InterruptedException { - synchronized (this) { - if (closed) - return false; // idempotent + synchronized(this) { + if(closed) + return false; // idempotent closed = true; } queue.put(POISON); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java index ee02b28404c..ccc73ff6160 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java @@ -78,7 +78,7 @@ public void readFully(byte[] b) throws IOException { } @Override - public void readFully(byte [] b, int off, int len) throws IOException { + public void readFully(byte[] b, int off, int len) throws IOException { if(len < 0) throw new IndexOutOfBoundsException(); @@ -127,12 +127,12 @@ public int readUnsignedByte() throws IOException { @Override public short readShort() throws IOException { if(_count - _pos >= 2) { - short ret = (short)baToShort(_buff, _pos); + short ret = (short) baToShort(_buff, _pos); _pos += 2; return ret; } readFully(_tmp, 0, 2); - return (short)baToShort(_tmp, 0); + return (short) baToShort(_tmp, 0); } @Override @@ -142,7 +142,7 @@ public int readUnsignedShort() throws IOException { @Override public char readChar() throws IOException { - return (char)readUnsignedShort(); + return (char) readUnsignedShort(); } @Override @@ -222,7 +222,7 @@ public long readDoubleArray(int len, double[] varr) throws IOException { @Override public long readSparseRows(int rlen, long nnz, SparseBlock rows) throws IOException { if(rows instanceof SparseBlockCSR) { - ((SparseBlockCSR)rows).initSparse(rlen, (int)nnz, this); + ((SparseBlockCSR) rows).initSparse(rlen, (int) nnz, this); return nnz; } @@ -256,7 +256,7 @@ private void refill() throws IOException { } private int getRefillLength() { - int pageOffset = (int)(_filePos & PAGE_MASK); + int pageOffset = (int) (_filePos & PAGE_MASK); if(pageOffset == 0) return _bufflen; return Math.min(_bufflen, PAGE_SIZE - pageOffset); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java index 9854a94586b..22b7b23706c 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java @@ -65,7 +65,7 @@ long getFlushedPosition() { public void write(int b) throws IOException { if(_count >= _bufflen) flushBuffer(); - _buff[_count++] = (byte)b; + _buff[_count++] = (byte) b; _position++; } @@ -109,7 +109,7 @@ public void close() throws IOException { public void writeBoolean(boolean v) throws IOException { if(_count >= _bufflen) flushBuffer(); - _buff[_count++] = (byte)(v ? 1 : 0); + _buff[_count++] = (byte) (v ? 1 : 0); _position++; } @@ -153,7 +153,7 @@ public void writeFloat(float v) throws IOException { public void writeByte(int v) throws IOException { if(_count + 1 > _bufflen) flushBuffer(); - _buff[_count++] = (byte)v; + _buff[_count++] = (byte) v; _position++; } @@ -194,18 +194,18 @@ public void writeUTF(String s) throws IOException { flushBuffer(); final char c = s.charAt(i); if(c >= 0x0001 && c <= 0x007F) { - _buff[_count++] = (byte)c; + _buff[_count++] = (byte) c; _position++; } else if(c >= 0x0800) { - _buff[_count++] = (byte)(0xE0 | ((c >> 12) & 0x0F)); - _buff[_count++] = (byte)(0x80 | ((c >> 6) & 0x3F)); - _buff[_count++] = (byte)(0x80 | (c & 0x3F)); + _buff[_count++] = (byte) (0xE0 | ((c >> 12) & 0x0F)); + _buff[_count++] = (byte) (0x80 | ((c >> 6) & 0x3F)); + _buff[_count++] = (byte) (0x80 | (c & 0x3F)); _position += 3; } else { - _buff[_count++] = (byte)(0xC0 | ((c >> 6) & 0x1F)); - _buff[_count++] = (byte)(0x80 | (c & 0x3F)); + _buff[_count++] = (byte) (0xC0 | ((c >> 6) & 0x1F)); + _buff[_count++] = (byte) (0x80 | (c & 0x3F)); _position += 2; } } @@ -213,7 +213,7 @@ else if(c >= 0x0800) { @Override public void writeDoubleArray(int len, double[] varr) throws IOException { - for(int i = 0; i < len; ) { + for(int i = 0; i < len;) { if(_count >= _bufflen) flushBuffer(); int lblen = Math.min(len - i, (_bufflen - _count) / 8); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java index ab28df0ef0f..6c13a062561 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java @@ -50,10 +50,10 @@ public interface OOCIOHandler { void registerSourceLocation(BlockKey key, SourceBlockDescriptor descriptor); /** - * Schedule an asynchronous read from an external source into the provided target stream. - * The returned future completes when either EOF is reached or the requested byte budget - * is exhausted. When the budget is reached and keepOpenOnLimit is true, the target stream - * is kept open and a continuation token is provided so the caller can resume. + * Schedule an asynchronous read from an external source into the provided target stream. The returned future + * completes when either EOF is reached or the requested byte budget is exhausted. When the budget is reached and + * keepOpenOnLimit is true, the target stream is kept open and a continuation token is provided so the caller can + * resume. */ CompletableFuture scheduleSourceRead(SourceReadRequest request); @@ -62,7 +62,8 @@ public interface OOCIOHandler { */ CompletableFuture continueSourceRead(SourceReadContinuation continuation, long maxBytesInFlight); - interface SourceReadContinuation {} + interface SourceReadContinuation { + } class SourceReadRequest { public final String path; @@ -75,9 +76,8 @@ class SourceReadRequest { public final boolean keepOpenOnLimit; public final OOCStream target; - public SourceReadRequest(String path, Types.FileFormat format, long rows, long cols, - int blen, long estNnz, long maxBytesInFlight, boolean keepOpenOnLimit, - OOCStream target) { + public SourceReadRequest(String path, Types.FileFormat format, long rows, long cols, int blen, long estNnz, + long maxBytesInFlight, boolean keepOpenOnLimit, OOCStream target) { this.path = path; this.format = format; this.rows = rows; @@ -113,9 +113,8 @@ class SourceBlockDescriptor { public final int recordLength; public final long serializedSize; - public SourceBlockDescriptor(String path, Types.FileFormat format, - MatrixIndexes indexes, long offset, int recordLength, - long serializedSize) { + public SourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, + int recordLength, long serializedSize) { this.path = path; this.format = format; this.indexes = indexes; @@ -129,8 +128,8 @@ class GroupSourceBlockDescriptor extends SourceBlockDescriptor { public final List blocks; public final int count; - public GroupSourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, int recordLength, - long serializedSize, List blocks) { + public GroupSourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, + int recordLength, long serializedSize, List blocks) { super(path, format, indexes, offset, recordLength, serializedSize); this.blocks = blocks; this.count = blocks.size(); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java index e9487f7bd45..55ac038205f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java @@ -81,7 +81,7 @@ public class OOCMatrixIOHandler implements OOCIOHandler { private final AtomicLong _readSeq = new AtomicLong(0); // Spill related structures - private final ConcurrentHashMap _spillLocations = new ConcurrentHashMap<>(); + private final ConcurrentHashMap _spillLocations = new ConcurrentHashMap<>(); private final ConcurrentHashMap _partitions = new ConcurrentHashMap<>(); private final ConcurrentHashMap _sourceLocations = new ConcurrentHashMap<>(); private final AtomicInteger _partitionCounter = new AtomicInteger(0); @@ -97,38 +97,21 @@ public class OOCMatrixIOHandler implements OOCIOHandler { @SuppressWarnings("unchecked") public OOCMatrixIOHandler() { this._spillDir = LocalFileUtils.getUniqueWorkingDir("ooc_stream"); - _writeExec = new ThreadPoolExecutor( - WRITER_SIZE, - WRITER_SIZE, - 0L, - TimeUnit.MILLISECONDS, + _writeExec = new ThreadPoolExecutor(WRITER_SIZE, WRITER_SIZE, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); - _readExec = new ThreadPoolExecutor( - READER_SIZE, - READER_SIZE, - 0L, - TimeUnit.MILLISECONDS, + _readExec = new ThreadPoolExecutor(READER_SIZE, READER_SIZE, 0L, TimeUnit.MILLISECONDS, new PriorityBlockingQueue<>()); - _srcReadExec = new ThreadPoolExecutor( - READER_SIZE, - READER_SIZE, - 0L, - TimeUnit.MILLISECONDS, - new ArrayBlockingQueue<>(100000)); - _deleteExec = new ThreadPoolExecutor( - 1, - 1, - 0L, - TimeUnit.MILLISECONDS, + _srcReadExec = new ThreadPoolExecutor(READER_SIZE, READER_SIZE, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); + _deleteExec = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); _q = new CloseableQueue[WRITER_SIZE]; _wCtr = new AtomicLong(0); - _started = new AtomicBoolean(false); + _started = new AtomicBoolean(false); } private synchronized void start() { - if (_started.compareAndSet(false, true)) { - for (int i = 0; i < WRITER_SIZE; i++) { + if(_started.compareAndSet(false, true)) { + for(int i = 0; i < WRITER_SIZE; i++) { final int finalIdx = i; _q[i] = new CloseableQueue<>(); _writeExec.submit(() -> evictTask(_q[finalIdx])); @@ -139,7 +122,7 @@ private synchronized void start() { @Override public void shutdown() { boolean started = _started.get(); - if (started) { + if(started) { try { for(int i = 0; i < WRITER_SIZE; i++) { if(_q[i] != null) @@ -159,7 +142,7 @@ public void shutdown() { _deleteExec.shutdownNow(); _spillLocations.clear(); _partitions.clear(); - if (started) + if(started) LocalFileUtils.deleteFileIfExists(_spillDir); } @@ -169,7 +152,7 @@ public CompletableFuture scheduleEviction(BlockEntry block) { CompletableFuture future = new CompletableFuture<>(); try { long q = _wCtr.getAndAdd(block.getSize()) / OVERFLOW; - int i = (int)(q % WRITER_SIZE); + int i = (int) (q % WRITER_SIZE); if(!_q[i].enqueueIfOpen(new Tuple2<>(block, future))) future.completeExceptionally(new DMLRuntimeException("OOC writer queue is closed")); } @@ -189,7 +172,8 @@ public OOCFuture scheduleRead(final BlockEntry block) { ReadTask task = new ReadTask(block, future, _readSeq.getAndIncrement(), pinnedPartitionId); _pendingReads.put(block.getKey(), task); _readExec.execute(task); - } catch (RejectedExecutionException e) { + } + catch(RejectedExecutionException e) { unpinPartitionForRead(pinnedPartitionId); _pendingReads.remove(block.getKey()); future.completeExceptionally(e); @@ -199,12 +183,12 @@ public OOCFuture scheduleRead(final BlockEntry block) { @Override public void prioritizeRead(BlockKey key, double priority) { - if (priority == 0) + if(priority == 0) return; ReadTask task = _pendingReads.get(key); - if (task == null) + if(task == null) return; - if (_readExec.getQueue().remove(task)) { + if(_readExec.getQueue().remove(task)) { task.addPriority(priority); _readExec.getQueue().offer(task); } @@ -228,8 +212,9 @@ public CompletableFuture scheduleSourceRead(SourceReadRequest } @Override - public CompletableFuture continueSourceRead(SourceReadContinuation continuation, long maxBytesInFlight) { - if (!(continuation instanceof SourceReadState state)) { + public CompletableFuture continueSourceRead(SourceReadContinuation continuation, + long maxBytesInFlight) { + if(!(continuation instanceof SourceReadState state)) { CompletableFuture failed = new CompletableFuture<>(); failed.completeExceptionally(new DMLRuntimeException("Unsupported continuation type: " + continuation)); return failed; @@ -240,8 +225,8 @@ public CompletableFuture continueSourceRead(SourceReadContinua private CompletableFuture submitSourceRead(SourceReadRequest request, SourceReadState state, long maxBytesInFlight) { if(request.format != Types.FileFormat.BINARY) - return CompletableFuture.failedFuture( - new DMLRuntimeException("Unsupported format for source read: " + request.format)); + return CompletableFuture + .failedFuture(new DMLRuntimeException("Unsupported format for source read: " + request.format)); return readBinarySourceParallel(request, state, maxBytesInFlight); } @@ -335,17 +320,18 @@ private CompletableFuture readBinarySourceParallel(SourceReadR return result; } - private void completeResult(CompletableFuture future, AtomicLong bytesRead, AtomicBoolean budgetHit, - AtomicReference error, SourceReadRequest request, Path[] files, AtomicLongArray filePositions, - AtomicIntegerArray completed, ConcurrentLinkedDeque descriptors) { + private void completeResult(CompletableFuture future, AtomicLong bytesRead, + AtomicBoolean budgetHit, AtomicReference error, SourceReadRequest request, Path[] files, + AtomicLongArray filePositions, AtomicIntegerArray completed, + ConcurrentLinkedDeque descriptors) { Throwable err = error.get(); - if (err != null) { + if(err != null) { future.completeExceptionally(err instanceof Exception ? err : new Exception(err)); return; } try { - if (budgetHit.get()) { + if(budgetHit.get()) { if(!request.keepOpenOnLimit) { closeTarget(request.target, false); } @@ -365,29 +351,29 @@ private void completeResult(CompletableFuture future, AtomicLo private void readSequenceFile(JobConf job, Path path, SourceReadRequest request, int fileIdx, AtomicLongArray filePositions, AtomicIntegerArray completed, AtomicBoolean stop, AtomicBoolean budgetHit, - AtomicLong bytesRead, long byteLimit, Object budgetLock, ConcurrentLinkedDeque descriptors) - throws IOException { + AtomicLong bytesRead, long byteLimit, Object budgetLock, + ConcurrentLinkedDeque descriptors) throws IOException { MatrixIndexes key = new MatrixIndexes(); try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { long pos = filePositions.get(fileIdx); - if (pos > 0) + if(pos > 0) reader.seek(pos); long ioStart = DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; while(!stop.get()) { long recordStart = reader.getPosition(); MatrixBlock value = new MatrixBlock(); - if (!reader.next(key, value)) + if(!reader.next(key, value)) break; long recordEnd = reader.getPosition(); long blockSize = value.getExactSerializedSize(); boolean shouldBreak = false; synchronized(budgetLock) { - if (stop.get()) + if(stop.get()) shouldBreak = true; - else if (bytesRead.get() + blockSize > byteLimit) { + else if(bytesRead.get() + blockSize > byteLimit) { stop.set(true); budgetHit.set(true); shouldBreak = true; @@ -398,7 +384,7 @@ else if (bytesRead.get() + blockSize > byteLimit) { MatrixIndexes outIdx = new MatrixIndexes(key); IndexedMatrixValue imv = new IndexedMatrixValue(outIdx, value); SourceBlockDescriptor descriptor = new SourceBlockDescriptor(path.toString(), request.format, outIdx, - recordStart, (int)(recordEnd - recordStart), blockSize); + recordStart, (int) (recordEnd - recordStart), blockSize); if(request.target instanceof SourceOOCStream src) src.enqueue(imv, descriptor); @@ -407,22 +393,23 @@ else if (bytesRead.get() + blockSize > byteLimit) { descriptors.add(descriptor); filePositions.set(fileIdx, reader.getPosition()); - if (DMLScript.OOC_LOG_EVENTS) { + if(DMLScript.OOC_LOG_EVENTS) { long currTime = System.nanoTime(); OOCEventLog.onDiskReadEvent(_srcReadCallerId, ioStart, currTime, blockSize); ioStart = currTime; } - if (shouldBreak) + if(shouldBreak) break; // Note that we knowingly go over limit, which could result in READER_SIZE*8MB overshoot } - if (!stop.get()) + if(!stop.get()) completed.set(fileIdx, 1); } } - private void closeTarget(org.apache.sysds.runtime.instructions.ooc.OOCStream target, boolean close) { + private void closeTarget(org.apache.sysds.runtime.instructions.ooc.OOCStream target, + boolean close) { if(close) { try { target.closeInput(); @@ -437,10 +424,10 @@ private void loadFromDisk(BlockEntry block) { String key = block.getKey().toFileKey(); SourceBlockDescriptor src = _sourceLocations.get(block.getKey()); - if (src != null) { + if(src != null) { long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; loadFromSource(block, src); - if (DMLScript.OOC_STATISTICS) { + if(DMLScript.OOC_STATISTICS) { Statistics.incrementOOCLoadFromDisk(); Statistics.accumulateOOCLoadFromDiskTime(System.nanoTime() - ioStart); Statistics.accumulateOOCLoadFromDiskBytes(block.getSize()); @@ -451,34 +438,36 @@ private void loadFromDisk(BlockEntry block) { long ioDuration = 0; // 1. find the blocks address (spill location) SpillLocation sloc = _spillLocations.get(key); - if (sloc == null) + if(sloc == null) throw new DMLRuntimeException("Failed to load spill location for: " + key); PartitionFile partFile = _partitions.get(sloc.partitionId); - if (partFile == null) + if(partFile == null) throw new DMLRuntimeException("Failed to load partition for: " + sloc.partitionId); String filename = partFile.filePath; SpillableObject obj; - try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { + try(RandomAccessFile raf = new RandomAccessFile(filename, "r")) { raf.seek(sloc.offset); DataInput dis = new OOCBufferedDataInputStream(raf); long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; obj = SpillableObjectRegistry.read(dis); - if (DMLScript.OOC_STATISTICS) + if(DMLScript.OOC_STATISTICS) ioDuration = System.nanoTime() - ioStart; - } catch (ClosedByInterruptException ignored) { + } + catch(ClosedByInterruptException ignored) { return; - } catch (IOException e) { + } + catch(IOException e) { throw new RuntimeException(e); } block.setDataUnsafe(obj); - if (DMLScript.OOC_STATISTICS) { + if(DMLScript.OOC_STATISTICS) { Statistics.incrementOOCLoadFromDisk(); Statistics.accumulateOOCLoadFromDiskTime(ioDuration); Statistics.accumulateOOCLoadFromDiskBytes(block.getSize()); @@ -486,22 +475,23 @@ private void loadFromDisk(BlockEntry block) { } private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { - if (src.format != Types.FileFormat.BINARY) + if(src.format != Types.FileFormat.BINARY) throw new DMLRuntimeException("Unsupported format for source read: " + src.format); JobConf job = new JobConf(ConfigurationManager.getCachedJobConf()); Path path = new Path(src.path); - if (src instanceof OOCIOHandler.GroupSourceBlockDescriptor gsrc) { + if(src instanceof OOCIOHandler.GroupSourceBlockDescriptor gsrc) { List values = new ArrayList<>(gsrc.count); try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { reader.seek(gsrc.offset); - for (int i = 0; i < gsrc.blocks.size(); i++) { + for(int i = 0; i < gsrc.blocks.size(); i++) { SourceBlockDescriptor d = gsrc.blocks.get(i); MatrixIndexes ix = new MatrixIndexes(); MatrixBlock mb = new MatrixBlock(); - if (!reader.next(ix, mb)) - throw new DMLRuntimeException("Failed to read source block at offset " + d.offset + " in " + d.path); + if(!reader.next(ix, mb)) + throw new DMLRuntimeException( + "Failed to read source block at offset " + d.offset + " in " + d.path); values.add(new IndexedMatrixValue(ix, mb)); } } @@ -516,8 +506,9 @@ private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { reader.seek(src.offset); - if (!reader.next(ix, mb)) - throw new DMLRuntimeException("Failed to read source block at offset " + src.offset + " in " + src.path); + if(!reader.next(ix, mb)) + throw new DMLRuntimeException( + "Failed to read source block at offset " + src.offset + " in " + src.path); } catch(IOException e) { throw new DMLRuntimeException(e); @@ -530,7 +521,7 @@ private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { private void evictTask(CloseableQueue>> q) { long byteCtr = 0; - while (!q.isFinished()) { + while(!q.isFinished()) { // --- 1. WRITE PHASE --- int partitionId = _partitionCounter.getAndIncrement(); @@ -572,17 +563,17 @@ private void evictTask(CloseableQueue } byteCtr += wrote; - if (byteCtr >= MAX_PARTITION_SIZE) { + if(byteCtr >= MAX_PARTITION_SIZE) { closePartition = true; byteCtr = 0; break; } - if (DMLScript.OOC_LOG_EVENTS) + if(DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskWriteEvent(_evictCallerId, ioStart, System.nanoTime(), wrote); } - if (!closePartition && q.close()) { + if(!closePartition && q.close()) { while((tpl = q.take()) != null) { long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; BlockEntry entry = tpl._1(); @@ -595,7 +586,7 @@ private void evictTask(CloseableQueue Statistics.accumulateOOCEvictionWriteTime(System.nanoTime() - ioStart); } - if (DMLScript.OOC_LOG_EVENTS) + if(DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskWriteEvent(_evictCallerId, ioStart, System.nanoTime(), wrote); } } @@ -617,13 +608,13 @@ private void evictTask(CloseableQueue } private long writeOut(int partitionId, BlockEntry entry, CompletableFuture future, - OOCBufferedDataOutputStream dos, - ConcurrentLinkedDeque>> flushQueue) throws IOException { + OOCBufferedDataOutputStream dos, ConcurrentLinkedDeque>> flushQueue) + throws IOException { String key = entry.getKey().toFileKey(); boolean alreadySpilled = _spillLocations.containsKey(key); - if (!alreadySpilled) { + if(!alreadySpilled) { long offsetBefore = dos.getPosition(); if(future.isCancelled()) @@ -656,9 +647,10 @@ private long writeOut(int partitionId, BlockEntry entry, CompletableFuture return 0; } - private void flushQueue(long offset, ConcurrentLinkedDeque>> flushQueue) { + private void flushQueue(long offset, + ConcurrentLinkedDeque>> flushQueue) { Tuple3> tmp; - while ((tmp = flushQueue.peek()) != null && tmp._2() <= offset) { + while((tmp = flushQueue.peek()) != null && tmp._2() <= offset) { flushQueue.poll(); tmp._3().complete(null); } @@ -777,12 +769,14 @@ public void run() { try { long ioStart = DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; loadFromDisk(_block); - if (DMLScript.OOC_LOG_EVENTS) + if(DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskReadEvent(_readCallerId, ioStart, System.nanoTime(), _block.getSize()); _future.complete(_block); - } catch (Throwable e) { + } + catch(Throwable e) { _future.completeExceptionally(e); - } finally { + } + finally { unpinPartitionForRead(_pinnedPartitionId); } } @@ -790,15 +784,12 @@ public void run() { @Override public int compareTo(ReadTask other) { int byPriority = Double.compare(other._priority, _priority); - if (byPriority != 0) + if(byPriority != 0) return byPriority; return Long.compare(_sequence, other._sequence); } } - - - private static class SpillLocation { // structure of spillLocation: file, offset final int partitionId; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java index a93b1d18f2a..3458838c071 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java @@ -24,8 +24,10 @@ import java.io.IOException; public interface SpillableObject { - boolean tryWrite(DataOutput out) throws IOException; - void read(DataInput in) throws IOException; + boolean tryWrite(DataOutput out) throws IOException; + + void read(DataInput in) throws IOException; + long size(); default void discard() { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java index d8ff79dd797..0d318d83d94 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java @@ -34,14 +34,16 @@ public interface OOCCacheScheduler { /** * Requests a single block from the cache. + * * @param key the requested key associated to the block * @return the available BlockEntry */ CompletableFuture request(BlockKey key); /** - * Tries to request a single block from the cache. - * Immediately returns the entry if present, otherwise null without scheduling reads. + * Tries to request a single block from the cache. Immediately returns the entry if present, otherwise null without + * scheduling reads. + * * @param key the requested key associated to the block * @return the available BlockEntry or null */ @@ -52,14 +54,16 @@ default BlockEntry tryRequest(BlockKey key) { /** * Requests a list of blocks from the cache that must be available at the same time. + * * @param keys the requested keys associated to the block * @return the list of available BlockEntries */ CompletableFuture> request(List keys); /** - * Tries to request a list of blocks from the cache that must be available at the same time. - * Immediately returns the list of entries if present, otherwise null without scheduling reads. + * Tries to request a list of blocks from the cache that must be available at the same time. Immediately returns the + * list of entries if present, otherwise null without scheduling reads. + * * @param keys the requested keys associated to the block * @return the list of available BlockEntries */ @@ -76,24 +80,25 @@ default BlockEntry tryRequest(BlockKey key) { List tryRequestAnyOf(List keys, int n, List selectionOut); /** - * Adds the given priority to any pending request accessing the key. - * Multi-requests are prioritized partially. + * Adds the given priority to any pending request accessing the key. Multi-requests are prioritized partially. */ void prioritize(BlockKey key, double priority); /** - * Places a new block in the cache. Note that objects are immutable and cannot be overwritten. - * The object data should now only be accessed via cache, as ownership has been transferred. - * @param key the associated key of the block + * Places a new block in the cache. Note that objects are immutable and cannot be overwritten. The object data + * should now only be accessed via cache, as ownership has been transferred. + * + * @param key the associated key of the block * @param data the block data * @param size the size of the data */ BlockKey put(BlockKey key, Object data, long size); /** - * Places a new block in the cache and returns a pinned handle. - * Note that objects are immutable and cannot be overwritten. - * @param key the associated key of the block + * Places a new block in the cache and returns a pinned handle. Note that objects are immutable and cannot be + * overwritten. + * + * @param key the associated key of the block * @param data the block data * @param size the size of the data */ @@ -101,8 +106,11 @@ default BlockEntry tryRequest(BlockKey key) { interface HandoverHandle { BlockKey getKey(); + boolean isCommitted(); + CompletableFuture getCompletionFuture(); + OOCStream.QueueCallback reclaim(); } @@ -131,27 +139,30 @@ BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.SourceBlockDescriptor descriptor); /** - * Notifies the cache that there is another reference to the same block key. - * This will prevent forget(key) from removing the block from cache. - * A block will only be forgotten after all referencing instances called forget(key). + * Notifies the cache that there is another reference to the same block key. This will prevent forget(key) from + * removing the block from cache. A block will only be forgotten after all referencing instances called forget(key). + * * @param key */ void addReference(BlockKey key); /** * Forgets a block from the cache. + * * @param key the associated key of the block */ void forget(BlockKey key); /** * Pins a BlockEntry in cache to prevent eviction. + * * @param entry the entry to be pinned */ void pin(BlockEntry entry); /** * Unpins a pinned block. + * * @param entry the entry to be unpinned */ void unpin(BlockEntry entry); @@ -192,8 +203,7 @@ BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, void updateLimits(long evictionLimit, long hardLimit); /** - * Creates a snapshot of the cache. - * Should only be used for debugging or diagnoses. + * Creates a snapshot of the cache. Should only be used for debugging or diagnoses. */ Collection snapshot(); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java index 96de368ccc9..9b0acc97b35 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java @@ -80,7 +80,7 @@ public class OOCLRUCacheScheduler implements OOCCacheScheduler { public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long hardLimit, long readBuffer) { this._ioHandler = ioHandler; this._cache = new LinkedHashMap<>(1024, 0.75f, true); - this._evictionCache = new HashMap<>(); + this._evictionCache = new HashMap<>(); this._deferredReadRequests = new DeferredReadQueue(); this._processingReadRequests = new ArrayDeque<>(); this._pendingHandovers = new ArrayDeque<>(); @@ -103,7 +103,7 @@ public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long har this._maintenanceNeedsIncr = new AtomicBoolean(false); this._callerId = DMLScript.OOC_LOG_EVENTS ? OOCEventLog.registerCaller("LRUCacheScheduler") : 0; - if (DMLScript.OOC_LOG_EVENTS) { + if(DMLScript.OOC_LOG_EVENTS) { OOCEventLog.putRunSetting("CacheEvictionLimit", _evictionLimit); OOCEventLog.putRunSetting("CacheHardLimit", _hardLimit); } @@ -111,7 +111,7 @@ public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long har @Override public CompletableFuture request(BlockKey key) { - if (!this._running) + if(!this._running) throw new IllegalStateException("Cache scheduler has been shut down."); Statistics.incrementOOCEvictionGet(); @@ -120,21 +120,21 @@ public CompletableFuture request(BlockKey key) { boolean couldPin = false; synchronized(this) { entry = _cache.get(key); - if (entry == null) + if(entry == null) entry = _evictionCache.get(key); - if (entry == null) + if(entry == null) throw new IllegalArgumentException("Could not find requested block with key " + key); synchronized(entry) { - if (entry.getState().isAvailable()) { - if (pinEntryWithAccounting(entry) == 0) + if(entry.getState().isAvailable()) { + if(pinEntryWithAccounting(entry) == 0) throw new IllegalStateException(); couldPin = true; } } } - if (couldPin) { + if(couldPin) { // Then we could pin the required entry and can terminate return CompletableFuture.completedFuture(entry); } @@ -184,7 +184,7 @@ public CompletableFuture> request(List keys) { } public CompletableFuture> request(List keys, boolean onlyIfAvailable) { - if (!this._running) + if(!this._running) throw new IllegalStateException("Cache scheduler has been shut down."); Statistics.incrementOOCEvictionGet(keys.size()); @@ -193,11 +193,11 @@ public CompletableFuture> request(List keys, boolean boolean allAvailable = true; synchronized(this) { - for (BlockKey key : keys) { + for(BlockKey key : keys) { BlockEntry entry = _cache.get(key); - if (entry == null) + if(entry == null) entry = _evictionCache.get(key); - if (entry == null) + if(entry == null) throw new IllegalArgumentException("Could not find requested block with key " + key); synchronized(entry) { @@ -217,7 +217,7 @@ public CompletableFuture> request(List keys, boolean } } - if (allAvailable) { + if(allAvailable) { // Then we could pin all entries return CompletableFuture.completedFuture(entries); } @@ -226,12 +226,12 @@ public CompletableFuture> request(List keys, boolean return null; // Schedule deferred read otherwise - final CompletableFuture> future = new CompletableFuture<>(); + final CompletableFuture> future = new CompletableFuture<>(); DeferredReadRequest request = new DeferredReadRequest(future, entries); - for (int i = 0; i < entries.size(); i++) { + for(int i = 0; i < entries.size(); i++) { BlockEntry entry = entries.get(i); synchronized(entry) { - if (entry.getState().isAvailable()) { + if(entry.getState().isAvailable()) { entry.addRetainHint(); request.markRetainHinted(i); } @@ -243,9 +243,9 @@ public CompletableFuture> request(List keys, boolean @Override public void prioritize(BlockKey key, double priority) { - if (!this._running) + if(!this._running) return; - if (priority == 0) + if(priority == 0) return; synchronized(this) { @@ -267,18 +267,18 @@ private void scheduleDeferredRead(DeferredReadRequest deferredReadRequest) { if(entry.getState().isAvailable()) readyCount++; BlockReadState state = _blockReads.get(entry.getKey()); - if (state != null) + if(state != null) score += state.priority; } - if (!deferredReadRequest.getEntries().isEmpty()) + if(!deferredReadRequest.getEntries().isEmpty()) score /= deferredReadRequest.getEntries().size(); - if (!deferredReadRequest.getEntries().isEmpty()) + if(!deferredReadRequest.getEntries().isEmpty()) score += ((double) readyCount) / deferredReadRequest.getEntries().size(); deferredReadRequest.setPriorityScore(score); _deferredReadRequests.add(deferredReadRequest); _deferredReadCountHint = _deferredReadRequests.size(); } - onCacheSizeChanged(true); // Apply pressure from deferred read demand. + onCacheSizeChanged(true); // Apply pressure from deferred read demand. onCacheSizeChanged(false); // Attempt to schedule deferred reads. } @@ -317,7 +317,8 @@ public void putSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.S } @Override - public BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.SourceBlockDescriptor descriptor) { + public BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, + OOCIOHandler.SourceBlockDescriptor descriptor) { return put(key, data, size, true, descriptor); } @@ -333,23 +334,24 @@ public void addReference(BlockKey key) { } } - private BlockEntry put(BlockKey key, Object data, long size, boolean pin, OOCIOHandler.SourceBlockDescriptor descriptor) { - if (!this._running) + private BlockEntry put(BlockKey key, Object data, long size, boolean pin, + OOCIOHandler.SourceBlockDescriptor descriptor) { + if(!this._running) throw new IllegalStateException(); - if (data == null) + if(data == null) throw new IllegalArgumentException(); - if (descriptor != null) + if(descriptor != null) _ioHandler.registerSourceLocation(key, descriptor); Statistics.incrementOOCEvictionPut(); BlockEntry entry = new BlockEntry(key, size, data); - if (descriptor != null) + if(descriptor != null) entry.setState(BlockState.WARM); - if (pin) + if(pin) entry.pin(); synchronized(this) { BlockEntry avail = _cache.putIfAbsent(key, entry); - if (avail != null || _evictionCache.containsKey(key)) + if(avail != null || _evictionCache.containsKey(key)) throw new IllegalStateException("Cannot overwrite existing entries: " + key); _cacheSize += size; if(pin) { @@ -364,7 +366,7 @@ private BlockEntry put(BlockKey key, Object data, long size, boolean pin, OOCIOH @Override public void forget(BlockKey key) { - if (!this._running) + if(!this._running) return; final MutableObject mEntry = new MutableObject<>(); BlockEntry entry; @@ -381,7 +383,7 @@ public void forget(BlockKey key) { return e; }); - if (mEntry.getValue() == null) { + if(mEntry.getValue() == null) { _evictionCache.compute(key, (k, e) -> { if(e == null) return null; @@ -395,10 +397,10 @@ public void forget(BlockKey key) { entry = mEntry.getValue(); - if (entry != null) { + if(entry != null) { synchronized(entry) { - shouldScheduleDeletion = entry.getState().isBackedByDisk() - || entry.getState() == BlockState.EVICTING; + shouldScheduleDeletion = entry.getState().isBackedByDisk() || + entry.getState() == BlockState.EVICTING; cacheSizeDelta = transitionMemState(entry, BlockState.REMOVED); if(entry.isPinned() && entry.getDataUnsafe() != null) _pinnedBytes -= entry.getSize(); @@ -408,9 +410,9 @@ public void forget(BlockKey key) { } } } - if (cacheSizeDelta != 0) + if(cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); - if (shouldScheduleDeletion) + if(shouldScheduleDeletion) _ioHandler.scheduleDeletion(entry); } @@ -424,11 +426,11 @@ public void pin(BlockEntry entry) { synchronized(this) { synchronized(entry) { int pinCount = pinEntryWithAccounting(entry); - if (pinCount == 0) + if(pinCount == 0) throw new IllegalStateException("Could not pin the requested entry: " + entry.getKey()); } // Access element in cache for Lru - //_cache.get(entry.getKey()); + // _cache.get(entry.getKey()); } } @@ -442,26 +444,26 @@ public void unpin(BlockEntry entry) { synchronized(entry) { if(!unpinEntryWithAccounting(entry)) return; - if (_cacheSize <= _evictionLimit) + if(_cacheSize <= _evictionLimit) return; // Nothing to do - if (entry.isPinned()) + if(entry.isPinned()) return; // Pin state changed so we cannot evict - if (entry.getState().isAvailable() && entry.getState().isBackedByDisk()) { - if (entry.getRetainHintCount() > 0) { + if(entry.getState().isAvailable() && entry.getState().isBackedByDisk()) { + if(entry.getRetainHintCount() > 0) { shouldCheckEviction = true; } else { - cacheSizeDelta = transitionMemState(entry, BlockState.COLD); + cacheSizeDelta = transitionMemState(entry, BlockState.COLD); long cleared = entry.clear(); - if (cleared != entry.getSize()) + if(cleared != entry.getSize()) throw new IllegalStateException(); _cache.remove(entry.getKey()); _evictionCache.put(entry.getKey(), entry); } } - else if (entry.getState() == BlockState.HOT) { - if (entry.getRetainHintCount() > 0) { + else if(entry.getState() == BlockState.HOT) { + if(entry.getRetainHintCount() > 0) { shouldCheckEviction = true; } else { @@ -470,9 +472,9 @@ else if (entry.getState() == BlockState.HOT) { } } } - if (cacheSizeDelta != 0) + if(cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); - else if (shouldCheckEviction) + else if(shouldCheckEviction) onCacheSizeChanged(true); } @@ -508,10 +510,11 @@ public synchronized void shutdown() { System.out.println("[WARN] Cache still holds " + _cache.size() + " / " + _evictionCache.size() + " blocks"); Set cachedStreams = _cache.keySet().stream().map(BlockKey::getStreamId).collect(Collectors.toSet()); - Set evictedStreams = _evictionCache.keySet().stream().map(BlockKey::getStreamId).collect(Collectors.toSet()); + Set evictedStreams = _evictionCache.keySet().stream().map(BlockKey::getStreamId) + .collect(Collectors.toSet()); cachedStreams.addAll(evictedStreams); - System.out.println("[WARN] Affected stream IDs: " + cachedStreams + ", Pinned: " + _cache.values().stream().mapToInt( - e -> e.isPinned() ? 1 : 0).sum()); + System.out.println("[WARN] Affected stream IDs: " + cachedStreams + ", Pinned: " + + _cache.values().stream().mapToInt(e -> e.isPinned() ? 1 : 0).sum()); } _cache.clear(); _evictionCache.clear(); @@ -575,7 +578,8 @@ private void runMaintenanceLoop() { do { _maintenanceRequested.set(false); onCacheSizeChangedInternal(_maintenanceNeedsIncr.getAndSet(false)); - } while(_maintenanceRequested.get()); + } + while(_maintenanceRequested.get()); } finally { _maintenanceRunning.set(false); @@ -591,7 +595,8 @@ private void onCacheSizeChangedInternal(boolean incr) { if(incr) onCacheSizeIncremented(); else - while(onCacheSizeDecremented()) {} + while(onCacheSizeDecremented()) { + } while(processPendingHandovers()) { onCacheSizeIncremented(); } @@ -601,18 +606,23 @@ private void onCacheSizeChangedInternal(boolean incr) { } private synchronized void sanityCheck() { - if (_cacheSize > _hardLimit * 1.1) { - if (!_warnThrottling) { + if(_cacheSize > _hardLimit * 1.1) { + if(!_warnThrottling) { _warnThrottling = true; - System.out.println("[WARN] Cache hard limit exceeded by over 10%: " + String.format("%.2f", _cacheSize/1000000.0) + "MB (-" + String.format("%.2f", _bytesUpForEviction/1000000.0) + "MB) > " + String.format("%.2f", _hardLimit/1000000.0) + "MB"); + System.out.println( + "[WARN] Cache hard limit exceeded by over 10%: " + String.format("%.2f", _cacheSize / 1000000.0) + + "MB (-" + String.format("%.2f", _bytesUpForEviction / 1000000.0) + "MB) > " + + String.format("%.2f", _hardLimit / 1000000.0) + "MB"); } } - else if (_warnThrottling && _cacheSize < _hardLimit) { + else if(_warnThrottling && _cacheSize < _hardLimit) { _warnThrottling = false; - System.out.println("[INFO] Cache within limit: " + String.format("%.2f", _cacheSize/1000000.0) + "MB (-" + String.format("%.2f", _bytesUpForEviction/1000000.0) + "MB) <= " + String.format("%.2f", _hardLimit/1000000.0) + "MB"); + System.out.println("[INFO] Cache within limit: " + String.format("%.2f", _cacheSize / 1000000.0) + "MB (-" + + String.format("%.2f", _bytesUpForEviction / 1000000.0) + "MB) <= " + + String.format("%.2f", _hardLimit / 1000000.0) + "MB"); } - if (!SANITY_CHECKS) + if(!SANITY_CHECKS) return; int pinned = 0; @@ -625,16 +635,16 @@ else if (_warnThrottling && _cacheSize < _hardLimit) { long actualPinnedEvictingBytes = 0; long actualWarmPinnedBytes = 0; long actualReadingReservedBytes = 0; - for (BlockEntry entry : _cache.values()) { - if (entry.isPinned()) { + for(BlockEntry entry : _cache.values()) { + if(entry.isPinned()) { pinned++; actualPinnedBytes += entry.getSize(); if(entry.getState() == BlockState.WARM) actualWarmPinnedBytes += entry.getSize(); } - if (entry.getState().isBackedByDisk()) + if(entry.getState().isBackedByDisk()) backedByDisk++; - if (entry.getState() == BlockState.EVICTING) { + if(entry.getState() == BlockState.EVICTING) { evicting++; upForEviction += entry.getSize(); if(entry.isPinned()) @@ -642,43 +652,44 @@ else if (_warnThrottling && _cacheSize < _hardLimit) { } if(entry.getState() == BlockState.READING) actualReadingReservedBytes += entry.getSize(); - if (!entry.getState().isAvailable()) + if(!entry.getState().isAvailable()) throw new IllegalStateException(); total++; actualCacheSize += entry.getSize(); } - for (BlockEntry entry : _evictionCache.values()) { - if (entry.getState().isAvailable()) + for(BlockEntry entry : _evictionCache.values()) { + if(entry.getState().isAvailable()) throw new IllegalStateException("Invalid eviction state: " + entry.getState()); - if (entry.getState() == BlockState.EVICTING && entry.isPinned()) + if(entry.getState() == BlockState.EVICTING && entry.isPinned()) actualPinnedEvictingBytes += entry.getSize(); - if (entry.getState() == BlockState.READING) + if(entry.getState() == BlockState.READING) actualCacheSize += entry.getSize(); - if (entry.getState() == BlockState.READING) + if(entry.getState() == BlockState.READING) actualReadingReservedBytes += entry.getSize(); - if (entry.isPinned()) { + if(entry.isPinned()) { actualPinnedBytes += entry.getSize(); if(entry.getState() == BlockState.WARM) actualWarmPinnedBytes += entry.getSize(); } } - if (actualCacheSize != _cacheSize) + if(actualCacheSize != _cacheSize) throw new IllegalStateException(actualCacheSize + " != " + _cacheSize); - if (upForEviction != _bytesUpForEviction) + if(upForEviction != _bytesUpForEviction) throw new IllegalStateException(upForEviction + " != " + _bytesUpForEviction); - if (actualPinnedBytes != _pinnedBytes) + if(actualPinnedBytes != _pinnedBytes) throw new IllegalStateException(actualPinnedBytes + " != " + _pinnedBytes); - if (actualPinnedEvictingBytes != _pinnedEvictingBytes) + if(actualPinnedEvictingBytes != _pinnedEvictingBytes) throw new IllegalStateException(actualPinnedEvictingBytes + " != " + _pinnedEvictingBytes); - if (_pinnedEvictingBytes > _bytesUpForEviction) + if(_pinnedEvictingBytes > _bytesUpForEviction) throw new IllegalStateException(_pinnedEvictingBytes + " > " + _bytesUpForEviction); if(actualWarmPinnedBytes != _warmPinnedBytes) throw new IllegalStateException(actualWarmPinnedBytes + " != " + _warmPinnedBytes); - if (actualReadingReservedBytes != _readingReservedBytes) + if(actualReadingReservedBytes != _readingReservedBytes) throw new IllegalStateException(actualReadingReservedBytes + " != " + _readingReservedBytes); System.out.println("=========="); - System.out.println("Limit: " + _evictionLimit/1000 + "KB"); - System.out.println("Memory: (" + _cacheSize/1000 + "KB - " + _bytesUpForEviction/1000 + "KB) / " + _hardLimit/1000 + "KB"); + System.out.println("Limit: " + _evictionLimit / 1000 + "KB"); + System.out.println("Memory: (" + _cacheSize / 1000 + "KB - " + _bytesUpForEviction / 1000 + "KB) / " + + _hardLimit / 1000 + "KB"); System.out.println("Pinned: " + pinned + " / " + total); System.out.println("Disk backed: " + backedByDisk + " / " + total); System.out.println("Evicting: " + evicting + " / " + total); @@ -695,10 +706,11 @@ private void onCacheSizeIncremented() { if(pressure <= _evictionLimit) return; // Nothing to do - long overshoot = Math.max((long)(0.1 * _evictionLimit), 10000000); + long overshoot = Math.max((long) (0.1 * _evictionLimit), 10000000); long lowLimit = _evictionLimit - _readBuffer - overshoot; - //System.out.println("[CACHE] Claiming " + (pressure + overshoot - _evictionLimit)/1000 + "kB (last claim was " + (System.currentTimeMillis() - _lastEvictRun) + "ms ago)"); + // System.out.println("[CACHE] Claiming " + (pressure + overshoot - _evictionLimit)/1000 + "kB (last claim + // was " + (System.currentTimeMillis() - _lastEvictRun) + "ms ago)"); // Scan for values that can be evicted Collection entries = _cache.values(); @@ -713,8 +725,8 @@ private void onCacheSizeIncremented() { break; synchronized(entry) { - //if(entry.isPinned()) - // continue; + // if(entry.isPinned()) + // continue; if(!allowRetainHint && entry.getRetainHintCount() > 0) continue; if(entry.getState() == BlockState.COLD || entry.getState() == BlockState.EVICTING) @@ -748,12 +760,12 @@ private void onCacheSizeIncremented() { _lastEvictRun = System.currentTimeMillis(); } - for (BlockEntry entry : upForEvictionNeedsWrite) + for(BlockEntry entry : upForEvictionNeedsWrite) evict(entry, true); - for (BlockEntry entry : upForEvictionNoWrite) + for(BlockEntry entry : upForEvictionNoWrite) evict(entry, false); - if (cacheSizeDelta != 0) + if(cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); } @@ -813,7 +825,7 @@ private boolean onCacheSizeDecremented() { throw new IllegalStateException(); req.setPinned(idx); } - else if (entry.getState() == BlockState.READING) { + else if(entry.getState() == BlockState.READING) { req.schedule(idx); registerWaiter(entry.getKey(), req, idx); reading = true; @@ -836,7 +848,7 @@ else if (entry.getState() == BlockState.READING) { if(allReserved) { _deferredReadRequests.poll(); _deferredReadCountHint = _deferredReadRequests.size(); - if (!toRead.isEmpty()) + if(!toRead.isEmpty()) _processingReadRequests.add(req); } @@ -943,17 +955,17 @@ private void onEvicted(final BlockEntry entry) { if(tmp != null && tmp != entry) throw new IllegalStateException(); tmp = _evictionCache.put(entry.getKey(), entry); - if (tmp != null) + if(tmp != null) throw new IllegalStateException(); sanityCheck(); } - if (cacheSizeDelta != 0) + if(cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); } private void clearRetainHints(DeferredReadRequest request) { - for (int i = 0; i < request.getEntries().size(); i++) { - if (!request.isRetainHinted(i)) + for(int i = 0; i < request.getEntries().size(); i++) { + if(!request.isRetainHinted(i)) continue; BlockEntry entry = request.getEntries().get(i); synchronized(entry) { @@ -963,12 +975,12 @@ private void clearRetainHints(DeferredReadRequest request) { } /** - * Cleanly transitions state of a BlockEntry and handles accounting. - * Requires both the scheduler object and the entry to be locked: + * Cleanly transitions state of a BlockEntry and handles accounting. Requires both the scheduler object and the + * entry to be locked: */ private long transitionMemState(BlockEntry entry, BlockState newState) { BlockState oldState = entry.getState(); - if (oldState == newState) + if(oldState == newState) return 0; long sz = entry.getSize(); @@ -976,7 +988,7 @@ private long transitionMemState(BlockEntry entry, BlockState newState) { boolean pinned = entry.isPinned(); // Remove old contribution - switch (oldState) { + switch(oldState) { case REMOVED: throw new IllegalStateException(); case HOT: @@ -1000,7 +1012,7 @@ private long transitionMemState(BlockEntry entry, BlockState newState) { } // Add new contribution - switch (newState) { + switch(newState) { case REMOVED: case COLD: break; @@ -1056,6 +1068,7 @@ private int pinEntryWithAccounting(BlockEntry entry) { /** * Requires scheduler lock and entry lock. + * * @return true if this call transitioned pin count to zero. */ private boolean unpinEntryWithAccounting(BlockEntry entry) { diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java b/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java index 1f731fc3aa5..f04534481c9 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java @@ -66,8 +66,8 @@ protected boolean isHashCol(int colID) { } /** - * Domain size of a dummycoded source column: the hash domain K from the meta cell for - * feature-hashed columns, otherwise the column's {@code numDistinct} (0 when unset). + * Domain size of a dummycoded source column: the hash domain K from the meta cell for feature-hashed columns, + * otherwise the column's {@code numDistinct} (0 when unset). * * @param meta transform meta frame * @param colID 1-based column id of the dummycoded source column @@ -144,13 +144,13 @@ public FrameBlock decode(final MatrixBlock in, final FrameBlock out, final int k try { final List> tasks = new ArrayList<>(); int blz = Math.max((in.getNumRows() + k) / k, 1000); - - for(int i = 0; i < in.getNumRows(); i += blz){ + + for(int i = 0; i < in.getNumRows(); i += blz) { final int start = i; final int end = Math.min(in.getNumRows(), i + blz); tasks.add(pool.submit(() -> decode(in, out, start, end))); } - + for(Future f : tasks) f.get(); return out; diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java index a286c03dce8..01a502154cd 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java @@ -72,10 +72,10 @@ public void decode(MatrixBlock in, FrameBlock out, int rl, int ru) { final double val = in.get(i, _srcCols[j] - 1); if(!Double.isNaN(val)){ final int key = (int) Math.round(val); - if(key == 0){ + if(key == 0) { a.set(i, _binMins[j][key]); } - else{ + else { double bmin = _binMins[j][key - 1]; double bmax = _binMaxs[j][key - 1]; double oval = bmin + (bmax - bmin) / 2 // bin center diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java index ee1a33c49fd..8aa0b60d990 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java @@ -34,7 +34,7 @@ /** * Simple atomic decoder for dummycoded columns. This decoder builds internally inverted column mappings from the given * frame meta data. - * + * */ public class DecoderDummycode extends Decoder { private static final long serialVersionUID = 4758831042891032129L; diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java index 8f6c45d63e8..fc123657032 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java @@ -64,15 +64,15 @@ public static Decoder createDecoder(String spec, String[] colnames, ValueType[] try { //parse transform specification JSONObject jSpec = new JSONObject(spec); - - //create decoders 'bin', 'recode', 'hash', 'dummy', and 'pass-through' + + // create decoders 'bin', 'recode', 'hash', 'dummy', and 'pass-through' List binIDs = TfMetaUtils.parseBinningColIDs(jSpec, colnames, minCol, maxCol); - List rcIDs = Arrays.asList(ArrayUtils.toObject( - TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.RECODE.toString(), minCol, maxCol))); - List hcIDs = Arrays.asList(ArrayUtils.toObject( - TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.HASH.toString(), minCol, maxCol))); - List dcIDs = Arrays.asList(ArrayUtils.toObject( - TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.DUMMYCODE.toString(), minCol, maxCol))); + List rcIDs = Arrays.asList(ArrayUtils + .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.RECODE.toString(), minCol, maxCol))); + List hcIDs = Arrays.asList(ArrayUtils + .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.HASH.toString(), minCol, maxCol))); + List dcIDs = Arrays.asList(ArrayUtils + .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.DUMMYCODE.toString(), minCol, maxCol))); // only specially treat the columns with both recode and dictionary rcIDs = unionDistinct(rcIDs, dcIDs); // hashing is a lossy, one-way transform with no inverse recode map, so hash columns @@ -106,29 +106,18 @@ public static Decoder createDecoder(String spec, String[] colnames, ValueType[] // collect all the decoders in one list. List ldecoders = new ArrayList<>(); - - if( !binIDs.isEmpty() ) { - ldecoders.add(new DecoderBin(schema, - ArrayUtils.toPrimitive(binIDs.toArray(new Integer[0])), + + if(!binIDs.isEmpty()) { + ldecoders.add(new DecoderBin(schema, ArrayUtils.toPrimitive(binIDs.toArray(new Integer[0])), ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); } if( !dcIDs.isEmpty() ) { ldecoders.add(new DecoderDummycode(schema, ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); } - if( !rcIDs.isEmpty() ) { - // recode on output (after dummycode rebuilds the categorical columns) when dummycoding is present - ldecoders.add(new DecoderRecode(schema, !dcIDs.isEmpty(), - ArrayUtils.toPrimitive(rcIDs.toArray(new Integer[0])))); - } - if( !ptIDs.isEmpty() ) { - ldecoders.add(new DecoderPassThrough(schema, - ArrayUtils.toPrimitive(ptIDs.toArray(new Integer[0])), - ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); - } - - //create composite decoder of all created decoders - //and initialize with given meta data (recode, dummy, bin) + + // create composite decoder of all created decoders + // and initialize with given meta data (recode, dummy, bin) decoder = new DecoderComposite(schema, ldecoders); decoder.setColnames(colnames); decoder.initMetaData(meta); @@ -147,7 +136,7 @@ else if( decoder instanceof DecoderRecode ) return DecoderType.Recode.ordinal(); else if( decoder instanceof DecoderPassThrough ) return DecoderType.PassThrough.ordinal(); - else if( decoder instanceof DecoderBin ) + else if(decoder instanceof DecoderBin) return DecoderType.Bin.ordinal(); throw new DMLRuntimeException("Unsupported decoder type: " + decoder.getClass().getCanonicalName()); @@ -158,10 +147,14 @@ public static Decoder createInstance(int type) { // create instance switch(dtype) { - case Bin: return new DecoderBin(); - case Dummycode: return new DecoderDummycode(null, null); - case PassThrough: return new DecoderPassThrough(null, null, null); - case Recode: return new DecoderRecode(null, false, null); + case Bin: + return new DecoderBin(); + case Dummycode: + return new DecoderDummycode(null, null); + case PassThrough: + return new DecoderPassThrough(null, null, null); + case Recode: + return new DecoderRecode(null, false, null); default: throw new DMLRuntimeException("Unsupported Encoder Type used: " + dtype); } diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java index 11dd2c7faa5..d8d624122a0 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java @@ -126,20 +126,20 @@ public void initMetaData(FrameBlock meta) { _rcMaps = new HashMap[_colList.length]; for( int j=0; j<_colList.length; j++ ) { HashMap map = new HashMap<>(); - for( int i=0; i= 0} - * both the minimum and maximum fraction digits are pinned to {@code decimal}, so values are - * printed with exactly that many decimals; otherwise the {@link DecimalFormat} defaults apply. + * Creates a non-grouping {@link DecimalFormat} for printing values. When {@code decimal >= 0} both the minimum and + * maximum fraction digits are pinned to {@code decimal}, so values are printed with exactly that many decimals; + * otherwise the {@link DecimalFormat} defaults apply. + * * @param decimal number of decimal places to print, -1 for default * @return a configured {@link DecimalFormat} */ private static DecimalFormat createDecimalFormat(int decimal) { DecimalFormat df = new DecimalFormat(); df.setGroupingUsed(false); - if (decimal >= 0) { + if(decimal >= 0) { df.setMinimumFractionDigits(decimal); df.setMaximumFractionDigits(decimal); } diff --git a/src/main/java/org/apache/sysds/utils/DoubleParser.java b/src/main/java/org/apache/sysds/utils/DoubleParser.java index c0122f8061f..2252b7270c8 100644 --- a/src/main/java/org/apache/sysds/utils/DoubleParser.java +++ b/src/main/java/org/apache/sysds/utils/DoubleParser.java @@ -200,7 +200,7 @@ public static double parseFloatingPointLiteral(String str, int offset, int endIn // : is the first character after numbers. // 0 is the first number. // we use the last position, since this is not allowed to be other values than a number. - if(str.charAt(endIndex - 1) > '9' || str.charAt(endIndex - 1) < '0') + if(str.charAt(endIndex - 1) > '9' || str.charAt(endIndex - 1) < '0') return Double.parseDouble(str); final double val = parseDecFloatLiteral(str, index, offset, endIndex); diff --git a/src/main/java/org/apache/sysds/utils/SettingsChecker.java b/src/main/java/org/apache/sysds/utils/SettingsChecker.java index c5f14a7043b..10d1a42b246 100644 --- a/src/main/java/org/apache/sysds/utils/SettingsChecker.java +++ b/src/main/java/org/apache/sysds/utils/SettingsChecker.java @@ -106,25 +106,24 @@ private static long maxMemMachineOSX() { } private static long maxMemMachineWin() { - //try modern powershell, otherwise wmic, log errors as warning but avoid crashes + // try modern powershell, otherwise wmic, log errors as warning but avoid crashes long tmp = maxMemMachineWin(true); - if( tmp < 0 ) + if(tmp < 0) tmp = maxMemMachineWin(false); return tmp; } - + private static long maxMemMachineWin(boolean modern) { int startIx = modern ? 3 : 1; - String command = modern ? - "powershell Get-CimInstance -ClassName Win32_PhysicalMemory | Select-Object Capacity" : - "wmic memorychip get capacity"; //in bytes + String command = modern ? "powershell Get-CimInstance -ClassName Win32_PhysicalMemory | Select-Object Capacity" : "wmic memorychip get capacity"; // in + // bytes try { Runtime rt = Runtime.getRuntime(); Process pr = rt.exec(command); String[] memStr = new String(pr.getInputStream().readAllBytes(), StandardCharsets.UTF_8).split("\n"); //skip header, and aggregate DIMM capacities long capacity = 0; - for( int i=startIx; i 0 ) capacity += Long.parseLong(tmp); diff --git a/src/test/java/org/apache/sysds/performance/Main.java b/src/test/java/org/apache/sysds/performance/Main.java index 0622e789baa..a941e90f724 100644 --- a/src/test/java/org/apache/sysds/performance/Main.java +++ b/src/test/java/org/apache/sysds/performance/Main.java @@ -243,10 +243,9 @@ private static void run17(String[] args) throws Exception { } /** - * Repeatedly read the same on-disk Delta frame table (written once as setup). - * Args: {@code 18 [mode] [targetFileSizeMB]} - * where mode is one of serial|parallel|both (default parallel) and an omitted - * target file size uses the adaptive default sizing. + * Repeatedly read the same on-disk Delta frame table (written once as setup). Args: + * {@code 18 [mode] [targetFileSizeMB]} where mode is one of serial|parallel|both (default parallel) + * and an omitted target file size uses the adaptive default sizing. */ private static void run18(String[] args) throws Exception { int rows = Integer.parseInt(args[1]); diff --git a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java index 36ea11b3e2f..4aac0685698 100644 --- a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java @@ -117,8 +117,8 @@ public abstract class AutomatedTestBase { public static final double GPU_TOLERANCE = 1e-9; /** - * Default deadline (ms) for federated worker/monitoring readiness waits and a few legacy - * {@code sleep()} calls. {@link FederatedWorkerUtils} enforces its own minimum floor. + * Default deadline (ms) for federated worker/monitoring readiness waits and a few legacy {@code sleep()} calls. + * {@link FederatedWorkerUtils} enforces its own minimum floor. */ public static final int FED_WORKER_WAIT = 3000; @@ -1761,9 +1761,9 @@ private static Process spawnLocalFedWorker(int port, String[] addArgs) { /** * Start a new JVM for a federated monitoring backend at the port. * - *

Returns once the backend's TCP port accepts connections (Netty's bind has completed), or - * throws a {@link RuntimeException} once the {@link FederatedWorkerUtils} readiness floor - * elapses. + *

+ * Returns once the backend's TCP port accepts connections (Netty's bind has completed), or throws a + * {@link RuntimeException} once the {@link FederatedWorkerUtils} readiness floor elapses. * * @param port Port to use for the JVM * @param addArgs Extra CLI args to append, or null @@ -1776,10 +1776,10 @@ protected Process startLocalFedMonitoring(int port, String[] addArgs) { /** * Start a new JVM for a federated monitoring backend at the port. * - *

Returns once the backend's TCP port accepts connections, or throws a - * {@link RuntimeException} after {@code timeoutMs} elapses. The monitoring server opens the - * port after Netty's {@code bind().sync()} returns; a successful TCP connect therefore signals - * that the HTTP listener is ready to accept requests. + *

+ * Returns once the backend's TCP port accepts connections, or throws a {@link RuntimeException} after + * {@code timeoutMs} elapses. The monitoring server opens the port after Netty's {@code bind().sync()} returns; a + * successful TCP connect therefore signals that the HTTP listener is ready to accept requests. * * @param port Port to use for the JVM * @param addArgs Extra CLI args to append, or null @@ -1798,8 +1798,9 @@ private static Process spawnLocalFedMonitoring(int port, String[] addArgs) { String separator = System.getProperty("file.separator"); String classpath = System.getProperty("java.class.path"); String path = System.getProperty("java.home") + separator + "bin" + separator + "java"; - String[] args = ArrayUtils.addAll(new String[] {path, "-cp", classpath, DMLScript.class.getName(), - "-fedMonitoring", Integer.toString(port)}, addArgs); + String[] args = ArrayUtils.addAll( + new String[] {path, "-cp", classpath, DMLScript.class.getName(), "-fedMonitoring", Integer.toString(port)}, + addArgs); try { return new ProcessBuilder(args).start(); } diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java index f14a614b583..0c7cd2046e5 100644 --- a/src/test/java/org/apache/sysds/test/TestUtils.java +++ b/src/test/java/org/apache/sysds/test/TestUtils.java @@ -2941,10 +2941,9 @@ public static void writeTestScalar(String file, double value) { } } - /** * Write scalar to file - * + * * @param file File to write to * @param value Value to write */ @@ -3519,9 +3518,9 @@ public static void shutdownThread(Thread t) { // Bounded join: workers are daemon threads, so even if one ignores the interrupt // we must not block cleanup (and the JVM) indefinitely waiting for it. t.join(THREAD_SHUTDOWN_JOIN_MS); - if( t.isAlive() ) - LOG.warn("Federated worker thread " + t.getName() - + " did not stop within " + THREAD_SHUTDOWN_JOIN_MS + "ms; leaving it as a daemon."); + if(t.isAlive()) + LOG.warn("Federated worker thread " + t.getName() + " did not stop within " + + THREAD_SHUTDOWN_JOIN_MS + "ms; leaving it as a daemon."); } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java b/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java index 07ec9752928..d76c741d870 100644 --- a/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java +++ b/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java @@ -67,10 +67,10 @@ public void setUp() { /** * Compile a DML script string into a runtime {@link Program} without executing it. * - * @param dmlScript the DML source - * @param args named command-line arguments ($name -> value), may be null - * @param mode the global execution mode (e.g. {@link ExecMode#HYBRID}) - * @param localMaxMem the local memory budget in bytes used for memory-based exec-type decisions + * @param dmlScript the DML source + * @param args named command-line arguments ($name -> value), may be null + * @param mode the global execution mode (e.g. {@link ExecMode#HYBRID}) + * @param localMaxMem the local memory budget in bytes used for memory-based exec-type decisions * @return the compiled runtime program */ protected Program compile(String dmlScript, Map args, ExecMode mode, long localMaxMem) { @@ -145,8 +145,7 @@ else if(pb instanceof FunctionProgramBlock) { /** All instructions whose opcode equals {@code opcode} (exact match). */ protected List getByOpcode(Program prog, String opcode) { - return getInstructions(prog).stream().filter(i -> opcode.equals(i.getOpcode())) - .collect(Collectors.toList()); + return getInstructions(prog).stream().filter(i -> opcode.equals(i.getOpcode())).collect(Collectors.toList()); } protected static boolean isSpark(Instruction inst) { @@ -169,13 +168,13 @@ protected void assertCP(Program prog, String opcode) { private void assertExecType(Program prog, String opcode, boolean expectSpark) { List matches = getByOpcode(prog, opcode); - Assert.assertFalse("Expected at least one '" + opcode + "' instruction but found none.\n" - + Explain.explain(prog), matches.isEmpty()); + Assert.assertFalse( + "Expected at least one '" + opcode + "' instruction but found none.\n" + Explain.explain(prog), + matches.isEmpty()); for(Instruction inst : matches) { boolean spark = isSpark(inst); - Assert.assertEquals("Instruction '" + opcode + "' expected exec type " - + (expectSpark ? "SPARK" : "CP") + " but was " + (spark ? "SPARK" : "CP") + ".\n" - + Explain.explain(prog), expectSpark, spark); + Assert.assertEquals("Instruction '" + opcode + "' expected exec type " + (expectSpark ? "SPARK" : "CP") + + " but was " + (spark ? "SPARK" : "CP") + ".\n" + Explain.explain(prog), expectSpark, spark); } } diff --git a/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java b/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java index 0b3889db908..7b45120d8f1 100644 --- a/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java +++ b/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java @@ -33,14 +33,13 @@ */ public class SparkTransitiveExecTypeCompileTest extends CompilerTestBase { - private static final String DML_HEADER = - "X = rand(rows=20000000, cols=8, seed=1);\n" + // ~1.2GB -> rand and colSums run on Spark - "v = colSums(X);\n"; // 1x8 Spark-resident vector (opcode uack+) + private static final String DML_HEADER = "X = rand(rows=20000000, cols=8, seed=1);\n" + // ~1.2GB -> rand and + // colSums run on Spark + "v = colSums(X);\n"; // 1x8 Spark-resident vector (opcode uack+) @Test public void singleConsumerUnaryPulledIntoSpark() { - String dml = DML_HEADER + - "r = round(v);\n" + // sole consumer of the Spark-resident vector -> pulled into Spark + String dml = DML_HEADER + "r = round(v);\n" + // sole consumer of the Spark-resident vector -> pulled into Spark "print(sum(r));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); @@ -50,62 +49,58 @@ public void singleConsumerUnaryPulledIntoSpark() { @Test public void multiConsumerUnaryStaysCP() { - String dml = DML_HEADER + - "a = round(v);\n" + // v now has two consumers (round + abs) ... - "b = abs(v);\n" + - "print(sum(a) + sum(b));\n"; + String dml = DML_HEADER + "a = round(v);\n" + // v now has two consumers (round + abs) ... + "b = abs(v);\n" + "print(sum(a) + sum(b));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uack+"); // input still has a Spark output ... - assertCP(prog, "round"); // ... but the multi-parent guard keeps both unaries in CP + assertCP(prog, "round"); // ... but the multi-parent guard keeps both unaries in CP assertCP(prog, "abs"); } // A tall, Spark-resident column vector that is still small enough (40 KB) to be CP by memory // estimate: rowSums over a very wide matrix runs on Spark, but its 1-column result fits in CP. - private static final String TALL_VECTOR_HEADER = - "X = rand(rows=5000, cols=200000, seed=1);\n" + // ~8GB -> rand and rowSums run on Spark - "c = rowSums(X);\n"; // 5000x1 Spark-resident vector (opcode uark+) + private static final String TALL_VECTOR_HEADER = "X = rand(rows=5000, cols=200000, seed=1);\n" + // ~8GB -> rand and + // rowSums run + // on Spark + "c = rowSums(X);\n"; // 5000x1 Spark-resident vector (opcode uark+) @Test public void cumulativeUnaryStaysCP() { - String dml = TALL_VECTOR_HEADER + - "r = cumsum(c);\n" + // sole consumer of the Spark-resident vector, CP by estimate ... + String dml = TALL_VECTOR_HEADER + "r = cumsum(c);\n" + // sole consumer of the Spark-resident vector, CP by + // estimate ... "print(as.scalar(r[2500,1]));\n"; // ... consume via indexing (avoids the sum(cumsum) rewrite) Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); - assertSpark(prog, "uark+"); // input genuinely has a Spark output + assertSpark(prog, "uark+"); // input genuinely has a Spark output assertCP(prog, "ucumk+"); // ... but cumulative ops are excluded from the transitive pull } @Test public void singleConsumerBinaryPulledIntoSpark() { - String dml = TALL_VECTOR_HEADER + - "r = c + 2.0;\n" + // matrix-scalar on the Spark-resident vector, sole consumer -> pulled into Spark + String dml = TALL_VECTOR_HEADER + "r = c + 2.0;\n" + // matrix-scalar on the Spark-resident vector, sole + // consumer -> pulled into Spark "print(as.scalar(r[2500,1]));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uark+"); // input genuinely has a Spark output (multi-block column vector) - assertSpark(prog, "+"); // matrix-scalar binary pulled into Spark (CP by estimate, single consumer) + assertSpark(prog, "+"); // matrix-scalar binary pulled into Spark (CP by estimate, single consumer) } @Test public void multiConsumerBinaryStaysCP() { - String dml = TALL_VECTOR_HEADER + - "a = c + 2.0;\n" + // c now has two consumers (+ and *) ... - "b = c * 3.0;\n" + - "print(as.scalar(a[2500,1]) + as.scalar(b[2500,1]));\n"; + String dml = TALL_VECTOR_HEADER + "a = c + 2.0;\n" + // c now has two consumers (+ and *) ... + "b = c * 3.0;\n" + "print(as.scalar(a[2500,1]) + as.scalar(b[2500,1]));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uark+"); // input still has a Spark output ... - assertCP(prog, "+"); // ... but the multi-parent guard keeps both binaries in CP + assertCP(prog, "+"); // ... but the multi-parent guard keeps both binaries in CP assertCP(prog, "*"); } @Test public void transitiveDisabledUnaryStaysCP() { - String dml = DML_HEADER + - "r = round(v);\n" + // pullable unary, but flag is off + String dml = DML_HEADER + "r = round(v);\n" + // pullable unary, but flag is off "print(sum(r));\n"; Program prog = compileWithTransitive(dml, false); @@ -115,8 +110,7 @@ public void transitiveDisabledUnaryStaysCP() { @Test public void transitiveDisabledBinaryStaysCP() { - String dml = TALL_VECTOR_HEADER + - "r = c + 2.0;\n" + // pullable matrix-scalar, but flag is off + String dml = TALL_VECTOR_HEADER + "r = c + 2.0;\n" + // pullable matrix-scalar, but flag is off "print(as.scalar(r[2500,1]));\n"; Program prog = compileWithTransitive(dml, false); diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java index 083a29f965b..22f867819c3 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java @@ -43,7 +43,8 @@ /** * Tests the {@code order} (sort) reorg operation on compressed matrices. A single column held in a single column group * is sorted ascending while staying compressed (via {@link org.apache.sysds.runtime.compress.lib.CLALibSort}); every - * other configuration falls back to a decompressed reorg. In all cases the result must match the uncompressed reference. + * other configuration falls back to a decompressed reorg. In all cases the result must match the uncompressed + * reference. */ public class CompressedSortTest { @@ -263,8 +264,7 @@ private void runCompressed(MatrixBlock mb, CompressionType ct) { assertEquals("Expected a single column group", 1, cmb.getColGroups().size()); MatrixBlock actual = cmb.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); - assertTrue("Expected the sorted result to stay compressed for " + ct, - actual instanceof CompressedMatrixBlock); + assertTrue("Expected the sorted result to stay compressed for " + ct, actual instanceof CompressedMatrixBlock); MatrixBlock expected = mb.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); TestUtils.compareMatrices(expected, CompressedMatrixBlock.getUncompressed(actual, "sort"), 0.0, "sort " + ct); diff --git a/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java index 833128ad9f0..49cbbdd248d 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java @@ -62,8 +62,8 @@ public static void setup() { } /** - * Build a compressed matrix backed by a single DDC column group spanning all {@code nCol} columns. This guarantees a - * single (non-uncompressed) column group, which is what triggers the mm-chain fast path for wide enough matrices. + * Build a compressed matrix backed by a single DDC column group spanning all {@code nCol} columns. This guarantees + * a single (non-uncompressed) column group, which is what triggers the mm-chain fast path for wide enough matrices. */ private static CompressedMatrixBlock singleDDC(int nRow, int nCol, int nVal, int seed) { Random r = new Random(seed); diff --git a/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java b/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java index 549010a78cb..c6afd5a3543 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java @@ -43,8 +43,8 @@ /** * Drive the solve opcode through {@link BinaryMatrixMatrixCPInstruction} with compressed inputs to cover the - * commons-math matrix-matrix branch that decompresses compressed left/right operands before solving. The - * script-level solve tests only ever see uncompressed inputs, so this branch is otherwise unreached. + * commons-math matrix-matrix branch that decompresses compressed left/right operands before solving. The script-level + * solve tests only ever see uncompressed inputs, so this branch is otherwise unreached. */ public class CompressedBinaryMatrixMatrixSolveTest { @@ -72,8 +72,8 @@ public void solveCompressedLeftCompressedRight() { assertTrue("A must compress to exercise the compressed-left path", aC instanceof CompressedMatrixBlock); CompressedMatrixBlock bC = CompressedMatrixBlockFactory.createConstant(n, 2, 1.0); - MatrixBlock expected = LibCommonsMath.matrixMatrixOperations( - CompressedMatrixBlock.getUncompressed(aC), CompressedMatrixBlock.getUncompressed(bC), SOLVE); + MatrixBlock expected = LibCommonsMath.matrixMatrixOperations(CompressedMatrixBlock.getUncompressed(aC), + CompressedMatrixBlock.getUncompressed(bC), SOLVE); MatrixBlock actual = runSolve(aC, bC); TestUtils.compareMatricesBitAvgDistance(expected, actual, 0, 0, SOLVE); @@ -94,8 +94,8 @@ public void solveCompressedLeftDenseRight() { assertTrue("A must compress to exercise the compressed-left path", aC instanceof CompressedMatrixBlock); MatrixBlock b = TestUtils.round(TestUtils.generateTestMatrixBlock(n, 1, -5, 5, 1.0, 7)); - MatrixBlock expected = LibCommonsMath.matrixMatrixOperations( - CompressedMatrixBlock.getUncompressed(aC), b, SOLVE); + MatrixBlock expected = LibCommonsMath.matrixMatrixOperations(CompressedMatrixBlock.getUncompressed(aC), b, + SOLVE); MatrixBlock actual = runSolve(aC, b); TestUtils.compareMatricesBitAvgDistance(expected, actual, 0, 0, SOLVE); @@ -119,7 +119,8 @@ private static BinaryMatrixMatrixCPInstruction solveInstruction() { } private static MatrixObject matrixObject(String name, MatrixBlock mb) { - MatrixCharacteristics mc = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), 1000, mb.getNonZeros()); + MatrixCharacteristics mc = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), 1000, + mb.getNonZeros()); MatrixObject mo = new MatrixObject(ValueType.FP64, "/dev/null/" + name, new MetaDataFormat(mc, FileFormat.BINARY), mb); return mo; diff --git a/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java b/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java index 8907c82f1d1..0d2f37e319c 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java @@ -50,7 +50,9 @@ public class OffsetClassInitConcurrencyTest { private static final String[] INIT_TARGETS = {PKG + "AOffset", PKG + "OffsetEmpty", PKG + "OffsetChar", PKG + "OffsetByte", PKG + "OffsetSingle", PKG + "OffsetTwo"}; - /** Whether a class-init cycle deadlocks depends on thread timing, so repeat to make a regression reliable to catch. */ + /** + * Whether a class-init cycle deadlocks depends on thread timing, so repeat to make a regression reliable to catch. + */ private static final int ROUNDS = 20; /** A real init deadlock never resolves; a healthy round finishes in milliseconds, so this bound is generous. */ diff --git a/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java b/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java index 3493da7d2b1..f7dffa9021c 100644 --- a/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java +++ b/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java @@ -35,12 +35,10 @@ public class SparkContextReferenceCountTest { /** - * Two DML executions sharing the JVM-wide singleton spark context (as happens - * with surefire parallel tests, threadCount>1). When the first execution - * finishes and calls close(), the shared context must stay alive because the - * second execution still has in-flight work. Before reference counting, - * close() stopped the context unconditionally, which cancelled the second - * execution's spark job and wedged it until the test watchdog. + * Two DML executions sharing the JVM-wide singleton spark context (as happens with surefire parallel tests, + * threadCount>1). When the first execution finishes and calls close(), the shared context must stay alive + * because the second execution still has in-flight work. Before reference counting, close() stopped the context + * unconditionally, which cancelled the second execution's spark job and wedged it until the test watchdog. */ @Test public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { @@ -64,16 +62,13 @@ public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { // context that B still uses SparkExecutionContext.exitSparkExecution(); ecA.close(); - assertFalse("shared context must stay alive while another execution is active", - sc.sc().isStopped()); - assertEquals("B's job must still run on the live context", - 10L, rdd.reduce(Integer::sum).longValue()); + assertFalse("shared context must stay alive while another execution is active", sc.sc().isStopped()); + assertEquals("B's job must still run on the live context", 10L, rdd.reduce(Integer::sum).longValue()); // B finishes last: releasing the final registration lets close() stop it SparkExecutionContext.exitSparkExecution(); ecB.close(); - assertTrue("shared context must be stopped once the last execution closes", - sc.sc().isStopped()); + assertTrue("shared context must be stopped once the last execution closes", sc.sc().isStopped()); } finally { // drain any remaining registrations and stop the context so a failed @@ -87,10 +82,9 @@ public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { } /** - * An unpaired close() (a caller that borrows the shared context but never - * registered via enterSparkExecution()) must not stop a context another - * execution still uses. This fails on the old unconditional-stop code, which - * tore the context down out from under the active execution. + * An unpaired close() (a caller that borrows the shared context but never registered via enterSparkExecution()) + * must not stop a context another execution still uses. This fails on the old unconditional-stop code, which tore + * the context down out from under the active execution. */ @Test public void unpairedCloseDoesNotStopAContextStillInUse() { @@ -106,14 +100,12 @@ public void unpairedCloseDoesNotStopAContextStillInUse() { // borrows the shared context): close() must not stop a context in use unregistered = ExecutionContextFactory.createSparkExecutionContext(); unregistered.close(); - assertFalse("unpaired close() must not stop a context still in use", - sc.sc().isStopped()); + assertFalse("unpaired close() must not stop a context still in use", sc.sc().isStopped()); // the registered execution finishing stops the context as the last user SparkExecutionContext.exitSparkExecution(); active.close(); - assertTrue("context must stop once the last registered execution closes", - sc.sc().isStopped()); + assertTrue("context must stop once the last registered execution closes", sc.sc().isStopped()); } finally { SparkExecutionContext.exitSparkExecution(); diff --git a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java index 2c854b4a81b..459936fa24c 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java @@ -78,17 +78,18 @@ public MatrixBlock getMatrixBlock(long id) { } /** - * Poll the federated worker until the matrix at {@code id} is observed as a - * {@link CompressedMatrixBlock}, or {@link #COMPRESS_TIMEOUT_MS} elapses. + * Poll the federated worker until the matrix at {@code id} is observed as a {@link CompressedMatrixBlock}, or + * {@link #COMPRESS_TIMEOUT_MS} elapses. * - *

Federated workers compress asynchronously after a PUT/READ_VAR (see - * {@code CompressedMatrixBlockFactory.compressAsync}), so a {@code getMatrixBlock} fired right - * after the operation can race against the in-flight compression and return the uncompressed - * block. Tests that need to observe the compressed form should poll instead of sleeping a fixed - * amount. + *

+ * Federated workers compress asynchronously after a PUT/READ_VAR (see + * {@code CompressedMatrixBlockFactory.compressAsync}), so a {@code getMatrixBlock} fired right after the operation + * can race against the in-flight compression and return the uncompressed block. Tests that need to observe the + * compressed form should poll instead of sleeping a fixed amount. * - *

On timeout this returns the most recent (uncompressed) read so the caller can produce a - * meaningful assertion failure naming the variable. + *

+ * On timeout this returns the most recent (uncompressed) read so the caller can produce a meaningful assertion + * failure naming the variable. * * @param id federated variable id * @return the matrix block, compressed if compression finished in time, otherwise the latest read diff --git a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java index 2b5ff327ef3..d3e4485bdf4 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java @@ -68,13 +68,11 @@ public void verifySameOrAlsoCompressedAsLocalCompress() { // federated. Compression on the worker is async; poll only when we expect compression to // match the local result, otherwise a single read is enough. final long id = putMatrixBlock(mb); - final MatrixBlock mbr = (mbcLocal instanceof CompressedMatrixBlock) - ? awaitCompressed(id) - : getMatrixBlock(id); + final MatrixBlock mbr = (mbcLocal instanceof CompressedMatrixBlock) ? awaitCompressed(id) : getMatrixBlock(id); if(mbcLocal instanceof CompressedMatrixBlock && !(mbr instanceof CompressedMatrixBlock)) - fail("Invalid result, the federated site did not compress the matrix block within " - + COMPRESS_TIMEOUT_MS + "ms"); + fail("Invalid result, the federated site did not compress the matrix block within " + COMPRESS_TIMEOUT_MS + + "ms"); TestUtils.compareMatricesBitAvgDistance(mbcLocal, mbr, 0, 0, "Not equivalent matrix block returned from federated site"); diff --git a/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java b/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java index 60587bf51a2..0de3b6db640 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java @@ -42,7 +42,7 @@ public void test100x100() { @Test public void testDecimalClampsFractionDigits() { - FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); + FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); f.ensureAllocatedColumns(1); f.set(0, 0, 5.244058388023880); // decimal=2 must print exactly two fraction digits, not DecimalFormat's default max of 3 @@ -53,10 +53,10 @@ public void testDecimalClampsFractionDigits() { @Test public void testDecimalPadsAndRounds() { - FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); + FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); f.ensureAllocatedColumns(2); - f.set(0, 0, 22.0); // integer-valued: padded up to the requested digits - f.set(1, 0, 5.244058388023880); // rounded at the last requested digit + f.set(0, 0, 22.0); // integer-valued: padded up to the requested digits + f.set(1, 0, 5.244058388023880); // rounded at the last requested digit String out = DataConverter.toString(f, false, " ", "\n", 2, 1, 4); assertTrue("expected 22.0000 padded: " + out, out.contains("22.0000\n")); assertTrue("expected 5.2441 rounded: " + out, out.contains("5.2441\n")); @@ -64,10 +64,10 @@ public void testDecimalPadsAndRounds() { @Test public void testNegativeDecimalUsesDefaultFormatting() { - FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); + FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); f.ensureAllocatedColumns(2); - f.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained - f.set(1, 0, 5.244058388023880); // default cap of three fraction digits + f.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained + f.set(1, 0, 5.244058388023880); // default cap of three fraction digits // decimal < 0 leaves DecimalFormat unconstrained (no min/max fraction digits set) String out = DataConverter.toString(f, false, " ", "\n", 2, 1, -1); assertTrue("expected unpadded 22: " + out, out.contains("22\n")); diff --git a/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java b/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java index 43a53879f17..7d67c04983b 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java @@ -142,8 +142,7 @@ public void safeCastWarnsOnlyOnce() { // the fallback warning must be logged exactly once across both conversions final List log = LoggingUtils.reinsert(appender); long warnings = log.stream() - .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")) - .count(); + .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")).count(); assertEquals(1, warnings); } @@ -153,8 +152,7 @@ public void strictThrowsWhenWarnCastDisabled() { setWarnCast(false); FrameBlock fb = mixedFrame(); - Exception e = assertThrows(DMLRuntimeException.class, - () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 1)); + Exception e = assertThrows(DMLRuntimeException.class, () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 1)); assertTrue(e.getMessage().contains("Failed to convert FrameBlock to MatrixBlock")); } @@ -164,8 +162,7 @@ public void strictThrowsParallelWhenWarnCastDisabled() { setWarnCast(false); FrameBlock fb = mixedFrame(); - Exception e = assertThrows(DMLRuntimeException.class, - () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 4)); + Exception e = assertThrows(DMLRuntimeException.class, () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 4)); assertTrue(e.getMessage().contains("Failed to convert FrameBlock to MatrixBlock")); } @@ -185,8 +182,7 @@ public void warnCastValidFrameConvertsWithoutFallback() { final List log = LoggingUtils.reinsert(appender); long warnings = log.stream() - .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")) - .count(); + .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")).count(); assertEquals(0, warnings); } diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java index ccba674707b..982941bb190 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java @@ -74,8 +74,8 @@ private void runDecode(String spec, int nCol, int nCat) { try { FrameBlock data = categoricalFrame(ROWS, nCol, nCat, 17); - MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), - data.getNumColumns(), null); + MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), data.getNumColumns(), + null); MatrixBlock encoded = encoder.encode(data, 1); Decoder decoder = buildDecoder(data, spec, encoder); @@ -122,8 +122,8 @@ public void singleThreadEqualsParallelManyCategories() { public void decoderIsComposite() { FrameBlock data = categoricalFrame(100, 2, 3, 1); String spec = "{recode:[C1], dummycode:[C2]}"; - MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), - data.getNumColumns(), null); + MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), data.getNumColumns(), + null); encoder.encode(data, 1); Decoder decoder = buildDecoder(data, spec, encoder); if(!(decoder instanceof DecoderComposite)) diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java index d9c540f54c5..412686c1e83 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java @@ -47,9 +47,9 @@ import org.junit.Test; /** - * Unit tests that drive the get_categorical_mask instruction directly to exercise the defensive code - * paths (distinct-count prefix in the metadata frame, default column metadata, non id-based specs and - * the unsupported opcode guard) that the script-level transform tests cannot reach. + * Unit tests that drive the get_categorical_mask instruction directly to exercise the defensive code paths + * (distinct-count prefix in the metadata frame, default column metadata, non id-based specs and the unsupported opcode + * guard) that the script-level transform tests cannot reach. */ public class GetCategoricalMaskInstructionTest { protected static final Log LOG = LogFactory.getLog(GetCategoricalMaskInstructionTest.class.getName()); @@ -210,7 +210,8 @@ public void imputeAndOmitAreAccepted() { // impute and omit do not change the output column count or categorical flag, so a spec that // only adds them on top of a recoded column must still succeed and mark that column categorical FrameBlock meta = new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}); - MatrixBlock res = run(meta, "{\"ids\": true, \"recode\": [1], \"impute\": [{\"id\": 1, \"method\": \"global_mode\"}], \"omit\": [1]}"); + MatrixBlock res = run(meta, + "{\"ids\": true, \"recode\": [1], \"impute\": [{\"id\": 1, \"method\": \"global_mode\"}], \"omit\": [1]}"); assertEquals(1, res.getNumRows()); assertEquals(1, res.getNumColumns()); @@ -230,8 +231,7 @@ public void unsupportedOpcodeThrows() { // any frame-scalar binary opcode other than get_categorical_mask must be rejected ExecutionContext ec = ExecutionContextFactory.createContext(); ec.setAutoCreateVars(true); - ec.setVariable("F", frameObject(new FrameBlock(new ValueType[] {ValueType.STRING}, - new String[][] {{"a"}}))); + ec.setVariable("F", frameObject(new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}))); assertThrowsMessage("Unsupported operation", () -> maskInstruction("+").processInstruction(ec)); } @@ -263,9 +263,9 @@ private static void assertMask(MatrixBlock res, double[] expected) { } /** - * Build a single-row metadata frame of nCol string columns. A positive distinct[i] is written to - * that column's metadata as the recode distinct count (the path real transformencode uses), while - * a zero leaves the column with default metadata (a continuous / non-dummycoded column). + * Build a single-row metadata frame of nCol string columns. A positive distinct[i] is written to that column's + * metadata as the recode distinct count (the path real transformencode uses), while a zero leaves the column with + * default metadata (a continuous / non-dummycoded column). */ private static FrameBlock metaWithDistinct(int nCol, int[] distinct) { ValueType[] schema = new ValueType[nCol]; @@ -290,7 +290,8 @@ private static MatrixBlock run(FrameBlock meta, String spec) { private static BinaryFrameScalarCPInstruction maskInstruction(String opcode) { String in1 = InstructionUtils.concatOperandParts("F", DataType.FRAME.name(), ValueType.STRING.name(), "false"); - String in2 = InstructionUtils.concatOperandParts("spec", DataType.SCALAR.name(), ValueType.STRING.name(), "true"); + String in2 = InstructionUtils.concatOperandParts("spec", DataType.SCALAR.name(), ValueType.STRING.name(), + "true"); String out = InstructionUtils.concatOperandParts("out", DataType.MATRIX.name(), ValueType.FP64.name(), "false"); String str = InstructionUtils.concatOperands("CP", opcode, in1, in2, out); return (BinaryFrameScalarCPInstruction) BinaryCPInstruction.parseInstruction(str); diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java index b2d31f43b83..645c4dd09bd 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -44,10 +44,10 @@ import org.junit.Test; /** - * Exact inverse correctness tests for the transform decoders. Recode and dummycode are lossless category encodings, so a - * decode of the encoded matrix must reconstruct the original categorical frame. These tests assert exact reconstruction - * for the dense path, the sparse path, and the parallel path so that the dummycode sparse binary search and the parallel - * block split are validated against ground truth rather than only against each other. + * Exact inverse correctness tests for the transform decoders. Recode and dummycode are lossless category encodings, so + * a decode of the encoded matrix must reconstruct the original categorical frame. These tests assert exact + * reconstruction for the dense path, the sparse path, and the parallel path so that the dummycode sparse binary search + * and the parallel block split are validated against ground truth rather than only against each other. */ public class TransformDecodeRoundTripTest { protected static final Log LOG = LogFactory.getLog(TransformDecodeRoundTripTest.class.getName()); @@ -59,9 +59,9 @@ public void setUp() { } private static FrameBlock categoricalFrame() { - final String[] values = new String[] { - "apple", "banana", "apple", "cherry", "banana", "date", "apple", "cherry", "date", "banana", "elderberry", - "apple", "fig", "banana", "cherry", "apple", "date", "fig", "elderberry", "banana"}; + final String[] values = new String[] {"apple", "banana", "apple", "cherry", "banana", "date", "apple", "cherry", + "date", "banana", "elderberry", "apple", "fig", "banana", "cherry", "apple", "date", "fig", "elderberry", + "banana"}; final FrameBlock f = new FrameBlock(new ValueType[] {ValueType.STRING}); f.ensureAllocatedColumns(values.length); for(int i = 0; i < values.length; i++) @@ -117,8 +117,8 @@ public void binWithDummycodeOnOtherColumnConsistency() { /** * Dummycode on an earlier column (1) shifts the bin column (2) to the right in the encoded matrix. The bin decoder - * must walk the dummycode domain sizes to recover the bin column's true source position. This drives the - * non-magic offset branch of the bin source-column mapping. + * must walk the dummycode domain sizes to recover the bin column's true source position. This drives the non-magic + * offset branch of the bin source-column mapping. */ @Test public void binAfterDummycodeOnEarlierColumnConsistency() { @@ -128,9 +128,9 @@ public void binAfterDummycodeOnEarlierColumnConsistency() { } /** - * Same right-shift as above, but the earlier column is feature-hashed before being dummycoded. The hash domain - * size K is stored as a plain integer in the single meta cell, so the bin source-column mapping reads it (instead - * of numDistinct) to compute the offset. + * Same right-shift as above, but the earlier column is feature-hashed before being dummycoded. The hash domain size + * K is stored as a plain integer in the single meta cell, so the bin source-column mapping reads it (instead of + * numDistinct) to compute the offset. */ @Test public void binAfterHashDummycodeOnEarlierColumnConsistency() { @@ -211,10 +211,10 @@ public void binDecodeZeroCodeUsesFirstBinBoundary() { } /** - * Spark broadcasts the decoder to executors via Java serialization without re-running initMetaData, so the - * decoder must round-trip all of its decode state through writeExternal/readExternal. Decode with a freshly - * deserialized decoder and assert it matches the in-memory decode. Covers plain bin and bin-with-dummycode - * (the latter exercises the serialized _srcCols/_dcCols source-column mapping). + * Spark broadcasts the decoder to executors via Java serialization without re-running initMetaData, so the decoder + * must round-trip all of its decode state through writeExternal/readExternal. Decode with a freshly deserialized + * decoder and assert it matches the in-memory decode. Covers plain bin and bin-with-dummycode (the latter exercises + * the serialized _srcCols/_dcCols source-column mapping). */ @Test public void binDecoderSurvivesSerialization() { @@ -481,8 +481,7 @@ public void parallelDecodeWrapsWorkerException() { fail("expected the parallel decode wrapper to propagate the worker failure"); } catch(DMLRuntimeException expected) { - assertNotNull("parallel decode wrapper must retain the worker exception as cause", - expected.getCause()); + assertNotNull("parallel decode wrapper must retain the worker exception as cause", expected.getCause()); } } catch(Exception e) { diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java index 54bd1679716..5745a35d506 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java index 8d7ad14539a..64012c06bbf 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java @@ -57,17 +57,15 @@ import io.delta.kernel.types.TimestampType; /** - * Targeted tests for the error/defensive branches of the native Delta matrix - * read/write code that the round-trip and interop tests do not reach: malformed - * per-file statistics, unsupported column types, unsupported stream operations, + * Targeted tests for the error/defensive branches of the native Delta matrix read/write code that the round-trip and + * interop tests do not reach: malformed per-file statistics, unsupported column types, unsupported stream operations, * bad table paths, and the non-dense writer input path. * - *

A few of these branches guard against inputs that the SystemDS writer and - * the Delta Kernel scan API never produce in a normal round trip (e.g. a - * statistics JSON without {@code numRecords}, or a column type code outside the - * supported set). They are exercised here by mocking the Delta Kernel data - * objects and invoking the (package-private) helpers reflectively, rather than - * widening their production visibility purely for testing. + *

+ * A few of these branches guard against inputs that the SystemDS writer and the Delta Kernel scan API never produce in + * a normal round trip (e.g. a statistics JSON without {@code numRecords}, or a column type code outside the supported + * set). They are exercised here by mocking the Delta Kernel data objects and invoking the (package-private) helpers + * reflectively, rather than widening their production visibility purely for testing. */ public class DeltaMatrixCoverageTest { @@ -89,8 +87,8 @@ public void qualifyRejectsUnknownFilesystemScheme() { @Test public void typeCodeReturnsNegativeForUnsupportedTypes() { - //non-numeric / unsupported Delta types must map to the sentinel -1 so the - //reader can reject them with a clear message rather than mis-decoding. + // non-numeric / unsupported Delta types must map to the sentinel -1 so the + // reader can reject them with a clear message rather than mis-decoding. assertEquals(-1, DeltaKernelUtils.typeCode(DateType.DATE)); assertEquals(-1, DeltaKernelUtils.typeCode(TimestampType.TIMESTAMP)); assertEquals(-1, DeltaKernelUtils.typeCode(BinaryType.BINARY)); @@ -112,17 +110,17 @@ public void writerRejectsStreamWrite() throws Exception { @Test public void numRecordsHandlesAbsentNullAndMalformedStats() throws Exception { - //no "stats" field at all -> -1 + // no "stats" field at all -> -1 assertEquals(-1, numRecords(addFileRow(new StructType().add("path", StringType.STRING), false, null))); - //stats column present but null-at -> -1 + // stats column present but null-at -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), true, null))); - //stats string explicitly null -> -1 + // stats string explicitly null -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, null))); - //malformed JSON -> JsonProcessingException -> -1 + // malformed JSON -> JsonProcessingException -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, "{not valid json"))); - //valid JSON but no numRecords field -> -1 + // valid JSON but no numRecords field -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, "{\"minValues\":{}}"))); - //well-formed stats -> the parsed count + // well-formed stats -> the parsed count assertEquals(1234L, numRecords(addFileRow(statsSchema(), false, "{\"numRecords\":1234}"))); } @@ -131,8 +129,8 @@ public void getDoubleValueRejectsUnknownTypeCode() throws Exception { Method m = ReaderDelta.class.getDeclaredMethod("getDoubleValue", ColumnVector.class, int.class, int.class); m.setAccessible(true); try { - //type code outside the supported T_* set; the switch default must throw - //before touching the (null) vector. + // type code outside the supported T_* set; the switch default must throw + // before touching the (null) vector. m.invoke(null, (ColumnVector) null, 0, 999); fail("expected a DMLRuntimeException for an unsupported type code"); } @@ -156,9 +154,9 @@ public void numericTypeCodeRejectsNonNumericType() throws Exception { @Test public void parallelReadWrapsFileFailure() throws Exception { - //a per-file decode failure in the parallel reader must surface as a single - //clear IOException (the awaitFileTasks catch), not a raw executor error. - //Provoke it by deleting one data file after the table (and its log) exist. + // a per-file decode failure in the parallel reader must surface as a single + // clear IOException (the awaitFileTasks catch), not a raw executor error. + // Provoke it by deleting one data file after the table (and its log) exist. MatrixBlock in = TestUtils.generateTestMatrixBlock(100_000, 8, -10, 10, 1.0, 13); in.recomputeNonZeros(); DMLConfig conf = new DMLConfig(); @@ -167,15 +165,14 @@ public void parallelReadWrapsFileFailure() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_fail_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); - //delete one parquet data file; the transaction log still references it, - //so the scan enumerates it but the decode task fails. + // delete one parquet data file; the transaction log still references it, + // so the scan enumerates it but the decode task fails. File victim; - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { - victim = s.filter(p -> p.toString().endsWith(".parquet")) - .findFirst().map(Path::toFile).orElse(null); + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { + victim = s.filter(p -> p.toString().endsWith(".parquet")).findFirst().map(Path::toFile).orElse(null); } assertTrue("expected at least one data file to delete", victim != null && victim.delete()); @@ -200,8 +197,8 @@ public void parallelReadWrapsFileFailure() throws Exception { @Test public void sparseFormatMatrixRoundTrips() throws Exception { - //a sparse-backed MatrixBlock takes the writer's non-contiguous path (no - //direct double[] view), exercising MatrixColumnVector.get via MatrixBlock. + // a sparse-backed MatrixBlock takes the writer's non-contiguous path (no + // direct double[] view), exercising MatrixColumnVector.get via MatrixBlock. MatrixBlock in = TestUtils.generateTestMatrixBlock(2000, 7, -5, 5, 0.05, 13); in.recomputeNonZeros(); in.examSparsity(); @@ -210,8 +207,8 @@ public void sparseFormatMatrixRoundTrips() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_sparse_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", in.getNumRows(), out.getNumRows()); assertEquals("cols", in.getNumColumns(), out.getNumColumns()); @@ -224,15 +221,15 @@ public void sparseFormatMatrixRoundTrips() throws Exception { @Test public void fillDenseHandlesNonContiguousBlock() throws Exception { - //the dense fill normally hits the contiguous fast path; force a multi-block - //(non-contiguous) dense block so the row-by-row fallback is exercised. Such - //blocks only arise for matrices beyond a single contiguous array, so we - //shrink the per-block allocation cap to provoke it on a tiny matrix. + // the dense fill normally hits the contiguous fast path; force a multi-block + // (non-contiguous) dense block so the row-by-row fallback is exercised. Such + // blocks only arise for matrices beyond a single contiguous array, so we + // shrink the per-block allocation cap to provoke it on a tiny matrix. int rows = 5, cols = 4; int savedMaxAlloc = DenseBlockLDRB.MAX_ALLOC; DenseBlock db; try { - DenseBlockLDRB.MAX_ALLOC = 2 * cols; //~2 rows per block -> multiple blocks + DenseBlockLDRB.MAX_ALLOC = 2 * cols; // ~2 rows per block -> multiple blocks db = new DenseBlockLFP64(new int[] {rows, cols}); } finally { @@ -240,14 +237,14 @@ public void fillDenseHandlesNonContiguousBlock() throws Exception { } assertTrue("expected a non-contiguous (multi-block) dense block", !db.isContiguous()); - //two row-major batches (3 rows + 2 rows) covering all 5 rows + // two row-major batches (3 rows + 2 rows) covering all 5 rows double[] b0 = new double[3 * cols]; double[] b1 = new double[2 * cols]; - for( int r = 0; r < 3; r++ ) - for( int c = 0; c < cols; c++ ) + for(int r = 0; r < 3; r++) + for(int c = 0; c < cols; c++) b0[r * cols + c] = cell(r, c); - for( int r = 0; r < 2; r++ ) - for( int c = 0; c < cols; c++ ) + for(int r = 0; r < 2; r++) + for(int c = 0; c < cols; c++) b1[r * cols + c] = cell(3 + r, c); java.util.ArrayList batches = new java.util.ArrayList<>(); batches.add(b0); @@ -258,8 +255,8 @@ public void fillDenseHandlesNonContiguousBlock() throws Exception { m.setAccessible(true); m.invoke(null, ret, batches); - for( int r = 0; r < rows; r++ ) - for( int c = 0; c < cols; c++ ) + for(int r = 0; r < rows; r++) + for(int c = 0; c < cols; c++) assertEquals("r" + r + " c" + c, cell(r, c), ret.getDenseBlock().get(r, c), 0.0); } @@ -276,8 +273,8 @@ private static StructType statsSchema() { } /** - * Build a mocked scan-file row whose AddFile child has the given schema, null - * flag and (when not null) stats string, matching what {@code numRecords} reads. + * Build a mocked scan-file row whose AddFile child has the given schema, null flag and (when not null) stats + * string, matching what {@code numRecords} reads. */ private static Row addFileRow(StructType addSchema, boolean statsNull, String statsValue) { Row outer = mock(Row.class); @@ -285,12 +282,12 @@ private static Row addFileRow(StructType addSchema, boolean statsNull, String st when(outer.getStruct(InternalScanFileUtils.ADD_FILE_ORDINAL)).thenReturn(add); when(add.getSchema()).thenReturn(addSchema); int statsOrd = addSchema.fieldNames().indexOf("stats"); - if( statsOrd >= 0 ) { + if(statsOrd >= 0) { when(add.isNullAt(statsOrd)).thenReturn(statsNull); - if( !statsNull ) + if(!statsNull) when(add.getString(statsOrd)).thenReturn(statsValue); } - return outer; //the scan-file row numRecords consumes (its AddFile child is 'add') + return outer; // the scan-file row numRecords consumes (its AddFile child is 'add') } private static long numRecords(Row scanFileRow) throws Exception { diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java index 54a3bcf6334..65c4ec21c0f 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java @@ -63,20 +63,19 @@ import io.delta.kernel.utils.CloseableIterator; /** - * Direct (no DML) round-trip tests for the native Delta Kernel based matrix - * reader/writer. Each test writes a MatrixBlock to a fresh local Delta table - * directory and reads it back, asserting dimensions and values match. + * Direct (no DML) round-trip tests for the native Delta Kernel based matrix reader/writer. Each test writes a + * MatrixBlock to a fresh local Delta table directory and reads it back, asserting dimensions and values match. */ public class DeltaMatrixReadWriteTest { - //small writer target file size (bytes) used to force a multi-file table - //layout cheaply, instead of brute-forcing huge row counts. + // small writer target file size (bytes) used to force a multi-file table + // layout cheaply, instead of brute-forcing huge row counts. private static final long SMALL_TARGET_FILE_SIZE = 256L * 1024; private static final int ROWS_MULTI_FILE = 100_000; private static MatrixBlock writeThenRead(MatrixBlock in) throws Exception { Path dir = Files.createTempDirectory("sysds_delta_"); - //WriterDelta creates the table at the given (empty) directory + // WriterDelta creates the table at the given (empty) directory String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { WriterDelta writer = new WriterDelta(); @@ -92,9 +91,9 @@ private static MatrixBlock writeThenRead(MatrixBlock in) throws Exception { @Test public void parallelReadMatchesSerialMultiFile() throws Exception { - //force a multi-file table cheaply via a small writer target file size - //(rather than a huge row count), so the parallel per-file path is - //actually exercised rather than falling back to serial. + // force a multi-file table cheaply via a small writer target file size + // (rather than a huge row count), so the parallel per-file path is + // actually exercised rather than falling back to serial. MatrixBlock in = TestUtils.generateTestMatrixBlock(ROWS_MULTI_FILE, 8, -10, 10, 1.0, 13); in.recomputeNonZeros(); @@ -104,21 +103,18 @@ public void parallelReadMatchesSerialMultiFile() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_par_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); - //sanity: confirm the table really is split across multiple files + // sanity: confirm the table really is split across multiple files long files; - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } - assertTrue("expected a multi-file Delta table to exercise the parallel path, got " + files, - files > 1); + assertTrue("expected a multi-file Delta table to exercise the parallel path, got " + files, files > 1); - MatrixBlock serial = new ReaderDelta() - .readMatrixFromHDFS(tablePath, -1, -1, -1, -1); - MatrixBlock parallel = new ReaderDeltaParallel() - .readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + MatrixBlock serial = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + MatrixBlock parallel = new ReaderDeltaParallel().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", serial.getNumRows(), parallel.getNumRows()); assertEquals("cols", serial.getNumColumns(), parallel.getNumColumns()); @@ -135,8 +131,8 @@ public void parallelReadMatchesSerialMultiFile() throws Exception { public void roundTripDenseSmall() throws Exception { MatrixBlock in = new MatrixBlock(3, 4, false); double v = 1.0; - for( int i=0; i<3; i++ ) - for( int j=0; j<4; j++ ) + for(int i = 0; i < 3; i++) + for(int j = 0; j < 4; j++) in.set(i, j, v++); in.recomputeNonZeros(); @@ -157,7 +153,7 @@ public void roundTripDenseRandom() throws Exception { @Test public void roundTripSparseRandom() throws Exception { - //values written are dense parquet, but exercise a sparse-ish input + // values written are dense parquet, but exercise a sparse-ish input MatrixBlock in = TestUtils.generateTestMatrixBlock(1200, 9, -5, 5, 0.1, 13); in.recomputeNonZeros(); MatrixBlock out = writeThenRead(in); @@ -168,7 +164,7 @@ public void roundTripSparseRandom() throws Exception { @Test public void roundTripMultiBatch() throws Exception { - //more rows than the writer batch size (4096) to exercise chunking + // more rows than the writer batch size (4096) to exercise chunking MatrixBlock in = TestUtils.generateTestMatrixBlock(10000, 5, 0, 100, 1.0, 1); MatrixBlock out = writeThenRead(in); assertEquals("rows", 10000, out.getNumRows()); @@ -182,8 +178,9 @@ public void readDiscoversUnknownDimensions() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); - //pass -1 dimensions: the reader must discover them from the table + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); + // pass -1 dimensions: the reader must discover them from the table MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 123, out.getNumRows()); assertEquals("cols", 6, out.getNumColumns()); @@ -212,22 +209,21 @@ public void emptyMatrixRoundTrip() throws Exception { @Test public void readNonDoubleNumericColumns() throws Exception { - //tables produced by external tools (or the frame writer) can carry - //long/int/boolean columns; the matrix reader must coerce them to double + // tables produced by external tools (or the frame writer) can carry + // long/int/boolean columns; the matrix reader must coerce them to double Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] longVals = {1, -2, 1_000_000_000L, 0}; - double[] intVals = {7, -8, 123456, 0}; + double[] intVals = {7, -8, 123456, 0}; double[] boolVals = {1, 0, 1, 0}; - writeTypedColumns(tablePath, - new DataType[] {LongType.LONG, IntegerType.INTEGER, BooleanType.BOOLEAN}, + writeTypedColumns(tablePath, new DataType[] {LongType.LONG, IntegerType.INTEGER, BooleanType.BOOLEAN}, new double[][] {longVals, intVals, boolVals}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 3, out.getNumColumns()); - for( int r=0; r<4; r++ ) { + for(int r = 0; r < 4; r++) { assertEquals("long col r" + r, longVals[r], out.get(r, 0), 0.0); assertEquals("int col r" + r, intVals[r], out.get(r, 1), 0.0); assertEquals("bool col r" + r, boolVals[r], out.get(r, 2), 0.0); @@ -240,14 +236,14 @@ public void readNonDoubleNumericColumns() throws Exception { @Test public void rewriteSamePathReplacesData() throws Exception { - //writing to a path that already holds a Delta table must fully replace it + // writing to a path that already holds a Delta table must fully replace it Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { MatrixBlock first = TestUtils.generateTestMatrixBlock(50, 8, 0, 100, 1.0, 1); new WriterDelta().writeMatrixToHDFS(first, tablePath, 50, 8, -1, first.getNonZeros()); - //second write has different dimensions and values + // second write has different dimensions and values MatrixBlock second = TestUtils.generateTestMatrixBlock(20, 3, -5, 5, 1.0, 2); new WriterDelta().writeMatrixToHDFS(second, tablePath, 20, 3, -1, second.getNonZeros()); @@ -263,10 +259,10 @@ public void rewriteSamePathReplacesData() throws Exception { @Test public void parallelBufferedPathMatchesSerial() throws Exception { - //the direct fast path is always taken for SystemDS-written tables (exact - //row stats, no deletion vectors); force the buffered fallback to exercise - //its per-file decode + serial concatenation and assert it matches serial. - //force a multi-file table cheaply via a small writer target file size. + // the direct fast path is always taken for SystemDS-written tables (exact + // row stats, no deletion vectors); force the buffered fallback to exercise + // its per-file decode + serial concatenation and assert it matches serial. + // force a multi-file table cheaply via a small writer target file size. MatrixBlock in = TestUtils.generateTestMatrixBlock(ROWS_MULTI_FILE, 8, -10, 10, 1.0, 23); in.recomputeNonZeros(); @@ -276,19 +272,22 @@ public void parallelBufferedPathMatchesSerial() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_buf_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); long files; - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } assertTrue("expected a multi-file Delta table, got " + files, files > 1); MatrixBlock serial = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); - //subclass that always declines the direct path -> readBuffered() + // subclass that always declines the direct path -> readBuffered() MatrixBlock buffered = new ReaderDeltaParallel() { - @Override protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { return false; } + @Override + protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { + return false; + } }.readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", serial.getNumRows(), buffered.getNumRows()); @@ -304,30 +303,30 @@ public void parallelBufferedPathMatchesSerial() throws Exception { @Test public void writerTargetFileSizeConfigProducesMoreFiles() throws Exception { - //a smaller configured target file size must make the writer roll more - //data files for the same matrix (the lever the parallel reader relies on). + // a smaller configured target file size must make the writer roll more + // data files for the same matrix (the lever the parallel reader relies on). MatrixBlock in = TestUtils.generateTestMatrixBlock(400_000, 16, -10, 10, 1.0, 7); in.recomputeNonZeros(); - //isolate the override in a fresh thread-local config (restored in finally) + // isolate the override in a fresh thread-local config (restored in finally) DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(1L * 1024 * 1024)); ConfigurationManager.setLocalConfig(conf); Path dir = Files.createTempDirectory("sysds_delta_cfg_"); try { - assertEquals("config getter reflects the override", - 1L * 1024 * 1024, ConfigurationManager.getDeltaWriterTargetFileSize()); + assertEquals("config getter reflects the override", 1L * 1024 * 1024, + ConfigurationManager.getDeltaWriterTargetFileSize()); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); long files; - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } assertTrue("expected >1 data file with a 1MB target, got " + files, files > 1); - //data still round-trips correctly with the custom layout + // data still round-trips correctly with the custom layout MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); TestUtils.compareMatrices(in, out, 1e-12, "small-target-roundtrip"); } @@ -339,21 +338,20 @@ public void writerTargetFileSizeConfigProducesMoreFiles() throws Exception { @Test public void readerBatchSizeConfigRoundTrips() throws Exception { - //a non-default reader batch size must not change the result (more, smaller - //batches exercise the per-batch extract/concatenate loop more often). + // a non-default reader batch size must not change the result (more, smaller + // batches exercise the per-batch extract/concatenate loop more often). MatrixBlock in = TestUtils.generateTestMatrixBlock(5000, 7, -10, 10, 1.0, 11); - //isolate the override in a fresh thread-local config (restored in finally) + // isolate the override in a fresh thread-local config (restored in finally) DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_READER_BATCH_SIZE, "128"); ConfigurationManager.setLocalConfig(conf); Path dir = Files.createTempDirectory("sysds_delta_bs_"); try { - assertEquals("config getter reflects the override", - 128, ConfigurationManager.getDeltaReaderBatchSize()); + assertEquals("config getter reflects the override", 128, ConfigurationManager.getDeltaReaderBatchSize()); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); TestUtils.compareMatrices(in, out, 1e-12, "small-batch-roundtrip"); } @@ -365,14 +363,13 @@ public void readerBatchSizeConfigRoundTrips() throws Exception { @Test public void factoryRoutesDeltaToParallelWhenEnabled() { - //the factory must pick the parallel reader iff parallel CP read is enabled + // the factory must pick the parallel reader iff parallel CP read is enabled CompilerConfig cc = ConfigurationManager.getCompilerConfig(); try { cc.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, true); ConfigurationManager.setLocalConfig(cc); MatrixReader par = MatrixReaderFactory.createMatrixReader(FileFormat.DELTA); - assertTrue("expected ReaderDeltaParallel when parallel read enabled", - par instanceof ReaderDeltaParallel); + assertTrue("expected ReaderDeltaParallel when parallel read enabled", par instanceof ReaderDeltaParallel); cc.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, false); ConfigurationManager.setLocalConfig(cc); @@ -387,20 +384,18 @@ public void factoryRoutesDeltaToParallelWhenEnabled() { @Test public void readFloatColumnsCoercedToDouble() throws Exception { - //float columns must be widened to double on read (exact-representable values) + // float columns must be widened to double on read (exact-representable values) Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] f0 = {1.5, -2.25, 0.0, 1024.5}; double[] f1 = {-0.5, 3.75, 100.125, -7.0}; - writeTypedColumns(tablePath, - new DataType[] {FloatType.FLOAT, FloatType.FLOAT}, - new double[][] {f0, f1}); + writeTypedColumns(tablePath, new DataType[] {FloatType.FLOAT, FloatType.FLOAT}, new double[][] {f0, f1}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 2, out.getNumColumns()); - for( int r=0; r<4; r++ ) { + for(int r = 0; r < 4; r++) { assertEquals("f0 r" + r, f0[r], out.get(r, 0), 0.0); assertEquals("f1 r" + r, f1[r], out.get(r, 1), 0.0); } @@ -412,21 +407,20 @@ public void readFloatColumnsCoercedToDouble() throws Exception { @Test public void readShortByteColumnsCoercedToDouble() throws Exception { - //short/byte columns must be coerced to double on read, exercising the - //T_SHORT / T_BYTE branches of ReaderDelta.getDoubleValue. + // short/byte columns must be coerced to double on read, exercising the + // T_SHORT / T_BYTE branches of ReaderDelta.getDoubleValue. Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] shortVals = {1, -2, 30000, 0}; - double[] byteVals = {7, -8, 120, 0}; - writeTypedColumns(tablePath, - new DataType[] {ShortType.SHORT, ByteType.BYTE}, + double[] byteVals = {7, -8, 120, 0}; + writeTypedColumns(tablePath, new DataType[] {ShortType.SHORT, ByteType.BYTE}, new double[][] {shortVals, byteVals}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 2, out.getNumColumns()); - for( int r=0; r<4; r++ ) { + for(int r = 0; r < 4; r++) { assertEquals("short col r" + r, shortVals[r], out.get(r, 0), 0.0); assertEquals("byte col r" + r, byteVals[r], out.get(r, 1), 0.0); } @@ -438,8 +432,8 @@ public void readShortByteColumnsCoercedToDouble() throws Exception { @Test public void writerRejectsDimensionMismatch() throws Exception { - //WriterDelta validates that the passed rlen/clen match the MatrixBlock - //and rejects a mismatch with an IOException. + // WriterDelta validates that the passed rlen/clen match the MatrixBlock + // and rejects a mismatch with an IOException. MatrixBlock in = TestUtils.generateTestMatrixBlock(10, 4, -1, 1, 1.0, 5); in.recomputeNonZeros(); Path dir = Files.createTempDirectory("sysds_delta_"); @@ -459,18 +453,18 @@ public void writerRejectsDimensionMismatch() throws Exception { @Test public void readNullCellsBecomeZero() throws Exception { - //nullable numeric columns with null cells must read back as 0.0 + // nullable numeric columns with null cells must read back as 0.0 Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - double[] vals = {3.0, 7.0, 9.0, 11.0}; - boolean[] nulls = {false, true, false, true}; + double[] vals = {3.0, 7.0, 9.0, 11.0}; + boolean[] nulls = {false, true, false, true}; writeNullableDoubleColumn(tablePath, vals, nulls); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 1, out.getNumColumns()); - for( int r=0; r<4; r++ ) + for(int r = 0; r < 4; r++) assertEquals("r" + r, nulls[r] ? 0.0 : vals[r], out.get(r, 0), 0.0); } finally { @@ -480,7 +474,7 @@ public void readNullCellsBecomeZero() throws Exception { @Test public void readStringColumnRejected() throws Exception { - //string columns cannot back an all-double matrix -> reader must reject them + // string columns cannot back an all-double matrix -> reader must reject them Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { @@ -503,9 +497,10 @@ public void readStringColumnRejected() throws Exception { private static void writeTypedColumns(String tablePath, DataType[] types, double[][] vals) throws Exception { Engine engine = DeltaKernelUtils.createEngine(); StructType schema = new StructType(); - for( int c=0; c singleton(FilteredColumnarBatch fcb) { return new CloseableIterator() { private boolean _done = false; - @Override public boolean hasNext() { return !_done; } - @Override public FilteredColumnarBatch next() { - if( _done ) throw new NoSuchElementException(); + + @Override + public boolean hasNext() { + return !_done; + } + + @Override + public FilteredColumnarBatch next() { + if(_done) + throw new NoSuchElementException(); _done = true; return fcb; } - @Override public void close() {} + + @Override + public void close() { + } }; } - /** Minimal in-memory columnar batch backed by per-column double[] values, with - * an optional per-column null mask ({@code nulls==null} => no nulls). */ + /** + * Minimal in-memory columnar batch backed by per-column double[] values, with an optional per-column null mask + * ({@code nulls==null} => no nulls). + */ private static class TypedBatch implements ColumnarBatch { private final StructType _schema; private final DataType[] _types; private final double[][] _vals; private final boolean[][] _nulls; + TypedBatch(StructType schema, DataType[] types, double[][] vals, boolean[][] nulls) { - _schema = schema; _types = types; _vals = vals; _nulls = nulls; + _schema = schema; + _types = types; + _vals = vals; + _nulls = nulls; } - @Override public StructType getSchema() { return _schema; } - @Override public int getSize() { return _vals[0].length; } - @Override public ColumnVector getColumnVector(int ordinal) { - return new TypedVector(_types[ordinal], _vals[ordinal], - _nulls == null ? null : _nulls[ordinal]); + + @Override + public StructType getSchema() { + return _schema; + } + + @Override + public int getSize() { + return _vals[0].length; + } + + @Override + public ColumnVector getColumnVector(int ordinal) { + return new TypedVector(_types[ordinal], _vals[ordinal], _nulls == null ? null : _nulls[ordinal]); } } @@ -568,28 +599,98 @@ private static class TypedVector implements ColumnVector { private final DataType _type; private final double[] _vals; private final boolean[] _nulls; - TypedVector(DataType type, double[] vals, boolean[] nulls) { _type = type; _vals = vals; _nulls = nulls; } - @Override public DataType getDataType() { return _type; } - @Override public int getSize() { return _vals.length; } - @Override public boolean isNullAt(int rowId) { return _nulls != null && _nulls[rowId]; } - @Override public double getDouble(int rowId) { return _vals[rowId]; } - @Override public float getFloat(int rowId) { return (float) _vals[rowId]; } - @Override public long getLong(int rowId) { return (long) _vals[rowId]; } - @Override public int getInt(int rowId) { return (int) _vals[rowId]; } - @Override public short getShort(int rowId) { return (short) _vals[rowId]; } - @Override public byte getByte(int rowId) { return (byte) _vals[rowId]; } - @Override public boolean getBoolean(int rowId) { return _vals[rowId] != 0; } - @Override public void close() {} + + TypedVector(DataType type, double[] vals, boolean[] nulls) { + _type = type; + _vals = vals; + _nulls = nulls; + } + + @Override + public DataType getDataType() { + return _type; + } + + @Override + public int getSize() { + return _vals.length; + } + + @Override + public boolean isNullAt(int rowId) { + return _nulls != null && _nulls[rowId]; + } + + @Override + public double getDouble(int rowId) { + return _vals[rowId]; + } + + @Override + public float getFloat(int rowId) { + return (float) _vals[rowId]; + } + + @Override + public long getLong(int rowId) { + return (long) _vals[rowId]; + } + + @Override + public int getInt(int rowId) { + return (int) _vals[rowId]; + } + + @Override + public short getShort(int rowId) { + return (short) _vals[rowId]; + } + + @Override + public byte getByte(int rowId) { + return (byte) _vals[rowId]; + } + + @Override + public boolean getBoolean(int rowId) { + return _vals[rowId] != 0; + } + + @Override + public void close() { + } } /** Column view exposing a String[] as a Delta string column. */ private static class StringVector implements ColumnVector { private final String[] _vals; - StringVector(String[] vals) { _vals = vals; } - @Override public DataType getDataType() { return StringType.STRING; } - @Override public int getSize() { return _vals.length; } - @Override public boolean isNullAt(int rowId) { return _vals[rowId] == null; } - @Override public String getString(int rowId) { return _vals[rowId]; } - @Override public void close() {} + + StringVector(String[] vals) { + _vals = vals; + } + + @Override + public DataType getDataType() { + return StringType.STRING; + } + + @Override + public int getSize() { + return _vals.length; + } + + @Override + public boolean isNullAt(int rowId) { + return _vals[rowId] == null; + } + + @Override + public String getString(int rowId) { + return _vals[rowId]; + } + + @Override + public void close() { + } } } diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java index 2d79b79f2dd..45194d12b5a 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java @@ -49,24 +49,22 @@ import org.junit.Test; /** - * Cross-engine interoperability tests for the native (Delta Kernel based) matrix - * reader/writer against the reference Delta implementation (Delta's Spark - * connector, {@code delta-spark}, pulled in test-only). + * Cross-engine interoperability tests for the native (Delta Kernel based) matrix reader/writer against the reference + * Delta implementation (Delta's Spark connector, {@code delta-spark}, pulled in test-only). * - *

The other Delta matrix tests round-trip exclusively through SystemDS' own - * Kernel-based read/write paths, so they cannot catch a table that SystemDS - * writes in a way other Delta engines reject (or vice versa). These tests close - * that gap by routing data through two independent engines: + *

+ * The other Delta matrix tests round-trip exclusively through SystemDS' own Kernel-based read/write paths, so they + * cannot catch a table that SystemDS writes in a way other Delta engines reject (or vice versa). These tests close that + * gap by routing data through two independent engines: *

    - *
  • SystemDS writes -> Spark/Delta reads (our output is spec-compliant), and
    • - *
  • Spark/Delta writes -> SystemDS reads, including a multi-file layout and a - * table with deletion vectors / a second commit that the SystemDS writer - * never produces itself.
    • + *
  • SystemDS writes -> Spark/Delta reads (our output is spec-compliant), and
    • + *
  • Spark/Delta writes -> SystemDS reads, including a multi-file layout and a table with deletion vectors / a + * second commit that the SystemDS writer never produces itself.
    • *
* - *

Row order is never assumed: every table carries a unique id in column 0 and - * comparisons are keyed by that id, since neither engine guarantees row order - * across files. + *

+ * Row order is never assumed: every table carries a unique id in column 0 and comparisons are keyed by that id, since + * neither engine guarantees row order across files. */ @net.jcip.annotations.NotThreadSafe public class DeltaMatrixSparkInteropTest { @@ -75,23 +73,19 @@ public class DeltaMatrixSparkInteropTest { @BeforeClass public static void startSpark() { - //each test class runs in its own fork (surefire reuseForks=false), so this - //is the only SparkSession in the JVM and gets the Delta extensions injected. + // each test class runs in its own fork (surefire reuseForks=false), so this + // is the only SparkSession in the JVM and gets the Delta extensions injected. SparkSession.clearActiveSession(); SparkSession.clearDefaultSession(); - spark = SparkSession.builder() - .appName("sysds-delta-interop") - .master("local[2]") - .config("spark.ui.enabled", "false") - .config("spark.sql.shuffle.partitions", "2") + spark = SparkSession.builder().appName("sysds-delta-interop").master("local[2]") + .config("spark.ui.enabled", "false").config("spark.sql.shuffle.partitions", "2") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") - .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") - .getOrCreate(); + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog").getOrCreate(); } @AfterClass public static void stopSpark() { - if( spark != null ) + if(spark != null) spark.stop(); SparkSession.clearActiveSession(); SparkSession.clearDefaultSession(); @@ -100,13 +94,13 @@ public static void stopSpark() { @Test public void systemdsWriteSparkReadMultiFile() throws Exception { - //SystemDS writes a (forced) multi-file Delta table; the reference Delta - //engine (Spark) must read every data file back with matching values. + // SystemDS writes a (forced) multi-file Delta table; the reference Delta + // engine (Spark) must read every data file back with matching values. int rows = 500, cols = 5; MatrixBlock in = indexedMatrix(rows, cols); - //small target file size -> multiple parquet data files (exercise that an - //external reader stitches all of our data files, not just the first). + // small target file size -> multiple parquet data files (exercise that an + // external reader stitches all of our data files, not just the first). DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(16L * 1024)); ConfigurationManager.setLocalConfig(conf); @@ -122,10 +116,10 @@ public void systemdsWriteSparkReadMultiFile() throws Exception { List read = df.collectAsList(); assertEquals(rows, read.size()); - for( Row r : read ) { + for(Row r : read) { int id = (int) Math.round(r.getDouble(0)); assertTrue("id in range: " + id, id >= 0 && id < rows); - for( int c = 0; c < cols; c++ ) + for(int c = 0; c < cols; c++) assertEquals("r" + id + " c" + c, in.get(id, c), r.getDouble(c), 1e-9); } } @@ -137,10 +131,10 @@ public void systemdsWriteSparkReadMultiFile() throws Exception { @Test public void sparkWriteSystemdsReadMultiFile() throws Exception { - //the reference Delta engine writes a multi-file table; both the serial and - //parallel SystemDS readers must reconstruct it (coercing long ids to double). + // the reference Delta engine writes a multi-file table; both the serial and + // parallel SystemDS readers must reconstruct it (coercing long ids to double). int rows = 600, cols = 4; - Dataset df = indexedDataFrame(rows, cols).repartition(3); //-> multiple data files + Dataset df = indexedDataFrame(rows, cols).repartition(3); // -> multiple data files Path dir = Files.createTempDirectory("sysds_delta_p2s_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { @@ -148,10 +142,10 @@ public void sparkWriteSystemdsReadMultiFile() throws Exception { assertTrue("spark should have written a multi-file table", countParquet(tablePath) > 1); Map expected = expectedById(rows, cols); - assertMatchesById(new ReaderDelta() - .readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, "serial"); - assertMatchesById(new ReaderDeltaParallel() - .readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, "parallel"); + assertMatchesById(new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, + "serial"); + assertMatchesById(new ReaderDeltaParallel().readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, + "parallel"); } finally { FileUtils.deleteQuietly(dir.toFile()); @@ -160,15 +154,15 @@ public void sparkWriteSystemdsReadMultiFile() throws Exception { @Test public void sparkDeletionVectorsSystemdsRead() throws Exception { - //a Delta table with deletion vectors + a second commit (the DELETE) is a - //layout the SystemDS writer never emits; the readers must honor the DV and - //return only the surviving rows. This exercises the hasDeletionVector path. + // a Delta table with deletion vectors + a second commit (the DELETE) is a + // layout the SystemDS writer never emits; the readers must honor the DV and + // return only the surviving rows. This exercises the hasDeletionVector path. int rows = 400, cols = 3, deleteBelow = 50; Path dir = Files.createTempDirectory("sysds_delta_dv_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - //enable deletion vectors for tables created in this block, then delete a - //row range so Delta records a DV rather than rewriting the data files. + // enable deletion vectors for tables created in this block, then delete a + // row range so Delta records a DV rather than rewriting the data files. spark.conf().set(DV_DEFAULT, "true"); indexedDataFrame(rows, cols).write().format("delta").save(tablePath); spark.sql("DELETE FROM delta.`" + tablePath + "` WHERE c0 < " + deleteBelow); @@ -185,21 +179,20 @@ public void sparkDeletionVectorsSystemdsRead() throws Exception { assertMatchesById(parallel, expected, cols, "parallel-dv"); } finally { - //fresh fork per test class, so simply clearing the override is enough + // fresh fork per test class, so simply clearing the override is enough spark.conf().unset(DV_DEFAULT); FileUtils.deleteQuietly(dir.toFile()); } } - private static final String DV_DEFAULT = - "spark.databricks.delta.properties.defaults.enableDeletionVectors"; + private static final String DV_DEFAULT = "spark.databricks.delta.properties.defaults.enableDeletionVectors"; /** Matrix whose column 0 is the row index and remaining columns are exact doubles. */ private static MatrixBlock indexedMatrix(int rows, int cols) { MatrixBlock mb = new MatrixBlock(rows, cols, false); - for( int r = 0; r < rows; r++ ) { + for(int r = 0; r < rows; r++) { mb.set(r, 0, r); - for( int c = 1; c < cols; c++ ) + for(int c = 1; c < cols; c++) mb.set(r, c, value(r, c)); } mb.recomputeNonZeros(); @@ -209,15 +202,15 @@ private static MatrixBlock indexedMatrix(int rows, int cols) { /** Spark DataFrame mirroring {@link #indexedMatrix} with columns c0..c(cols-1) as doubles. */ private static Dataset indexedDataFrame(int rows, int cols) { StructField[] fields = new StructField[cols]; - for( int c = 0; c < cols; c++ ) + for(int c = 0; c < cols; c++) fields[c] = DataTypes.createStructField("c" + c, DataTypes.DoubleType, false); StructType schema = DataTypes.createStructType(fields); List data = new ArrayList<>(rows); - for( int r = 0; r < rows; r++ ) { + for(int r = 0; r < rows; r++) { Object[] vals = new Object[cols]; vals[0] = (double) r; - for( int c = 1; c < cols; c++ ) + for(int c = 1; c < cols; c++) vals[c] = value(r, c); data.add(RowFactory.create(vals)); } @@ -231,10 +224,10 @@ private static double value(int row, int col) { private static Map expectedById(int rows, int cols) { Map exp = new HashMap<>(rows); - for( int r = 0; r < rows; r++ ) { + for(int r = 0; r < rows; r++) { double[] row = new double[cols]; row[0] = r; - for( int c = 1; c < cols; c++ ) + for(int c = 1; c < cols; c++) row[c] = value(r, c); exp.put(r, row); } @@ -246,25 +239,25 @@ private static void assertMatchesById(MatrixBlock out, Map ex assertEquals(tag + " rows", expected.size(), out.getNumRows()); assertEquals(tag + " cols", cols, out.getNumColumns()); boolean[] seen = new boolean[expected.size() == 0 ? 0 : maxId(expected) + 1]; - for( int r = 0; r < out.getNumRows(); r++ ) { + for(int r = 0; r < out.getNumRows(); r++) { int id = (int) Math.round(out.get(r, 0)); double[] exp = expected.get(id); assertTrue(tag + ": unexpected/duplicate id " + id, exp != null && id < seen.length && !seen[id]); seen[id] = true; - for( int c = 0; c < cols; c++ ) + for(int c = 0; c < cols; c++) assertEquals(tag + " id" + id + " c" + c, exp[c], out.get(r, c), 1e-9); } } private static int maxId(Map expected) { int m = 0; - for( int id : expected.keySet() ) + for(int id : expected.keySet()) m = Math.max(m, id); return m; } private static long countParquet(String tablePath) throws Exception { - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { return s.filter(p -> p.toString().endsWith(".parquet")).count(); } } diff --git a/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java b/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java index 472b61d8cd6..ae19b291882 100644 --- a/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java +++ b/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -26,11 +26,10 @@ /** * Tests the single-column (unweighted) branch of {@link MatrixBlock#pickValue(double, boolean)} and - * {@link MatrixBlock#median()}. The values are assumed to be sorted in ascending order, mirroring the contract used - * by the quantile pick instructions. The unweighted branch uses the same ceil-based rank as the two-column weighted - * branch (with an implicit weight of 1 per value), so a single column yields the same quantile as the equivalent - * (value, weight) representation. The two-column (weighted) branch is exercised separately through the compressed - * sort tests. + * {@link MatrixBlock#median()}. The values are assumed to be sorted in ascending order, mirroring the contract used by + * the quantile pick instructions. The unweighted branch uses the same ceil-based rank as the two-column weighted branch + * (with an implicit weight of 1 per value), so a single column yields the same quantile as the equivalent (value, + * weight) representation. The two-column (weighted) branch is exercised separately through the compressed sort tests. */ public class QuantilePickTest { diff --git a/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java b/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java index 5c9ed821e78..05aca41ebc7 100644 --- a/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java +++ b/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java @@ -30,7 +30,7 @@ public class TensorToStringTest { @Test public void testDecimalClampsFractionDigits() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 1}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 1}); tb.allocateBlock(); tb.set(0, 0, 5.244058388023880); // decimal=2 must print exactly two fraction digits, not DecimalFormat's default max of 3 @@ -41,10 +41,10 @@ public void testDecimalClampsFractionDigits() { @Test public void testDecimalPadsAndRounds() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 2}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 2}); tb.allocateBlock(); - tb.set(0, 0, 22.0); // integer-valued: padded up to the requested digits - tb.set(0, 1, 5.244058388023880); // rounded at the last requested digit + tb.set(0, 0, 22.0); // integer-valued: padded up to the requested digits + tb.set(0, 1, 5.244058388023880); // rounded at the last requested digit String out = DataConverter.toString(tb, false, " ", "\n", "[", "]", 1, 2, 4); assertTrue("expected 22.0000 padded: " + out, out.contains("22.0000")); assertTrue("expected 5.2441 rounded: " + out, out.contains("5.2441")); @@ -52,10 +52,10 @@ public void testDecimalPadsAndRounds() { @Test public void testNegativeDecimalUsesDefaultFormatting() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 2}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 2}); tb.allocateBlock(); - tb.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained - tb.set(0, 1, 5.244058388023880); // default cap of three fraction digits + tb.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained + tb.set(0, 1, 5.244058388023880); // default cap of three fraction digits // decimal < 0 leaves DecimalFormat unconstrained (no min/max fraction digits set) String out = DataConverter.toString(tb, false, " ", "\n", "[", "]", 1, 2, -1); assertTrue("expected unpadded 22: " + out, out.contains("22")); diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java index 810e2614e7c..426de81263f 100644 --- a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java +++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java @@ -52,18 +52,12 @@ public class QuantileTest extends AutomatedTestBase public void setUp() { TestUtils.clearAssertionInformation(); - addTestConfiguration(TEST_NAME1, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"})); - addTestConfiguration(TEST_NAME2, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"})); - addTestConfiguration(TEST_NAME3, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] {"R"})); - addTestConfiguration(TEST_NAME4, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] {"R"})); - addTestConfiguration(TEST_NAME5, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] {"R"})); - addTestConfiguration(TEST_NAME6, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] {"R"})); + addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"})); + addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"})); + addTestConfiguration(TEST_NAME3, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] {"R"})); + addTestConfiguration(TEST_NAME4, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] {"R"})); + addTestConfiguration(TEST_NAME5, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] {"R"})); + addTestConfiguration(TEST_NAME6, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] {"R"})); } @Test diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java index e70aedcabe4..35cfb2982fc 100644 --- a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java +++ b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java @@ -59,7 +59,8 @@ private void runSTEPGlmTest(ExecType instType) { // runTest executes the script; fails if the DML script invokes stop() runTest(true, false, null, -1); - } finally { + } + finally { rtplatform = platformOld; } } diff --git a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java index 5de429a3c53..d8cfc4d9fd7 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java @@ -91,10 +91,12 @@ public void testBackendPerformance() throws InterruptedException { taskFutures.forEach(res -> { try { Assert.assertEquals("Stats parsed correctly", res.get().statusCode(), 200); - } catch (InterruptedException e) { + } + catch(InterruptedException e) { Thread.currentThread().interrupt(); Assert.fail("Interrupted while fetching statistics: " + e.getMessage()); - } catch (ExecutionException e) { + } + catch(ExecutionException e) { Assert.fail("Failed to fetch statistics: " + e.getMessage()); } }); diff --git a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java index f8acdd07930..1cab99ee1ab 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java @@ -375,9 +375,8 @@ public void federatedLogicalTest(String testname, Type op_type, ExecMode execMod int port2 = single_fed_worker ? 0 : getRandomAvailablePort(); int port3 = single_fed_worker ? 0 : getRandomAvailablePort(); int port4 = single_fed_worker ? 0 : getRandomAvailablePort(); - Process[] workers = startLocalFedWorkers(single_fed_worker - ? new int[] {port1} - : new int[] {port1, port2, port3, port4}); + Process[] workers = startLocalFedWorkers( + single_fed_worker ? new int[] {port1} : new int[] {port1, port2, port3, port4}); try { if(!isAlive(workers)) diff --git a/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java b/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java index dbbf199f8fa..9d2bb583a4f 100644 --- a/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java +++ b/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java @@ -72,27 +72,26 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod } if(et == ExecType.SPARK) { - rtplatform = ExecMode.SPARK; + rtplatform = ExecMode.SPARK; } else { // rtplatform = (et==ExecType.MR)? ExecMode.HADOOP : ExecMode.SINGLE_NODE; - rtplatform = ExecMode.HYBRID; + rtplatform = ExecMode.HYBRID; } if( rtplatform == ExecMode.SPARK ) DMLScript.USE_LOCAL_SPARK_CONFIG = true; config.addVariable("rows", rows); - config.addVariable("cols", cols); + config.addVariable("cols", cols); - long rowstart=816, rowend=1229, colstart=967, colend=1009; - // long rowstart=2, rowend=4, colstart=9, colend=10; + long rowstart = 816, rowend = 1229, colstart = 967, colend = 1009; + // long rowstart=2, rowend=4, colstart=9, colend=10; /* - Random rand=new Random(System.currentTimeMillis()); - rowstart=(long)(rand.nextDouble()*((double)rows))+1; - rowend=(long)(rand.nextDouble()*((double)(rows-rowstart+1)))+rowstart; - colstart=(long)(rand.nextDouble()*((double)cols))+1; - colend=(long)(rand.nextDouble()*((double)(cols-colstart+1)))+colstart; - */ + * Random rand=new Random(System.currentTimeMillis()); rowstart=(long)(rand.nextDouble()*((double)rows))+1; + * rowend=(long)(rand.nextDouble()*((double)(rows-rowstart+1)))+rowstart; + * colstart=(long)(rand.nextDouble()*((double)cols))+1; + * colend=(long)(rand.nextDouble()*((double)(cols-colstart+1)))+colstart; + */ config.addVariable("rowstart", rowstart); config.addVariable("rowend", rowend); config.addVariable("colstart", colstart); @@ -119,17 +118,20 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod double sparsity=1.0;//rand.nextDouble(); double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, System.currentTimeMillis()); writeInputMatrix("A", A, true); - - sparsity=0.1;//rand.nextDouble(); - double[][] B = getRandomMatrix((int)(rowend-rowstart+1), (int)(colend-colstart+1), min, max, sparsity, System.currentTimeMillis()); + + sparsity = 0.1;// rand.nextDouble(); + double[][] B = getRandomMatrix((int) (rowend - rowstart + 1), (int) (colend - colstart + 1), min, max, + sparsity, System.currentTimeMillis()); writeInputMatrix("B", B, true); - - sparsity=0.5;//rand.nextDouble(); - double[][] C = getRandomMatrix((int)(rowend), (int)(cols-colstart+1), min, max, sparsity, System.currentTimeMillis()); + + sparsity = 0.5;// rand.nextDouble(); + double[][] C = getRandomMatrix((int) (rowend), (int) (cols - colstart + 1), min, max, sparsity, + System.currentTimeMillis()); writeInputMatrix("C", C, true); - - sparsity=0.01;//rand.nextDouble(); - double[][] D = getRandomMatrix(rows, (int)(colend-colstart+1), min, max, sparsity, System.currentTimeMillis()); + + sparsity = 0.01;// rand.nextDouble(); + double[][] D = getRandomMatrix(rows, (int) (colend - colstart + 1), min, max, sparsity, + System.currentTimeMillis()); writeInputMatrix("D", D, true); /* @@ -138,9 +140,9 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod * While loop iteration - 10 jobs * Final output write - 1 job */ - //boolean exceptionExpected = false; - //int expectedNumberOfJobs = 12; - //runTest(exceptionExpected, null, expectedNumberOfJobs); + // boolean exceptionExpected = false; + // int expectedNumberOfJobs = 12; + // runTest(exceptionExpected, null, expectedNumberOfJobs); boolean exceptionExpected = false; int expectedNumberOfJobs = -1; runTest(true, exceptionExpected, null, expectedNumberOfJobs); diff --git a/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java b/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java index a1b51ee9d81..fad1cc123c8 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java @@ -81,8 +81,9 @@ public void testDoubleScalarWrite() { fullDMLScriptName = HOME + "ScalarComputeWrite.dml"; runTest(true, false, null, -1); - double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1,1)).doubleValue(); - Assert.assertEquals("Computation test for Doubles failed: Values not equal: " + double_scalar + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar); + double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)).doubleValue(); + Assert.assertEquals("Computation test for Doubles failed: Values not equal: " + double_scalar + + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar); } @Test @@ -122,8 +123,7 @@ public void testIntScalarRead() { programArgs = new String[]{"-args", String.valueOf(int_scalar), output("a.scalar")}; runTest(true, false, null, -1); - int int_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)) - .get(new CellIndex(1,1)).intValue(); + int int_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)).intValue(); Assert.assertEquals("Values not equal: " + int_scalar + Opcodes.NOTEQUAL.toString() + int_out_scalar, int_scalar, int_out_scalar); @@ -143,8 +143,8 @@ public void testDoubleScalarRead() { programArgs = new String[]{ "-args", String.valueOf(double_scalar), output("a.scalar") }; runTest(true, false, null, -1); - double double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)) - .get(new CellIndex(1,1)).doubleValue(); + double double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)) + .doubleValue(); Assert.assertEquals("Values not equal: " + double_scalar + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar, 0); diff --git a/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java b/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java index a4013c3672d..20ae7aa63ea 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java @@ -35,14 +35,14 @@ /** * End-to-end DML test of the native Delta read/write path. * - *

The write and the read are run as two separate SystemDS executions - * on purpose. If they shared a single script/process, SystemDS would reuse the - * still-materialized in-memory matrix for the subsequent read and never invoke - * {@link org.apache.sysds.runtime.io.ReaderDelta} at all (verified: the cache - * reports 0 HDFS hits in that case). Splitting the executions forces a genuine - * read from disk, and we additionally assert via {@link CacheStatistics} that - * the read run actually performed HDFS reads (the Delta table + the text - * reference) rather than serving the matrix from cache.

+ *

+ * The write and the read are run as two separate SystemDS executions on purpose. If they shared a single + * script/process, SystemDS would reuse the still-materialized in-memory matrix for the subsequent read and never invoke + * {@link org.apache.sysds.runtime.io.ReaderDelta} at all (verified: the cache reports 0 HDFS hits in that case). + * Splitting the executions forces a genuine read from disk, and we additionally assert via {@link CacheStatistics} that + * the read run actually performed HDFS reads (the Delta table + the text reference) rather than serving the matrix from + * cache. + *

*/ public class DeltaReadWriteTest extends AutomatedTestBase { @@ -54,10 +54,8 @@ public class DeltaReadWriteTest extends AutomatedTestBase { @Override public void setUp() { TestUtils.clearAssertionInformation(); - addTestConfiguration(WRITE_NAME, - new TestConfiguration(TEST_CLASS_DIR, WRITE_NAME, new String[] { "ref" })); - addTestConfiguration(READ_NAME, - new TestConfiguration(TEST_CLASS_DIR, READ_NAME, new String[] { "R" })); + addTestConfiguration(WRITE_NAME, new TestConfiguration(TEST_CLASS_DIR, WRITE_NAME, new String[] {"ref"})); + addTestConfiguration(READ_NAME, new TestConfiguration(TEST_CLASS_DIR, READ_NAME, new String[] {"R"})); } @Test @@ -84,17 +82,16 @@ private void runDeltaRoundTrip(int rows, int cols, double sparsity) { String deltaPath = output("deltaTable"); String refPath = output("ref"); fullDMLScriptName = HOME + WRITE_NAME + ".dml"; - programArgs = new String[] { "-stats", "-args", - String.valueOf(rows), String.valueOf(cols), String.valueOf(sparsity), - deltaPath, refPath }; + programArgs = new String[] {"-stats", "-args", String.valueOf(rows), String.valueOf(cols), + String.valueOf(sparsity), deltaPath, refPath}; runTest(true, false, null, -1); // the write run must have materialized two matrices to disk (the Delta // table under test + the text reference); WriterDelta genuinely hitting // HDFS is what produces these write-side cache statistics. long hdfsWrites = CacheStatistics.getHDFSWrites(); - assertTrue("expected >= 2 HDFS writes in the write run (delta + reference), got " - + hdfsWrites, hdfsWrites >= 2); + assertTrue("expected >= 2 HDFS writes in the write run (delta + reference), got " + hdfsWrites, + hdfsWrites >= 2); // and a real Delta table (transaction log) must have been created assertTrue("missing Delta transaction log under " + deltaPath, new File(deltaPath, "_delta_log").isDirectory()); @@ -102,19 +99,18 @@ private void runDeltaRoundTrip(int rows, int cols, double sparsity) { // ---- phase 2: fresh execution reads the Delta table and compares ---- getAndLoadTestConfiguration(READ_NAME); fullDMLScriptName = HOME + READ_NAME + ".dml"; - programArgs = new String[] { "-stats", "-args", - deltaPath, refPath, output("R") }; + programArgs = new String[] {"-stats", "-args", deltaPath, refPath, output("R")}; runTest(true, false, null, -1); // the read run must have materialized two matrices from disk (the Delta // table under test + the text reference); a cached/short-circuited read // would report fewer HDFS hits and fail here. long hdfsReads = CacheStatistics.getHDFSHits(); - assertTrue("expected >= 2 HDFS reads in the read run (delta + reference), got " - + hdfsReads, hdfsReads >= 2); + assertTrue("expected >= 2 HDFS reads in the read run (delta + reference), got " + hdfsReads, + hdfsReads >= 2); HashMap R = readDMLMatrixFromOutputDir("R"); - //text-cell output omits exact zeros, so a missing cell means 0.0 + // text-cell output omits exact zeros, so a missing cell means 0.0 double diff = R.getOrDefault(new CellIndex(1, 1), 0.0); double nrow = R.getOrDefault(new CellIndex(1, 2), 0.0); double ncol = R.getOrDefault(new CellIndex(1, 3), 0.0); diff --git a/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java index cc1412b1606..a844321c249 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java @@ -49,10 +49,9 @@ public class FrameParquetSchemaTest extends AutomatedTestBase { @Override public void setUp() { - addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[]{"Rout"})); + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"Rout"})); } - /** * Test for sequential writer and reader * diff --git a/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java b/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java index dfb3d8a19de..6e6e4665f5e 100644 --- a/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java @@ -42,12 +42,8 @@ */ @net.jcip.annotations.NotThreadSafe public class JMLConnectionTest extends AutomatedTestBase { - public static final String META = "{\"data_type\": \"matrix\",\n" + - " \"value_type\": \"double\", \n" + - " \"rows\": 1,\n" + - " \"cols\": 1,\n" + - " \"nnz\": 1,\n" + - " \"format\": \"csv\"}"; + public static final String META = "{\"data_type\": \"matrix\",\n" + " \"value_type\": \"double\", \n" + + " \"rows\": 1,\n" + " \"cols\": 1,\n" + " \"nnz\": 1,\n" + " \"format\": \"csv\"}"; private final static String TEST_NAME = "JMLConnectionTest"; private final static String TEST_DIR = "functions/jmlc/"; @@ -99,12 +95,14 @@ public void testConnectionInvalidInName() throws DMLException { conn.gatherMemStats(false); Assert.assertFalse(DMLScript.STATISTICS); - try (conn) { - conn.prepareScript("printx('hello')", new String[]{"$inScalar1", null}, new String[]{null}); + try(conn) { + conn.prepareScript("printx('hello')", new String[] {"$inScalar1", null}, new String[] {null}); throw new AssertionError("Test should have thrown a LanguageException"); - } catch (LanguageException e) { + } + catch(LanguageException e) { Assert.assertTrue(e.getMessage().startsWith("Invalid variable names")); - } finally { + } + finally { DMLScript.STATISTICS = oldStat; DMLScript.JMLC_MEM_STATISTICS = oldJMLCStat; } @@ -112,21 +110,24 @@ public void testConnectionInvalidInName() throws DMLException { @Test public void testConnectionParseLanguageException() { - try (Connection conn = new Connection()) { - conn.prepareScript("printx('hello')", new String[]{}, new String[]{}); + try(Connection conn = new Connection()) { + conn.prepareScript("printx('hello')", new String[] {}, new String[] {}); throw new AssertionError("Test should have thrown a DMLException"); - } catch (DMLException e) { + } + catch(DMLException e) { Throwable cause = e.getCause(); - Assert.assertTrue(cause.getMessage().startsWith("ERROR: [line 1:0] -> printx('hello') -- function printx is undefined in namespace .builtinNS")); + Assert.assertTrue(cause.getMessage().startsWith( + "ERROR: [line 1:0] -> printx('hello') -- function printx is undefined in namespace .builtinNS")); } } @Test public void testConnectionParseException() { - try (Connection conn = new Connection()) { - conn.prepareScript("print('hello'", new String[]{}, new String[]{}); + try(Connection conn = new Connection()) { + conn.prepareScript("print('hello'", new String[] {}, new String[] {}); throw new AssertionError("Test should have thrown a ParseException"); - } catch (Exception e) { + } + catch(Exception e) { Assert.assertEquals("ParseException", e.getClass().getSimpleName()); } } @@ -144,10 +145,11 @@ public void testConnectionClose() { @Test public void testReadScriptHDFS() { - try (Connection conn = new Connection()) { + try(Connection conn = new Connection()) { conn.readScript("hdfs://localhost:9000/Test"); - } catch (IOException e) { - Assert.assertEquals("ConnectException",e.getClass().getSimpleName()); + } + catch(IOException e) { + Assert.assertEquals("ConnectException", e.getClass().getSimpleName()); } } diff --git a/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java b/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java index 4852220861e..2178884ef5b 100644 --- a/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java @@ -111,17 +111,16 @@ public void federatedReuse(String test) { // Run reference dml script with normal matrix. Reuse of ba+*. fullDMLScriptName = HOME + test + "Reference.dml"; - programArgs = new String[] {"-stats", "-lineage", "reuse_full", - "-nvargs", "X1=" + input("X1"), "X2=" + input("X2"), "Y1=" + input("Y1"), - "Y2=" + input("Y2"), "Z=" + expected("Z")}; + programArgs = new String[] {"-stats", "-lineage", "reuse_full", "-nvargs", "X1=" + input("X1"), + "X2=" + input("X2"), "Y1=" + input("Y1"), "Y2=" + input("Y2"), "Z=" + expected("Z")}; runTest(true, false, null, -1); long mmCount = Statistics.getCPHeavyHitterCount(Opcodes.MMULT.toString()); // Run actual dml script with federated matrix // The fed workers reuse ba+* fullDMLScriptName = HOME + test + ".dml"; - programArgs = new String[] {"-stats","-lineage", "reuse_full", - "-nvargs", "X1=" + TestUtils.federatedAddress(port1, input("X1")), + programArgs = new String[] {"-stats", "-lineage", "reuse_full", "-nvargs", + "X1=" + TestUtils.federatedAddress(port1, input("X1")), "X2=" + TestUtils.federatedAddress(port2, input("X2")), "Y1=" + TestUtils.federatedAddress(port1, input("Y1")), "Y2=" + TestUtils.federatedAddress(port2, input("Y2")), "r=" + rows, "c=" + cols, "Z=" + output("Z")}; @@ -129,12 +128,12 @@ public void federatedReuse(String test) { long mmCount_fed = Statistics.getCPHeavyHitterCount(Opcodes.MMULT.toString()); long fedMMCount = Statistics.getCPHeavyHitterCount("fed_ba+*"); - // compare results + // compare results compareResults(1e-9); // compare matrix multiplication count - // #federated execution of ba+* = #threads times #non-federated execution of ba+* (after reuse) - Assert.assertTrue("Violated reuse count: "+mmCount_fed+" == "+mmCount*2, - mmCount_fed == mmCount * 2); // #threads = 2 + // #federated execution of ba+* = #threads times #non-federated execution of ba+* (after reuse) + Assert.assertTrue("Violated reuse count: " + mmCount_fed + " == " + mmCount * 2, + mmCount_fed == mmCount * 2); // #threads = 2 switch(test) { case TEST_NAME1: // If the o/p is federated, fed_ba+* will be called everytime diff --git a/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java b/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java index eca3628a89b..c86eb0f4941 100644 --- a/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java @@ -121,9 +121,8 @@ private void runTriUDFReuse(ExecMode execMode) { // Run reference dml script with normal matrix fullDMLScriptName = HOME + TEST_NAME + "Reference.dml"; - programArgs = new String[] {"-lineage", "reuse_full", "-stats", "100", "-args", - input("X1"), input("X2"), input("X3"), input("X4"), - Boolean.toString(rowPartitioned).toUpperCase(), expected("S")}; + programArgs = new String[] {"-lineage", "reuse_full", "-stats", "100", "-args", input("X1"), input("X2"), + input("X3"), input("X4"), Boolean.toString(rowPartitioned).toUpperCase(), expected("S")}; runTest(null); // Run actual dml script with federated matrix diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java index 18ca2fbc454..3ffdfe1d30b 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java @@ -272,82 +272,79 @@ protected void toStringTestHelper(ExecMode platform, String testName, String exp } @Test - public void testPrintWithDecimal(){ + public void testPrintWithDecimal() { String testName = "ToString12"; String decimalPoints = "2"; String value = "22"; String expectedOutput = "22.00\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - @Test - public void testPrintWithDecimal2(){ + public void testPrintWithDecimal2() { String testName = "ToString12"; String decimalPoints = "2"; String value = "5.244058388023880"; String expectedOutput = "5.24\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - @Test - public void testPrintWithDecimal3(){ + public void testPrintWithDecimal3() { String testName = "ToString12"; String decimalPoints = "10"; String value = "5.244058388023880"; String expectedOutput = "5.2440583880\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - @Test - public void testPrintWithDecimal4(){ + public void testPrintWithDecimal4() { String testName = "ToString12"; String decimalPoints = "4"; String value = "5.244058388023880"; String expectedOutput = "5.2441\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - @Test - public void testPrintWithDecimal5(){ + public void testPrintWithDecimal5() { String testName = "ToString12"; String decimalPoints = "10"; String value = "0.000000008023880"; String expectedOutput = "0.0000000080\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - protected void toStringTestHelper2(ExecMode platform, String testName, String expectedOutput, String decimalPoints, String value) { + protected void toStringTestHelper2(ExecMode platform, String testName, String expectedOutput, String decimalPoints, + String value) { ExecMode platformOld = rtplatform; - + rtplatform = platform; boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - if (rtplatform == ExecMode.SPARK) + if(rtplatform == ExecMode.SPARK) DMLScript.USE_LOCAL_SPARK_CONFIG = true; try { // Create and load test configuration getAndLoadTestConfiguration(testName); String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testName + ".dml"; - programArgs = new String[]{"-args", output(OUTPUT_NAME), value, decimalPoints}; + programArgs = new String[] {"-args", output(OUTPUT_NAME), value, decimalPoints}; // Run DML and R scripts runTest(true, false, null, -1); diff --git a/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java index 770c5b7c5bf..26143dc16ee 100644 --- a/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java @@ -76,10 +76,9 @@ public ReshapeTest(int rlen, int clen, int rows, int cols, boolean rowWise) { @Parameterized.Parameters(name = "{0}x{1} {2}x{3} rowWise {4}") public static Iterable getParams() { - int[][][] dims = { - {{1000, 1000}, {1, 1000000}}, // single row/col - {{3000, 4000}, {1500, 8000}}, // partialBlocks - {{2400, 1400}, {800, 4200}} // fullBlocks + int[][][] dims = {{{1000, 1000}, {1, 1000000}}, // single row/col + {{3000, 4000}, {1500, 8000}}, // partialBlocks + {{2400, 1400}, {800, 4200}} // fullBlocks }; ArrayList params = new ArrayList<>(); @@ -117,7 +116,8 @@ public void runTestMatrixReshapeOOC() { double[][] X = getRandomMatrix(rlen, clen, 0, 1, 1, 7); MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(Types.FileFormat.BINARY); - writer.writeMatrixToHDFS(DataConverter.convertToMatrixBlock(X), input(INPUT_NAME), rlen, clen, 1000, rlen * clen); + writer.writeMatrixToHDFS(DataConverter.convertToMatrixBlock(X), input(INPUT_NAME), rlen, clen, 1000, + rlen * clen); HDFSTool.writeMetaDataFile(input(INPUT_NAME + ".mtd"), Types.ValueType.FP64, new MatrixCharacteristics(rlen, clen, blen, rlen * clen), Types.FileFormat.BINARY); @@ -143,8 +143,8 @@ public void runTestMatrixReshapeOOC() { runTest(true, false, null, -1); // compare results - MatrixBlock actual = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME), - Types.FileFormat.BINARY, rows, cols, blen); + MatrixBlock actual = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME), Types.FileFormat.BINARY, rows, + cols, blen); MatrixBlock expected = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME + "_target"), Types.FileFormat.BINARY, rows, cols, blen); diff --git a/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java index 5f42db7d733..49a52587cde 100644 --- a/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java @@ -335,9 +335,9 @@ private void runTestMatrixReshape( ReshapeType type, boolean rowwise, boolean sp String.valueOf(trows), String.valueOf(tcols), output("Y") }; fullRScriptName = HOME + TEST_NAME + ".R"; - rCmd = "Rscript" + " " + fullRScriptName + " " + - inputDir() + " " + trows + " " + tcols + " " + expectedDir(); - + rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + trows + " " + tcols + " " + + expectedDir(); + double[][] X = getRandomMatrix(rows, cols, 0, 1, sparsity, 7); writeInputMatrix("X", X, true); diff --git a/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java index dcdafddcd47..69d6958f8a6 100644 --- a/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java @@ -94,9 +94,9 @@ private void runVectorReshape(boolean sparse, ExecType et) String.valueOf(rows2), String.valueOf(cols2), output("R") }; fullRScriptName = HOME + TEST_NAME + ".R"; - rCmd = "Rscript" + " " + fullRScriptName + " " + - inputDir() + " " + rows2 + " " + cols2 + " " + expectedDir(); - + rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + rows2 + " " + cols2 + " " + + expectedDir(); + double sparsity = sparse ? sparsitySparse : sparsityDense; double[][] X = getRandomMatrix(rows1, cols1, 0, 1, sparsity, 7); writeInputMatrixWithMTD("X", X, true); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java index 60b491b8141..b16554045e4 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java @@ -151,8 +151,8 @@ private void runTestMatrixChainDP(String testName) { overwriteCurrentConfig(dmlConfig); } catch(FileNotFoundException fnfe) { - Assert.fail("Could not find DML config file: " + - getCurConfigFile().getPath() + " . " + fnfe.getMessage()); + Assert.fail( + "Could not find DML config file: " + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); } catch(IOException ioe) { Assert.fail("Could not overwrite the DML configuration file. " + ioe.getMessage()); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java index bf9acd9e52a..96c479e206d 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java @@ -123,8 +123,8 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { overwriteCurrentConfig(dmlConfig); } catch(FileNotFoundException fnfe) { - Assert.fail("Could not find DML config file: " + - getCurConfigFile().getPath() + " . " + fnfe.getMessage()); + Assert.fail( + "Could not find DML config file: " + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); } catch(IOException ioe) { Assert.fail("Could not overwrite the DML configuration file. " + ioe.getMessage()); @@ -132,8 +132,7 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + TEST_NAME + ".dml"; - programArgs = new String[] {"-explain", "hops", "-stats", - "-args", input("X"), input("Y"), output("R")}; + programArgs = new String[] {"-explain", "hops", "-stats", "-args", input("X"), input("Y"), output("R")}; fullRScriptName = HOME + TEST_NAME + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java index e8e885f905f..15b80e49618 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java @@ -74,7 +74,7 @@ public void testRewriteQuantizationFusedCompressionNoRewrite() { /** * Unified method to test both scalar and matrix scale factors. - * + * * @param testname Test name * @param rewrites Whether to enable fusion rewrites * @param isScalar Whether the scale factor is a scalar or a matrix diff --git a/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java b/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java index 30681f373e4..39266f5f3d3 100644 --- a/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java +++ b/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -106,7 +106,8 @@ public void testHash2() throws Exception { @Test public void testHash3() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.INT64,ValueType.UINT8}, 32); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, + new ValueType[] {ValueType.UINT8, ValueType.INT64, ValueType.UINT8}, 32); MatrixBlock expected = new MatrixBlock(1, 7, new double[] {1, 1, 1, 0, 1, 1, 1}); String spec = "{\"ids\": true, \"dummycode\": [1,3], \"hash\": [1,3], \"K\": 3}"; @@ -114,11 +115,11 @@ public void testHash3() throws Exception { } - @Test public void testHybrid1() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.INT64,ValueType.UINT8, ValueType.BOOLEAN}, 32); - MatrixBlock expected = new MatrixBlock(1, 9, new double[] {1, 1, 1, 0, 1, 1, 1,1,1}); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, + new ValueType[] {ValueType.UINT8, ValueType.INT64, ValueType.UINT8, ValueType.BOOLEAN}, 32); + MatrixBlock expected = new MatrixBlock(1, 9, new double[] {1, 1, 1, 0, 1, 1, 1, 1, 1}); String spec = "{\"ids\": true, \"dummycode\": [1,3,4], \"hash\": [1,3], \"K\": 3}"; runTransformTest(fb, spec, expected); @@ -127,8 +128,9 @@ public void testHybrid1() throws Exception { @Test public void testHybrid2() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.BOOLEAN,ValueType.UINT8, ValueType.BOOLEAN}, 32); - MatrixBlock expected = new MatrixBlock(1, 10, new double[] {1, 1, 1, 1,1, 1, 1, 1,1,1}); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, + new ValueType[] {ValueType.UINT8, ValueType.BOOLEAN, ValueType.UINT8, ValueType.BOOLEAN}, 32); + MatrixBlock expected = new MatrixBlock(1, 10, new double[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}); String spec = "{\"ids\": true, \"dummycode\": [1,2,3,4], \"hash\": [1,3], \"K\": 3}"; runTransformTest(fb, spec, expected); @@ -139,7 +141,7 @@ private void runTransformTest(FrameBlock fb, String spec, MatrixBlock expected) try { getAndLoadTestConfiguration(TEST_NAME1); - + String inF = input("F-In"); String inS = input("spec"); diff --git a/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java b/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java index 8c4ba6ae8ad..cd28649dc42 100644 --- a/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java +++ b/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java @@ -283,7 +283,8 @@ private String[][] readTwoColumnStringCSV(String s) { out[1][i] = in.getString(i, 1); } return out; - } catch (IOException e) { + } + catch(IOException e) { throw new RuntimeException(e); } } diff --git a/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java b/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java index d3d71d820d6..ae13cbd510f 100644 --- a/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java +++ b/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java @@ -92,7 +92,7 @@ private void runVectorizationTest( String testName, boolean rewrites ) runTest(true, false, null, -1); runRScript(true); - //compare results + // compare results HashMap dmlfile = readDMLMatrixFromOutputDir("R"); HashMap rfile = readRMatrixFromExpectedDir("R"); TestUtils.compareMatrices(dmlfile, rfile, 1e-14, "DML", "R"); From 056c7f9bfda818bb5f9a31573c29a89e0b6a772e Mon Sep 17 00:00:00 2001 From: bruno Date: Tue, 1 Sep 2026 14:30:50 +0200 Subject: [PATCH 129/132] Revert "dev/format-changed.sh" This reverts commit 55e6bf6bf64f3d495f8f5d2ba3c246c5fe0fb2ed. --- .../java/org/apache/sysds/api/DMLScript.java | 10 +- .../org/apache/sysds/common/Builtins.java | 376 +++++++++++++----- .../java/org/apache/sysds/hops/BinaryOp.java | 4 +- src/main/java/org/apache/sysds/hops/Hop.java | 14 +- .../java/org/apache/sysds/hops/UnaryOp.java | 9 +- .../sysds/hops/estim/EstimationUtils.java | 12 +- .../sysds/hops/rewrite/ProgramRewriter.java | 6 +- ...riteMatrixMultChainOptimizationSparse.java | 19 +- .../parser/BuiltinFunctionExpression.java | 24 ++ .../apache/sysds/parser/DMLTranslator.java | 87 +++- .../apache/sysds/parser/DataExpression.java | 206 +++++++--- .../compress/CompressedMatrixBlock.java | 2 +- .../runtime/compress/colgroup/AColGroup.java | 17 +- .../compress/colgroup/AColGroupValue.java | 1 + .../runtime/compress/colgroup/ASDCZero.java | 6 +- .../compress/colgroup/ColGroupDDC.java | 4 +- .../compress/colgroup/ColGroupEmpty.java | 7 +- .../colgroup/ColGroupLinearFunctional.java | 2 +- .../compress/colgroup/ColGroupOLE.java | 2 +- .../compress/colgroup/ColGroupRLE.java | 4 +- .../compress/colgroup/ColGroupSDCFOR.java | 4 +- .../colgroup/ColGroupUncompressed.java | 13 +- .../colgroup/ColGroupUncompressedArray.java | 2 +- .../dictionary/AIdentityDictionary.java | 4 +- .../colgroup/dictionary/DeltaDictionary.java | 4 +- .../colgroup/dictionary/IDictionary.java | 6 +- .../dictionary/IdentityDictionary.java | 4 +- .../dictionary/IdentityDictionarySlice.java | 4 +- .../compress/colgroup/mapping/AMapToData.java | 2 +- .../compress/colgroup/offset/AOffset.java | 10 +- .../compress/colgroup/offset/OffsetEmpty.java | 1 - .../compress/lib/CLALibBinaryCellOp.java | 6 +- .../runtime/compress/lib/CLALibMMChain.java | 2 +- .../compress/lib/CLALibRemoveEmpty.java | 15 +- .../runtime/compress/lib/CLALibSort.java | 8 +- .../controlprogram/caching/FrameObject.java | 4 +- .../controlprogram/caching/MatrixObject.java | 2 +- .../context/SparkExecutionContext.java | 34 +- .../federated/FederatedWorker.java | 3 +- .../frame/data/columns/ArrayFactory.java | 22 +- .../frame/data/lib/MatrixBlockFromFrame.java | 4 +- .../runtime/functionobjects/Builtin.java | 139 ++++--- .../instructions/cp/BinaryCPInstruction.java | 2 +- .../cp/BinaryFrameScalarCPInstruction.java | 10 +- .../cp/BinaryMatrixMatrixCPInstruction.java | 4 +- .../cp/ParameterizedBuiltinCPInstruction.java | 7 +- .../instructions/ooc/ReorgOOCInstruction.java | 8 +- .../ooc/ReshapeOOCInstruction.java | 69 ++-- .../spark/QuantilePickSPInstruction.java | 3 +- .../spark/data/IndexedMatrixValue.java | 7 +- .../sysds/runtime/io/DeltaKernelUtils.java | 257 ++++++------ .../apache/sysds/runtime/io/ReaderDelta.java | 107 +++-- .../sysds/runtime/io/ReaderDeltaParallel.java | 96 +++-- .../apache/sysds/runtime/io/WriterDelta.java | 74 ++-- .../runtime/matrix/data/LibMatrixReorg.java | 34 +- .../runtime/matrix/data/MatrixBlock.java | 34 +- .../sysds/runtime/ooc/cache/OOCFuture.java | 9 +- .../runtime/ooc/cache/io/CloseableQueue.java | 38 +- .../cache/io/OOCBufferedDataInputStream.java | 12 +- .../cache/io/OOCBufferedDataOutputStream.java | 20 +- .../runtime/ooc/cache/io/OOCIOHandler.java | 25 +- .../ooc/cache/io/OOCMatrixIOHandler.java | 161 ++++---- .../runtime/ooc/cache/io/SpillableObject.java | 6 +- .../ooc/cache/legacy/OOCCacheScheduler.java | 44 +- .../cache/legacy/OOCLRUCacheScheduler.java | 207 +++++----- .../runtime/transform/decode/Decoder.java | 10 +- .../runtime/transform/decode/DecoderBin.java | 4 +- .../transform/decode/DecoderDummycode.java | 2 +- .../transform/decode/DecoderFactory.java | 53 +-- .../transform/decode/DecoderRecode.java | 18 +- .../sysds/runtime/util/CommonThreadPool.java | 8 +- .../sysds/runtime/util/DataConverter.java | 9 +- .../org/apache/sysds/utils/DoubleParser.java | 2 +- .../apache/sysds/utils/SettingsChecker.java | 13 +- .../org/apache/sysds/performance/Main.java | 7 +- .../apache/sysds/test/AutomatedTestBase.java | 23 +- .../java/org/apache/sysds/test/TestUtils.java | 9 +- .../component/compile/CompilerTestBase.java | 21 +- .../SparkTransitiveExecTypeCompileTest.java | 50 ++- .../compress/CompressedSortTest.java | 6 +- .../compress/lib/CLALibMMChainTest.java | 4 +- ...CompressedBinaryMatrixMatrixSolveTest.java | 15 +- .../OffsetClassInitConcurrencyTest.java | 4 +- .../SparkContextReferenceCountTest.java | 32 +- .../component/federated/FedWorkerBase.java | 19 +- .../federated/FedWorkerMatrixCompress.java | 8 +- .../component/frame/FrameToStringTest.java | 14 +- .../frame/MatrixFromFrameSafeCastTest.java | 12 +- .../frame/transform/DecoderCompositeTest.java | 8 +- .../GetCategoricalMaskInstructionTest.java | 21 +- .../TransformDecodeRoundTripTest.java | 39 +- .../frame/transform/TransformDecodeTest.java | 4 +- .../component/io/DeltaMatrixCoverageTest.java | 97 ++--- .../io/DeltaMatrixReadWriteTest.java | 333 ++++++---------- .../io/DeltaMatrixSparkInteropTest.java | 105 ++--- .../component/matrix/QuantilePickTest.java | 13 +- .../component/tensor/TensorToStringTest.java | 14 +- .../functions/binary/matrix/QuantileTest.java | 18 +- .../builtin/part2/BuiltinSTEPGlmTest.java | 3 +- .../FederatedBackendPerformanceTest.java | 6 +- .../part4/FederatedLogicalTest.java | 5 +- .../functions/indexing/LeftIndexingTest.java | 48 ++- .../sysds/test/functions/io/ScalarIOTest.java | 12 +- .../io/delta/DeltaReadWriteTest.java | 40 +- .../io/parquet/FrameParquetSchemaTest.java | 3 +- .../functions/jmlc/JMLConnectionTest.java | 42 +- .../functions/lineage/FedFullReuseTest.java | 17 +- .../functions/lineage/FedUDFReuseTest.java | 5 +- .../test/functions/misc/ToStringTest.java | 33 +- .../sysds/test/functions/ooc/ReshapeTest.java | 14 +- .../functions/reorg/MatrixReshapeTest.java | 6 +- .../functions/reorg/VectorReshapeTest.java | 6 +- .../rewrite/RewriteMatrixChainDPTest.java | 4 +- .../RewriteMatrixMultChainOptSparseTest.java | 7 +- ...writeQuantizationFusedCompressionTest.java | 2 +- .../transform/GetCategoricalMaskTest.java | 20 +- .../TransformFrameEncodeBagOfWords.java | 3 +- .../vect/LeftIndexingChainUpdateTest.java | 2 +- 118 files changed, 1958 insertions(+), 1654 deletions(-) diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java b/src/main/java/org/apache/sysds/api/DMLScript.java index 0bb1e9b462d..a7a175bb7b6 100644 --- a/src/main/java/org/apache/sysds/api/DMLScript.java +++ b/src/main/java/org/apache/sysds/api/DMLScript.java @@ -508,9 +508,9 @@ private static void execute(String dmlScriptStr, String fnameOptConfig, Map inHops1 = new ArrayList<>(); + inHops1.add(expr); + inHops1.add(expr2); + currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), + OpOpDnn.valueOf(source.getOpCode().name()), inHops1); + setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); + break; + } + case AVG_POOL: + case MAX_POOL: { + currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), + OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForPoolingForwardIM2COL(expr, source, 1, hops)); + setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); + break; + } + case AVG_POOL_BACKWARD: + case MAX_POOL_BACKWARD: { + currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), + OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForConvOpPoolingCOL2IM(expr, source, 1, hops)); + setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); + break; + } + case CONV2D: + case CONV2D_BACKWARD_FILTER: + case CONV2D_BACKWARD_DATA: { + currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), + OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForConvOp(expr, source, 1, hops)); + setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); + break; + } + + case ROW_COUNT_DISTINCT: + currBuiltinOp = new AggUnaryOp(target.getName(), + DataType.MATRIX, target.getValueType(), AggOp.COUNT_DISTINCT, Direction.Row, expr); + break; + + case COL_COUNT_DISTINCT: + currBuiltinOp = new AggUnaryOp(target.getName(), + DataType.MATRIX, target.getValueType(), AggOp.COUNT_DISTINCT, Direction.Col, expr); + break; + + case GET_CATEGORICAL_MASK: + currBuiltinOp = new BinaryOp(target.getName(), DataType.MATRIX, ValueType.FP64, OpOp2.GET_CATEGORICAL_MASK, expr, expr2); + break; + default: + throw new ParseException("Unsupported builtin function type: "+source.getOpCode()); + } + + boolean isConvolution = source.getOpCode() == Builtins.CONV2D || source.getOpCode() == Builtins.CONV2D_BACKWARD_DATA || + source.getOpCode() == Builtins.CONV2D_BACKWARD_FILTER || + source.getOpCode() == Builtins.MAX_POOL || source.getOpCode() == Builtins.MAX_POOL_BACKWARD || + source.getOpCode() == Builtins.AVG_POOL || source.getOpCode() == Builtins.AVG_POOL_BACKWARD; + if( !isConvolution) { // Since the dimension of output doesnot match that of input variable for these operations setIdentifierParams(currBuiltinOp, source.getOutput()); } diff --git a/src/main/java/org/apache/sysds/parser/DataExpression.java b/src/main/java/org/apache/sysds/parser/DataExpression.java index 3d3a90b4f6f..68a3d1b7ffe 100644 --- a/src/main/java/org/apache/sysds/parser/DataExpression.java +++ b/src/main/java/org/apache/sysds/parser/DataExpression.java @@ -1176,72 +1176,52 @@ else if( getVarParam(READNNZPARAM) != null ) { boolean isHDF5 = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.HDF5.toString())); - // handle all csv default parameters - handleCSVDefaultParam(DELIM_DELIMITER, ValueType.STRING, conditional); - handleCSVDefaultParam(DELIM_FILL_VALUE, ValueType.FP64, conditional); - handleCSVDefaultParam(DELIM_HAS_HEADER_ROW, ValueType.BOOLEAN, conditional); - handleCSVDefaultParam(DELIM_FILL, ValueType.BOOLEAN, conditional); - handleCSVDefaultParam(DELIM_NA_STRINGS, ValueType.STRING, conditional); - } - - boolean isLIBSVM = false; - isLIBSVM = (formatTypeString != null && - formatTypeString.equalsIgnoreCase(FileFormat.LIBSVM.toString())); - if(isLIBSVM) { - // Handle libsvm file format - shouldReadMTD = true; - - // only allow IO_FILENAME, READROWPARAM, READCOLPARAM - // as valid parameters - if(!inferredFormatType) { - for(String key : _varParams.keySet()) { - if(!(key.equals(IO_FILENAME) || key.equals(FORMAT_TYPE) || key.equals(READROWPARAM) || - key.equals(READCOLPARAM) || key.equals(READNNZPARAM) || key.equals(DATATYPEPARAM) || - key.equals(VALUETYPEPARAM) || key.equals(DELIM_DELIMITER) || - key.equals(LIBSVM_INDEX_DELIM))) { - String msg = "Only parameters allowed are: " + IO_FILENAME + "," + READROWPARAM + "," - + READCOLPARAM + DELIM_DELIMITER + "," + LIBSVM_INDEX_DELIM; - - raiseValidateError( - "Invalid parameter " + key + " in read statement: " + toString() + ". " + msg, - conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - } - } - // handle all default parameters - handleCSVDefaultParam(DELIM_DELIMITER, ValueType.STRING, conditional); - handleCSVDefaultParam(LIBSVM_INDEX_DELIM, ValueType.STRING, conditional); - } + boolean isCOG = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.COG.toString())); - boolean isHDF5 = (formatTypeString != null && - formatTypeString.equalsIgnoreCase(FileFormat.HDF5.toString())); + // Delta tables are self-describing (schema + dimensions discovered from the + // transaction log at read time), so dimensions are optional like CSV. + boolean isDelta = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.DELTA.toString())); - boolean isCOG = (formatTypeString != null && - formatTypeString.equalsIgnoreCase(FileFormat.COG.toString())); - - // Delta tables are self-describing (schema + dimensions discovered from the - // transaction log at read time), so dimensions are optional like CSV. - boolean isDelta = (formatTypeString != null && - formatTypeString.equalsIgnoreCase(FileFormat.DELTA.toString())); - - dataTypeString = (getVarParam(DATATYPEPARAM) == null) ? null : getVarParam(DATATYPEPARAM).toString(); - - if(dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE) || - dataTypeString.equalsIgnoreCase(Statement.FRAME_DATA_TYPE)) { - - boolean isMatrix = false; - if(dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE)) + dataTypeString = (getVarParam(DATATYPEPARAM) == null) ? null : getVarParam(DATATYPEPARAM).toString(); + + if ( dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE) + || dataTypeString.equalsIgnoreCase(Statement.FRAME_DATA_TYPE)) { + + boolean isMatrix = false; + if ( dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE)) isMatrix = true; + + // set data type + getOutput().setDataType(isMatrix ? DataType.MATRIX : DataType.FRAME); + + // set number non-zeros + Expression ennz = getVarParam("nnz"); + long nnz = -1; + if( ennz != null ) { + nnz = Long.valueOf(ennz.toString()); + getOutput().setNnz(nnz); + } - // set data type - getOutput().setDataType(isMatrix ? DataType.MATRIX : DataType.FRAME); - - // set number non-zeros - Expression ennz = getVarParam("nnz"); - long nnz = -1; - if(ennz != null) { - nnz = Long.valueOf(ennz.toString()); - getOutput().setNnz(nnz); + // Following dimension checks must be done when data type = MATRIX_DATA_TYPE + // initialize size of target data identifier to UNKNOWN + getOutput().setDimensions(-1, -1); + + if (!isCSV && !isLIBSVM && !isHDF5 && !isCOG && !isDelta && ConfigurationManager.getCompilerConfig() + .getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) //skip check for csv/libsvm/delta format / jmlc api + && (getVarParam(READROWPARAM) == null || getVarParam(READCOLPARAM) == null) ) { + raiseValidateError("Missing or incomplete dimension information in read statement: " + + mtdFileName, conditional, LanguageErrorCodes.INVALID_PARAMETERS); + } + + if (getVarParam(READROWPARAM) instanceof ConstIdentifier + && getVarParam(READCOLPARAM) instanceof ConstIdentifier) + { + // these are strings that are long values + Long dim1 = (getVarParam(READROWPARAM) == null) ? null : Long.valueOf( getVarParam(READROWPARAM).toString()); + Long dim2 = (getVarParam(READCOLPARAM) == null) ? null : Long.valueOf( getVarParam(READCOLPARAM).toString()); + if ( !isCSV && !isDelta && (dim1 < 0 || dim2 < 0) && ConfigurationManager + .getCompilerConfig().getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) ) { + raiseValidateError("Invalid dimension information in read statement", conditional, LanguageErrorCodes.INVALID_PARAMETERS); } // set dim1 and dim2 values @@ -1272,10 +1252,104 @@ else if( getVarParam(READNNZPARAM) != null ) { catch(Exception ex) { raiseValidateError("Invalid format '" + fmt+ "' in statement: " + toString(), conditional); } - else if(getVarParam(FORMAT_TYPE) instanceof StringIdentifier) // literal format - raiseValidateError("Invalid format " + getVarParam(FORMAT_TYPE) + " in statement: " + toString(), - conditional); - break; + + if (getVarParam(ROWBLOCKCOUNTPARAM) instanceof ConstIdentifier && getVarParam(COLUMNBLOCKCOUNTPARAM) instanceof ConstIdentifier) { + Integer rowBlockCount = (getVarParam(ROWBLOCKCOUNTPARAM) == null) ? + null : Integer.valueOf(getVarParam(ROWBLOCKCOUNTPARAM).toString()); + getOutput().setBlocksize(rowBlockCount != null ? rowBlockCount : -1); + } + + // block dimensions must be -1x-1 when format="text" + // NOTE MB: disabled validate of default blocksize for inputs w/ format="binary" + // because we automatically introduce reblocks if blocksizes don't match + if ( (getOutput().getFileFormat().isTextFormat() || !isMatrix) && getOutput().getBlocksize() != -1 ){ + raiseValidateError("Invalid block dimensions (" + getOutput().getBlocksize() + ") when format=" + getVarParam(FORMAT_TYPE) + " in \"" + this.toString() + "\".", conditional); + } + + } + else if ( dataTypeString.equalsIgnoreCase(Statement.SCALAR_DATA_TYPE)) { + getOutput().setDataType(DataType.SCALAR); + getOutput().setNnz(-1L); + } + else if ( dataTypeString.equalsIgnoreCase(DataType.LIST.name())) { + getOutput().setDataType(DataType.LIST); + } + else{ + raiseValidateError("Unknown Data Type " + dataTypeString + ". Valid values: " + + Statement.SCALAR_DATA_TYPE +", " + Statement.MATRIX_DATA_TYPE+", " + Statement.FRAME_DATA_TYPE + +", " + DataType.LIST.name().toLowerCase(), conditional, LanguageErrorCodes.INVALID_PARAMETERS); + } + + // handle value type parameter + if (getVarParam(VALUETYPEPARAM) != null && !(getVarParam(VALUETYPEPARAM) instanceof StringIdentifier)){ + raiseValidateError("for read method, parameter " + VALUETYPEPARAM + " can only be a string. " + + "Valid values are: " + Statement.DOUBLE_VALUE_TYPE +", " + Statement.INT_VALUE_TYPE + ", " + Statement.BOOLEAN_VALUE_TYPE + ", " + Statement.STRING_VALUE_TYPE, conditional); + } + // Identify the value type (used only for read method) + String valueTypeString = getVarParam(VALUETYPEPARAM) == null ? null : getVarParam(VALUETYPEPARAM).toString(); + if (valueTypeString != null) { + if (valueTypeString.equalsIgnoreCase(Statement.DOUBLE_VALUE_TYPE)) + getOutput().setValueType(ValueType.FP64); + else if (valueTypeString.equalsIgnoreCase(Statement.STRING_VALUE_TYPE)) + getOutput().setValueType(ValueType.STRING); + else if (valueTypeString.equalsIgnoreCase(Statement.INT_VALUE_TYPE)) + getOutput().setValueType(ValueType.INT64); + else if (valueTypeString.equalsIgnoreCase(Statement.BOOLEAN_VALUE_TYPE)) + getOutput().setValueType(ValueType.BOOLEAN); + else if (valueTypeString.equalsIgnoreCase(ValueType.UNKNOWN.name())) + getOutput().setValueType(ValueType.UNKNOWN); + else { + raiseValidateError("Unknown Value Type " + valueTypeString + + ". Valid values are: " + Statement.DOUBLE_VALUE_TYPE +", " + Statement.INT_VALUE_TYPE + ", " + Statement.BOOLEAN_VALUE_TYPE + ", " + Statement.STRING_VALUE_TYPE, conditional); + } + } else { + getOutput().setValueType(ValueType.FP64); + } + + break; + + case WRITE: + + // for CSV format, if no delimiter specified THEN set default "," + if (getVarParam(FORMAT_TYPE) == null || checkFormatType(FileFormat.CSV) ){ + if (getVarParam(DELIM_DELIMITER) == null) { + addVarParam(DELIM_DELIMITER, new StringIdentifier(DEFAULT_DELIM_DELIMITER, this)); + } + if (getVarParam(DELIM_HAS_HEADER_ROW) == null) { + addVarParam(DELIM_HAS_HEADER_ROW, new BooleanIdentifier(DEFAULT_DELIM_HAS_HEADER_ROW, this)); + } + if (getVarParam(DELIM_SPARSE) == null) { + addVarParam(DELIM_SPARSE, new BooleanIdentifier(DEFAULT_DELIM_SPARSE, this)); + } + } + + // for LIBSVM format, add the default separators if not specified + if (getVarParam(FORMAT_TYPE) == null || checkFormatType(FileFormat.LIBSVM)) { + if(getVarParam(DELIM_DELIMITER) == null) { + addVarParam(DELIM_DELIMITER, new StringIdentifier(DEFAULT_DELIM_DELIMITER, this)); + } + if(getVarParam(LIBSVM_INDEX_DELIM) == null) { + addVarParam(LIBSVM_INDEX_DELIM, new StringIdentifier(DEFAULT_LIBSVM_INDEX_DELIM, this)); + } + if(getVarParam(DELIM_SPARSE) == null) { + addVarParam(DELIM_SPARSE, new BooleanIdentifier(DEFAULT_DELIM_SPARSE, this)); + } + } + + //validate read filename + if (getVarParam(FORMAT_TYPE) == null || FileFormat.isTextFormat(getVarParam(FORMAT_TYPE).toString()) + || checkFormatType(FileFormat.DELTA)) //delta: columnar, no block layout + getOutput().setBlocksize(-1); + else if (checkFormatType(FileFormat.BINARY, FileFormat.COMPRESSED, FileFormat.UNKNOWN)) { + if( getVarParam(ROWBLOCKCOUNTPARAM)!=null ) + getOutput().setBlocksize(Integer.parseInt(getVarParam(ROWBLOCKCOUNTPARAM).toString())); + else + getOutput().setBlocksize(ConfigurationManager.getBlocksize()); + } + else if( getVarParam(FORMAT_TYPE) instanceof StringIdentifier ) //literal format + raiseValidateError("Invalid format " + getVarParam(FORMAT_TYPE) + + " in statement: " + toString(), conditional); + break; case RAND: diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java index 042e0dc0328..d0ba5363939 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java @@ -484,7 +484,7 @@ public static CompressedMatrixBlock read(DataInput in) throws IOException { long nonZeros = in.readLong(); boolean overlappingColGroups = in.readBoolean(); List groups = ColGroupIO.readGroups(in, rlen); - CompressedMatrixBlock ret = new CompressedMatrixBlock(rlen, clen, nonZeros, overlappingColGroups, groups); + CompressedMatrixBlock ret = new CompressedMatrixBlock(rlen, clen, nonZeros, overlappingColGroups, groups); LOG.debug("Compressed read serialization time: " + t.stop()); return ret; } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java index 66d4e78cb0f..354325e293b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java @@ -402,8 +402,7 @@ public final AColGroup rightMultByMatrix(MatrixBlock right) { * @param cru The right hand side column upper * @param nRows The number of rows in this column group */ - public void rightDecompressingMult(MatrixBlock right, MatrixBlock ret, int rl, int ru, int nRows, int crl, - int cru) { + public void rightDecompressingMult(MatrixBlock right, MatrixBlock ret, int rl, int ru, int nRows, int crl, int cru) { throw new NotImplementedException( "not supporting right Decompressing Multiply on class: " + this.getClass().getSimpleName()); } @@ -978,9 +977,9 @@ public AColGroup[] splitReshapePushDown(final int multiplier, final int nRow, fi /** * Sort the values of the column group according to double comparison operations and return as another compressed * group. - * + * * This sorting assumes that the column group is sorted independently of everything else. - * + * * @return The sorted group */ public abstract AColGroup sort(); @@ -997,9 +996,9 @@ public String toString() { /** * Return a new column group containing only the selected rows in the given boolean vector. - * + * * Whenever possible only modify the index structure, not the dictionary of the column groups. - * + * * @param selectV The selection vector * @param rOut The number of rows in the output * @return The new column group @@ -1008,9 +1007,9 @@ public String toString() { /** * Return a new column group containing only the selected columns in the given boolean vector. - * + * * Whenever possible only modify the column index, and reduce the dictionaries of the column groups. - * + * * @param selectV The selection vector * @return The new column group, or {@code null} if no column of this group is selected */ @@ -1046,7 +1045,7 @@ public AColGroup removeEmptyCols(boolean[] selectV) { /** * Using the selection of columns, slice out those and return in a new column group with the given column indexes. * Ideally this method should only modify the dictionaries. - * + * * @param newColumnIDs the new column indexes * @param selectedColumns The selected columns of this column group (guaranteed < current number of columns) * @return A new Column group diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java index d610c1b586c..d825b91f089 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java @@ -210,6 +210,7 @@ public void clear() { counts = null; } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java index 794d90c0d11..30de5e120c5 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java @@ -212,8 +212,8 @@ public void decompressToSparseBlock(SparseBlock sb, int rl, int ru, int offR, in // TODO make sparse decompression where the iterator is known in argument decompressToSparseBlockSparseDictionary(sb, rl, ru, offR, offC, mb.getSparseBlock()); else - decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, - mb.getDenseBlockValues(), it); + decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, mb.getDenseBlockValues(), + it); } else decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, _dict.getValues(), it); @@ -240,7 +240,7 @@ public void decompressToDenseBlockDenseDictionary(DenseBlock db, int rl, int ru, } public abstract void decompressToSparseBlockDenseDictionaryWithProvidedIterator(SparseBlock db, int rl, int ru, - int offR, int offC, double[] values, AIterator it); + int offR, int offC, double[] values, AIterator it); public abstract void decompressToDenseBlockDenseDictionaryWithProvidedIterator(DenseBlock db, int rl, int ru, int offR, int offC, double[] values, AIterator it); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java index d643cae440c..b316e48474a 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java @@ -674,8 +674,8 @@ private void defaultRightDecompressingMult(MatrixBlock right, MatrixBlock ret, i } } - final void vectMM(double aa, double[] b, double[] c, int endT, int jd, int crl, int cru, int offOut, int k, - int vLen, DoubleVector vVec) { + final void vectMM(double aa, double[] b, double[] c, int endT, int jd, int crl, int cru, int offOut, int k, int vLen, + DoubleVector vVec) { vVec = vVec.broadcast(aa); final int offj = k * jd; final int end = endT + offj; diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java index d5ad55772c7..64114a054ab 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java @@ -478,13 +478,14 @@ public AColGroup combineWithSameIndex(int nRow, int nCol, List right) return new ColGroupEmpty(combinedIndex); } - @Override - public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut){ return this; } + @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ return new ColGroupEmpty(newColumnIDs); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java index e0bea3c3696..fa8aa104ffb 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java @@ -747,7 +747,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java index b4f0c144a73..a251d828b5f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java @@ -738,7 +738,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java index 43df7fa3b94..347cea9c0da 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java @@ -1195,9 +1195,9 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { throw new NotImplementedException("Unimplemented method 'removeEmptyRows'"); } - + @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java index 4566106a3e2..815ecacf378 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java @@ -634,8 +634,8 @@ protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList s for(int i = 0; i < selectedColumns.size(); i++) { ref[i] = _reference[selectedColumns.get(i)]; } - return ColGroupSDCFOR.create(newColumnIDs, _numRows, _dict.sliceColumns(selectedColumns, getNumCols()), - _indexes, _data, null, ref); + return ColGroupSDCFOR.create(newColumnIDs, _numRows, _dict.sliceColumns(selectedColumns, getNumCols()), _indexes, _data, null, + ref); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java index 9797087f8c3..611add6480f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java @@ -85,7 +85,7 @@ public class ColGroupUncompressed extends AColGroup { /** * Do not use this constructor of column group uncompressed, instead use the create constructor. - * + * * @param mb The contained data. * @param colIndexes Column indexes for this Columngroup */ @@ -96,10 +96,9 @@ protected ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes) { /** * Do not use this constructor of column group quantization-fused uncompressed, instead use the create constructor. - * + * * @param mb The contained data. - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire - * matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix * @param colIndexes Column indexes for this Columngroup */ protected ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes, double[] scaleFactors) { @@ -139,8 +138,7 @@ public static AColGroup create(MatrixBlock mb, IColIndex colIndexes) { * * @param mb The MB / data to contain in the uncompressed column * @param colIndexes The column indexes for the group - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire - * matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix * @return An Uncompressed Column group */ public static AColGroup createQuantized(MatrixBlock mb, IColIndex colIndexes, double[] scaleFactors) { @@ -159,8 +157,7 @@ public static AColGroup createQuantized(MatrixBlock mb, IColIndex colIndexes, do * @param rawBlock The uncompressed block; uncompressed data must be present at the time that the constructor is * called * @param transposed Says if the input matrix raw block have been transposed. - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire - * matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix * @return AColGroup. */ public static AColGroup createQuantized(IColIndex colIndexes, MatrixBlock rawBlock, boolean transposed, diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java index de8a740ceb2..51e26a3f9d2 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java @@ -290,7 +290,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java index 6e66ef6ef9b..a7e715b59b8 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java @@ -76,8 +76,8 @@ public double[] productAllRowsToDoubleWithDefault(double[] defaultTuple) { return ret; } - @Override - public int[] sort() { + @Override + public int[] sort(){ throw new NotImplementedException(); } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java index 7ebba2f1a76..9a0412145f0 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java @@ -138,8 +138,8 @@ public IDictionary clone() { throw new NotImplementedException(); } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ throw new NotImplementedException(); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java index b5e1a99355b..c8ddfc4883a 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java @@ -1055,7 +1055,7 @@ public IDictionary rightMMPreAggSparse(int numVals, SparseBlock b, IColIndex thi /** * Slice out the selected columns given of this encoded group. - * + * * @param selectedColumns The columns to slice out and return as a new matrix. * @param nCol The number of columns in this dictionary. * @return The returned matrix @@ -1064,9 +1064,9 @@ public IDictionary rightMMPreAggSparse(int numVals, SparseBlock b, IColIndex thi /** * Sort the values of this dictionary via an index of how the values mapped previously. - * + * * In practice this design means we can reuse the previous dictionary for the resulting column group - * + * * @return The sorted index. */ public int[] sort(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java index 4337da7307f..c2540de959a 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java @@ -541,8 +541,8 @@ public String getString(int colIndexes) { return "IdentityMatrix of size: " + nRowCol + " with empty: " + withEmpty; } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ return getMBDict().sliceColumns(selectedColumns, nCol); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java index 47628b43d2a..c7f642edfd0 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java @@ -311,8 +311,8 @@ public String getString(int colIndexes) { return toString(); } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ return getMBDict().sliceColumns(selectedColumns, nCol); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java index 6d516713689..83a74972db7 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java @@ -1064,7 +1064,7 @@ public AMapToData removeEmpty(final boolean[] selectV, final int rOut) { /** * Use the offsets of the select vector to choose which values to keep. - * + * * @param select The row indexes to keep * @return A New MapToData */ diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java index bf8ee7f9ee1..f65876b7f37 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java @@ -56,11 +56,11 @@ public abstract class AOffset implements Serializable { protected static final Log LOG = LogFactory.getLog(AOffset.class.getName()); /** - * Lazy holder for the cached empty slice. The empty slice is built on first use rather than in AOffset's static - * initializer: instantiating the OffsetEmpty subclass from AOffset's {@code } forms a superclass/subclass - * class-initialization cycle that deadlocks when several threads first touch the offset classes concurrently (e.g. - * parallel tests). Deferring it to first use guarantees AOffset is already initialized by the time OffsetEmpty is - * loaded, so no cycle exists. + * Lazy holder for the cached empty slice. The empty slice is built on first use rather than in AOffset's + * static initializer: instantiating the OffsetEmpty subclass from AOffset's {@code } forms a + * superclass/subclass class-initialization cycle that deadlocks when several threads first touch the offset + * classes concurrently (e.g. parallel tests). Deferring it to first use guarantees AOffset is already + * initialized by the time OffsetEmpty is loaded, so no cycle exists. */ private static final class EmptySliceHolder { static final OffsetSliceInfo EMPTY_SLICE = new OffsetSliceInfo(-1, -1, new OffsetEmpty()); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java index 37ff41cf817..866168ded2f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java @@ -76,7 +76,6 @@ public int getOffsetToLast() { public long getInMemorySize() { return estimateInMemorySize(); } - @Override public boolean equals(AOffset b) { return b instanceof OffsetEmpty; diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java index 7953322350e..d981ab87838 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java @@ -139,8 +139,7 @@ private static boolean isDoubleCompressedOpApplicable(CompressedMatrixBlock m1, m1.getColGroups().get(0) instanceof ColGroupDDC && !((CompressedMatrixBlock) that).isOverlapping() && ((CompressedMatrixBlock) that).getColGroups().get(0) instanceof ColGroupDDC && ((IMapToDataGroup) m1.getColGroups().get(0)) - .getMapToData() == ((IMapToDataGroup) ((CompressedMatrixBlock) that).getColGroups().get(0)) - .getMapToData(); + .getMapToData() == ((IMapToDataGroup) ((CompressedMatrixBlock) that).getColGroups().get(0)).getMapToData(); } private static CompressedMatrixBlock doubleCompressedBinaryOp(BinaryOperator op, CompressedMatrixBlock m1, @@ -1063,8 +1062,7 @@ public Long call() { return _ret.recomputeNonZeros(_rl, _ru - 1); } - private final void processBlock(final int rl, final int ru, final List groups, - final AIterator[] its) { + private final void processBlock(final int rl, final int ru, final List groups, final AIterator[] its) { decompressToTmpBlock(rl, ru, tmp.getSparseBlock(), groups, its); // decompressing multiple column groups can leave the temp rows with unsorted column indices, so sort // before reading them in stored order into the (column-sorted) output sparse block. diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java index a91b75ae73c..cc7953f8c5d 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java @@ -96,7 +96,7 @@ public static MatrixBlock mmChain(CompressedMatrixBlock x, MatrixBlock v, Matrix if(x.isEmpty()) return returnEmpty(x, out); - if(ctype == ChainType.XtXv && x.getColGroups().size() < 5 && x.getNumColumns() > 30) { + if(ctype == ChainType.XtXv && x.getColGroups().size() < 5 && x.getNumColumns()> 30){ MatrixBlock tmp = CLALibTSMM.leftMultByTransposeSelf(x, k); return tmp.aggregateBinaryOperations(tmp, v, out, InstructionUtils.getMatMultOperator(k)); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java index 802eddffcb8..3755e4040e7 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java @@ -36,7 +36,7 @@ public class CLALibRemoveEmpty { /** * CP rmempty operation (single input, single output matrix) - * + * * @param in The input matrix * @param ret The output matrix * @param rows If we are removing based on rows, or columns. @@ -66,13 +66,13 @@ private static MatrixBlock rmEmptyCols(CompressedMatrixBlock in, MatrixBlock ret int cOut = (int) select.getNonZeros(); if(cOut == -1) cOut = (int) select.recomputeNonZeros(); - if(cOut == 0) { + if(cOut == 0){ ret.reset(in.getNumRows(), !emptyReturn ? 0 : 1); return ret; } - final boolean[] selectV = DataConverter.convertToBooleanVector( - CompressedMatrixBlock.getUncompressed(select, "decompressing selection in rmempty")); + final boolean[] selectV = DataConverter + .convertToBooleanVector(CompressedMatrixBlock.getUncompressed(select, "decompressing selection in rmempty")); final List inG = in.getColGroups(); final List retG = new ArrayList<>(inG.size()); @@ -102,17 +102,18 @@ private static MatrixBlock rmEmptyRows(CompressedMatrixBlock in, MatrixBlock ret int rOut = (int) select.getNonZeros(); if(rOut == -1) rOut = (int) select.recomputeNonZeros(); - if(rOut == 0) { + if(rOut == 0){ ret.reset(!emptyReturn ? 0 : 1, in.getNumColumns()); return ret; } - // TODO: add optimization to avoid linear scan and make selectV indexes, if selection is small relative to - // number + // TODO: add optimization to avoid linear scan and make selectV indexes, if selection is small relative to number // of rows // TODO: add decompress to boolean vector. final boolean[] selectV = DataConverter.convertToBooleanVector(select); + + final List inG = in.getColGroups(); final List retG = new ArrayList<>(inG.size()); try { diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java index 5ae7bd5103b..b94f11ae723 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java @@ -40,10 +40,10 @@ private CLALibSort() { /** * Sort (order) a compressed matrix in place of the {@code order} built-in, while keeping the result compressed. * - * The compressed fast-path only supports the case the user can benefit from: a single column held in a single - * column group, sorted ascending and returning the sorted values (not the index permutation). For everything else - * (multiple columns, multiple column groups, descending order, index return, or a column-group encoding without a - * sort implementation) this returns {@code null} so the caller can fall back to a decompressed reorg. + * The compressed fast-path only supports the case the user can benefit from: a single column held in a single column + * group, sorted ascending and returning the sorted values (not the index permutation). For everything else (multiple + * columns, multiple column groups, descending order, index return, or a column-group encoding without a sort + * implementation) this returns {@code null} so the caller can fall back to a decompressed reorg. * * @param mb the compressed matrix to sort * @param fn the sort specification carried by the reorg operator diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java index 9ccaa474f39..87d14dbf87e 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java @@ -208,8 +208,8 @@ protected FrameBlock readBlobFromHDFS(String fname, long[] dims) throws IOExcept if(data == null) throw new IOException("Unable to load frame from file: " + fname); - // Delta and CSV discover dimensions (and Delta also schema) at read time, so - // refresh the cached metadata to reflect the materialized frame block. + //Delta and CSV discover dimensions (and Delta also schema) at read time, so + //refresh the cached metadata to reflect the materialized frame block. if(iimd.getFileFormat() == FileFormat.CSV || iimd.getFileFormat() == FileFormat.DELTA) { _metaData = _metaData instanceof MetaDataFormat ? new MetaDataFormat(data.getDataCharacteristics(), iimd.getFileFormat()) : new MetaData(data.getDataCharacteristics()); diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java index 4331da2b426..28fa70f7741 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java @@ -454,7 +454,7 @@ protected MatrixBlock readBlobFromHDFS(String fname, long[] dims) throws IOExcep rlen, clen, blen, mc.getNonZeros(), getFileFormatProperties()); if(iimd.getFileFormat() == FileFormat.CSV || iimd.getFileFormat() == FileFormat.DELTA) { - // dimensions/nnz are discovered at read time for these self-describing formats + //dimensions/nnz are discovered at read time for these self-describing formats _metaData = _metaData instanceof MetaDataFormat ? new MetaDataFormat(newData.getDataCharacteristics(), iimd.getFileFormat()) : new MetaData(newData.getDataCharacteristics()); } diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java index fbae4925c66..b52f3777e1f 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java @@ -122,9 +122,9 @@ public class SparkExecutionContext extends ExecutionContext //singleton spark context (as there can be only one spark context per JVM) private static JavaSparkContext _spctx = null; - // registered users of the singleton context (guarded by the - // SparkExecutionContext.class monitor); maintained by enterSparkExecution()/ - // exitSparkExecution(), and close() only stops the context once it hits zero + //registered users of the singleton context (guarded by the + //SparkExecutionContext.class monitor); maintained by enterSparkExecution()/ + //exitSparkExecution(), and close() only stops the context once it hits zero private static int _activeExecutions = 0; //registry of parallelized RDDs to enforce that at any time, we spent at most @@ -175,8 +175,8 @@ public synchronized static JavaSparkContext getSparkContextStatic() { initSparkContext(); if(_spctx.sc().isStopped()){ _spctx = null; - // the previous context was stopped; reset the active-execution count so a - // stale registration cannot skip a future legitimate stop of the new one + //the previous context was stopped; reset the active-execution count so a + //stale registration cannot skip a future legitimate stop of the new one _activeExecutions = 0; initSparkContext(); } @@ -196,15 +196,16 @@ public synchronized static boolean isSparkContextCreated() { public static void resetSparkContextStatic() { synchronized(SparkExecutionContext.class) { _spctx = null; - // force-discarding the shared context: drop the active-execution count so - // a stale registration cannot skip a future legitimate stop + //force-discarding the shared context: drop the active-execution count so + //a stale registration cannot skip a future legitimate stop _activeExecutions = 0; } } /** - * Registers an active user of the shared spark context. Must be balanced by a later {@link #exitSparkExecution()} - * so a concurrent execution cannot stop the context while this one still has in-flight jobs. + * Registers an active user of the shared spark context. Must be balanced by a + * later {@link #exitSparkExecution()} so a concurrent execution cannot stop the + * context while this one still has in-flight jobs. */ public static void enterSparkExecution() { synchronized(SparkExecutionContext.class) { @@ -213,8 +214,9 @@ public static void enterSparkExecution() { } /** - * Releases an active user previously registered via {@link #enterSparkExecution()}. Only adjusts the count; the - * actual teardown is left to {@link #close()}, which stops the context once no registered execution remains. + * Releases an active user previously registered via {@link #enterSparkExecution()}. + * Only adjusts the count; the actual teardown is left to {@link #close()}, which + * stops the context once no registered execution remains. */ public static void exitSparkExecution() { synchronized(SparkExecutionContext.class) { @@ -225,13 +227,13 @@ public static void exitSparkExecution() { public void close() { synchronized(SparkExecutionContext.class) { - // keep the shared context alive while a registered execution still uses - // it; close() never changes the count, so an unpaired close() (a caller - // that never entered) cannot stop a context another execution is using + //keep the shared context alive while a registered execution still uses + //it; close() never changes the count, so an unpaired close() (a caller + //that never entered) cannot stop a context another execution is using if(_activeExecutions > 0) { if(LOG.isDebugEnabled()) - LOG.debug( - "Keeping shared spark context alive; " + _activeExecutions + " execution(s) still active"); + LOG.debug("Keeping shared spark context alive; " + _activeExecutions + + " execution(s) still active"); return; } if(_spctx != null) { diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java index c502817e026..682cc8e3fff 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java @@ -95,7 +95,8 @@ private void run() { int par_conn = ConfigurationManager.getDMLConfig().getIntValue(DMLConfig.FEDERATED_PAR_CONN); final int EVENT_LOOP_THREADS = (par_conn > 0) ? par_conn : InfrastructureAnalyzer.getLocalParallelism(); // Daemon event loops so a leaked in-JVM (test) worker cannot block JVM exit. - NioEventLoopGroup bossGroup = new NioEventLoopGroup(1, new DefaultThreadFactory("fed-worker-boss", true)); + NioEventLoopGroup bossGroup = new NioEventLoopGroup(1, + new DefaultThreadFactory("fed-worker-boss", true)); ThreadPoolExecutor workerTPE = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 10, TimeUnit.SECONDS, new SynchronousQueue(true), new DefaultThreadFactory("fed-worker-pool", true)); NioEventLoopGroup workerGroup = new NioEventLoopGroup(EVENT_LOOP_THREADS, workerTPE); diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java index ebf05972b87..80a5d699dfa 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java @@ -125,15 +125,13 @@ public static RaggedArray create(T[] col, int m) { /** * Wrap a fully populated raw typed column array into an {@link Array} of the given value type. The runtime type of - * {@code col} must match the primitive backing type of {@code vt} (e.g. {@code double[]} for - * {@link ValueType#FP64}, {@code String[]} for {@link ValueType#STRING}). + * {@code col} must match the primitive backing type of {@code vt} (e.g. {@code double[]} for {@link ValueType#FP64}, + * {@code String[]} for {@link ValueType#STRING}). * - *

- * For {@link ValueType#BOOLEAN} this mirrors {@link #allocateBoolean(int)}: a {@code boolean[]} longer than - * {@link #bitSetSwitchPoint} is bit-packed into a compact {@link BitSetArray} (so a bulk decoder that fills a plain - * {@code boolean[]} still ends up with the same representation as every other frame allocation path), while shorter - * columns stay a plain {@link BooleanArray}. - *

+ *

For {@link ValueType#BOOLEAN} this mirrors {@link #allocateBoolean(int)}: a {@code boolean[]} longer than + * {@link #bitSetSwitchPoint} is bit-packed into a compact {@link BitSetArray} (so a bulk decoder that fills a + * plain {@code boolean[]} still ends up with the same representation as every other frame allocation path), + * while shorter columns stay a plain {@link BooleanArray}.

* * @param vt the value type of the column * @param col the backing array to wrap @@ -170,10 +168,10 @@ public static Array create(ValueType vt, Object col) { /** * Allocate the raw backing array for a column of the given value type: the inverse of - * {@link #create(ValueType, Object)}. Returns {@code double[]} for {@link ValueType#FP64}, {@code int[]} for - * INT32/UINT/HASH32, {@code long[]} for INT64/HASH64, {@code String[]} for STRING, etc. The runtime array type - * matches what {@link #create(ValueType, Object)} expects, so a bulk decoder can fill this primitive array directly - * and then wrap it via {@code create(vt, backing)}. + * {@link #create(ValueType, Object)}. Returns {@code double[]} for {@link ValueType#FP64}, + * {@code int[]} for INT32/UINT/HASH32, {@code long[]} for INT64/HASH64, {@code String[]} for STRING, etc. The + * runtime array type matches what {@link #create(ValueType, Object)} expects, so a bulk decoder can fill this + * primitive array directly and then wrap it via {@code create(vt, backing)}. * * @param vt the value type of the column * @param nRow the number of rows to allocate diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java b/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java index 95be95117e2..9ff58065d97 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java @@ -41,7 +41,7 @@ public class MatrixBlockFromFrame { public static Boolean WARNED_FOR_FAILED_CAST = false; - private MatrixBlockFromFrame() { + private MatrixBlockFromFrame(){ // private constructor for code coverage. } @@ -115,7 +115,7 @@ private static long convert(FrameBlock frame, MatrixBlock mb, int n, int rl, int return convertStrict(frame, mb, n, rl, ru); } catch(NumberFormatException | DMLRuntimeException e) { - synchronized(WARNED_FOR_FAILED_CAST) { + synchronized(WARNED_FOR_FAILED_CAST){ if(!WARNED_FOR_FAILED_CAST) { LOG.error( "Failed to convert to Matrix because of number format errors, falling back to NaN on incompatible cells", diff --git a/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java b/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java index c12a187cf17..eed2c58f78c 100644 --- a/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java +++ b/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java @@ -30,13 +30,32 @@ import jdk.incubator.vector.VectorSpecies; - public enum BuiltinCode { - AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, ATAN, LOG, LOG_NZ, MIN, MAX, ABS, SIGN, SQRT, EXP, PLOGP, - PRINT, PRINTF, NROW, NCOL, LENGTH, LINEAGE, ROUND, MAXINDEX, MININDEX, STOP, CEIL, FLOOR, CUMSUM, ROWCUMSUM, - CUMPROD, CUMMIN, CUMMAX, CUMSUMPROD, INVERSE, SPROP, SIGMOID, EVAL, LIST, TYPEOF, APPLY_SCHEMA, DETECTSCHEMA, - ISNA, ISNAN, ISINF, DROP_INVALID_TYPE, DROP_INVALID_LENGTH, VALUE_SWAP, FRAME_ROW_REPLICATE, - GET_CATEGORICAL_MASK, MAP, COUNT_DISTINCT, COUNT_DISTINCT_APPROX, UNIQUE - } +/** + * Class with pre-defined set of objects. This class can not be instantiated elsewhere. + * + * Notes on commons.math FastMath: + * * FastMath uses lookup tables and interpolation instead of native calls. + * * The memory overhead for those tables is roughly 48KB in total (acceptable) + * * Micro and application benchmarks showed significantly (30%-3x) performance improvements + * for most operations; without loss of accuracy. + * * atan / sqrt were 20% slower in FastMath and hence, we use Math there + * * round / abs were equivalent in FastMath and hence, we use Math there + * * Finally, there is just one argument against FastMath - The comparison heavily depends + * on the JVM. For example, currently the IBM JDK JIT compiles to HW instructions for sqrt + * which makes this operation very efficient; as soon as other operations like log/exp are + * similarly compiled, we should rerun the micro benchmarks, and switch back if necessary. + * + */ +public class Builtin extends ValueFunction +{ + private static final long serialVersionUID = 3836744687789840574L; + + public enum BuiltinCode { AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, ATAN, LOG, LOG_NZ, MIN, + MAX, ABS, SIGN, SQRT, EXP, PLOGP, PRINT, PRINTF, NROW, NCOL, LENGTH, LINEAGE, ROUND, MAXINDEX, MININDEX, + STOP, CEIL, FLOOR, CUMSUM, ROWCUMSUM, CUMPROD, CUMMIN, CUMMAX, CUMSUMPROD, INVERSE, SPROP, SIGMOID, EVAL, LIST, + TYPEOF, APPLY_SCHEMA, DETECTSCHEMA, ISNA, ISNAN, ISINF, DROP_INVALID_TYPE, + DROP_INVALID_LENGTH, VALUE_SWAP, FRAME_ROW_REPLICATE, GET_CATEGORICAL_MASK, + MAP, COUNT_DISTINCT, COUNT_DISTINCT_APPROX, UNIQUE} private static final VectorSpecies SPECIES = DoubleVector.SPECIES_PREFERRED; private static final int vLen = SPECIES.length(); @@ -49,59 +68,59 @@ public enum BuiltinCode { static public HashMap String2BuiltinCode; static { String2BuiltinCode = new HashMap<>(); - String2BuiltinCode.put("autoDiff", BuiltinCode.AUTODIFF); - String2BuiltinCode.put("sin", BuiltinCode.SIN); - String2BuiltinCode.put("cos", BuiltinCode.COS); - String2BuiltinCode.put("tan", BuiltinCode.TAN); - String2BuiltinCode.put("sinh", BuiltinCode.SINH); - String2BuiltinCode.put("cosh", BuiltinCode.COSH); - String2BuiltinCode.put("tanh", BuiltinCode.TANH); - String2BuiltinCode.put("asin", BuiltinCode.ASIN); - String2BuiltinCode.put("acos", BuiltinCode.ACOS); - String2BuiltinCode.put("atan", BuiltinCode.ATAN); - String2BuiltinCode.put("log", BuiltinCode.LOG); - String2BuiltinCode.put("log_nz", BuiltinCode.LOG_NZ); - String2BuiltinCode.put("min", BuiltinCode.MIN); - String2BuiltinCode.put("max", BuiltinCode.MAX); - String2BuiltinCode.put("maxindex", BuiltinCode.MAXINDEX); - String2BuiltinCode.put("minindex", BuiltinCode.MININDEX); - String2BuiltinCode.put("abs", BuiltinCode.ABS); - String2BuiltinCode.put("sign", BuiltinCode.SIGN); - String2BuiltinCode.put("sqrt", BuiltinCode.SQRT); - String2BuiltinCode.put("exp", BuiltinCode.EXP); - String2BuiltinCode.put("plogp", BuiltinCode.PLOGP); - String2BuiltinCode.put("print", BuiltinCode.PRINT); - String2BuiltinCode.put("printf", BuiltinCode.PRINTF); - String2BuiltinCode.put("eval", BuiltinCode.EVAL); - String2BuiltinCode.put("list", BuiltinCode.LIST); - String2BuiltinCode.put("nrow", BuiltinCode.NROW); - String2BuiltinCode.put("ncol", BuiltinCode.NCOL); - String2BuiltinCode.put("length", BuiltinCode.LENGTH); - String2BuiltinCode.put("round", BuiltinCode.ROUND); - String2BuiltinCode.put("stop", BuiltinCode.STOP); - String2BuiltinCode.put("ceil", BuiltinCode.CEIL); - String2BuiltinCode.put("floor", BuiltinCode.FLOOR); - String2BuiltinCode.put("ucumk+", BuiltinCode.CUMSUM); - String2BuiltinCode.put("urowcumk+", BuiltinCode.ROWCUMSUM); - String2BuiltinCode.put("ucum*", BuiltinCode.CUMPROD); - String2BuiltinCode.put("ucumk+*", BuiltinCode.CUMSUMPROD); - String2BuiltinCode.put("ucummin", BuiltinCode.CUMMIN); - String2BuiltinCode.put("ucummax", BuiltinCode.CUMMAX); - String2BuiltinCode.put("inverse", BuiltinCode.INVERSE); - String2BuiltinCode.put("sprop", BuiltinCode.SPROP); - String2BuiltinCode.put("sigmoid", BuiltinCode.SIGMOID); - String2BuiltinCode.put("typeOf", BuiltinCode.TYPEOF); - String2BuiltinCode.put("detectSchema", BuiltinCode.DETECTSCHEMA); - String2BuiltinCode.put("isna", BuiltinCode.ISNA); - String2BuiltinCode.put("isnan", BuiltinCode.ISNAN); - String2BuiltinCode.put("isinf", BuiltinCode.ISINF); - String2BuiltinCode.put("dropInvalidType", BuiltinCode.DROP_INVALID_TYPE); - String2BuiltinCode.put("freplicate", BuiltinCode.FRAME_ROW_REPLICATE); - String2BuiltinCode.put("dropInvalidLength", BuiltinCode.DROP_INVALID_LENGTH); - String2BuiltinCode.put("_map", BuiltinCode.MAP); - String2BuiltinCode.put("valueSwap", BuiltinCode.VALUE_SWAP); - String2BuiltinCode.put("applySchema", BuiltinCode.APPLY_SCHEMA); - String2BuiltinCode.put("get_categorical_mask", BuiltinCode.GET_CATEGORICAL_MASK); + String2BuiltinCode.put( "autoDiff" , BuiltinCode.AUTODIFF); + String2BuiltinCode.put( "sin" , BuiltinCode.SIN); + String2BuiltinCode.put( "cos" , BuiltinCode.COS); + String2BuiltinCode.put( "tan" , BuiltinCode.TAN); + String2BuiltinCode.put( "sinh" , BuiltinCode.SINH); + String2BuiltinCode.put( "cosh" , BuiltinCode.COSH); + String2BuiltinCode.put( "tanh" , BuiltinCode.TANH); + String2BuiltinCode.put( "asin" , BuiltinCode.ASIN); + String2BuiltinCode.put( "acos" , BuiltinCode.ACOS); + String2BuiltinCode.put( "atan" , BuiltinCode.ATAN); + String2BuiltinCode.put( "log" , BuiltinCode.LOG); + String2BuiltinCode.put( "log_nz" , BuiltinCode.LOG_NZ); + String2BuiltinCode.put( "min" , BuiltinCode.MIN); + String2BuiltinCode.put( "max" , BuiltinCode.MAX); + String2BuiltinCode.put( "maxindex", BuiltinCode.MAXINDEX); + String2BuiltinCode.put( "minindex", BuiltinCode.MININDEX); + String2BuiltinCode.put( "abs" , BuiltinCode.ABS); + String2BuiltinCode.put( "sign" , BuiltinCode.SIGN); + String2BuiltinCode.put( "sqrt" , BuiltinCode.SQRT); + String2BuiltinCode.put( "exp" , BuiltinCode.EXP); + String2BuiltinCode.put( "plogp" , BuiltinCode.PLOGP); + String2BuiltinCode.put( "print" , BuiltinCode.PRINT); + String2BuiltinCode.put( "printf" , BuiltinCode.PRINTF); + String2BuiltinCode.put( "eval" , BuiltinCode.EVAL); + String2BuiltinCode.put( "list" , BuiltinCode.LIST); + String2BuiltinCode.put( "nrow" , BuiltinCode.NROW); + String2BuiltinCode.put( "ncol" , BuiltinCode.NCOL); + String2BuiltinCode.put( "length" , BuiltinCode.LENGTH); + String2BuiltinCode.put( "round" , BuiltinCode.ROUND); + String2BuiltinCode.put( "stop" , BuiltinCode.STOP); + String2BuiltinCode.put( "ceil" , BuiltinCode.CEIL); + String2BuiltinCode.put( "floor" , BuiltinCode.FLOOR); + String2BuiltinCode.put( "ucumk+" , BuiltinCode.CUMSUM); + String2BuiltinCode.put( "urowcumk+" , BuiltinCode.ROWCUMSUM); + String2BuiltinCode.put( "ucum*" , BuiltinCode.CUMPROD); + String2BuiltinCode.put( "ucumk+*", BuiltinCode.CUMSUMPROD); + String2BuiltinCode.put( "ucummin", BuiltinCode.CUMMIN); + String2BuiltinCode.put( "ucummax", BuiltinCode.CUMMAX); + String2BuiltinCode.put( "inverse", BuiltinCode.INVERSE); + String2BuiltinCode.put( "sprop", BuiltinCode.SPROP); + String2BuiltinCode.put( "sigmoid", BuiltinCode.SIGMOID); + String2BuiltinCode.put( "typeOf", BuiltinCode.TYPEOF); + String2BuiltinCode.put( "detectSchema", BuiltinCode.DETECTSCHEMA); + String2BuiltinCode.put( "isna", BuiltinCode.ISNA); + String2BuiltinCode.put( "isnan", BuiltinCode.ISNAN); + String2BuiltinCode.put( "isinf", BuiltinCode.ISINF); + String2BuiltinCode.put( "dropInvalidType", BuiltinCode.DROP_INVALID_TYPE); + String2BuiltinCode.put( "freplicate", BuiltinCode.FRAME_ROW_REPLICATE); + String2BuiltinCode.put( "dropInvalidLength", BuiltinCode.DROP_INVALID_LENGTH); + String2BuiltinCode.put( "_map", BuiltinCode.MAP); + String2BuiltinCode.put( "valueSwap", BuiltinCode.VALUE_SWAP); + String2BuiltinCode.put( "applySchema", BuiltinCode.APPLY_SCHEMA); + String2BuiltinCode.put( "get_categorical_mask", BuiltinCode.GET_CATEGORICAL_MASK); } protected Builtin(BuiltinCode bf) { diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java index 08d28512d5c..86184f47be6 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java @@ -59,7 +59,7 @@ else if (in1.getDataType() == DataType.TENSOR && in2.getDataType() == DataType.T return new BinaryTensorTensorCPInstruction(operator, in1, in2, out, opcode, str); else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.FRAME) return new BinaryFrameFrameCPInstruction(operator, in1, in2, out, opcode, str); - else if(in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.SCALAR) + else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.SCALAR) return new BinaryFrameScalarCPInstruction(operator, in1, in2, out, opcode, str); else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.MATRIX) return new BinaryFrameMatrixCPInstruction(operator, in1, in2, out, opcode, str); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java index de76fca18b8..193894fd9bc 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java @@ -37,8 +37,8 @@ public class BinaryFrameScalarCPInstruction extends BinaryCPInstruction { // private static final Log LOG = LogFactory.getLog(BinaryFrameFrameCPInstruction.class.getName()); - private static final TfMethod[] UNSUPPORTED_MASK_METHODS = new TfMethod[] {TfMethod.BIN, TfMethod.WORD_EMBEDDING, - TfMethod.BAG_OF_WORDS, TfMethod.UDF}; + private static final TfMethod[] UNSUPPORTED_MASK_METHODS = new TfMethod[] {TfMethod.BIN, + TfMethod.WORD_EMBEDDING, TfMethod.BAG_OF_WORDS, TfMethod.UDF}; protected BinaryFrameScalarCPInstruction(MultiThreadedOperator op, CPOperand in1, CPOperand in2, CPOperand out, String opcode, String istr) { @@ -96,9 +96,9 @@ public void processGetCategorical(ExecutionContext ec, FrameBlock f, ScalarObjec } /** - * Accumulates, per input column, how many output columns it expands to (lengths) and whether those output columns - * are categorical (categorical). The arrays are allocated lazily: a column that no method touches keeps the - * implicit default of a single, non-categorical output column. + * Accumulates, per input column, how many output columns it expands to (lengths) and whether those + * output columns are categorical (categorical). The arrays are allocated lazily: a column that no + * method touches keeps the implicit default of a single, non-categorical output column. */ private static final class CategoricalMask { private final FrameBlock f; diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java index 2c8093c3717..d76dbe0d45e 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java @@ -80,10 +80,10 @@ public void processInstruction(ExecutionContext ec) { retBlock = inBlock1; } else { - if(LibCommonsMath.isSupportedMatrixMatrixOperation(getOpcode())) { + if(LibCommonsMath.isSupportedMatrixMatrixOperation(getOpcode()) ){ if(compressedLeft) inBlock1 = CompressedMatrixBlock.getUncompressed(inBlock1, getOpcode()); - + if(compressedRight) inBlock2 = CompressedMatrixBlock.getUncompressed(inBlock2, getOpcode()); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java index 97ae151ecc0..e53958ac4b8 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java @@ -350,10 +350,9 @@ else if(opcode.equalsIgnoreCase(Opcodes.TRANSFORMDECODE.toString())) { String[] colnames = meta.getColumnNames(); // compute transformdecode - Decoder decoder = DecoderFactory.createDecoder(getParameterMap().get("spec"), colnames, null, meta, - data.getNumColumns()); - FrameBlock fbout = decoder.decode(data, new FrameBlock(decoder.getSchema()), - InfrastructureAnalyzer.getLocalParallelism()); + Decoder decoder = DecoderFactory + .createDecoder(getParameterMap().get("spec"), colnames, null, meta, data.getNumColumns()); + FrameBlock fbout = decoder.decode(data, new FrameBlock(decoder.getSchema()), InfrastructureAnalyzer.getLocalParallelism()); fbout.setColumnNames(Arrays.copyOfRange(colnames, 0, fbout.getNumColumns())); // release locks diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java index 04353806ca3..40a677e5d71 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java @@ -45,8 +45,8 @@ protected ReorgOOCInstruction(ReorgOperator op, CPOperand in1, CPOperand out, St this(op, in1, out, null, null, null, opcode, istr); } - private ReorgOOCInstruction(Operator op, CPOperand in, CPOperand out, CPOperand col, CPOperand desc, - CPOperand ixret, String opcode, String istr) { + private ReorgOOCInstruction(Operator op, CPOperand in, CPOperand out, CPOperand col, CPOperand desc, CPOperand ixret, + String opcode, String istr) { super(OOCType.Reorg, op, in, out, opcode, istr); _col = col; _desc = desc; @@ -76,8 +76,8 @@ else if(opcode.equalsIgnoreCase(Opcodes.SORT.toString())) { CPOperand desc = new CPOperand(parts[3]); CPOperand ixret = new CPOperand(parts[4]); int k = Integer.parseInt(parts[6]); - return new ReorgOOCInstruction(new ReorgOperator(new SortIndex(1, false, false), k), in, out, col, desc, - ixret, opcode, str); + return new ReorgOOCInstruction(new ReorgOperator(new SortIndex(1, false, false), k), + in, out, col, desc, ixret, opcode, str); } else throw new NotImplementedException(); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java index 091c6785b3c..7590438b949 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java @@ -128,20 +128,17 @@ public void processInstruction(ExecutionContext ec) { long r = tmp.getIndexes().getRowIndex(); long c = tmp.getIndexes().getColumnIndex(); // adapt index to new position in row - return new IndexedMatrixValue(new MatrixIndexes(1, (r - 1) * numBlocksPerRowIn + c), - tmp.getValue()); + return new IndexedMatrixValue(new MatrixIndexes(1, (r - 1) * numBlocksPerRowIn + c), tmp.getValue()); }); } else { f.join(); - reshapeFullColBlocks(rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, - singleRowBlocks, qOut); + reshapeFullColBlocks(rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, singleRowBlocks, qOut); } } else { f.join(); - reshapePartialColBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, - numBlocksPerColOut, singleRowBlocks, qOut); + reshapePartialColBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, singleRowBlocks, qOut); } } else { @@ -169,20 +166,17 @@ public void processInstruction(ExecutionContext ec) { long r = tmp.getIndexes().getRowIndex(); long c = tmp.getIndexes().getColumnIndex(); // adapt index to new position in col - return new IndexedMatrixValue(new MatrixIndexes((c - 1) * numBlocksPerColIn + r, 1), - tmp.getValue()); + return new IndexedMatrixValue(new MatrixIndexes((c - 1) * numBlocksPerColIn + r, 1), tmp.getValue()); }); } else { f.join(); - reshapeFullRowBlocks(rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, - singleColBlocks, qOut); + reshapeFullRowBlocks(rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, singleColBlocks, qOut); } } else { f.join(); - reshapePartialRowBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, - numBlocksPerColOut, singleColBlocks, qOut); + reshapePartialRowBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, singleColBlocks, qOut); } } } @@ -232,8 +226,7 @@ private void reshapeFullColBlocks(long rows, long cols, int blen, int numBlocksP } private void reshapePartialColBlocks(long rlen, long clen, long rows, long cols, int blen, int numBlocksPerRowIn, - int numBlocksPerRowOut, int numBlocksPerColOut, OOCStream singleRowBlocks, - OOCStream qOut) { + int numBlocksPerRowOut, int numBlocksPerColOut, OOCStream singleRowBlocks, OOCStream qOut) { // use cache for accessing input rows by index CachingStream singleRowBlockCache = new CachingStream(singleRowBlocks); singleRowBlockCache.incrSubscriberCount(1); @@ -244,16 +237,14 @@ private void reshapePartialColBlocks(long rlen, long clen, long rows, long cols, int r = 0; // allocate row of output blocks - MatrixBlock[] outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, - true); + MatrixBlock[] outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut,true); int offsetOut = 0; int localColsOut = (cols > blen) ? blen : (int) cols; // iterate through input rows and add to row of output blocks for(int i = 1; i <= rlen; i++) { for(int j = 1; j <= numBlocksPerRowIn; j++) { - try(OOCStream.QueueCallback qcb = singleRowBlockCache - .findCached(new MatrixIndexes(i, j))) { + try(OOCStream.QueueCallback qcb = singleRowBlockCache.findCached(new MatrixIndexes(i, j))) { MatrixBlock blk = (MatrixBlock) qcb.get().getValue(); int offsetIn = 0; @@ -287,12 +278,10 @@ else if(remIn == remOut) { if(r == outputBlockRow[0].getNumRows()) { // enqueue filled output blocks and allocate new ones for(int b = 0; b < outputBlockRow.length; b++) - qOut.enqueue( - new IndexedMatrixValue(new MatrixIndexes(br + 1, b + 1), outputBlockRow[b])); + qOut.enqueue(new IndexedMatrixValue(new MatrixIndexes(br + 1, b + 1), outputBlockRow[b])); br++; // allocate new block row - outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, - numBlocksPerColOut, true); + outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, true); r = 0; } bc = 0; @@ -352,8 +341,7 @@ private void reshapeFullRowBlocks(long rows, long cols, int blen, int numBlocksP } private void reshapePartialRowBlocks(long rlen, long clen, long rows, long cols, int blen, int numBlocksPerRowOut, - int numBlocksPerColIn, int numBlocksPerColOut, OOCStream singleColBlocks, - OOCStream qOut) { + int numBlocksPerColIn, int numBlocksPerColOut, OOCStream singleColBlocks, OOCStream qOut) { // use cache for accessing input cols by index CachingStream singleRowBlockCache = new CachingStream(singleColBlocks); singleRowBlockCache.incrSubscriberCount(1); @@ -364,16 +352,14 @@ private void reshapePartialRowBlocks(long rlen, long clen, long rows, long cols, int c = 0; // allocate col of output blocks - MatrixBlock[] outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, - false); + MatrixBlock[] outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, false); int offsetOut = 0; int localRowsOut = (rows > blen) ? blen : (int) rows; // iterate through input cols and add to col of output blocks for(int j = 1; j <= clen; j++) { for(int i = 1; i <= numBlocksPerColIn; i++) { - try(OOCStream.QueueCallback qcb = singleRowBlockCache - .findCached(new MatrixIndexes(i, j))) { + try(OOCStream.QueueCallback qcb = singleRowBlockCache.findCached(new MatrixIndexes(i, j))) { MatrixBlock blk = (MatrixBlock) qcb.get().getValue(); int offsetIn = 0; @@ -408,13 +394,11 @@ else if(remIn == remOut) { // enqueue filled output blocks and allocate new ones for(int b = 0; b < outputBlockCol.length; b++) { outputBlockCol[b].recomputeNonZeros(); - qOut.enqueue( - new IndexedMatrixValue(new MatrixIndexes(b + 1, bc + 1), outputBlockCol[b])); + qOut.enqueue(new IndexedMatrixValue(new MatrixIndexes(b + 1, bc + 1), outputBlockCol[b])); } bc++; // allocate new block col - outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, - numBlocksPerColOut, false); + outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, false); c = 0; } br = 0; @@ -427,20 +411,16 @@ else if(remIn == remOut) { qOut.closeInput(); } - private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int blen, int numBlocksPerRow, - int numBlocksPerCol, boolean isBlockRowSlice) { + private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int blen, int numBlocksPerRow, int numBlocksPerCol, boolean isBlockRowSlice) { int num = isBlockRowSlice ? numBlocksPerRow : numBlocksPerCol; MatrixBlock[] res = new MatrixBlock[num]; // full inner blocks, adjust for outer blocks - int localRows = ((!isBlockRowSlice || idx == numBlocksPerCol - 1) && rows % blen != 0) ? (int) rows % - blen : blen; - int localCols = ((isBlockRowSlice || idx == numBlocksPerRow - 1) && cols % blen != 0) ? (int) cols % - blen : blen; + int localRows = ((!isBlockRowSlice || idx == numBlocksPerCol - 1) && rows % blen != 0) ? (int) rows % blen : blen; + int localCols = ((isBlockRowSlice || idx == numBlocksPerRow - 1) && cols % blen != 0) ? (int) cols % blen : blen; for(int k = 0; k < num - 1; k++) { - res[k] = isBlockRowSlice ? new MatrixBlock(localRows, blen, false) : new MatrixBlock(blen, localCols, - false); + res[k] = isBlockRowSlice ? new MatrixBlock(localRows, blen, false) : new MatrixBlock(blen, localCols, false); res[k].allocateDenseBlock(); } res[num - 1] = new MatrixBlock(localRows, localCols, false); @@ -448,13 +428,10 @@ private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int ble return res; } - private void setOutputEntries(MatrixBlock src, MatrixBlock dest, int idx, int srcOffset, int destOffset, int length, - boolean rowWise) { + private void setOutputEntries(MatrixBlock src, MatrixBlock dest, int idx, int srcOffset, int destOffset, int length, boolean rowWise) { if(rowWise) - ((DenseBlockFP64) dest.getDenseBlock()).setPartialRow(src.getDenseBlock(), idx, srcOffset, destOffset, - length); + ((DenseBlockFP64) dest.getDenseBlock()).setPartialRow(src.getDenseBlock(), idx, srcOffset, destOffset, length); else - ((DenseBlockFP64) dest.getDenseBlock()).setPartialCol(src.getDenseBlock(), idx, srcOffset, destOffset, - length); + ((DenseBlockFP64) dest.getDenseBlock()).setPartialCol(src.getDenseBlock(), idx, srcOffset, destOffset, length); } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java index e25219b80ba..75f84882478 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java @@ -114,7 +114,8 @@ public void processInstruction(ExecutionContext ec) { case VALUEPICK: { if( input2.isScalar() ) { ScalarObject quantile = ec.getScalarInput(input2); - double[] wt = getWeightedQuantileSummary(in, mc, new double[] {quantile.getDoubleValue()}, true); + double[] wt = getWeightedQuantileSummary(in, mc, + new double[] {quantile.getDoubleValue()}, true); ec.setScalarOutput(output.getName(), new DoubleObject(wt[3])); } else { diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java index f007558ebdc..2f83caa5526 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java @@ -30,7 +30,8 @@ import org.apache.sysds.runtime.matrix.data.MatrixValue; import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; -public class IndexedMatrixValue implements SpillableObject, Serializable { +public class IndexedMatrixValue implements SpillableObject, Serializable +{ private static final long serialVersionUID = 6723389820806752110L; private MatrixIndexes _indexes = null; @@ -109,8 +110,8 @@ public void discard() { @Override public void read(DataInput dataInput) throws IOException { - _indexes = new MatrixIndexes(); - _value = new MatrixBlock(); + _indexes = new MatrixIndexes(); + _value = new MatrixBlock(); _indexes.readFields(dataInput); _value.readFields(dataInput); } diff --git a/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java index cc8491d4515..c3b9351d3d3 100644 --- a/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java +++ b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java @@ -87,10 +87,11 @@ import io.delta.kernel.utils.FileStatus; /** - * Shared helpers for the native (Spark-free) Delta Lake read/write paths used by both the matrix and frame - * readers/writers. Centralizes engine creation, path qualification, the scan loop (snapshot -> data files -> - * logical columnar batches, honoring deletion vectors), and the write transaction (logical data -> parquet -> - * commit). + * Shared helpers for the native (Spark-free) Delta Lake read/write paths used + * by both the matrix and frame readers/writers. Centralizes engine creation, + * path qualification, the scan loop (snapshot -> data files -> logical + * columnar batches, honoring deletion vectors), and the write transaction + * (logical data -> parquet -> commit). */ public class DeltaKernelUtils { @@ -101,29 +102,23 @@ public class DeltaKernelUtils { /** Reused thread-safe JSON reader for the per-file Delta stats (numRecords). */ private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); - /** - * Delta Kernel config key: number of rows per parquet read batch, overridable via - * {@link org.apache.sysds.conf.DMLConfig#DELTA_READER_BATCH_SIZE}. - */ + /** Delta Kernel config key: number of rows per parquet read batch, overridable via + * {@link org.apache.sysds.conf.DMLConfig#DELTA_READER_BATCH_SIZE}. */ private static final String CONF_READER_BATCH_SIZE = "delta.kernel.default.parquet.reader.batch-size"; - /** - * Delta Kernel config key: target size (bytes) at which the writer rolls a new data file, overridable via - * {@link org.apache.sysds.conf.DMLConfig#DELTA_WRITER_TARGET_FILE_SIZE}. - */ + /** Delta Kernel config key: target size (bytes) at which the writer rolls a new data file, overridable via + * {@link org.apache.sysds.conf.DMLConfig#DELTA_WRITER_TARGET_FILE_SIZE}. */ private static final String CONF_WRITER_TARGET_FILE_SIZE = "delta.kernel.default.parquet.writer.targetMaxFileSize"; - /** - * Internal Delta column type codes shared by the matrix and frame readers to dispatch boxing-free primitive column - * access. - */ - public static final int T_DOUBLE = 0; - public static final int T_FLOAT = 1; - public static final int T_LONG = 2; - public static final int T_INT = 3; - public static final int T_SHORT = 4; - public static final int T_BYTE = 5; + /** Internal Delta column type codes shared by the matrix and frame readers to + * dispatch boxing-free primitive column access. */ + public static final int T_DOUBLE = 0; + public static final int T_FLOAT = 1; + public static final int T_LONG = 2; + public static final int T_INT = 3; + public static final int T_SHORT = 4; + public static final int T_BYTE = 5; public static final int T_BOOLEAN = 6; - public static final int T_STRING = 7; + public static final int T_STRING = 7; /** * Parquet physical type each {@code T_*} column is stored as, indexed by type code (delta int/short/byte columns @@ -133,22 +128,22 @@ public class DeltaKernelUtils { PrimitiveTypeName.INT64, PrimitiveTypeName.INT32, PrimitiveTypeName.INT32, PrimitiveTypeName.INT32, PrimitiveTypeName.BOOLEAN, PrimitiveTypeName.BINARY}; - // derived configuration cached to avoid copying the (large) base conf on every - // engine creation (createEngine is called once per data file in parallel reads); - // rebuilt whenever the base conf or the relevant SystemDS settings change. + //derived configuration cached to avoid copying the (large) base conf on every + //engine creation (createEngine is called once per data file in parallel reads); + //rebuilt whenever the base conf or the relevant SystemDS settings change. private static Configuration cachedConf; private static Configuration cachedConfBase; private static int cachedBatchSize; private static long cachedTargetFileSize; - private DeltaKernelUtils() { - } + private DeltaKernelUtils() {} /** - * Consumes a whole columnar batch. {@code selected} is {@code null} when all {@code size} rows are live; otherwise - * {@code selected[r]} indicates whether row {@code r} survived the deletion/selection vector. Batch-level - * consumption lets callers extract data column-at-a-time (cache friendly, boxing free) instead of paying a per-row - * callback. + * Consumes a whole columnar batch. {@code selected} is {@code null} when all + * {@code size} rows are live; otherwise {@code selected[r]} indicates whether + * row {@code r} survived the deletion/selection vector. Batch-level consumption + * lets callers extract data column-at-a-time (cache friendly, boxing free) + * instead of paying a per-row callback. */ @FunctionalInterface public interface BatchConsumer { @@ -156,29 +151,22 @@ public interface BatchConsumer { } /** - * Map a Delta Kernel {@link DataType} to an internal type code (see the {@code T_*} constants). Returned once per - * column so the per-cell read loop can switch on a primitive int instead of repeating {@code instanceof} checks. + * Map a Delta Kernel {@link DataType} to an internal type code (see the + * {@code T_*} constants). Returned once per column so the per-cell read loop + * can switch on a primitive int instead of repeating {@code instanceof} checks. * * @param dt the Delta column data type * @return the matching {@code T_*} code, or {@code -1} if the type is not supported */ public static int typeCode(DataType dt) { - if(dt instanceof DoubleType) - return T_DOUBLE; - if(dt instanceof FloatType) - return T_FLOAT; - if(dt instanceof LongType) - return T_LONG; - if(dt instanceof IntegerType) - return T_INT; - if(dt instanceof ShortType) - return T_SHORT; - if(dt instanceof ByteType) - return T_BYTE; - if(dt instanceof BooleanType) - return T_BOOLEAN; - if(dt instanceof StringType) - return T_STRING; + if( dt instanceof DoubleType ) return T_DOUBLE; + if( dt instanceof FloatType ) return T_FLOAT; + if( dt instanceof LongType ) return T_LONG; + if( dt instanceof IntegerType ) return T_INT; + if( dt instanceof ShortType ) return T_SHORT; + if( dt instanceof ByteType ) return T_BYTE; + if( dt instanceof BooleanType ) return T_BOOLEAN; + if( dt instanceof StringType ) return T_STRING; return -1; } @@ -441,10 +429,8 @@ private static void decodeColumnInto(ColumnReader creader, int maxDef, int nrow, } } - /** - * Floor on the adaptive writer target file size. Below this the per-file metadata/open overhead (and tiny-file - * proliferation) outweighs the extra read parallelism. - */ + /** Floor on the adaptive writer target file size. Below this the per-file metadata/open + * overhead (and tiny-file proliferation) outweighs the extra read parallelism. */ public static final long ADAPTIVE_WRITER_MIN_FILE_SIZE = 4L * 1024 * 1024; private static Configuration buildConf(Configuration base, int batchSize, long targetFileSize) { @@ -458,8 +444,9 @@ private static synchronized Configuration deltaConf() { Configuration base = ConfigurationManager.getCachedJobConf(); int batchSize = ConfigurationManager.getDeltaReaderBatchSize(); long targetFileSize = ConfigurationManager.getDeltaWriterTargetFileSize(); - if(cachedConf == null || cachedConfBase != base || cachedBatchSize != batchSize || - cachedTargetFileSize != targetFileSize) { + if(cachedConf == null || cachedConfBase != base + || cachedBatchSize != batchSize || cachedTargetFileSize != targetFileSize) + { cachedConf = buildConf(base, batchSize, targetFileSize); cachedConfBase = base; cachedBatchSize = batchSize; @@ -473,11 +460,12 @@ public static Engine createEngine() { } /** - * Compute the parquet target data-file size (bytes) for writing a table of the given estimated size. With adaptive - * sizing enabled the writer aims for roughly one data file per expected parallel reader (so the native per-file - * parallel read can use all threads): never above the configured target, and never below - * {@code ADAPTIVE_WRITER_MIN_FILE_SIZE} unless the configured target is itself smaller than that floor (in which - * case the configured target wins). + * Compute the parquet target data-file size (bytes) for writing a table of the given + * estimated size. With adaptive sizing enabled the writer aims for roughly one data + * file per expected parallel reader (so the native per-file parallel read can use all + * threads): never above the configured target, and never below + * {@code ADAPTIVE_WRITER_MIN_FILE_SIZE} unless the configured target is itself smaller + * than that floor (in which case the configured target wins). * * @param estimatedBytes estimate of the table's size (the block in-memory size is a fine proxy) * @return the target max parquet data-file size in bytes @@ -488,7 +476,7 @@ public static long adaptiveWriterTargetFileSize(long estimatedBytes) { return configured; int par = Math.max(1, OptimizerUtils.getParallelBinaryReadParallelism()); long perReader = Math.max(1, estimatedBytes / par); - // never above the configured cap, never below the floor (unless the cap itself is lower) + //never above the configured cap, never below the floor (unless the cap itself is lower) long target = Math.min(configured, Math.max(ADAPTIVE_WRITER_MIN_FILE_SIZE, perReader)); if(LOG.isDebugEnabled()) LOG.debug("Delta adaptive file size: est=" + estimatedBytes + "B par=" + par + " -> target=" + target @@ -497,24 +485,24 @@ public static long adaptiveWriterTargetFileSize(long estimatedBytes) { } /** - * Create an engine for writing a table of the given estimated size, configured with an adaptive target data-file - * size (see {@link #adaptiveWriterTargetFileSize(long)}). A fresh (uncached) configuration is built since writes - * happen once per table, not per data file. + * Create an engine for writing a table of the given estimated size, configured with an + * adaptive target data-file size (see {@link #adaptiveWriterTargetFileSize(long)}). A fresh + * (uncached) configuration is built since writes happen once per table, not per data file. * * @param estimatedBytes estimate of the table's size (the block in-memory size is a fine proxy) * @return a Delta Kernel engine for the write */ public static Engine createWriteEngine(long estimatedBytes) { - // the reader batch size is irrelevant on the write path but is set to keep the - // conf shape identical to deltaConf(); only the target file size matters here. + //the reader batch size is irrelevant on the write path but is set to keep the + //conf shape identical to deltaConf(); only the target file size matters here. Configuration c = buildConf(ConfigurationManager.getCachedJobConf(), ConfigurationManager.getDeltaReaderBatchSize(), adaptiveWriterTargetFileSize(estimatedBytes)); return DefaultEngine.create(c); } /** - * Resolve a (possibly relative) path to a fully-qualified URI so the kernel's default engine can locate the table - * on the right filesystem. + * Resolve a (possibly relative) path to a fully-qualified URI so the + * kernel's default engine can locate the table on the right filesystem. * * @param fname input path * @return fully-qualified table path @@ -531,9 +519,11 @@ public static String qualify(String fname) { } /** - * Opened latest snapshot of a Delta table: the logical schema plus everything needed to (re)read its data files, - * including the list of per-data-file scan rows. Delta Kernel scan-file rows are self-contained (the kernel's - * distributed design serializes them to workers), so they can be retained and read independently / in parallel. + * Opened latest snapshot of a Delta table: the logical schema plus everything + * needed to (re)read its data files, including the list of per-data-file scan + * rows. Delta Kernel scan-file rows are self-contained (the kernel's + * distributed design serializes them to workers), so they can be retained and + * read independently / in parallel. */ public static final class ScanHandle { public final StructType schema; @@ -541,18 +531,19 @@ public static final class ScanHandle { public final StructType physicalReadSchema; public final List scanFiles; /** - * Per-file record counts taken from the Delta {@code numRecords} statistic, aligned with {@link #scanFiles}; - * {@code -1} where the statistic is absent. + * Per-file record counts taken from the Delta {@code numRecords} statistic, + * aligned with {@link #scanFiles}; {@code -1} where the statistic is absent. */ public final long[] numRecords; /** - * Per-file flag indicating a deletion vector is present (so the live row count differs from - * {@link #numRecords}), aligned with {@link #scanFiles}. + * Per-file flag indicating a deletion vector is present (so the live row + * count differs from {@link #numRecords}), aligned with {@link #scanFiles}. */ public final boolean[] hasDeletionVector; - private ScanHandle(StructType schema, Row scanState, StructType physicalReadSchema, List scanFiles, - long[] numRecords, boolean[] hasDeletionVector) { + private ScanHandle(StructType schema, Row scanState, StructType physicalReadSchema, + List scanFiles, long[] numRecords, boolean[] hasDeletionVector) + { this.schema = schema; this.scanState = scanState; this.physicalReadSchema = physicalReadSchema; @@ -562,12 +553,13 @@ private ScanHandle(StructType schema, Row scanState, StructType physicalReadSche } /** - * @return true iff every data file carries a {@code numRecords} statistic and none has a deletion vector, i.e. - * exact per-file row offsets can be derived from metadata without reading the data. + * @return true iff every data file carries a {@code numRecords} statistic + * and none has a deletion vector, i.e. exact per-file row offsets + * can be derived from metadata without reading the data. */ public boolean hasExactRowCounts() { - for(int i = 0; i < numRecords.length; i++) - if(numRecords[i] < 0 || hasDeletionVector[i]) + for( int i=0; i scanFileIter = (scan instanceof ScanImpl) ? ((ScanImpl) scan) - .getScanFiles(engine, true) : scan.getScanFiles(engine); + //request the scan files WITH per-file statistics (numRecords) so callers can + //pre-size output and place rows without reading the data; harmless extra + //column for the data-read path. Fall back to the stats-less iterator if the + //concrete scan does not support it. + CloseableIterator scanFileIter = (scan instanceof ScanImpl) + ? ((ScanImpl) scan).getScanFiles(engine, true) + : scan.getScanFiles(engine); List files = new ArrayList<>(); List recs = new ArrayList<>(); List dvs = new ArrayList<>(); - try(CloseableIterator scanFiles = scanFileIter) { - while(scanFiles.hasNext()) { + try( CloseableIterator scanFiles = scanFileIter ) { + while( scanFiles.hasNext() ) { FilteredColumnarBatch scanFileBatch = scanFiles.next(); - try(CloseableIterator scanFileRows = scanFileBatch.getRows()) { - while(scanFileRows.hasNext()) { + try( CloseableIterator scanFileRows = scanFileBatch.getRows() ) { + while( scanFileRows.hasNext() ) { Row scanFileRow = scanFileRows.next(); files.add(scanFileRow); recs.add(numRecords(scanFileRow)); @@ -615,7 +609,7 @@ public static ScanHandle openScan(Engine engine, String tablePath) throws IOExce } long[] numRecords = new long[recs.size()]; boolean[] hasDv = new boolean[dvs.size()]; - for(int i = 0; i < numRecords.length; i++) { + for( int i=0; i physicalData = engine.getParquetHandler() .readParquetFiles(Utils.singletonCloseableIterator(dataFile), physicalReadSchema, Optional.empty()); - try(CloseableIterator logicalData = Scan.transformPhysicalData(engine, scanState, - scanFileRow, physicalData)) { - while(logicalData.hasNext()) + try( CloseableIterator logicalData = + Scan.transformPhysicalData(engine, scanState, scanFileRow, physicalData) ) + { + while( logicalData.hasNext() ) consumeBatch(logicalData.next(), consumer); } } /** - * Scan the latest snapshot of a Delta table sequentially, invoking the batch consumer for every data batch. The - * consumer is created lazily from the table schema (so callers can size buffers / derive per-column types up - * front). + * Scan the latest snapshot of a Delta table sequentially, invoking the batch + * consumer for every data batch. The consumer is created lazily from the table + * schema (so callers can size buffers / derive per-column types up front). * * @param engine delta kernel engine * @param tablePath fully-qualified table path @@ -679,10 +677,11 @@ public static void readScanFile(Engine engine, Row scanState, StructType physica * @throws IOException on read failure */ public static StructType scan(Engine engine, String tablePath, Function consumerFactory) - throws IOException { + throws IOException + { ScanHandle h = openScan(engine, tablePath); BatchConsumer consumer = consumerFactory.apply(h.schema); - for(Row scanFileRow : h.scanFiles) + for( Row scanFileRow : h.scanFiles ) readScanFile(engine, h.scanState, h.physicalReadSchema, scanFileRow, consumer); return h.schema; } @@ -691,27 +690,28 @@ private static void consumeBatch(FilteredColumnarBatch fcb, BatchConsumer consum ColumnarBatch batch = fcb.getData(); int ncol = batch.getSchema().length(); ColumnVector[] cols = new ColumnVector[ncol]; - for(int c = 0; c < ncol; c++) + for( int c=0; c all rows live) + //materialize the deletion/selection mask once (null => all rows live) Optional selVector = fcb.getSelectionVector(); boolean[] selected = null; - if(selVector.isPresent()) { + if( selVector.isPresent() ) { ColumnVector sv = selVector.get(); selected = new boolean[size]; - for(int r = 0; r < size; r++) + for( int r=0; r logicalData) throws IOException { - // replace any existing table at the path (the other SystemDS writers delete - // the output first; the caching layer does not do it on our behalf) + CloseableIterator logicalData) throws IOException + { + //replace any existing table at the path (the other SystemDS writers delete + //the output first; the caching layer does not do it on our behalf) HDFSTool.deleteFileIfExistOnHDFS(tablePath); Table table = Table.forPath(engine, tablePath); - TransactionBuilder txnBuilder = table.createTransactionBuilder(engine, ENGINE_INFO, Operation.CREATE_TABLE) + TransactionBuilder txnBuilder = table + .createTransactionBuilder(engine, ENGINE_INFO, Operation.CREATE_TABLE) .withSchema(engine, schema); Transaction txn = txnBuilder.build(engine); Row txnState = txn.getTransactionState(engine); - CloseableIterator physicalData = Transaction.transformLogicalData(engine, txnState, - logicalData, Collections.emptyMap()); - DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, Collections.emptyMap()); + CloseableIterator physicalData = + Transaction.transformLogicalData(engine, txnState, logicalData, Collections.emptyMap()); + DataWriteContext writeContext = + Transaction.getWriteContext(engine, txnState, Collections.emptyMap()); CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles(writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns()); - CloseableIterator appendActions = Transaction.generateAppendActions(engine, txnState, dataFiles, - writeContext); + CloseableIterator appendActions = + Transaction.generateAppendActions(engine, txnState, dataFiles, writeContext); txn.commit(engine, CloseableIterable.inMemoryIterable(appendActions)); } } diff --git a/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java b/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java index 55d8f8f7c2d..58a98741975 100644 --- a/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java +++ b/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java @@ -33,27 +33,29 @@ import io.delta.kernel.types.StructType; /** - * Single-threaded native Delta Lake reader for matrices, built on the Spark-free Delta Kernel library. It opens the - * latest snapshot of a Delta table directory, reads its parquet data files through the kernel's default engine - * (honoring deletion vectors), and materializes the numeric columns into a dense {@link MatrixBlock}. + * Single-threaded native Delta Lake reader for matrices, built on the + * Spark-free Delta Kernel library. It opens the latest snapshot of a Delta + * table directory, reads its parquet data files through the kernel's default + * engine (honoring deletion vectors), and materializes the numeric columns + * into a dense {@link MatrixBlock}. * - *

- * Only numeric columns (double/float/long/int/short/byte/boolean) are supported, matching the all-double nature of a - * SystemDS matrix. Dimensions do not need to be known up front: the row count is discovered while scanning and the - * column count is taken from the table schema. - *

+ *

Only numeric columns (double/float/long/int/short/byte/boolean) are + * supported, matching the all-double nature of a SystemDS matrix. Dimensions + * do not need to be known up front: the row count is discovered while scanning + * and the column count is taken from the table schema.

*/ public class ReaderDelta extends MatrixReader { @Override public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int blen, long estnnz) - throws IOException, DMLRuntimeException { + throws IOException, DMLRuntimeException + { Engine engine = DeltaKernelUtils.createEngine(); String tablePath = DeltaKernelUtils.qualify(fname); - // Scan column-at-a-time into one row-major buffer per batch (no per-row - // allocation, no boxing, no per-cell set()). Buffers are concatenated into - // the dense output via bulk array copies below. + //Scan column-at-a-time into one row-major buffer per batch (no per-row + //allocation, no boxing, no per-cell set()). Buffers are concatenated into + //the dense output via bulk array copies below. ArrayList batches = new ArrayList<>(); int[] nrowH = new int[1]; StructType schema = DeltaKernelUtils.scan(engine, tablePath, sch -> { @@ -70,7 +72,7 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl long lestnnz = (estnnz >= 0) ? estnnz : (long) nrow * ncol; MatrixBlock ret = createOutputMatrixBlock(nrow, ncol, Math.max(nrow, 1), lestnnz, true, false); - if(nrow > 0 && ncol > 0) + if( nrow > 0 && ncol > 0 ) fillDense(ret, batches); ret.recomputeNonZeros(); ret.examSparsity(); @@ -81,14 +83,14 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl static int[] columnTypes(StructType schema) { int ncol = schema.length(); int[] types = new int[ncol]; - for(int c = 0; c < ncol; c++) + for( int c=0; c batches) { DenseBlock db = ret.getDenseBlock(); - if(db.isContiguous()) { + if( db.isContiguous() ) { double[] dv = db.valuesAt(0); int off = 0; - for(double[] buf : batches) { + for( double[] buf : batches ) { System.arraycopy(buf, 0, dv, off, buf.length); off += buf.length; } } else { - // rare large multi-block fallback: route each row through the block API + //rare large multi-block fallback: route each row through the block API int ncol = ret.getNumColumns(); int r = 0; - for(double[] buf : batches) { + for( double[] buf : batches ) { int rowsInBuf = buf.length / ncol; - for(int i = 0; i < rowsInBuf; i++, r++) - for(int c = 0; c < ncol; c++) + for( int i=0; i - * The expensive part of a Delta read is the parquet decode, which the kernel performs per data file; parallelizing - * across files is therefore the natural way to bridge the gap to the (near-raw) binary reader. A table backed by a - * single data file (the default for tables <= the parquet target file size) cannot be split this way, so the reader - * transparently falls back to the sequential {@link ReaderDelta} path in that case. - *

+ *

The expensive part of a Delta read is the parquet decode, which the kernel + * performs per data file; parallelizing across files is therefore the natural + * way to bridge the gap to the (near-raw) binary reader. A table backed by a + * single data file (the default for tables <= the parquet target file size) + * cannot be split this way, so the reader transparently falls back to the + * sequential {@link ReaderDelta} path in that case.

*/ public class ReaderDeltaParallel extends ReaderDelta { @@ -56,27 +57,28 @@ public ReaderDeltaParallel() { @Override public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int blen, long estnnz) - throws IOException, DMLRuntimeException { + throws IOException, DMLRuntimeException + { Engine engine = DeltaKernelUtils.createEngine(); String tablePath = DeltaKernelUtils.qualify(fname); DeltaKernelUtils.ScanHandle handle = DeltaKernelUtils.openScan(engine, tablePath); final int nfiles = handle.scanFiles.size(); - // nothing to gain from parallelism for single-file (or empty) tables - if(_numThreads <= 1 || nfiles <= 1) + //nothing to gain from parallelism for single-file (or empty) tables + if( _numThreads <= 1 || nfiles <= 1 ) return super.readMatrixFromHDFS(fname, rlen, clen, blen, estnnz); final int ncol = handle.schema.length(); final int[] types = columnTypes(handle.schema); - // fast path: exact per-file row counts are known from metadata and the dense - // output fits a single contiguous array -> pre-size once and let each thread - // decode directly into its slice (no intermediate buffers, no serial copy). - if(useDirectPath(handle)) { + //fast path: exact per-file row counts are known from metadata and the dense + //output fits a single contiguous array -> pre-size once and let each thread + //decode directly into its slice (no intermediate buffers, no serial copy). + if( useDirectPath(handle) ) { long total = 0; - for(long r : handle.numRecords) + for( long r : handle.numRecords ) total += r; - if(total > 0 && (long) total * ncol <= Integer.MAX_VALUE) + if( total > 0 && (long) total * ncol <= Integer.MAX_VALUE ) return readDirect(fname, handle, ncol, types, (int) total, estnnz); } @@ -84,9 +86,11 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl } /** - * Whether the metadata-driven direct-write fast path can be used for this table (exact per-file row counts and no - * deletion vectors). Visible for testing: the buffered fallback is otherwise only reachable for tables lacking row - * statistics or carrying deletion vectors, which the SystemDS Delta writer never produces. + * Whether the metadata-driven direct-write fast path can be used for this + * table (exact per-file row counts and no deletion vectors). Visible for + * testing: the buffered fallback is otherwise only reachable for tables + * lacking row statistics or carrying deletion vectors, which the SystemDS + * Delta writer never produces. * * @param handle the opened scan handle * @return true if the direct path is applicable @@ -96,37 +100,38 @@ protected boolean useDirectPath(DeltaKernelUtils.ScanHandle handle) { } /** - * Fast path: each thread decodes one data file straight into the final dense array at a metadata-derived row - * offset. Single allocation, fully parallel. + * Fast path: each thread decodes one data file straight into the final dense + * array at a metadata-derived row offset. Single allocation, fully parallel. */ - private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, int ncol, int[] types, int nrow, - long estnnz) throws IOException { + private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, + int ncol, int[] types, int nrow, long estnnz) throws IOException + { final int nfiles = handle.scanFiles.size(); final int[] rowOffset = new int[nfiles]; int acc = 0; - for(int i = 0; i < nfiles; i++) { + for( int i=0; i> tasks = new ArrayList<>(nfiles); - for(int i = 0; i < nfiles; i++) { + for( int i=0; i { int[] cur = new int[] {base}; Engine eng = DeltaKernelUtils.createEngine(); DeltaKernelUtils.readScanFile(eng, handle.scanState, handle.physicalReadSchema, scanFileRow, (cols, size, selected) -> { - if(cur[0] + DeltaKernelUtils.countSelected(size, selected) > limit) + if( cur[0] + DeltaKernelUtils.countSelected(size, selected) > limit ) throw new DMLRuntimeException("Delta file produced more rows than its " + "numRecords statistic; refusing parallel direct read of " + fname); cur[0] += extractBatchInto(cols, size, selected, types, ncol, dv, cur[0]); @@ -142,17 +147,19 @@ private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, } /** - * Fallback path: decode each file in parallel into per-file buffers (used when row counts are unknown, deletion - * vectors are present, or the matrix exceeds a single contiguous array), then concatenate in file order. + * Fallback path: decode each file in parallel into per-file buffers (used when + * row counts are unknown, deletion vectors are present, or the matrix exceeds a + * single contiguous array), then concatenate in file order. */ - private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handle, int ncol, int[] types, - long estnnz) throws IOException { + private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handle, + int ncol, int[] types, long estnnz) throws IOException + { final int nfiles = handle.scanFiles.size(); @SuppressWarnings("unchecked") final ArrayList[] fileBufs = new ArrayList[nfiles]; final int[] fileRows = new int[nfiles]; ArrayList> tasks = new ArrayList<>(nfiles); - for(int i = 0; i < nfiles; i++) { + for( int i=0; i { @@ -172,15 +179,15 @@ private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handl awaitFileTasks(tasks, fname); int nrow = 0; - for(int i = 0; i < nfiles; i++) + for( int i=0; i ordered = new ArrayList<>(); - for(int i = 0; i < nfiles; i++) + for( int i=0; i= 0) ? estnnz : (long) nrow * ncol; MatrixBlock ret = createOutputMatrixBlock(nrow, ncol, Math.max(nrow, 1), lestnnz, true, false); - if(nrow > 0 && ncol > 0) + if( nrow > 0 && ncol > 0 ) fillDense(ret, ordered); ret.recomputeNonZeros(_numThreads); ret.examSparsity(); @@ -188,15 +195,16 @@ private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handl } /** - * Run one decode task per data file on the shared common thread pool and await completion. Full parallelism is - * requested (the task count, one per data file, naturally caps concurrency); this avoids the per-thread pool-size - * caching in {@code CommonThreadPool.get(k)} that could otherwise throttle this reader to a smaller pool created - * earlier on the same thread. + * Run one decode task per data file on the shared common thread pool and await + * completion. Full parallelism is requested (the task count, one per data file, + * naturally caps concurrency); this avoids the per-thread pool-size caching in + * {@code CommonThreadPool.get(k)} that could otherwise throttle this reader to a + * smaller pool created earlier on the same thread. */ private void awaitFileTasks(List> tasks, String fname) throws IOException { ExecutorService pool = CommonThreadPool.get(_numThreads); try { - for(Future f : pool.invokeAll(tasks)) + for( Future f : pool.invokeAll(tasks) ) f.get(); } catch(Exception ex) { diff --git a/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java b/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java index 602b540c407..55ea8a54297 100644 --- a/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java +++ b/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java @@ -39,54 +39,60 @@ import io.delta.kernel.utils.CloseableIterator; /** - * Single-threaded native Delta Lake writer for matrices, built on the Spark-free Delta Kernel library. It creates a - * Delta table at the target directory with an all-double schema {@code c0..c(n-1)} (replacing any existing table at - * that path), streams the {@link MatrixBlock} rows as columnar batches into parquet data files via the kernel's default - * engine, and commits the corresponding add-file actions to the transaction log. + * Single-threaded native Delta Lake writer for matrices, built on the + * Spark-free Delta Kernel library. It creates a Delta table at the target + * directory with an all-double schema {@code c0..c(n-1)} (replacing any existing + * table at that path), streams the {@link MatrixBlock} rows as columnar batches + * into parquet data files via the kernel's default engine, and commits the + * corresponding add-file actions to the transaction log. */ public class WriterDelta extends MatrixWriter { @Override public void writeMatrixToHDFS(MatrixBlock src, String fname, long rlen, long clen, int blen, long nnz, boolean diag) - throws IOException { - if(src.getNumRows() != rlen || src.getNumColumns() != clen) - throw new IOException("Matrix dimensions mismatch with metadata: (" + src.getNumRows() + "x" - + src.getNumColumns() + ") vs (" + rlen + "x" + clen + ")."); + throws IOException + { + if( src.getNumRows() != rlen || src.getNumColumns() != clen ) + throw new IOException("Matrix dimensions mismatch with metadata: (" + + src.getNumRows() + "x" + src.getNumColumns() + ") vs (" + rlen + "x" + clen + ")."); int ncol = (int) clen; int nrow = (int) rlen; int batchRows = ConfigurationManager.getDeltaWriterBatchSize(); - // fast path: a contiguous dense block lets the column views read straight - // from the backing double[] (avoids per-cell MatrixBlock.get dispatch). - double[] dense = (!src.isInSparseFormat() && src.getDenseBlock() != null && - src.getDenseBlock().isContiguous()) ? src.getDenseBlockValues() : null; - // size data files adaptively (toward one file per parallel reader) for faster parallel reads. - // Delta writes every cell as a double, so size by the dense footprint rather than the (possibly - // sparse) in-memory size, which would understate the on-disk table for sparse inputs. + //fast path: a contiguous dense block lets the column views read straight + //from the backing double[] (avoids per-cell MatrixBlock.get dispatch). + double[] dense = (!src.isInSparseFormat() && src.getDenseBlock() != null + && src.getDenseBlock().isContiguous()) ? src.getDenseBlockValues() : null; + //size data files adaptively (toward one file per parallel reader) for faster parallel reads. + //Delta writes every cell as a double, so size by the dense footprint rather than the (possibly + //sparse) in-memory size, which would understate the on-disk table for sparse inputs. long estimatedBytes = (long) nrow * ncol * 8L; Engine engine = DeltaKernelUtils.createWriteEngine(estimatedBytes); - DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), buildSchema(ncol), - new MatrixBatchIterator(src, dense, nrow, ncol, batchRows)); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), + buildSchema(ncol), new MatrixBatchIterator(src, dense, nrow, ncol, batchRows)); } @Override - public void writeEmptyMatrixToHDFS(String fname, long rlen, long clen, int blen) throws IOException { - // empty table: create with schema but no data files + public void writeEmptyMatrixToHDFS(String fname, long rlen, long clen, int blen) + throws IOException + { + //empty table: create with schema but no data files Engine engine = DeltaKernelUtils.createEngine(); - DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), buildSchema((int) clen), - CloseableIterable.emptyIterable().iterator()); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), + buildSchema((int) clen), CloseableIterable.emptyIterable().iterator()); } private static StructType buildSchema(int ncol) { StructType schema = new StructType(); - for(int c = 0; c < ncol; c++) + for( int c=0; c stream, long rlen, long clen, - int blen) throws IOException { + public long writeMatrixFromStream(String fname, OOCStream stream, long rlen, long clen, int blen) + throws IOException + { throw new UnsupportedOperationException("Out-of-core stream write is not supported for the Delta format."); } @@ -116,18 +122,18 @@ public boolean hasNext() { @Override public FilteredColumnarBatch next() { - if(!hasNext()) + if( !hasNext() ) throw new NoSuchElementException(); int size = Math.min(_batchRows, _nrow - _pos); ColumnarBatch batch = new MatrixColumnarBatch(_mb, _dense, _schema, _pos, size, _ncol); _pos += size; - // no selection vector: all rows in the batch are written + //no selection vector: all rows in the batch are written return new FilteredColumnarBatch(batch, Optional.empty()); } @Override public void close() { - // nothing to release + //nothing to release } } @@ -156,7 +162,7 @@ public StructType getSchema() { @Override public ColumnVector getColumnVector(int ordinal) { - if(ordinal < 0 || ordinal >= _ncol) + if( ordinal < 0 || ordinal >= _ncol ) throw new IndexOutOfBoundsException("column ordinal " + ordinal); return new MatrixColumnVector(_mb, _dense, _rowStart, _size, _ncol, ordinal); } @@ -202,14 +208,16 @@ public boolean isNullAt(int rowId) { @Override public double getDouble(int rowId) { - // dense contiguous single block => index fits in int (getDenseBlockValues - // is only handed over for single-block dense matrices) - return (_dense != null) ? _dense[(_rowStart + rowId) * _ncol + _col] : _mb.get(_rowStart + rowId, _col); + //dense contiguous single block => index fits in int (getDenseBlockValues + //is only handed over for single-block dense matrices) + return (_dense != null) + ? _dense[(_rowStart + rowId) * _ncol + _col] + : _mb.get(_rowStart + rowId, _col); } @Override public void close() { - // nothing to release + //nothing to release } } } diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java index 0bf9f7d87ba..5f478979104 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java @@ -977,7 +977,7 @@ public static MatrixBlock rmempty(MatrixBlock in, MatrixBlock ret, boolean rows, if(ret == null) ret = new MatrixBlock(); MatrixBlock ret2 = rmemptyEarlyAbort(in, ret, rows, emptyReturn, select); - if(ret2 != null) + if(ret2 != null ) return ret2; // core removeEmpty return rmemptyUnsafe(in, ret, rows, emptyReturn, select); @@ -985,7 +985,7 @@ public static MatrixBlock rmempty(MatrixBlock in, MatrixBlock ret, boolean rows, public static MatrixBlock rmemptyUnsafe(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, MatrixBlock select) { - if(rows) + if( rows ) return removeEmptyRows(in, ret, select, emptyReturn); else // cols return removeEmptyColumns(in, ret, select, emptyReturn); @@ -999,15 +999,14 @@ public static MatrixBlock rmemptyUnsafe(MatrixBlock in, MatrixBlock ret, boolean * @param rows If removing based on rows, or columns * @param emptyReturn Return a row/column of zeros for empty input * @param select An optional selection vector - * @return The early-abort result, or {@code null} if no early termination applies and the caller must continue. For - * the select-all case the returned block is the input {@code in} itself (a shallow alias, not a copy). + * @return The early-abort result, or {@code null} if no early termination applies and the caller must continue. + * For the select-all case the returned block is the input {@code in} itself (a shallow alias, not a copy). */ - public static MatrixBlock rmemptyEarlyAbort(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, - MatrixBlock select) { - // check for empty inputs - // (the semantics of removeEmpty are that for an empty m-by-n matrix, the output - // is an empty 1-by-n or m-by-1 matrix because we don't allow matrices with dims 0) - if(in.isEmptyBlock(false) && select == null) { + public static MatrixBlock rmemptyEarlyAbort(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, MatrixBlock select){ + //check for empty inputs + //(the semantics of removeEmpty are that for an empty m-by-n matrix, the output + //is an empty 1-by-n or m-by-1 matrix because we don't allow matrices with dims 0) + if( in.isEmptyBlock(false) && select == null ) { int n = emptyReturn ? 1 : 0; if( rows ) ret.reset(n, in.clen, in.sparse); @@ -3652,7 +3651,7 @@ private static MatrixBlock removeEmptyRows(MatrixBlock in, MatrixBlock ret, Matr /** * Remove selected rows, based on the boolean array given. Note this function is internal use only, and require a * boolean vector to be constructed first. - * + * * @param in Input to remove rows from * @param ret Output to assign the result into * @param emptyReturn If the output is allowed to be empty. @@ -3673,9 +3672,9 @@ public static MatrixBlock removeEmptyRows(MatrixBlock in, MatrixBlock ret, boole ret.reset(rlen2, n, sp); if( in.isEmptyBlock(false) ) return ret; - - if(SHALLOW_COPY_REORG && m == rlen2 && selectNull) { - // the condition m==rlen2 is not enough with non-empty 1-row input but empty + + if( SHALLOW_COPY_REORG && m == rlen2 && selectNull ) { + // the condition m==rlen2 is not enough with non-empty 1-row input but empty // 1-row select vector because if emptyReturn should output a single empty row ret.sparse = in.sparse; if( ret.sparse ) @@ -3715,9 +3714,10 @@ else if( !in.sparse && !ret.sparse ) //DENSE <- DENSE ci++; } } - - // check sparsity - ret.nonZeros = (selectNull) ? in.nonZeros : ret.recomputeNonZeros(); + + //check sparsity + ret.nonZeros = (selectNull) ? + in.nonZeros : ret.recomputeNonZeros(); ret.examSparsity(); return ret; diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java index c450ecbe1f5..7525dab2f7f 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java @@ -4756,13 +4756,13 @@ public static double computeIQMCorrection(double sum, double sum_wt, } /** - * Pick the quantiles out of this matrix. If this matrix contains two columns it is weighted quantile picking. If a - * single column it is unweighted. - * + * Pick the quantiles out of this matrix. If this matrix contains two columns it is weighted quantile picking. + * If a single column it is unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantiles The quantiles to pick - * @param ret The result matrix + * @param ret The result matrix * @return The result matrix */ public final MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret) { @@ -4792,11 +4792,11 @@ public MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret, boolean av } /** - * Pick the median value from this matrix. If this matrix has two columns it is weighted picking using the weight - * column, otherwise it is unweighted over the single column. - * + * Pick the median value from this matrix. If this matrix has two columns it is weighted picking using the + * weight column, otherwise it is unweighted over the single column. + * * Note the values are assumed to be sorted. - * + * * @return The median value */ public double median() { @@ -4807,11 +4807,10 @@ public double median() { } /** - * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is - * unweighted. - * + * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantile The quantile to pick * @return The quantile */ @@ -4820,13 +4819,12 @@ public final double pickValue(double quantile){ } /** - * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is - * unweighted. - * + * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantile The quantile to pick - * @param average If the quantile is averaged. + * @param average If the quantile is averaged. * @return The quantile */ public final double pickValue(double quantile, boolean average) { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java index 9ea39bd1e2f..9e3494a7d48 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java @@ -34,8 +34,8 @@ import java.util.function.Function; /** - * Small future implementation for OOC hot paths. It supports multiple synchronous subscribers without the - * completion-stage support of {@link java.util.concurrent.CompletableFuture}. + * Small future implementation for OOC hot paths. It supports multiple synchronous subscribers without + * the completion-stage support of {@link java.util.concurrent.CompletableFuture}. */ public class OOCFuture { private Subscriber _subscribers; @@ -240,7 +240,7 @@ private static void accept(Function mapper, Consu if(resultError == null) { try { @SuppressWarnings("unchecked") - R mapped = mapper == null ? (R) value : mapper.apply(value); + R mapped = mapper == null ? (R)value : mapper.apply(value); result = mapped; } catch(Throwable t) { @@ -345,7 +345,8 @@ public T get() throws InterruptedException, ExecutionException { } @Override - public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + public T get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { try { return mapper.apply(source.get(timeout, unit)); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java index df981f40223..94411242df1 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java @@ -25,22 +25,20 @@ public class CloseableQueue { private final BlockingQueue queue = new LinkedBlockingQueue<>(); - private final Object POISON = new Object(); // sentinel + private final Object POISON = new Object(); // sentinel private volatile boolean closed = false; - public CloseableQueue() { - } + public CloseableQueue() { } /** * Enqueue if the queue is not closed. - * * @return false if already closed */ public boolean enqueueIfOpen(T task) throws InterruptedException { - if(task == null) + if (task == null) throw new IllegalArgumentException("null tasks not allowed"); - synchronized(this) { - if(closed) + synchronized (this) { + if (closed) return false; queue.put(task); } @@ -49,12 +47,12 @@ public boolean enqueueIfOpen(T task) throws InterruptedException { @SuppressWarnings("unchecked") public T take() throws InterruptedException { - if(closed && queue.isEmpty()) + if (closed && queue.isEmpty()) return null; Object x = queue.take(); - if(x == POISON) + if (x == POISON) return null; return (T) x; @@ -62,31 +60,33 @@ public T take() throws InterruptedException { /** * Poll with max timeout. - * - * @return item, or null if: - timeout, or - queue has been closed and this consumer reached its poison pill + * @return item, or null if: + * - timeout, or + * - queue has been closed and this consumer reached its poison pill */ @SuppressWarnings("unchecked") public T poll(long timeout, TimeUnit unit) throws InterruptedException { - if(closed && queue.isEmpty()) + if (closed && queue.isEmpty()) return null; Object x = queue.poll(timeout, unit); - if(x == null) - return null; // timeout + if (x == null) + return null; // timeout - if(x == POISON) + if (x == POISON) return null; return (T) x; } /** - * Close queue for N consumers. Each consumer will receive exactly one poison pill and then should stop. + * Close queue for N consumers. + * Each consumer will receive exactly one poison pill and then should stop. */ public boolean close() throws InterruptedException { - synchronized(this) { - if(closed) - return false; // idempotent + synchronized (this) { + if (closed) + return false; // idempotent closed = true; } queue.put(POISON); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java index ccc73ff6160..ee02b28404c 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java @@ -78,7 +78,7 @@ public void readFully(byte[] b) throws IOException { } @Override - public void readFully(byte[] b, int off, int len) throws IOException { + public void readFully(byte [] b, int off, int len) throws IOException { if(len < 0) throw new IndexOutOfBoundsException(); @@ -127,12 +127,12 @@ public int readUnsignedByte() throws IOException { @Override public short readShort() throws IOException { if(_count - _pos >= 2) { - short ret = (short) baToShort(_buff, _pos); + short ret = (short)baToShort(_buff, _pos); _pos += 2; return ret; } readFully(_tmp, 0, 2); - return (short) baToShort(_tmp, 0); + return (short)baToShort(_tmp, 0); } @Override @@ -142,7 +142,7 @@ public int readUnsignedShort() throws IOException { @Override public char readChar() throws IOException { - return (char) readUnsignedShort(); + return (char)readUnsignedShort(); } @Override @@ -222,7 +222,7 @@ public long readDoubleArray(int len, double[] varr) throws IOException { @Override public long readSparseRows(int rlen, long nnz, SparseBlock rows) throws IOException { if(rows instanceof SparseBlockCSR) { - ((SparseBlockCSR) rows).initSparse(rlen, (int) nnz, this); + ((SparseBlockCSR)rows).initSparse(rlen, (int)nnz, this); return nnz; } @@ -256,7 +256,7 @@ private void refill() throws IOException { } private int getRefillLength() { - int pageOffset = (int) (_filePos & PAGE_MASK); + int pageOffset = (int)(_filePos & PAGE_MASK); if(pageOffset == 0) return _bufflen; return Math.min(_bufflen, PAGE_SIZE - pageOffset); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java index 22b7b23706c..9854a94586b 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java @@ -65,7 +65,7 @@ long getFlushedPosition() { public void write(int b) throws IOException { if(_count >= _bufflen) flushBuffer(); - _buff[_count++] = (byte) b; + _buff[_count++] = (byte)b; _position++; } @@ -109,7 +109,7 @@ public void close() throws IOException { public void writeBoolean(boolean v) throws IOException { if(_count >= _bufflen) flushBuffer(); - _buff[_count++] = (byte) (v ? 1 : 0); + _buff[_count++] = (byte)(v ? 1 : 0); _position++; } @@ -153,7 +153,7 @@ public void writeFloat(float v) throws IOException { public void writeByte(int v) throws IOException { if(_count + 1 > _bufflen) flushBuffer(); - _buff[_count++] = (byte) v; + _buff[_count++] = (byte)v; _position++; } @@ -194,18 +194,18 @@ public void writeUTF(String s) throws IOException { flushBuffer(); final char c = s.charAt(i); if(c >= 0x0001 && c <= 0x007F) { - _buff[_count++] = (byte) c; + _buff[_count++] = (byte)c; _position++; } else if(c >= 0x0800) { - _buff[_count++] = (byte) (0xE0 | ((c >> 12) & 0x0F)); - _buff[_count++] = (byte) (0x80 | ((c >> 6) & 0x3F)); - _buff[_count++] = (byte) (0x80 | (c & 0x3F)); + _buff[_count++] = (byte)(0xE0 | ((c >> 12) & 0x0F)); + _buff[_count++] = (byte)(0x80 | ((c >> 6) & 0x3F)); + _buff[_count++] = (byte)(0x80 | (c & 0x3F)); _position += 3; } else { - _buff[_count++] = (byte) (0xC0 | ((c >> 6) & 0x1F)); - _buff[_count++] = (byte) (0x80 | (c & 0x3F)); + _buff[_count++] = (byte)(0xC0 | ((c >> 6) & 0x1F)); + _buff[_count++] = (byte)(0x80 | (c & 0x3F)); _position += 2; } } @@ -213,7 +213,7 @@ else if(c >= 0x0800) { @Override public void writeDoubleArray(int len, double[] varr) throws IOException { - for(int i = 0; i < len;) { + for(int i = 0; i < len; ) { if(_count >= _bufflen) flushBuffer(); int lblen = Math.min(len - i, (_bufflen - _count) / 8); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java index 6c13a062561..ab28df0ef0f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java @@ -50,10 +50,10 @@ public interface OOCIOHandler { void registerSourceLocation(BlockKey key, SourceBlockDescriptor descriptor); /** - * Schedule an asynchronous read from an external source into the provided target stream. The returned future - * completes when either EOF is reached or the requested byte budget is exhausted. When the budget is reached and - * keepOpenOnLimit is true, the target stream is kept open and a continuation token is provided so the caller can - * resume. + * Schedule an asynchronous read from an external source into the provided target stream. + * The returned future completes when either EOF is reached or the requested byte budget + * is exhausted. When the budget is reached and keepOpenOnLimit is true, the target stream + * is kept open and a continuation token is provided so the caller can resume. */ CompletableFuture scheduleSourceRead(SourceReadRequest request); @@ -62,8 +62,7 @@ public interface OOCIOHandler { */ CompletableFuture continueSourceRead(SourceReadContinuation continuation, long maxBytesInFlight); - interface SourceReadContinuation { - } + interface SourceReadContinuation {} class SourceReadRequest { public final String path; @@ -76,8 +75,9 @@ class SourceReadRequest { public final boolean keepOpenOnLimit; public final OOCStream target; - public SourceReadRequest(String path, Types.FileFormat format, long rows, long cols, int blen, long estNnz, - long maxBytesInFlight, boolean keepOpenOnLimit, OOCStream target) { + public SourceReadRequest(String path, Types.FileFormat format, long rows, long cols, + int blen, long estNnz, long maxBytesInFlight, boolean keepOpenOnLimit, + OOCStream target) { this.path = path; this.format = format; this.rows = rows; @@ -113,8 +113,9 @@ class SourceBlockDescriptor { public final int recordLength; public final long serializedSize; - public SourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, - int recordLength, long serializedSize) { + public SourceBlockDescriptor(String path, Types.FileFormat format, + MatrixIndexes indexes, long offset, int recordLength, + long serializedSize) { this.path = path; this.format = format; this.indexes = indexes; @@ -128,8 +129,8 @@ class GroupSourceBlockDescriptor extends SourceBlockDescriptor { public final List blocks; public final int count; - public GroupSourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, - int recordLength, long serializedSize, List blocks) { + public GroupSourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, int recordLength, + long serializedSize, List blocks) { super(path, format, indexes, offset, recordLength, serializedSize); this.blocks = blocks; this.count = blocks.size(); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java index 55ac038205f..e9487f7bd45 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java @@ -81,7 +81,7 @@ public class OOCMatrixIOHandler implements OOCIOHandler { private final AtomicLong _readSeq = new AtomicLong(0); // Spill related structures - private final ConcurrentHashMap _spillLocations = new ConcurrentHashMap<>(); + private final ConcurrentHashMap _spillLocations = new ConcurrentHashMap<>(); private final ConcurrentHashMap _partitions = new ConcurrentHashMap<>(); private final ConcurrentHashMap _sourceLocations = new ConcurrentHashMap<>(); private final AtomicInteger _partitionCounter = new AtomicInteger(0); @@ -97,21 +97,38 @@ public class OOCMatrixIOHandler implements OOCIOHandler { @SuppressWarnings("unchecked") public OOCMatrixIOHandler() { this._spillDir = LocalFileUtils.getUniqueWorkingDir("ooc_stream"); - _writeExec = new ThreadPoolExecutor(WRITER_SIZE, WRITER_SIZE, 0L, TimeUnit.MILLISECONDS, + _writeExec = new ThreadPoolExecutor( + WRITER_SIZE, + WRITER_SIZE, + 0L, + TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); - _readExec = new ThreadPoolExecutor(READER_SIZE, READER_SIZE, 0L, TimeUnit.MILLISECONDS, + _readExec = new ThreadPoolExecutor( + READER_SIZE, + READER_SIZE, + 0L, + TimeUnit.MILLISECONDS, new PriorityBlockingQueue<>()); - _srcReadExec = new ThreadPoolExecutor(READER_SIZE, READER_SIZE, 0L, TimeUnit.MILLISECONDS, + _srcReadExec = new ThreadPoolExecutor( + READER_SIZE, + READER_SIZE, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(100000)); + _deleteExec = new ThreadPoolExecutor( + 1, + 1, + 0L, + TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); - _deleteExec = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); _q = new CloseableQueue[WRITER_SIZE]; _wCtr = new AtomicLong(0); - _started = new AtomicBoolean(false); + _started = new AtomicBoolean(false); } private synchronized void start() { - if(_started.compareAndSet(false, true)) { - for(int i = 0; i < WRITER_SIZE; i++) { + if (_started.compareAndSet(false, true)) { + for (int i = 0; i < WRITER_SIZE; i++) { final int finalIdx = i; _q[i] = new CloseableQueue<>(); _writeExec.submit(() -> evictTask(_q[finalIdx])); @@ -122,7 +139,7 @@ private synchronized void start() { @Override public void shutdown() { boolean started = _started.get(); - if(started) { + if (started) { try { for(int i = 0; i < WRITER_SIZE; i++) { if(_q[i] != null) @@ -142,7 +159,7 @@ public void shutdown() { _deleteExec.shutdownNow(); _spillLocations.clear(); _partitions.clear(); - if(started) + if (started) LocalFileUtils.deleteFileIfExists(_spillDir); } @@ -152,7 +169,7 @@ public CompletableFuture scheduleEviction(BlockEntry block) { CompletableFuture future = new CompletableFuture<>(); try { long q = _wCtr.getAndAdd(block.getSize()) / OVERFLOW; - int i = (int) (q % WRITER_SIZE); + int i = (int)(q % WRITER_SIZE); if(!_q[i].enqueueIfOpen(new Tuple2<>(block, future))) future.completeExceptionally(new DMLRuntimeException("OOC writer queue is closed")); } @@ -172,8 +189,7 @@ public OOCFuture scheduleRead(final BlockEntry block) { ReadTask task = new ReadTask(block, future, _readSeq.getAndIncrement(), pinnedPartitionId); _pendingReads.put(block.getKey(), task); _readExec.execute(task); - } - catch(RejectedExecutionException e) { + } catch (RejectedExecutionException e) { unpinPartitionForRead(pinnedPartitionId); _pendingReads.remove(block.getKey()); future.completeExceptionally(e); @@ -183,12 +199,12 @@ public OOCFuture scheduleRead(final BlockEntry block) { @Override public void prioritizeRead(BlockKey key, double priority) { - if(priority == 0) + if (priority == 0) return; ReadTask task = _pendingReads.get(key); - if(task == null) + if (task == null) return; - if(_readExec.getQueue().remove(task)) { + if (_readExec.getQueue().remove(task)) { task.addPriority(priority); _readExec.getQueue().offer(task); } @@ -212,9 +228,8 @@ public CompletableFuture scheduleSourceRead(SourceReadRequest } @Override - public CompletableFuture continueSourceRead(SourceReadContinuation continuation, - long maxBytesInFlight) { - if(!(continuation instanceof SourceReadState state)) { + public CompletableFuture continueSourceRead(SourceReadContinuation continuation, long maxBytesInFlight) { + if (!(continuation instanceof SourceReadState state)) { CompletableFuture failed = new CompletableFuture<>(); failed.completeExceptionally(new DMLRuntimeException("Unsupported continuation type: " + continuation)); return failed; @@ -225,8 +240,8 @@ public CompletableFuture continueSourceRead(SourceReadContinua private CompletableFuture submitSourceRead(SourceReadRequest request, SourceReadState state, long maxBytesInFlight) { if(request.format != Types.FileFormat.BINARY) - return CompletableFuture - .failedFuture(new DMLRuntimeException("Unsupported format for source read: " + request.format)); + return CompletableFuture.failedFuture( + new DMLRuntimeException("Unsupported format for source read: " + request.format)); return readBinarySourceParallel(request, state, maxBytesInFlight); } @@ -320,18 +335,17 @@ private CompletableFuture readBinarySourceParallel(SourceReadR return result; } - private void completeResult(CompletableFuture future, AtomicLong bytesRead, - AtomicBoolean budgetHit, AtomicReference error, SourceReadRequest request, Path[] files, - AtomicLongArray filePositions, AtomicIntegerArray completed, - ConcurrentLinkedDeque descriptors) { + private void completeResult(CompletableFuture future, AtomicLong bytesRead, AtomicBoolean budgetHit, + AtomicReference error, SourceReadRequest request, Path[] files, AtomicLongArray filePositions, + AtomicIntegerArray completed, ConcurrentLinkedDeque descriptors) { Throwable err = error.get(); - if(err != null) { + if (err != null) { future.completeExceptionally(err instanceof Exception ? err : new Exception(err)); return; } try { - if(budgetHit.get()) { + if (budgetHit.get()) { if(!request.keepOpenOnLimit) { closeTarget(request.target, false); } @@ -351,29 +365,29 @@ private void completeResult(CompletableFuture future, AtomicLo private void readSequenceFile(JobConf job, Path path, SourceReadRequest request, int fileIdx, AtomicLongArray filePositions, AtomicIntegerArray completed, AtomicBoolean stop, AtomicBoolean budgetHit, - AtomicLong bytesRead, long byteLimit, Object budgetLock, - ConcurrentLinkedDeque descriptors) throws IOException { + AtomicLong bytesRead, long byteLimit, Object budgetLock, ConcurrentLinkedDeque descriptors) + throws IOException { MatrixIndexes key = new MatrixIndexes(); try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { long pos = filePositions.get(fileIdx); - if(pos > 0) + if (pos > 0) reader.seek(pos); long ioStart = DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; while(!stop.get()) { long recordStart = reader.getPosition(); MatrixBlock value = new MatrixBlock(); - if(!reader.next(key, value)) + if (!reader.next(key, value)) break; long recordEnd = reader.getPosition(); long blockSize = value.getExactSerializedSize(); boolean shouldBreak = false; synchronized(budgetLock) { - if(stop.get()) + if (stop.get()) shouldBreak = true; - else if(bytesRead.get() + blockSize > byteLimit) { + else if (bytesRead.get() + blockSize > byteLimit) { stop.set(true); budgetHit.set(true); shouldBreak = true; @@ -384,7 +398,7 @@ else if(bytesRead.get() + blockSize > byteLimit) { MatrixIndexes outIdx = new MatrixIndexes(key); IndexedMatrixValue imv = new IndexedMatrixValue(outIdx, value); SourceBlockDescriptor descriptor = new SourceBlockDescriptor(path.toString(), request.format, outIdx, - recordStart, (int) (recordEnd - recordStart), blockSize); + recordStart, (int)(recordEnd - recordStart), blockSize); if(request.target instanceof SourceOOCStream src) src.enqueue(imv, descriptor); @@ -393,23 +407,22 @@ else if(bytesRead.get() + blockSize > byteLimit) { descriptors.add(descriptor); filePositions.set(fileIdx, reader.getPosition()); - if(DMLScript.OOC_LOG_EVENTS) { + if (DMLScript.OOC_LOG_EVENTS) { long currTime = System.nanoTime(); OOCEventLog.onDiskReadEvent(_srcReadCallerId, ioStart, currTime, blockSize); ioStart = currTime; } - if(shouldBreak) + if (shouldBreak) break; // Note that we knowingly go over limit, which could result in READER_SIZE*8MB overshoot } - if(!stop.get()) + if (!stop.get()) completed.set(fileIdx, 1); } } - private void closeTarget(org.apache.sysds.runtime.instructions.ooc.OOCStream target, - boolean close) { + private void closeTarget(org.apache.sysds.runtime.instructions.ooc.OOCStream target, boolean close) { if(close) { try { target.closeInput(); @@ -424,10 +437,10 @@ private void loadFromDisk(BlockEntry block) { String key = block.getKey().toFileKey(); SourceBlockDescriptor src = _sourceLocations.get(block.getKey()); - if(src != null) { + if (src != null) { long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; loadFromSource(block, src); - if(DMLScript.OOC_STATISTICS) { + if (DMLScript.OOC_STATISTICS) { Statistics.incrementOOCLoadFromDisk(); Statistics.accumulateOOCLoadFromDiskTime(System.nanoTime() - ioStart); Statistics.accumulateOOCLoadFromDiskBytes(block.getSize()); @@ -438,36 +451,34 @@ private void loadFromDisk(BlockEntry block) { long ioDuration = 0; // 1. find the blocks address (spill location) SpillLocation sloc = _spillLocations.get(key); - if(sloc == null) + if (sloc == null) throw new DMLRuntimeException("Failed to load spill location for: " + key); PartitionFile partFile = _partitions.get(sloc.partitionId); - if(partFile == null) + if (partFile == null) throw new DMLRuntimeException("Failed to load partition for: " + sloc.partitionId); String filename = partFile.filePath; SpillableObject obj; - try(RandomAccessFile raf = new RandomAccessFile(filename, "r")) { + try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { raf.seek(sloc.offset); DataInput dis = new OOCBufferedDataInputStream(raf); long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; obj = SpillableObjectRegistry.read(dis); - if(DMLScript.OOC_STATISTICS) + if (DMLScript.OOC_STATISTICS) ioDuration = System.nanoTime() - ioStart; - } - catch(ClosedByInterruptException ignored) { + } catch (ClosedByInterruptException ignored) { return; - } - catch(IOException e) { + } catch (IOException e) { throw new RuntimeException(e); } block.setDataUnsafe(obj); - if(DMLScript.OOC_STATISTICS) { + if (DMLScript.OOC_STATISTICS) { Statistics.incrementOOCLoadFromDisk(); Statistics.accumulateOOCLoadFromDiskTime(ioDuration); Statistics.accumulateOOCLoadFromDiskBytes(block.getSize()); @@ -475,23 +486,22 @@ private void loadFromDisk(BlockEntry block) { } private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { - if(src.format != Types.FileFormat.BINARY) + if (src.format != Types.FileFormat.BINARY) throw new DMLRuntimeException("Unsupported format for source read: " + src.format); JobConf job = new JobConf(ConfigurationManager.getCachedJobConf()); Path path = new Path(src.path); - if(src instanceof OOCIOHandler.GroupSourceBlockDescriptor gsrc) { + if (src instanceof OOCIOHandler.GroupSourceBlockDescriptor gsrc) { List values = new ArrayList<>(gsrc.count); try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { reader.seek(gsrc.offset); - for(int i = 0; i < gsrc.blocks.size(); i++) { + for (int i = 0; i < gsrc.blocks.size(); i++) { SourceBlockDescriptor d = gsrc.blocks.get(i); MatrixIndexes ix = new MatrixIndexes(); MatrixBlock mb = new MatrixBlock(); - if(!reader.next(ix, mb)) - throw new DMLRuntimeException( - "Failed to read source block at offset " + d.offset + " in " + d.path); + if (!reader.next(ix, mb)) + throw new DMLRuntimeException("Failed to read source block at offset " + d.offset + " in " + d.path); values.add(new IndexedMatrixValue(ix, mb)); } } @@ -506,9 +516,8 @@ private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { reader.seek(src.offset); - if(!reader.next(ix, mb)) - throw new DMLRuntimeException( - "Failed to read source block at offset " + src.offset + " in " + src.path); + if (!reader.next(ix, mb)) + throw new DMLRuntimeException("Failed to read source block at offset " + src.offset + " in " + src.path); } catch(IOException e) { throw new DMLRuntimeException(e); @@ -521,7 +530,7 @@ private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { private void evictTask(CloseableQueue>> q) { long byteCtr = 0; - while(!q.isFinished()) { + while (!q.isFinished()) { // --- 1. WRITE PHASE --- int partitionId = _partitionCounter.getAndIncrement(); @@ -563,17 +572,17 @@ private void evictTask(CloseableQueue } byteCtr += wrote; - if(byteCtr >= MAX_PARTITION_SIZE) { + if (byteCtr >= MAX_PARTITION_SIZE) { closePartition = true; byteCtr = 0; break; } - if(DMLScript.OOC_LOG_EVENTS) + if (DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskWriteEvent(_evictCallerId, ioStart, System.nanoTime(), wrote); } - if(!closePartition && q.close()) { + if (!closePartition && q.close()) { while((tpl = q.take()) != null) { long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; BlockEntry entry = tpl._1(); @@ -586,7 +595,7 @@ private void evictTask(CloseableQueue Statistics.accumulateOOCEvictionWriteTime(System.nanoTime() - ioStart); } - if(DMLScript.OOC_LOG_EVENTS) + if (DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskWriteEvent(_evictCallerId, ioStart, System.nanoTime(), wrote); } } @@ -608,13 +617,13 @@ private void evictTask(CloseableQueue } private long writeOut(int partitionId, BlockEntry entry, CompletableFuture future, - OOCBufferedDataOutputStream dos, ConcurrentLinkedDeque>> flushQueue) - throws IOException { + OOCBufferedDataOutputStream dos, + ConcurrentLinkedDeque>> flushQueue) throws IOException { String key = entry.getKey().toFileKey(); boolean alreadySpilled = _spillLocations.containsKey(key); - if(!alreadySpilled) { + if (!alreadySpilled) { long offsetBefore = dos.getPosition(); if(future.isCancelled()) @@ -647,10 +656,9 @@ private long writeOut(int partitionId, BlockEntry entry, CompletableFuture return 0; } - private void flushQueue(long offset, - ConcurrentLinkedDeque>> flushQueue) { + private void flushQueue(long offset, ConcurrentLinkedDeque>> flushQueue) { Tuple3> tmp; - while((tmp = flushQueue.peek()) != null && tmp._2() <= offset) { + while ((tmp = flushQueue.peek()) != null && tmp._2() <= offset) { flushQueue.poll(); tmp._3().complete(null); } @@ -769,14 +777,12 @@ public void run() { try { long ioStart = DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; loadFromDisk(_block); - if(DMLScript.OOC_LOG_EVENTS) + if (DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskReadEvent(_readCallerId, ioStart, System.nanoTime(), _block.getSize()); _future.complete(_block); - } - catch(Throwable e) { + } catch (Throwable e) { _future.completeExceptionally(e); - } - finally { + } finally { unpinPartitionForRead(_pinnedPartitionId); } } @@ -784,12 +790,15 @@ public void run() { @Override public int compareTo(ReadTask other) { int byPriority = Double.compare(other._priority, _priority); - if(byPriority != 0) + if (byPriority != 0) return byPriority; return Long.compare(_sequence, other._sequence); } } + + + private static class SpillLocation { // structure of spillLocation: file, offset final int partitionId; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java index 3458838c071..a93b1d18f2a 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java @@ -24,10 +24,8 @@ import java.io.IOException; public interface SpillableObject { - boolean tryWrite(DataOutput out) throws IOException; - - void read(DataInput in) throws IOException; - + boolean tryWrite(DataOutput out) throws IOException; + void read(DataInput in) throws IOException; long size(); default void discard() { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java index 0d318d83d94..d8ff79dd797 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java @@ -34,16 +34,14 @@ public interface OOCCacheScheduler { /** * Requests a single block from the cache. - * * @param key the requested key associated to the block * @return the available BlockEntry */ CompletableFuture request(BlockKey key); /** - * Tries to request a single block from the cache. Immediately returns the entry if present, otherwise null without - * scheduling reads. - * + * Tries to request a single block from the cache. + * Immediately returns the entry if present, otherwise null without scheduling reads. * @param key the requested key associated to the block * @return the available BlockEntry or null */ @@ -54,16 +52,14 @@ default BlockEntry tryRequest(BlockKey key) { /** * Requests a list of blocks from the cache that must be available at the same time. - * * @param keys the requested keys associated to the block * @return the list of available BlockEntries */ CompletableFuture> request(List keys); /** - * Tries to request a list of blocks from the cache that must be available at the same time. Immediately returns the - * list of entries if present, otherwise null without scheduling reads. - * + * Tries to request a list of blocks from the cache that must be available at the same time. + * Immediately returns the list of entries if present, otherwise null without scheduling reads. * @param keys the requested keys associated to the block * @return the list of available BlockEntries */ @@ -80,25 +76,24 @@ default BlockEntry tryRequest(BlockKey key) { List tryRequestAnyOf(List keys, int n, List selectionOut); /** - * Adds the given priority to any pending request accessing the key. Multi-requests are prioritized partially. + * Adds the given priority to any pending request accessing the key. + * Multi-requests are prioritized partially. */ void prioritize(BlockKey key, double priority); /** - * Places a new block in the cache. Note that objects are immutable and cannot be overwritten. The object data - * should now only be accessed via cache, as ownership has been transferred. - * - * @param key the associated key of the block + * Places a new block in the cache. Note that objects are immutable and cannot be overwritten. + * The object data should now only be accessed via cache, as ownership has been transferred. + * @param key the associated key of the block * @param data the block data * @param size the size of the data */ BlockKey put(BlockKey key, Object data, long size); /** - * Places a new block in the cache and returns a pinned handle. Note that objects are immutable and cannot be - * overwritten. - * - * @param key the associated key of the block + * Places a new block in the cache and returns a pinned handle. + * Note that objects are immutable and cannot be overwritten. + * @param key the associated key of the block * @param data the block data * @param size the size of the data */ @@ -106,11 +101,8 @@ default BlockEntry tryRequest(BlockKey key) { interface HandoverHandle { BlockKey getKey(); - boolean isCommitted(); - CompletableFuture getCompletionFuture(); - OOCStream.QueueCallback reclaim(); } @@ -139,30 +131,27 @@ BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.SourceBlockDescriptor descriptor); /** - * Notifies the cache that there is another reference to the same block key. This will prevent forget(key) from - * removing the block from cache. A block will only be forgotten after all referencing instances called forget(key). - * + * Notifies the cache that there is another reference to the same block key. + * This will prevent forget(key) from removing the block from cache. + * A block will only be forgotten after all referencing instances called forget(key). * @param key */ void addReference(BlockKey key); /** * Forgets a block from the cache. - * * @param key the associated key of the block */ void forget(BlockKey key); /** * Pins a BlockEntry in cache to prevent eviction. - * * @param entry the entry to be pinned */ void pin(BlockEntry entry); /** * Unpins a pinned block. - * * @param entry the entry to be unpinned */ void unpin(BlockEntry entry); @@ -203,7 +192,8 @@ BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, void updateLimits(long evictionLimit, long hardLimit); /** - * Creates a snapshot of the cache. Should only be used for debugging or diagnoses. + * Creates a snapshot of the cache. + * Should only be used for debugging or diagnoses. */ Collection snapshot(); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java index 9b0acc97b35..96de368ccc9 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java @@ -80,7 +80,7 @@ public class OOCLRUCacheScheduler implements OOCCacheScheduler { public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long hardLimit, long readBuffer) { this._ioHandler = ioHandler; this._cache = new LinkedHashMap<>(1024, 0.75f, true); - this._evictionCache = new HashMap<>(); + this._evictionCache = new HashMap<>(); this._deferredReadRequests = new DeferredReadQueue(); this._processingReadRequests = new ArrayDeque<>(); this._pendingHandovers = new ArrayDeque<>(); @@ -103,7 +103,7 @@ public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long har this._maintenanceNeedsIncr = new AtomicBoolean(false); this._callerId = DMLScript.OOC_LOG_EVENTS ? OOCEventLog.registerCaller("LRUCacheScheduler") : 0; - if(DMLScript.OOC_LOG_EVENTS) { + if (DMLScript.OOC_LOG_EVENTS) { OOCEventLog.putRunSetting("CacheEvictionLimit", _evictionLimit); OOCEventLog.putRunSetting("CacheHardLimit", _hardLimit); } @@ -111,7 +111,7 @@ public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long har @Override public CompletableFuture request(BlockKey key) { - if(!this._running) + if (!this._running) throw new IllegalStateException("Cache scheduler has been shut down."); Statistics.incrementOOCEvictionGet(); @@ -120,21 +120,21 @@ public CompletableFuture request(BlockKey key) { boolean couldPin = false; synchronized(this) { entry = _cache.get(key); - if(entry == null) + if (entry == null) entry = _evictionCache.get(key); - if(entry == null) + if (entry == null) throw new IllegalArgumentException("Could not find requested block with key " + key); synchronized(entry) { - if(entry.getState().isAvailable()) { - if(pinEntryWithAccounting(entry) == 0) + if (entry.getState().isAvailable()) { + if (pinEntryWithAccounting(entry) == 0) throw new IllegalStateException(); couldPin = true; } } } - if(couldPin) { + if (couldPin) { // Then we could pin the required entry and can terminate return CompletableFuture.completedFuture(entry); } @@ -184,7 +184,7 @@ public CompletableFuture> request(List keys) { } public CompletableFuture> request(List keys, boolean onlyIfAvailable) { - if(!this._running) + if (!this._running) throw new IllegalStateException("Cache scheduler has been shut down."); Statistics.incrementOOCEvictionGet(keys.size()); @@ -193,11 +193,11 @@ public CompletableFuture> request(List keys, boolean boolean allAvailable = true; synchronized(this) { - for(BlockKey key : keys) { + for (BlockKey key : keys) { BlockEntry entry = _cache.get(key); - if(entry == null) + if (entry == null) entry = _evictionCache.get(key); - if(entry == null) + if (entry == null) throw new IllegalArgumentException("Could not find requested block with key " + key); synchronized(entry) { @@ -217,7 +217,7 @@ public CompletableFuture> request(List keys, boolean } } - if(allAvailable) { + if (allAvailable) { // Then we could pin all entries return CompletableFuture.completedFuture(entries); } @@ -226,12 +226,12 @@ public CompletableFuture> request(List keys, boolean return null; // Schedule deferred read otherwise - final CompletableFuture> future = new CompletableFuture<>(); + final CompletableFuture> future = new CompletableFuture<>(); DeferredReadRequest request = new DeferredReadRequest(future, entries); - for(int i = 0; i < entries.size(); i++) { + for (int i = 0; i < entries.size(); i++) { BlockEntry entry = entries.get(i); synchronized(entry) { - if(entry.getState().isAvailable()) { + if (entry.getState().isAvailable()) { entry.addRetainHint(); request.markRetainHinted(i); } @@ -243,9 +243,9 @@ public CompletableFuture> request(List keys, boolean @Override public void prioritize(BlockKey key, double priority) { - if(!this._running) + if (!this._running) return; - if(priority == 0) + if (priority == 0) return; synchronized(this) { @@ -267,18 +267,18 @@ private void scheduleDeferredRead(DeferredReadRequest deferredReadRequest) { if(entry.getState().isAvailable()) readyCount++; BlockReadState state = _blockReads.get(entry.getKey()); - if(state != null) + if (state != null) score += state.priority; } - if(!deferredReadRequest.getEntries().isEmpty()) + if (!deferredReadRequest.getEntries().isEmpty()) score /= deferredReadRequest.getEntries().size(); - if(!deferredReadRequest.getEntries().isEmpty()) + if (!deferredReadRequest.getEntries().isEmpty()) score += ((double) readyCount) / deferredReadRequest.getEntries().size(); deferredReadRequest.setPriorityScore(score); _deferredReadRequests.add(deferredReadRequest); _deferredReadCountHint = _deferredReadRequests.size(); } - onCacheSizeChanged(true); // Apply pressure from deferred read demand. + onCacheSizeChanged(true); // Apply pressure from deferred read demand. onCacheSizeChanged(false); // Attempt to schedule deferred reads. } @@ -317,8 +317,7 @@ public void putSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.S } @Override - public BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, - OOCIOHandler.SourceBlockDescriptor descriptor) { + public BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.SourceBlockDescriptor descriptor) { return put(key, data, size, true, descriptor); } @@ -334,24 +333,23 @@ public void addReference(BlockKey key) { } } - private BlockEntry put(BlockKey key, Object data, long size, boolean pin, - OOCIOHandler.SourceBlockDescriptor descriptor) { - if(!this._running) + private BlockEntry put(BlockKey key, Object data, long size, boolean pin, OOCIOHandler.SourceBlockDescriptor descriptor) { + if (!this._running) throw new IllegalStateException(); - if(data == null) + if (data == null) throw new IllegalArgumentException(); - if(descriptor != null) + if (descriptor != null) _ioHandler.registerSourceLocation(key, descriptor); Statistics.incrementOOCEvictionPut(); BlockEntry entry = new BlockEntry(key, size, data); - if(descriptor != null) + if (descriptor != null) entry.setState(BlockState.WARM); - if(pin) + if (pin) entry.pin(); synchronized(this) { BlockEntry avail = _cache.putIfAbsent(key, entry); - if(avail != null || _evictionCache.containsKey(key)) + if (avail != null || _evictionCache.containsKey(key)) throw new IllegalStateException("Cannot overwrite existing entries: " + key); _cacheSize += size; if(pin) { @@ -366,7 +364,7 @@ private BlockEntry put(BlockKey key, Object data, long size, boolean pin, @Override public void forget(BlockKey key) { - if(!this._running) + if (!this._running) return; final MutableObject mEntry = new MutableObject<>(); BlockEntry entry; @@ -383,7 +381,7 @@ public void forget(BlockKey key) { return e; }); - if(mEntry.getValue() == null) { + if (mEntry.getValue() == null) { _evictionCache.compute(key, (k, e) -> { if(e == null) return null; @@ -397,10 +395,10 @@ public void forget(BlockKey key) { entry = mEntry.getValue(); - if(entry != null) { + if (entry != null) { synchronized(entry) { - shouldScheduleDeletion = entry.getState().isBackedByDisk() || - entry.getState() == BlockState.EVICTING; + shouldScheduleDeletion = entry.getState().isBackedByDisk() + || entry.getState() == BlockState.EVICTING; cacheSizeDelta = transitionMemState(entry, BlockState.REMOVED); if(entry.isPinned() && entry.getDataUnsafe() != null) _pinnedBytes -= entry.getSize(); @@ -410,9 +408,9 @@ public void forget(BlockKey key) { } } } - if(cacheSizeDelta != 0) + if (cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); - if(shouldScheduleDeletion) + if (shouldScheduleDeletion) _ioHandler.scheduleDeletion(entry); } @@ -426,11 +424,11 @@ public void pin(BlockEntry entry) { synchronized(this) { synchronized(entry) { int pinCount = pinEntryWithAccounting(entry); - if(pinCount == 0) + if (pinCount == 0) throw new IllegalStateException("Could not pin the requested entry: " + entry.getKey()); } // Access element in cache for Lru - // _cache.get(entry.getKey()); + //_cache.get(entry.getKey()); } } @@ -444,26 +442,26 @@ public void unpin(BlockEntry entry) { synchronized(entry) { if(!unpinEntryWithAccounting(entry)) return; - if(_cacheSize <= _evictionLimit) + if (_cacheSize <= _evictionLimit) return; // Nothing to do - if(entry.isPinned()) + if (entry.isPinned()) return; // Pin state changed so we cannot evict - if(entry.getState().isAvailable() && entry.getState().isBackedByDisk()) { - if(entry.getRetainHintCount() > 0) { + if (entry.getState().isAvailable() && entry.getState().isBackedByDisk()) { + if (entry.getRetainHintCount() > 0) { shouldCheckEviction = true; } else { - cacheSizeDelta = transitionMemState(entry, BlockState.COLD); + cacheSizeDelta = transitionMemState(entry, BlockState.COLD); long cleared = entry.clear(); - if(cleared != entry.getSize()) + if (cleared != entry.getSize()) throw new IllegalStateException(); _cache.remove(entry.getKey()); _evictionCache.put(entry.getKey(), entry); } } - else if(entry.getState() == BlockState.HOT) { - if(entry.getRetainHintCount() > 0) { + else if (entry.getState() == BlockState.HOT) { + if (entry.getRetainHintCount() > 0) { shouldCheckEviction = true; } else { @@ -472,9 +470,9 @@ else if(entry.getState() == BlockState.HOT) { } } } - if(cacheSizeDelta != 0) + if (cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); - else if(shouldCheckEviction) + else if (shouldCheckEviction) onCacheSizeChanged(true); } @@ -510,11 +508,10 @@ public synchronized void shutdown() { System.out.println("[WARN] Cache still holds " + _cache.size() + " / " + _evictionCache.size() + " blocks"); Set cachedStreams = _cache.keySet().stream().map(BlockKey::getStreamId).collect(Collectors.toSet()); - Set evictedStreams = _evictionCache.keySet().stream().map(BlockKey::getStreamId) - .collect(Collectors.toSet()); + Set evictedStreams = _evictionCache.keySet().stream().map(BlockKey::getStreamId).collect(Collectors.toSet()); cachedStreams.addAll(evictedStreams); - System.out.println("[WARN] Affected stream IDs: " + cachedStreams + ", Pinned: " - + _cache.values().stream().mapToInt(e -> e.isPinned() ? 1 : 0).sum()); + System.out.println("[WARN] Affected stream IDs: " + cachedStreams + ", Pinned: " + _cache.values().stream().mapToInt( + e -> e.isPinned() ? 1 : 0).sum()); } _cache.clear(); _evictionCache.clear(); @@ -578,8 +575,7 @@ private void runMaintenanceLoop() { do { _maintenanceRequested.set(false); onCacheSizeChangedInternal(_maintenanceNeedsIncr.getAndSet(false)); - } - while(_maintenanceRequested.get()); + } while(_maintenanceRequested.get()); } finally { _maintenanceRunning.set(false); @@ -595,8 +591,7 @@ private void onCacheSizeChangedInternal(boolean incr) { if(incr) onCacheSizeIncremented(); else - while(onCacheSizeDecremented()) { - } + while(onCacheSizeDecremented()) {} while(processPendingHandovers()) { onCacheSizeIncremented(); } @@ -606,23 +601,18 @@ private void onCacheSizeChangedInternal(boolean incr) { } private synchronized void sanityCheck() { - if(_cacheSize > _hardLimit * 1.1) { - if(!_warnThrottling) { + if (_cacheSize > _hardLimit * 1.1) { + if (!_warnThrottling) { _warnThrottling = true; - System.out.println( - "[WARN] Cache hard limit exceeded by over 10%: " + String.format("%.2f", _cacheSize / 1000000.0) - + "MB (-" + String.format("%.2f", _bytesUpForEviction / 1000000.0) + "MB) > " - + String.format("%.2f", _hardLimit / 1000000.0) + "MB"); + System.out.println("[WARN] Cache hard limit exceeded by over 10%: " + String.format("%.2f", _cacheSize/1000000.0) + "MB (-" + String.format("%.2f", _bytesUpForEviction/1000000.0) + "MB) > " + String.format("%.2f", _hardLimit/1000000.0) + "MB"); } } - else if(_warnThrottling && _cacheSize < _hardLimit) { + else if (_warnThrottling && _cacheSize < _hardLimit) { _warnThrottling = false; - System.out.println("[INFO] Cache within limit: " + String.format("%.2f", _cacheSize / 1000000.0) + "MB (-" - + String.format("%.2f", _bytesUpForEviction / 1000000.0) + "MB) <= " - + String.format("%.2f", _hardLimit / 1000000.0) + "MB"); + System.out.println("[INFO] Cache within limit: " + String.format("%.2f", _cacheSize/1000000.0) + "MB (-" + String.format("%.2f", _bytesUpForEviction/1000000.0) + "MB) <= " + String.format("%.2f", _hardLimit/1000000.0) + "MB"); } - if(!SANITY_CHECKS) + if (!SANITY_CHECKS) return; int pinned = 0; @@ -635,16 +625,16 @@ else if(_warnThrottling && _cacheSize < _hardLimit) { long actualPinnedEvictingBytes = 0; long actualWarmPinnedBytes = 0; long actualReadingReservedBytes = 0; - for(BlockEntry entry : _cache.values()) { - if(entry.isPinned()) { + for (BlockEntry entry : _cache.values()) { + if (entry.isPinned()) { pinned++; actualPinnedBytes += entry.getSize(); if(entry.getState() == BlockState.WARM) actualWarmPinnedBytes += entry.getSize(); } - if(entry.getState().isBackedByDisk()) + if (entry.getState().isBackedByDisk()) backedByDisk++; - if(entry.getState() == BlockState.EVICTING) { + if (entry.getState() == BlockState.EVICTING) { evicting++; upForEviction += entry.getSize(); if(entry.isPinned()) @@ -652,44 +642,43 @@ else if(_warnThrottling && _cacheSize < _hardLimit) { } if(entry.getState() == BlockState.READING) actualReadingReservedBytes += entry.getSize(); - if(!entry.getState().isAvailable()) + if (!entry.getState().isAvailable()) throw new IllegalStateException(); total++; actualCacheSize += entry.getSize(); } - for(BlockEntry entry : _evictionCache.values()) { - if(entry.getState().isAvailable()) + for (BlockEntry entry : _evictionCache.values()) { + if (entry.getState().isAvailable()) throw new IllegalStateException("Invalid eviction state: " + entry.getState()); - if(entry.getState() == BlockState.EVICTING && entry.isPinned()) + if (entry.getState() == BlockState.EVICTING && entry.isPinned()) actualPinnedEvictingBytes += entry.getSize(); - if(entry.getState() == BlockState.READING) + if (entry.getState() == BlockState.READING) actualCacheSize += entry.getSize(); - if(entry.getState() == BlockState.READING) + if (entry.getState() == BlockState.READING) actualReadingReservedBytes += entry.getSize(); - if(entry.isPinned()) { + if (entry.isPinned()) { actualPinnedBytes += entry.getSize(); if(entry.getState() == BlockState.WARM) actualWarmPinnedBytes += entry.getSize(); } } - if(actualCacheSize != _cacheSize) + if (actualCacheSize != _cacheSize) throw new IllegalStateException(actualCacheSize + " != " + _cacheSize); - if(upForEviction != _bytesUpForEviction) + if (upForEviction != _bytesUpForEviction) throw new IllegalStateException(upForEviction + " != " + _bytesUpForEviction); - if(actualPinnedBytes != _pinnedBytes) + if (actualPinnedBytes != _pinnedBytes) throw new IllegalStateException(actualPinnedBytes + " != " + _pinnedBytes); - if(actualPinnedEvictingBytes != _pinnedEvictingBytes) + if (actualPinnedEvictingBytes != _pinnedEvictingBytes) throw new IllegalStateException(actualPinnedEvictingBytes + " != " + _pinnedEvictingBytes); - if(_pinnedEvictingBytes > _bytesUpForEviction) + if (_pinnedEvictingBytes > _bytesUpForEviction) throw new IllegalStateException(_pinnedEvictingBytes + " > " + _bytesUpForEviction); if(actualWarmPinnedBytes != _warmPinnedBytes) throw new IllegalStateException(actualWarmPinnedBytes + " != " + _warmPinnedBytes); - if(actualReadingReservedBytes != _readingReservedBytes) + if (actualReadingReservedBytes != _readingReservedBytes) throw new IllegalStateException(actualReadingReservedBytes + " != " + _readingReservedBytes); System.out.println("=========="); - System.out.println("Limit: " + _evictionLimit / 1000 + "KB"); - System.out.println("Memory: (" + _cacheSize / 1000 + "KB - " + _bytesUpForEviction / 1000 + "KB) / " - + _hardLimit / 1000 + "KB"); + System.out.println("Limit: " + _evictionLimit/1000 + "KB"); + System.out.println("Memory: (" + _cacheSize/1000 + "KB - " + _bytesUpForEviction/1000 + "KB) / " + _hardLimit/1000 + "KB"); System.out.println("Pinned: " + pinned + " / " + total); System.out.println("Disk backed: " + backedByDisk + " / " + total); System.out.println("Evicting: " + evicting + " / " + total); @@ -706,11 +695,10 @@ private void onCacheSizeIncremented() { if(pressure <= _evictionLimit) return; // Nothing to do - long overshoot = Math.max((long) (0.1 * _evictionLimit), 10000000); + long overshoot = Math.max((long)(0.1 * _evictionLimit), 10000000); long lowLimit = _evictionLimit - _readBuffer - overshoot; - // System.out.println("[CACHE] Claiming " + (pressure + overshoot - _evictionLimit)/1000 + "kB (last claim - // was " + (System.currentTimeMillis() - _lastEvictRun) + "ms ago)"); + //System.out.println("[CACHE] Claiming " + (pressure + overshoot - _evictionLimit)/1000 + "kB (last claim was " + (System.currentTimeMillis() - _lastEvictRun) + "ms ago)"); // Scan for values that can be evicted Collection entries = _cache.values(); @@ -725,8 +713,8 @@ private void onCacheSizeIncremented() { break; synchronized(entry) { - // if(entry.isPinned()) - // continue; + //if(entry.isPinned()) + // continue; if(!allowRetainHint && entry.getRetainHintCount() > 0) continue; if(entry.getState() == BlockState.COLD || entry.getState() == BlockState.EVICTING) @@ -760,12 +748,12 @@ private void onCacheSizeIncremented() { _lastEvictRun = System.currentTimeMillis(); } - for(BlockEntry entry : upForEvictionNeedsWrite) + for (BlockEntry entry : upForEvictionNeedsWrite) evict(entry, true); - for(BlockEntry entry : upForEvictionNoWrite) + for (BlockEntry entry : upForEvictionNoWrite) evict(entry, false); - if(cacheSizeDelta != 0) + if (cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); } @@ -825,7 +813,7 @@ private boolean onCacheSizeDecremented() { throw new IllegalStateException(); req.setPinned(idx); } - else if(entry.getState() == BlockState.READING) { + else if (entry.getState() == BlockState.READING) { req.schedule(idx); registerWaiter(entry.getKey(), req, idx); reading = true; @@ -848,7 +836,7 @@ else if(entry.getState() == BlockState.READING) { if(allReserved) { _deferredReadRequests.poll(); _deferredReadCountHint = _deferredReadRequests.size(); - if(!toRead.isEmpty()) + if (!toRead.isEmpty()) _processingReadRequests.add(req); } @@ -955,17 +943,17 @@ private void onEvicted(final BlockEntry entry) { if(tmp != null && tmp != entry) throw new IllegalStateException(); tmp = _evictionCache.put(entry.getKey(), entry); - if(tmp != null) + if (tmp != null) throw new IllegalStateException(); sanityCheck(); } - if(cacheSizeDelta != 0) + if (cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); } private void clearRetainHints(DeferredReadRequest request) { - for(int i = 0; i < request.getEntries().size(); i++) { - if(!request.isRetainHinted(i)) + for (int i = 0; i < request.getEntries().size(); i++) { + if (!request.isRetainHinted(i)) continue; BlockEntry entry = request.getEntries().get(i); synchronized(entry) { @@ -975,12 +963,12 @@ private void clearRetainHints(DeferredReadRequest request) { } /** - * Cleanly transitions state of a BlockEntry and handles accounting. Requires both the scheduler object and the - * entry to be locked: + * Cleanly transitions state of a BlockEntry and handles accounting. + * Requires both the scheduler object and the entry to be locked: */ private long transitionMemState(BlockEntry entry, BlockState newState) { BlockState oldState = entry.getState(); - if(oldState == newState) + if (oldState == newState) return 0; long sz = entry.getSize(); @@ -988,7 +976,7 @@ private long transitionMemState(BlockEntry entry, BlockState newState) { boolean pinned = entry.isPinned(); // Remove old contribution - switch(oldState) { + switch (oldState) { case REMOVED: throw new IllegalStateException(); case HOT: @@ -1012,7 +1000,7 @@ private long transitionMemState(BlockEntry entry, BlockState newState) { } // Add new contribution - switch(newState) { + switch (newState) { case REMOVED: case COLD: break; @@ -1068,7 +1056,6 @@ private int pinEntryWithAccounting(BlockEntry entry) { /** * Requires scheduler lock and entry lock. - * * @return true if this call transitioned pin count to zero. */ private boolean unpinEntryWithAccounting(BlockEntry entry) { diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java b/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java index f04534481c9..1f731fc3aa5 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java @@ -66,8 +66,8 @@ protected boolean isHashCol(int colID) { } /** - * Domain size of a dummycoded source column: the hash domain K from the meta cell for feature-hashed columns, - * otherwise the column's {@code numDistinct} (0 when unset). + * Domain size of a dummycoded source column: the hash domain K from the meta cell for + * feature-hashed columns, otherwise the column's {@code numDistinct} (0 when unset). * * @param meta transform meta frame * @param colID 1-based column id of the dummycoded source column @@ -144,13 +144,13 @@ public FrameBlock decode(final MatrixBlock in, final FrameBlock out, final int k try { final List> tasks = new ArrayList<>(); int blz = Math.max((in.getNumRows() + k) / k, 1000); - - for(int i = 0; i < in.getNumRows(); i += blz) { + + for(int i = 0; i < in.getNumRows(); i += blz){ final int start = i; final int end = Math.min(in.getNumRows(), i + blz); tasks.add(pool.submit(() -> decode(in, out, start, end))); } - + for(Future f : tasks) f.get(); return out; diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java index 01a502154cd..a286c03dce8 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java @@ -72,10 +72,10 @@ public void decode(MatrixBlock in, FrameBlock out, int rl, int ru) { final double val = in.get(i, _srcCols[j] - 1); if(!Double.isNaN(val)){ final int key = (int) Math.round(val); - if(key == 0) { + if(key == 0){ a.set(i, _binMins[j][key]); } - else { + else{ double bmin = _binMins[j][key - 1]; double bmax = _binMaxs[j][key - 1]; double oval = bmin + (bmax - bmin) / 2 // bin center diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java index 8aa0b60d990..ee1a33c49fd 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java @@ -34,7 +34,7 @@ /** * Simple atomic decoder for dummycoded columns. This decoder builds internally inverted column mappings from the given * frame meta data. - * + * */ public class DecoderDummycode extends Decoder { private static final long serialVersionUID = 4758831042891032129L; diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java index fc123657032..8f6c45d63e8 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java @@ -64,15 +64,15 @@ public static Decoder createDecoder(String spec, String[] colnames, ValueType[] try { //parse transform specification JSONObject jSpec = new JSONObject(spec); - - // create decoders 'bin', 'recode', 'hash', 'dummy', and 'pass-through' + + //create decoders 'bin', 'recode', 'hash', 'dummy', and 'pass-through' List binIDs = TfMetaUtils.parseBinningColIDs(jSpec, colnames, minCol, maxCol); - List rcIDs = Arrays.asList(ArrayUtils - .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.RECODE.toString(), minCol, maxCol))); - List hcIDs = Arrays.asList(ArrayUtils - .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.HASH.toString(), minCol, maxCol))); - List dcIDs = Arrays.asList(ArrayUtils - .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.DUMMYCODE.toString(), minCol, maxCol))); + List rcIDs = Arrays.asList(ArrayUtils.toObject( + TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.RECODE.toString(), minCol, maxCol))); + List hcIDs = Arrays.asList(ArrayUtils.toObject( + TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.HASH.toString(), minCol, maxCol))); + List dcIDs = Arrays.asList(ArrayUtils.toObject( + TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.DUMMYCODE.toString(), minCol, maxCol))); // only specially treat the columns with both recode and dictionary rcIDs = unionDistinct(rcIDs, dcIDs); // hashing is a lossy, one-way transform with no inverse recode map, so hash columns @@ -106,18 +106,29 @@ public static Decoder createDecoder(String spec, String[] colnames, ValueType[] // collect all the decoders in one list. List ldecoders = new ArrayList<>(); - - if(!binIDs.isEmpty()) { - ldecoders.add(new DecoderBin(schema, ArrayUtils.toPrimitive(binIDs.toArray(new Integer[0])), + + if( !binIDs.isEmpty() ) { + ldecoders.add(new DecoderBin(schema, + ArrayUtils.toPrimitive(binIDs.toArray(new Integer[0])), ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); } if( !dcIDs.isEmpty() ) { ldecoders.add(new DecoderDummycode(schema, ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); } - - // create composite decoder of all created decoders - // and initialize with given meta data (recode, dummy, bin) + if( !rcIDs.isEmpty() ) { + // recode on output (after dummycode rebuilds the categorical columns) when dummycoding is present + ldecoders.add(new DecoderRecode(schema, !dcIDs.isEmpty(), + ArrayUtils.toPrimitive(rcIDs.toArray(new Integer[0])))); + } + if( !ptIDs.isEmpty() ) { + ldecoders.add(new DecoderPassThrough(schema, + ArrayUtils.toPrimitive(ptIDs.toArray(new Integer[0])), + ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); + } + + //create composite decoder of all created decoders + //and initialize with given meta data (recode, dummy, bin) decoder = new DecoderComposite(schema, ldecoders); decoder.setColnames(colnames); decoder.initMetaData(meta); @@ -136,7 +147,7 @@ else if( decoder instanceof DecoderRecode ) return DecoderType.Recode.ordinal(); else if( decoder instanceof DecoderPassThrough ) return DecoderType.PassThrough.ordinal(); - else if(decoder instanceof DecoderBin) + else if( decoder instanceof DecoderBin ) return DecoderType.Bin.ordinal(); throw new DMLRuntimeException("Unsupported decoder type: " + decoder.getClass().getCanonicalName()); @@ -147,14 +158,10 @@ public static Decoder createInstance(int type) { // create instance switch(dtype) { - case Bin: - return new DecoderBin(); - case Dummycode: - return new DecoderDummycode(null, null); - case PassThrough: - return new DecoderPassThrough(null, null, null); - case Recode: - return new DecoderRecode(null, false, null); + case Bin: return new DecoderBin(); + case Dummycode: return new DecoderDummycode(null, null); + case PassThrough: return new DecoderPassThrough(null, null, null); + case Recode: return new DecoderRecode(null, false, null); default: throw new DMLRuntimeException("Unsupported Encoder Type used: " + dtype); } diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java index d8d624122a0..11dd2c7faa5 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java @@ -126,20 +126,20 @@ public void initMetaData(FrameBlock meta) { _rcMaps = new HashMap[_colList.length]; for( int j=0; j<_colList.length; j++ ) { HashMap map = new HashMap<>(); - for(int i = 0; i < meta.getNumRows(); i++) { - try { + for( int i=0; i= 0} both the minimum and - * maximum fraction digits are pinned to {@code decimal}, so values are printed with exactly that many decimals; - * otherwise the {@link DecimalFormat} defaults apply. - * + * Creates a non-grouping {@link DecimalFormat} for printing values. When {@code decimal >= 0} + * both the minimum and maximum fraction digits are pinned to {@code decimal}, so values are + * printed with exactly that many decimals; otherwise the {@link DecimalFormat} defaults apply. * @param decimal number of decimal places to print, -1 for default * @return a configured {@link DecimalFormat} */ private static DecimalFormat createDecimalFormat(int decimal) { DecimalFormat df = new DecimalFormat(); df.setGroupingUsed(false); - if(decimal >= 0) { + if (decimal >= 0) { df.setMinimumFractionDigits(decimal); df.setMaximumFractionDigits(decimal); } diff --git a/src/main/java/org/apache/sysds/utils/DoubleParser.java b/src/main/java/org/apache/sysds/utils/DoubleParser.java index 2252b7270c8..c0122f8061f 100644 --- a/src/main/java/org/apache/sysds/utils/DoubleParser.java +++ b/src/main/java/org/apache/sysds/utils/DoubleParser.java @@ -200,7 +200,7 @@ public static double parseFloatingPointLiteral(String str, int offset, int endIn // : is the first character after numbers. // 0 is the first number. // we use the last position, since this is not allowed to be other values than a number. - if(str.charAt(endIndex - 1) > '9' || str.charAt(endIndex - 1) < '0') + if(str.charAt(endIndex - 1) > '9' || str.charAt(endIndex - 1) < '0') return Double.parseDouble(str); final double val = parseDecFloatLiteral(str, index, offset, endIndex); diff --git a/src/main/java/org/apache/sysds/utils/SettingsChecker.java b/src/main/java/org/apache/sysds/utils/SettingsChecker.java index 10d1a42b246..c5f14a7043b 100644 --- a/src/main/java/org/apache/sysds/utils/SettingsChecker.java +++ b/src/main/java/org/apache/sysds/utils/SettingsChecker.java @@ -106,24 +106,25 @@ private static long maxMemMachineOSX() { } private static long maxMemMachineWin() { - // try modern powershell, otherwise wmic, log errors as warning but avoid crashes + //try modern powershell, otherwise wmic, log errors as warning but avoid crashes long tmp = maxMemMachineWin(true); - if(tmp < 0) + if( tmp < 0 ) tmp = maxMemMachineWin(false); return tmp; } - + private static long maxMemMachineWin(boolean modern) { int startIx = modern ? 3 : 1; - String command = modern ? "powershell Get-CimInstance -ClassName Win32_PhysicalMemory | Select-Object Capacity" : "wmic memorychip get capacity"; // in - // bytes + String command = modern ? + "powershell Get-CimInstance -ClassName Win32_PhysicalMemory | Select-Object Capacity" : + "wmic memorychip get capacity"; //in bytes try { Runtime rt = Runtime.getRuntime(); Process pr = rt.exec(command); String[] memStr = new String(pr.getInputStream().readAllBytes(), StandardCharsets.UTF_8).split("\n"); //skip header, and aggregate DIMM capacities long capacity = 0; - for(int i = startIx; i < memStr.length; i++) { + for( int i=startIx; i 0 ) capacity += Long.parseLong(tmp); diff --git a/src/test/java/org/apache/sysds/performance/Main.java b/src/test/java/org/apache/sysds/performance/Main.java index a941e90f724..0622e789baa 100644 --- a/src/test/java/org/apache/sysds/performance/Main.java +++ b/src/test/java/org/apache/sysds/performance/Main.java @@ -243,9 +243,10 @@ private static void run17(String[] args) throws Exception { } /** - * Repeatedly read the same on-disk Delta frame table (written once as setup). Args: - * {@code 18 [mode] [targetFileSizeMB]} where mode is one of serial|parallel|both (default parallel) - * and an omitted target file size uses the adaptive default sizing. + * Repeatedly read the same on-disk Delta frame table (written once as setup). + * Args: {@code 18 [mode] [targetFileSizeMB]} + * where mode is one of serial|parallel|both (default parallel) and an omitted + * target file size uses the adaptive default sizing. */ private static void run18(String[] args) throws Exception { int rows = Integer.parseInt(args[1]); diff --git a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java index 4aac0685698..36ea11b3e2f 100644 --- a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java @@ -117,8 +117,8 @@ public abstract class AutomatedTestBase { public static final double GPU_TOLERANCE = 1e-9; /** - * Default deadline (ms) for federated worker/monitoring readiness waits and a few legacy {@code sleep()} calls. - * {@link FederatedWorkerUtils} enforces its own minimum floor. + * Default deadline (ms) for federated worker/monitoring readiness waits and a few legacy + * {@code sleep()} calls. {@link FederatedWorkerUtils} enforces its own minimum floor. */ public static final int FED_WORKER_WAIT = 3000; @@ -1761,9 +1761,9 @@ private static Process spawnLocalFedWorker(int port, String[] addArgs) { /** * Start a new JVM for a federated monitoring backend at the port. * - *

- * Returns once the backend's TCP port accepts connections (Netty's bind has completed), or throws a - * {@link RuntimeException} once the {@link FederatedWorkerUtils} readiness floor elapses. + *

Returns once the backend's TCP port accepts connections (Netty's bind has completed), or + * throws a {@link RuntimeException} once the {@link FederatedWorkerUtils} readiness floor + * elapses. * * @param port Port to use for the JVM * @param addArgs Extra CLI args to append, or null @@ -1776,10 +1776,10 @@ protected Process startLocalFedMonitoring(int port, String[] addArgs) { /** * Start a new JVM for a federated monitoring backend at the port. * - *

- * Returns once the backend's TCP port accepts connections, or throws a {@link RuntimeException} after - * {@code timeoutMs} elapses. The monitoring server opens the port after Netty's {@code bind().sync()} returns; a - * successful TCP connect therefore signals that the HTTP listener is ready to accept requests. + *

Returns once the backend's TCP port accepts connections, or throws a + * {@link RuntimeException} after {@code timeoutMs} elapses. The monitoring server opens the + * port after Netty's {@code bind().sync()} returns; a successful TCP connect therefore signals + * that the HTTP listener is ready to accept requests. * * @param port Port to use for the JVM * @param addArgs Extra CLI args to append, or null @@ -1798,9 +1798,8 @@ private static Process spawnLocalFedMonitoring(int port, String[] addArgs) { String separator = System.getProperty("file.separator"); String classpath = System.getProperty("java.class.path"); String path = System.getProperty("java.home") + separator + "bin" + separator + "java"; - String[] args = ArrayUtils.addAll( - new String[] {path, "-cp", classpath, DMLScript.class.getName(), "-fedMonitoring", Integer.toString(port)}, - addArgs); + String[] args = ArrayUtils.addAll(new String[] {path, "-cp", classpath, DMLScript.class.getName(), + "-fedMonitoring", Integer.toString(port)}, addArgs); try { return new ProcessBuilder(args).start(); } diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java index 0c7cd2046e5..f14a614b583 100644 --- a/src/test/java/org/apache/sysds/test/TestUtils.java +++ b/src/test/java/org/apache/sysds/test/TestUtils.java @@ -2941,9 +2941,10 @@ public static void writeTestScalar(String file, double value) { } } + /** * Write scalar to file - * + * * @param file File to write to * @param value Value to write */ @@ -3518,9 +3519,9 @@ public static void shutdownThread(Thread t) { // Bounded join: workers are daemon threads, so even if one ignores the interrupt // we must not block cleanup (and the JVM) indefinitely waiting for it. t.join(THREAD_SHUTDOWN_JOIN_MS); - if(t.isAlive()) - LOG.warn("Federated worker thread " + t.getName() + " did not stop within " - + THREAD_SHUTDOWN_JOIN_MS + "ms; leaving it as a daemon."); + if( t.isAlive() ) + LOG.warn("Federated worker thread " + t.getName() + + " did not stop within " + THREAD_SHUTDOWN_JOIN_MS + "ms; leaving it as a daemon."); } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java b/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java index d76c741d870..07ec9752928 100644 --- a/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java +++ b/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java @@ -67,10 +67,10 @@ public void setUp() { /** * Compile a DML script string into a runtime {@link Program} without executing it. * - * @param dmlScript the DML source - * @param args named command-line arguments ($name -> value), may be null - * @param mode the global execution mode (e.g. {@link ExecMode#HYBRID}) - * @param localMaxMem the local memory budget in bytes used for memory-based exec-type decisions + * @param dmlScript the DML source + * @param args named command-line arguments ($name -> value), may be null + * @param mode the global execution mode (e.g. {@link ExecMode#HYBRID}) + * @param localMaxMem the local memory budget in bytes used for memory-based exec-type decisions * @return the compiled runtime program */ protected Program compile(String dmlScript, Map args, ExecMode mode, long localMaxMem) { @@ -145,7 +145,8 @@ else if(pb instanceof FunctionProgramBlock) { /** All instructions whose opcode equals {@code opcode} (exact match). */ protected List getByOpcode(Program prog, String opcode) { - return getInstructions(prog).stream().filter(i -> opcode.equals(i.getOpcode())).collect(Collectors.toList()); + return getInstructions(prog).stream().filter(i -> opcode.equals(i.getOpcode())) + .collect(Collectors.toList()); } protected static boolean isSpark(Instruction inst) { @@ -168,13 +169,13 @@ protected void assertCP(Program prog, String opcode) { private void assertExecType(Program prog, String opcode, boolean expectSpark) { List matches = getByOpcode(prog, opcode); - Assert.assertFalse( - "Expected at least one '" + opcode + "' instruction but found none.\n" + Explain.explain(prog), - matches.isEmpty()); + Assert.assertFalse("Expected at least one '" + opcode + "' instruction but found none.\n" + + Explain.explain(prog), matches.isEmpty()); for(Instruction inst : matches) { boolean spark = isSpark(inst); - Assert.assertEquals("Instruction '" + opcode + "' expected exec type " + (expectSpark ? "SPARK" : "CP") - + " but was " + (spark ? "SPARK" : "CP") + ".\n" + Explain.explain(prog), expectSpark, spark); + Assert.assertEquals("Instruction '" + opcode + "' expected exec type " + + (expectSpark ? "SPARK" : "CP") + " but was " + (spark ? "SPARK" : "CP") + ".\n" + + Explain.explain(prog), expectSpark, spark); } } diff --git a/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java b/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java index 7b45120d8f1..0b3889db908 100644 --- a/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java +++ b/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java @@ -33,13 +33,14 @@ */ public class SparkTransitiveExecTypeCompileTest extends CompilerTestBase { - private static final String DML_HEADER = "X = rand(rows=20000000, cols=8, seed=1);\n" + // ~1.2GB -> rand and - // colSums run on Spark - "v = colSums(X);\n"; // 1x8 Spark-resident vector (opcode uack+) + private static final String DML_HEADER = + "X = rand(rows=20000000, cols=8, seed=1);\n" + // ~1.2GB -> rand and colSums run on Spark + "v = colSums(X);\n"; // 1x8 Spark-resident vector (opcode uack+) @Test public void singleConsumerUnaryPulledIntoSpark() { - String dml = DML_HEADER + "r = round(v);\n" + // sole consumer of the Spark-resident vector -> pulled into Spark + String dml = DML_HEADER + + "r = round(v);\n" + // sole consumer of the Spark-resident vector -> pulled into Spark "print(sum(r));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); @@ -49,58 +50,62 @@ public void singleConsumerUnaryPulledIntoSpark() { @Test public void multiConsumerUnaryStaysCP() { - String dml = DML_HEADER + "a = round(v);\n" + // v now has two consumers (round + abs) ... - "b = abs(v);\n" + "print(sum(a) + sum(b));\n"; + String dml = DML_HEADER + + "a = round(v);\n" + // v now has two consumers (round + abs) ... + "b = abs(v);\n" + + "print(sum(a) + sum(b));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uack+"); // input still has a Spark output ... - assertCP(prog, "round"); // ... but the multi-parent guard keeps both unaries in CP + assertCP(prog, "round"); // ... but the multi-parent guard keeps both unaries in CP assertCP(prog, "abs"); } // A tall, Spark-resident column vector that is still small enough (40 KB) to be CP by memory // estimate: rowSums over a very wide matrix runs on Spark, but its 1-column result fits in CP. - private static final String TALL_VECTOR_HEADER = "X = rand(rows=5000, cols=200000, seed=1);\n" + // ~8GB -> rand and - // rowSums run - // on Spark - "c = rowSums(X);\n"; // 5000x1 Spark-resident vector (opcode uark+) + private static final String TALL_VECTOR_HEADER = + "X = rand(rows=5000, cols=200000, seed=1);\n" + // ~8GB -> rand and rowSums run on Spark + "c = rowSums(X);\n"; // 5000x1 Spark-resident vector (opcode uark+) @Test public void cumulativeUnaryStaysCP() { - String dml = TALL_VECTOR_HEADER + "r = cumsum(c);\n" + // sole consumer of the Spark-resident vector, CP by - // estimate ... + String dml = TALL_VECTOR_HEADER + + "r = cumsum(c);\n" + // sole consumer of the Spark-resident vector, CP by estimate ... "print(as.scalar(r[2500,1]));\n"; // ... consume via indexing (avoids the sum(cumsum) rewrite) Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); - assertSpark(prog, "uark+"); // input genuinely has a Spark output + assertSpark(prog, "uark+"); // input genuinely has a Spark output assertCP(prog, "ucumk+"); // ... but cumulative ops are excluded from the transitive pull } @Test public void singleConsumerBinaryPulledIntoSpark() { - String dml = TALL_VECTOR_HEADER + "r = c + 2.0;\n" + // matrix-scalar on the Spark-resident vector, sole - // consumer -> pulled into Spark + String dml = TALL_VECTOR_HEADER + + "r = c + 2.0;\n" + // matrix-scalar on the Spark-resident vector, sole consumer -> pulled into Spark "print(as.scalar(r[2500,1]));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uark+"); // input genuinely has a Spark output (multi-block column vector) - assertSpark(prog, "+"); // matrix-scalar binary pulled into Spark (CP by estimate, single consumer) + assertSpark(prog, "+"); // matrix-scalar binary pulled into Spark (CP by estimate, single consumer) } @Test public void multiConsumerBinaryStaysCP() { - String dml = TALL_VECTOR_HEADER + "a = c + 2.0;\n" + // c now has two consumers (+ and *) ... - "b = c * 3.0;\n" + "print(as.scalar(a[2500,1]) + as.scalar(b[2500,1]));\n"; + String dml = TALL_VECTOR_HEADER + + "a = c + 2.0;\n" + // c now has two consumers (+ and *) ... + "b = c * 3.0;\n" + + "print(as.scalar(a[2500,1]) + as.scalar(b[2500,1]));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uark+"); // input still has a Spark output ... - assertCP(prog, "+"); // ... but the multi-parent guard keeps both binaries in CP + assertCP(prog, "+"); // ... but the multi-parent guard keeps both binaries in CP assertCP(prog, "*"); } @Test public void transitiveDisabledUnaryStaysCP() { - String dml = DML_HEADER + "r = round(v);\n" + // pullable unary, but flag is off + String dml = DML_HEADER + + "r = round(v);\n" + // pullable unary, but flag is off "print(sum(r));\n"; Program prog = compileWithTransitive(dml, false); @@ -110,7 +115,8 @@ public void transitiveDisabledUnaryStaysCP() { @Test public void transitiveDisabledBinaryStaysCP() { - String dml = TALL_VECTOR_HEADER + "r = c + 2.0;\n" + // pullable matrix-scalar, but flag is off + String dml = TALL_VECTOR_HEADER + + "r = c + 2.0;\n" + // pullable matrix-scalar, but flag is off "print(as.scalar(r[2500,1]));\n"; Program prog = compileWithTransitive(dml, false); diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java index 22f867819c3..083a29f965b 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java @@ -43,8 +43,7 @@ /** * Tests the {@code order} (sort) reorg operation on compressed matrices. A single column held in a single column group * is sorted ascending while staying compressed (via {@link org.apache.sysds.runtime.compress.lib.CLALibSort}); every - * other configuration falls back to a decompressed reorg. In all cases the result must match the uncompressed - * reference. + * other configuration falls back to a decompressed reorg. In all cases the result must match the uncompressed reference. */ public class CompressedSortTest { @@ -264,7 +263,8 @@ private void runCompressed(MatrixBlock mb, CompressionType ct) { assertEquals("Expected a single column group", 1, cmb.getColGroups().size()); MatrixBlock actual = cmb.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); - assertTrue("Expected the sorted result to stay compressed for " + ct, actual instanceof CompressedMatrixBlock); + assertTrue("Expected the sorted result to stay compressed for " + ct, + actual instanceof CompressedMatrixBlock); MatrixBlock expected = mb.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); TestUtils.compareMatrices(expected, CompressedMatrixBlock.getUncompressed(actual, "sort"), 0.0, "sort " + ct); diff --git a/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java index 49cbbdd248d..833128ad9f0 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java @@ -62,8 +62,8 @@ public static void setup() { } /** - * Build a compressed matrix backed by a single DDC column group spanning all {@code nCol} columns. This guarantees - * a single (non-uncompressed) column group, which is what triggers the mm-chain fast path for wide enough matrices. + * Build a compressed matrix backed by a single DDC column group spanning all {@code nCol} columns. This guarantees a + * single (non-uncompressed) column group, which is what triggers the mm-chain fast path for wide enough matrices. */ private static CompressedMatrixBlock singleDDC(int nRow, int nCol, int nVal, int seed) { Random r = new Random(seed); diff --git a/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java b/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java index c6afd5a3543..549010a78cb 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java @@ -43,8 +43,8 @@ /** * Drive the solve opcode through {@link BinaryMatrixMatrixCPInstruction} with compressed inputs to cover the - * commons-math matrix-matrix branch that decompresses compressed left/right operands before solving. The script-level - * solve tests only ever see uncompressed inputs, so this branch is otherwise unreached. + * commons-math matrix-matrix branch that decompresses compressed left/right operands before solving. The + * script-level solve tests only ever see uncompressed inputs, so this branch is otherwise unreached. */ public class CompressedBinaryMatrixMatrixSolveTest { @@ -72,8 +72,8 @@ public void solveCompressedLeftCompressedRight() { assertTrue("A must compress to exercise the compressed-left path", aC instanceof CompressedMatrixBlock); CompressedMatrixBlock bC = CompressedMatrixBlockFactory.createConstant(n, 2, 1.0); - MatrixBlock expected = LibCommonsMath.matrixMatrixOperations(CompressedMatrixBlock.getUncompressed(aC), - CompressedMatrixBlock.getUncompressed(bC), SOLVE); + MatrixBlock expected = LibCommonsMath.matrixMatrixOperations( + CompressedMatrixBlock.getUncompressed(aC), CompressedMatrixBlock.getUncompressed(bC), SOLVE); MatrixBlock actual = runSolve(aC, bC); TestUtils.compareMatricesBitAvgDistance(expected, actual, 0, 0, SOLVE); @@ -94,8 +94,8 @@ public void solveCompressedLeftDenseRight() { assertTrue("A must compress to exercise the compressed-left path", aC instanceof CompressedMatrixBlock); MatrixBlock b = TestUtils.round(TestUtils.generateTestMatrixBlock(n, 1, -5, 5, 1.0, 7)); - MatrixBlock expected = LibCommonsMath.matrixMatrixOperations(CompressedMatrixBlock.getUncompressed(aC), b, - SOLVE); + MatrixBlock expected = LibCommonsMath.matrixMatrixOperations( + CompressedMatrixBlock.getUncompressed(aC), b, SOLVE); MatrixBlock actual = runSolve(aC, b); TestUtils.compareMatricesBitAvgDistance(expected, actual, 0, 0, SOLVE); @@ -119,8 +119,7 @@ private static BinaryMatrixMatrixCPInstruction solveInstruction() { } private static MatrixObject matrixObject(String name, MatrixBlock mb) { - MatrixCharacteristics mc = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), 1000, - mb.getNonZeros()); + MatrixCharacteristics mc = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), 1000, mb.getNonZeros()); MatrixObject mo = new MatrixObject(ValueType.FP64, "/dev/null/" + name, new MetaDataFormat(mc, FileFormat.BINARY), mb); return mo; diff --git a/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java b/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java index 0d2f37e319c..8907c82f1d1 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java @@ -50,9 +50,7 @@ public class OffsetClassInitConcurrencyTest { private static final String[] INIT_TARGETS = {PKG + "AOffset", PKG + "OffsetEmpty", PKG + "OffsetChar", PKG + "OffsetByte", PKG + "OffsetSingle", PKG + "OffsetTwo"}; - /** - * Whether a class-init cycle deadlocks depends on thread timing, so repeat to make a regression reliable to catch. - */ + /** Whether a class-init cycle deadlocks depends on thread timing, so repeat to make a regression reliable to catch. */ private static final int ROUNDS = 20; /** A real init deadlock never resolves; a healthy round finishes in milliseconds, so this bound is generous. */ diff --git a/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java b/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java index f7dffa9021c..3493da7d2b1 100644 --- a/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java +++ b/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java @@ -35,10 +35,12 @@ public class SparkContextReferenceCountTest { /** - * Two DML executions sharing the JVM-wide singleton spark context (as happens with surefire parallel tests, - * threadCount>1). When the first execution finishes and calls close(), the shared context must stay alive - * because the second execution still has in-flight work. Before reference counting, close() stopped the context - * unconditionally, which cancelled the second execution's spark job and wedged it until the test watchdog. + * Two DML executions sharing the JVM-wide singleton spark context (as happens + * with surefire parallel tests, threadCount>1). When the first execution + * finishes and calls close(), the shared context must stay alive because the + * second execution still has in-flight work. Before reference counting, + * close() stopped the context unconditionally, which cancelled the second + * execution's spark job and wedged it until the test watchdog. */ @Test public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { @@ -62,13 +64,16 @@ public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { // context that B still uses SparkExecutionContext.exitSparkExecution(); ecA.close(); - assertFalse("shared context must stay alive while another execution is active", sc.sc().isStopped()); - assertEquals("B's job must still run on the live context", 10L, rdd.reduce(Integer::sum).longValue()); + assertFalse("shared context must stay alive while another execution is active", + sc.sc().isStopped()); + assertEquals("B's job must still run on the live context", + 10L, rdd.reduce(Integer::sum).longValue()); // B finishes last: releasing the final registration lets close() stop it SparkExecutionContext.exitSparkExecution(); ecB.close(); - assertTrue("shared context must be stopped once the last execution closes", sc.sc().isStopped()); + assertTrue("shared context must be stopped once the last execution closes", + sc.sc().isStopped()); } finally { // drain any remaining registrations and stop the context so a failed @@ -82,9 +87,10 @@ public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { } /** - * An unpaired close() (a caller that borrows the shared context but never registered via enterSparkExecution()) - * must not stop a context another execution still uses. This fails on the old unconditional-stop code, which tore - * the context down out from under the active execution. + * An unpaired close() (a caller that borrows the shared context but never + * registered via enterSparkExecution()) must not stop a context another + * execution still uses. This fails on the old unconditional-stop code, which + * tore the context down out from under the active execution. */ @Test public void unpairedCloseDoesNotStopAContextStillInUse() { @@ -100,12 +106,14 @@ public void unpairedCloseDoesNotStopAContextStillInUse() { // borrows the shared context): close() must not stop a context in use unregistered = ExecutionContextFactory.createSparkExecutionContext(); unregistered.close(); - assertFalse("unpaired close() must not stop a context still in use", sc.sc().isStopped()); + assertFalse("unpaired close() must not stop a context still in use", + sc.sc().isStopped()); // the registered execution finishing stops the context as the last user SparkExecutionContext.exitSparkExecution(); active.close(); - assertTrue("context must stop once the last registered execution closes", sc.sc().isStopped()); + assertTrue("context must stop once the last registered execution closes", + sc.sc().isStopped()); } finally { SparkExecutionContext.exitSparkExecution(); diff --git a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java index 459936fa24c..2c854b4a81b 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java @@ -78,18 +78,17 @@ public MatrixBlock getMatrixBlock(long id) { } /** - * Poll the federated worker until the matrix at {@code id} is observed as a {@link CompressedMatrixBlock}, or - * {@link #COMPRESS_TIMEOUT_MS} elapses. + * Poll the federated worker until the matrix at {@code id} is observed as a + * {@link CompressedMatrixBlock}, or {@link #COMPRESS_TIMEOUT_MS} elapses. * - *

- * Federated workers compress asynchronously after a PUT/READ_VAR (see - * {@code CompressedMatrixBlockFactory.compressAsync}), so a {@code getMatrixBlock} fired right after the operation - * can race against the in-flight compression and return the uncompressed block. Tests that need to observe the - * compressed form should poll instead of sleeping a fixed amount. + *

Federated workers compress asynchronously after a PUT/READ_VAR (see + * {@code CompressedMatrixBlockFactory.compressAsync}), so a {@code getMatrixBlock} fired right + * after the operation can race against the in-flight compression and return the uncompressed + * block. Tests that need to observe the compressed form should poll instead of sleeping a fixed + * amount. * - *

- * On timeout this returns the most recent (uncompressed) read so the caller can produce a meaningful assertion - * failure naming the variable. + *

On timeout this returns the most recent (uncompressed) read so the caller can produce a + * meaningful assertion failure naming the variable. * * @param id federated variable id * @return the matrix block, compressed if compression finished in time, otherwise the latest read diff --git a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java index d3e4485bdf4..2b5ff327ef3 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java @@ -68,11 +68,13 @@ public void verifySameOrAlsoCompressedAsLocalCompress() { // federated. Compression on the worker is async; poll only when we expect compression to // match the local result, otherwise a single read is enough. final long id = putMatrixBlock(mb); - final MatrixBlock mbr = (mbcLocal instanceof CompressedMatrixBlock) ? awaitCompressed(id) : getMatrixBlock(id); + final MatrixBlock mbr = (mbcLocal instanceof CompressedMatrixBlock) + ? awaitCompressed(id) + : getMatrixBlock(id); if(mbcLocal instanceof CompressedMatrixBlock && !(mbr instanceof CompressedMatrixBlock)) - fail("Invalid result, the federated site did not compress the matrix block within " + COMPRESS_TIMEOUT_MS - + "ms"); + fail("Invalid result, the federated site did not compress the matrix block within " + + COMPRESS_TIMEOUT_MS + "ms"); TestUtils.compareMatricesBitAvgDistance(mbcLocal, mbr, 0, 0, "Not equivalent matrix block returned from federated site"); diff --git a/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java b/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java index 0de3b6db640..60587bf51a2 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java @@ -42,7 +42,7 @@ public void test100x100() { @Test public void testDecimalClampsFractionDigits() { - FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); + FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); f.ensureAllocatedColumns(1); f.set(0, 0, 5.244058388023880); // decimal=2 must print exactly two fraction digits, not DecimalFormat's default max of 3 @@ -53,10 +53,10 @@ public void testDecimalClampsFractionDigits() { @Test public void testDecimalPadsAndRounds() { - FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); + FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); f.ensureAllocatedColumns(2); - f.set(0, 0, 22.0); // integer-valued: padded up to the requested digits - f.set(1, 0, 5.244058388023880); // rounded at the last requested digit + f.set(0, 0, 22.0); // integer-valued: padded up to the requested digits + f.set(1, 0, 5.244058388023880); // rounded at the last requested digit String out = DataConverter.toString(f, false, " ", "\n", 2, 1, 4); assertTrue("expected 22.0000 padded: " + out, out.contains("22.0000\n")); assertTrue("expected 5.2441 rounded: " + out, out.contains("5.2441\n")); @@ -64,10 +64,10 @@ public void testDecimalPadsAndRounds() { @Test public void testNegativeDecimalUsesDefaultFormatting() { - FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); + FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); f.ensureAllocatedColumns(2); - f.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained - f.set(1, 0, 5.244058388023880); // default cap of three fraction digits + f.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained + f.set(1, 0, 5.244058388023880); // default cap of three fraction digits // decimal < 0 leaves DecimalFormat unconstrained (no min/max fraction digits set) String out = DataConverter.toString(f, false, " ", "\n", 2, 1, -1); assertTrue("expected unpadded 22: " + out, out.contains("22\n")); diff --git a/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java b/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java index 7d67c04983b..43a53879f17 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java @@ -142,7 +142,8 @@ public void safeCastWarnsOnlyOnce() { // the fallback warning must be logged exactly once across both conversions final List log = LoggingUtils.reinsert(appender); long warnings = log.stream() - .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")).count(); + .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")) + .count(); assertEquals(1, warnings); } @@ -152,7 +153,8 @@ public void strictThrowsWhenWarnCastDisabled() { setWarnCast(false); FrameBlock fb = mixedFrame(); - Exception e = assertThrows(DMLRuntimeException.class, () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 1)); + Exception e = assertThrows(DMLRuntimeException.class, + () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 1)); assertTrue(e.getMessage().contains("Failed to convert FrameBlock to MatrixBlock")); } @@ -162,7 +164,8 @@ public void strictThrowsParallelWhenWarnCastDisabled() { setWarnCast(false); FrameBlock fb = mixedFrame(); - Exception e = assertThrows(DMLRuntimeException.class, () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 4)); + Exception e = assertThrows(DMLRuntimeException.class, + () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 4)); assertTrue(e.getMessage().contains("Failed to convert FrameBlock to MatrixBlock")); } @@ -182,7 +185,8 @@ public void warnCastValidFrameConvertsWithoutFallback() { final List log = LoggingUtils.reinsert(appender); long warnings = log.stream() - .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")).count(); + .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")) + .count(); assertEquals(0, warnings); } diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java index 982941bb190..ccba674707b 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java @@ -74,8 +74,8 @@ private void runDecode(String spec, int nCol, int nCat) { try { FrameBlock data = categoricalFrame(ROWS, nCol, nCat, 17); - MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), data.getNumColumns(), - null); + MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), + data.getNumColumns(), null); MatrixBlock encoded = encoder.encode(data, 1); Decoder decoder = buildDecoder(data, spec, encoder); @@ -122,8 +122,8 @@ public void singleThreadEqualsParallelManyCategories() { public void decoderIsComposite() { FrameBlock data = categoricalFrame(100, 2, 3, 1); String spec = "{recode:[C1], dummycode:[C2]}"; - MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), data.getNumColumns(), - null); + MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), + data.getNumColumns(), null); encoder.encode(data, 1); Decoder decoder = buildDecoder(data, spec, encoder); if(!(decoder instanceof DecoderComposite)) diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java index 412686c1e83..d9c540f54c5 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java @@ -47,9 +47,9 @@ import org.junit.Test; /** - * Unit tests that drive the get_categorical_mask instruction directly to exercise the defensive code paths - * (distinct-count prefix in the metadata frame, default column metadata, non id-based specs and the unsupported opcode - * guard) that the script-level transform tests cannot reach. + * Unit tests that drive the get_categorical_mask instruction directly to exercise the defensive code + * paths (distinct-count prefix in the metadata frame, default column metadata, non id-based specs and + * the unsupported opcode guard) that the script-level transform tests cannot reach. */ public class GetCategoricalMaskInstructionTest { protected static final Log LOG = LogFactory.getLog(GetCategoricalMaskInstructionTest.class.getName()); @@ -210,8 +210,7 @@ public void imputeAndOmitAreAccepted() { // impute and omit do not change the output column count or categorical flag, so a spec that // only adds them on top of a recoded column must still succeed and mark that column categorical FrameBlock meta = new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}); - MatrixBlock res = run(meta, - "{\"ids\": true, \"recode\": [1], \"impute\": [{\"id\": 1, \"method\": \"global_mode\"}], \"omit\": [1]}"); + MatrixBlock res = run(meta, "{\"ids\": true, \"recode\": [1], \"impute\": [{\"id\": 1, \"method\": \"global_mode\"}], \"omit\": [1]}"); assertEquals(1, res.getNumRows()); assertEquals(1, res.getNumColumns()); @@ -231,7 +230,8 @@ public void unsupportedOpcodeThrows() { // any frame-scalar binary opcode other than get_categorical_mask must be rejected ExecutionContext ec = ExecutionContextFactory.createContext(); ec.setAutoCreateVars(true); - ec.setVariable("F", frameObject(new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}))); + ec.setVariable("F", frameObject(new FrameBlock(new ValueType[] {ValueType.STRING}, + new String[][] {{"a"}}))); assertThrowsMessage("Unsupported operation", () -> maskInstruction("+").processInstruction(ec)); } @@ -263,9 +263,9 @@ private static void assertMask(MatrixBlock res, double[] expected) { } /** - * Build a single-row metadata frame of nCol string columns. A positive distinct[i] is written to that column's - * metadata as the recode distinct count (the path real transformencode uses), while a zero leaves the column with - * default metadata (a continuous / non-dummycoded column). + * Build a single-row metadata frame of nCol string columns. A positive distinct[i] is written to + * that column's metadata as the recode distinct count (the path real transformencode uses), while + * a zero leaves the column with default metadata (a continuous / non-dummycoded column). */ private static FrameBlock metaWithDistinct(int nCol, int[] distinct) { ValueType[] schema = new ValueType[nCol]; @@ -290,8 +290,7 @@ private static MatrixBlock run(FrameBlock meta, String spec) { private static BinaryFrameScalarCPInstruction maskInstruction(String opcode) { String in1 = InstructionUtils.concatOperandParts("F", DataType.FRAME.name(), ValueType.STRING.name(), "false"); - String in2 = InstructionUtils.concatOperandParts("spec", DataType.SCALAR.name(), ValueType.STRING.name(), - "true"); + String in2 = InstructionUtils.concatOperandParts("spec", DataType.SCALAR.name(), ValueType.STRING.name(), "true"); String out = InstructionUtils.concatOperandParts("out", DataType.MATRIX.name(), ValueType.FP64.name(), "false"); String str = InstructionUtils.concatOperands("CP", opcode, in1, in2, out); return (BinaryFrameScalarCPInstruction) BinaryCPInstruction.parseInstruction(str); diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java index 645c4dd09bd..b2d31f43b83 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -44,10 +44,10 @@ import org.junit.Test; /** - * Exact inverse correctness tests for the transform decoders. Recode and dummycode are lossless category encodings, so - * a decode of the encoded matrix must reconstruct the original categorical frame. These tests assert exact - * reconstruction for the dense path, the sparse path, and the parallel path so that the dummycode sparse binary search - * and the parallel block split are validated against ground truth rather than only against each other. + * Exact inverse correctness tests for the transform decoders. Recode and dummycode are lossless category encodings, so a + * decode of the encoded matrix must reconstruct the original categorical frame. These tests assert exact reconstruction + * for the dense path, the sparse path, and the parallel path so that the dummycode sparse binary search and the parallel + * block split are validated against ground truth rather than only against each other. */ public class TransformDecodeRoundTripTest { protected static final Log LOG = LogFactory.getLog(TransformDecodeRoundTripTest.class.getName()); @@ -59,9 +59,9 @@ public void setUp() { } private static FrameBlock categoricalFrame() { - final String[] values = new String[] {"apple", "banana", "apple", "cherry", "banana", "date", "apple", "cherry", - "date", "banana", "elderberry", "apple", "fig", "banana", "cherry", "apple", "date", "fig", "elderberry", - "banana"}; + final String[] values = new String[] { + "apple", "banana", "apple", "cherry", "banana", "date", "apple", "cherry", "date", "banana", "elderberry", + "apple", "fig", "banana", "cherry", "apple", "date", "fig", "elderberry", "banana"}; final FrameBlock f = new FrameBlock(new ValueType[] {ValueType.STRING}); f.ensureAllocatedColumns(values.length); for(int i = 0; i < values.length; i++) @@ -117,8 +117,8 @@ public void binWithDummycodeOnOtherColumnConsistency() { /** * Dummycode on an earlier column (1) shifts the bin column (2) to the right in the encoded matrix. The bin decoder - * must walk the dummycode domain sizes to recover the bin column's true source position. This drives the non-magic - * offset branch of the bin source-column mapping. + * must walk the dummycode domain sizes to recover the bin column's true source position. This drives the + * non-magic offset branch of the bin source-column mapping. */ @Test public void binAfterDummycodeOnEarlierColumnConsistency() { @@ -128,9 +128,9 @@ public void binAfterDummycodeOnEarlierColumnConsistency() { } /** - * Same right-shift as above, but the earlier column is feature-hashed before being dummycoded. The hash domain size - * K is stored as a plain integer in the single meta cell, so the bin source-column mapping reads it (instead of - * numDistinct) to compute the offset. + * Same right-shift as above, but the earlier column is feature-hashed before being dummycoded. The hash domain + * size K is stored as a plain integer in the single meta cell, so the bin source-column mapping reads it (instead + * of numDistinct) to compute the offset. */ @Test public void binAfterHashDummycodeOnEarlierColumnConsistency() { @@ -211,10 +211,10 @@ public void binDecodeZeroCodeUsesFirstBinBoundary() { } /** - * Spark broadcasts the decoder to executors via Java serialization without re-running initMetaData, so the decoder - * must round-trip all of its decode state through writeExternal/readExternal. Decode with a freshly deserialized - * decoder and assert it matches the in-memory decode. Covers plain bin and bin-with-dummycode (the latter exercises - * the serialized _srcCols/_dcCols source-column mapping). + * Spark broadcasts the decoder to executors via Java serialization without re-running initMetaData, so the + * decoder must round-trip all of its decode state through writeExternal/readExternal. Decode with a freshly + * deserialized decoder and assert it matches the in-memory decode. Covers plain bin and bin-with-dummycode + * (the latter exercises the serialized _srcCols/_dcCols source-column mapping). */ @Test public void binDecoderSurvivesSerialization() { @@ -481,7 +481,8 @@ public void parallelDecodeWrapsWorkerException() { fail("expected the parallel decode wrapper to propagate the worker failure"); } catch(DMLRuntimeException expected) { - assertNotNull("parallel decode wrapper must retain the worker exception as cause", expected.getCause()); + assertNotNull("parallel decode wrapper must retain the worker exception as cause", + expected.getCause()); } } catch(Exception e) { diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java index 5745a35d506..54bd1679716 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java index 64012c06bbf..8d7ad14539a 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java @@ -57,15 +57,17 @@ import io.delta.kernel.types.TimestampType; /** - * Targeted tests for the error/defensive branches of the native Delta matrix read/write code that the round-trip and - * interop tests do not reach: malformed per-file statistics, unsupported column types, unsupported stream operations, + * Targeted tests for the error/defensive branches of the native Delta matrix + * read/write code that the round-trip and interop tests do not reach: malformed + * per-file statistics, unsupported column types, unsupported stream operations, * bad table paths, and the non-dense writer input path. * - *

- * A few of these branches guard against inputs that the SystemDS writer and the Delta Kernel scan API never produce in - * a normal round trip (e.g. a statistics JSON without {@code numRecords}, or a column type code outside the supported - * set). They are exercised here by mocking the Delta Kernel data objects and invoking the (package-private) helpers - * reflectively, rather than widening their production visibility purely for testing. + *

A few of these branches guard against inputs that the SystemDS writer and + * the Delta Kernel scan API never produce in a normal round trip (e.g. a + * statistics JSON without {@code numRecords}, or a column type code outside the + * supported set). They are exercised here by mocking the Delta Kernel data + * objects and invoking the (package-private) helpers reflectively, rather than + * widening their production visibility purely for testing. */ public class DeltaMatrixCoverageTest { @@ -87,8 +89,8 @@ public void qualifyRejectsUnknownFilesystemScheme() { @Test public void typeCodeReturnsNegativeForUnsupportedTypes() { - // non-numeric / unsupported Delta types must map to the sentinel -1 so the - // reader can reject them with a clear message rather than mis-decoding. + //non-numeric / unsupported Delta types must map to the sentinel -1 so the + //reader can reject them with a clear message rather than mis-decoding. assertEquals(-1, DeltaKernelUtils.typeCode(DateType.DATE)); assertEquals(-1, DeltaKernelUtils.typeCode(TimestampType.TIMESTAMP)); assertEquals(-1, DeltaKernelUtils.typeCode(BinaryType.BINARY)); @@ -110,17 +112,17 @@ public void writerRejectsStreamWrite() throws Exception { @Test public void numRecordsHandlesAbsentNullAndMalformedStats() throws Exception { - // no "stats" field at all -> -1 + //no "stats" field at all -> -1 assertEquals(-1, numRecords(addFileRow(new StructType().add("path", StringType.STRING), false, null))); - // stats column present but null-at -> -1 + //stats column present but null-at -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), true, null))); - // stats string explicitly null -> -1 + //stats string explicitly null -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, null))); - // malformed JSON -> JsonProcessingException -> -1 + //malformed JSON -> JsonProcessingException -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, "{not valid json"))); - // valid JSON but no numRecords field -> -1 + //valid JSON but no numRecords field -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, "{\"minValues\":{}}"))); - // well-formed stats -> the parsed count + //well-formed stats -> the parsed count assertEquals(1234L, numRecords(addFileRow(statsSchema(), false, "{\"numRecords\":1234}"))); } @@ -129,8 +131,8 @@ public void getDoubleValueRejectsUnknownTypeCode() throws Exception { Method m = ReaderDelta.class.getDeclaredMethod("getDoubleValue", ColumnVector.class, int.class, int.class); m.setAccessible(true); try { - // type code outside the supported T_* set; the switch default must throw - // before touching the (null) vector. + //type code outside the supported T_* set; the switch default must throw + //before touching the (null) vector. m.invoke(null, (ColumnVector) null, 0, 999); fail("expected a DMLRuntimeException for an unsupported type code"); } @@ -154,9 +156,9 @@ public void numericTypeCodeRejectsNonNumericType() throws Exception { @Test public void parallelReadWrapsFileFailure() throws Exception { - // a per-file decode failure in the parallel reader must surface as a single - // clear IOException (the awaitFileTasks catch), not a raw executor error. - // Provoke it by deleting one data file after the table (and its log) exist. + //a per-file decode failure in the parallel reader must surface as a single + //clear IOException (the awaitFileTasks catch), not a raw executor error. + //Provoke it by deleting one data file after the table (and its log) exist. MatrixBlock in = TestUtils.generateTestMatrixBlock(100_000, 8, -10, 10, 1.0, 13); in.recomputeNonZeros(); DMLConfig conf = new DMLConfig(); @@ -165,14 +167,15 @@ public void parallelReadWrapsFileFailure() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_fail_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, - in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); - // delete one parquet data file; the transaction log still references it, - // so the scan enumerates it but the decode task fails. + //delete one parquet data file; the transaction log still references it, + //so the scan enumerates it but the decode task fails. File victim; - try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { - victim = s.filter(p -> p.toString().endsWith(".parquet")).findFirst().map(Path::toFile).orElse(null); + try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + victim = s.filter(p -> p.toString().endsWith(".parquet")) + .findFirst().map(Path::toFile).orElse(null); } assertTrue("expected at least one data file to delete", victim != null && victim.delete()); @@ -197,8 +200,8 @@ public void parallelReadWrapsFileFailure() throws Exception { @Test public void sparseFormatMatrixRoundTrips() throws Exception { - // a sparse-backed MatrixBlock takes the writer's non-contiguous path (no - // direct double[] view), exercising MatrixColumnVector.get via MatrixBlock. + //a sparse-backed MatrixBlock takes the writer's non-contiguous path (no + //direct double[] view), exercising MatrixColumnVector.get via MatrixBlock. MatrixBlock in = TestUtils.generateTestMatrixBlock(2000, 7, -5, 5, 0.05, 13); in.recomputeNonZeros(); in.examSparsity(); @@ -207,8 +210,8 @@ public void sparseFormatMatrixRoundTrips() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_sparse_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, - in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", in.getNumRows(), out.getNumRows()); assertEquals("cols", in.getNumColumns(), out.getNumColumns()); @@ -221,15 +224,15 @@ public void sparseFormatMatrixRoundTrips() throws Exception { @Test public void fillDenseHandlesNonContiguousBlock() throws Exception { - // the dense fill normally hits the contiguous fast path; force a multi-block - // (non-contiguous) dense block so the row-by-row fallback is exercised. Such - // blocks only arise for matrices beyond a single contiguous array, so we - // shrink the per-block allocation cap to provoke it on a tiny matrix. + //the dense fill normally hits the contiguous fast path; force a multi-block + //(non-contiguous) dense block so the row-by-row fallback is exercised. Such + //blocks only arise for matrices beyond a single contiguous array, so we + //shrink the per-block allocation cap to provoke it on a tiny matrix. int rows = 5, cols = 4; int savedMaxAlloc = DenseBlockLDRB.MAX_ALLOC; DenseBlock db; try { - DenseBlockLDRB.MAX_ALLOC = 2 * cols; // ~2 rows per block -> multiple blocks + DenseBlockLDRB.MAX_ALLOC = 2 * cols; //~2 rows per block -> multiple blocks db = new DenseBlockLFP64(new int[] {rows, cols}); } finally { @@ -237,14 +240,14 @@ public void fillDenseHandlesNonContiguousBlock() throws Exception { } assertTrue("expected a non-contiguous (multi-block) dense block", !db.isContiguous()); - // two row-major batches (3 rows + 2 rows) covering all 5 rows + //two row-major batches (3 rows + 2 rows) covering all 5 rows double[] b0 = new double[3 * cols]; double[] b1 = new double[2 * cols]; - for(int r = 0; r < 3; r++) - for(int c = 0; c < cols; c++) + for( int r = 0; r < 3; r++ ) + for( int c = 0; c < cols; c++ ) b0[r * cols + c] = cell(r, c); - for(int r = 0; r < 2; r++) - for(int c = 0; c < cols; c++) + for( int r = 0; r < 2; r++ ) + for( int c = 0; c < cols; c++ ) b1[r * cols + c] = cell(3 + r, c); java.util.ArrayList batches = new java.util.ArrayList<>(); batches.add(b0); @@ -255,8 +258,8 @@ public void fillDenseHandlesNonContiguousBlock() throws Exception { m.setAccessible(true); m.invoke(null, ret, batches); - for(int r = 0; r < rows; r++) - for(int c = 0; c < cols; c++) + for( int r = 0; r < rows; r++ ) + for( int c = 0; c < cols; c++ ) assertEquals("r" + r + " c" + c, cell(r, c), ret.getDenseBlock().get(r, c), 0.0); } @@ -273,8 +276,8 @@ private static StructType statsSchema() { } /** - * Build a mocked scan-file row whose AddFile child has the given schema, null flag and (when not null) stats - * string, matching what {@code numRecords} reads. + * Build a mocked scan-file row whose AddFile child has the given schema, null + * flag and (when not null) stats string, matching what {@code numRecords} reads. */ private static Row addFileRow(StructType addSchema, boolean statsNull, String statsValue) { Row outer = mock(Row.class); @@ -282,12 +285,12 @@ private static Row addFileRow(StructType addSchema, boolean statsNull, String st when(outer.getStruct(InternalScanFileUtils.ADD_FILE_ORDINAL)).thenReturn(add); when(add.getSchema()).thenReturn(addSchema); int statsOrd = addSchema.fieldNames().indexOf("stats"); - if(statsOrd >= 0) { + if( statsOrd >= 0 ) { when(add.isNullAt(statsOrd)).thenReturn(statsNull); - if(!statsNull) + if( !statsNull ) when(add.getString(statsOrd)).thenReturn(statsValue); } - return outer; // the scan-file row numRecords consumes (its AddFile child is 'add') + return outer; //the scan-file row numRecords consumes (its AddFile child is 'add') } private static long numRecords(Row scanFileRow) throws Exception { diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java index 65c4ec21c0f..54a3bcf6334 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java @@ -63,19 +63,20 @@ import io.delta.kernel.utils.CloseableIterator; /** - * Direct (no DML) round-trip tests for the native Delta Kernel based matrix reader/writer. Each test writes a - * MatrixBlock to a fresh local Delta table directory and reads it back, asserting dimensions and values match. + * Direct (no DML) round-trip tests for the native Delta Kernel based matrix + * reader/writer. Each test writes a MatrixBlock to a fresh local Delta table + * directory and reads it back, asserting dimensions and values match. */ public class DeltaMatrixReadWriteTest { - // small writer target file size (bytes) used to force a multi-file table - // layout cheaply, instead of brute-forcing huge row counts. + //small writer target file size (bytes) used to force a multi-file table + //layout cheaply, instead of brute-forcing huge row counts. private static final long SMALL_TARGET_FILE_SIZE = 256L * 1024; private static final int ROWS_MULTI_FILE = 100_000; private static MatrixBlock writeThenRead(MatrixBlock in) throws Exception { Path dir = Files.createTempDirectory("sysds_delta_"); - // WriterDelta creates the table at the given (empty) directory + //WriterDelta creates the table at the given (empty) directory String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { WriterDelta writer = new WriterDelta(); @@ -91,9 +92,9 @@ private static MatrixBlock writeThenRead(MatrixBlock in) throws Exception { @Test public void parallelReadMatchesSerialMultiFile() throws Exception { - // force a multi-file table cheaply via a small writer target file size - // (rather than a huge row count), so the parallel per-file path is - // actually exercised rather than falling back to serial. + //force a multi-file table cheaply via a small writer target file size + //(rather than a huge row count), so the parallel per-file path is + //actually exercised rather than falling back to serial. MatrixBlock in = TestUtils.generateTestMatrixBlock(ROWS_MULTI_FILE, 8, -10, 10, 1.0, 13); in.recomputeNonZeros(); @@ -103,18 +104,21 @@ public void parallelReadMatchesSerialMultiFile() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_par_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, - in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); - // sanity: confirm the table really is split across multiple files + //sanity: confirm the table really is split across multiple files long files; - try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { + try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } - assertTrue("expected a multi-file Delta table to exercise the parallel path, got " + files, files > 1); + assertTrue("expected a multi-file Delta table to exercise the parallel path, got " + files, + files > 1); - MatrixBlock serial = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); - MatrixBlock parallel = new ReaderDeltaParallel().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + MatrixBlock serial = new ReaderDelta() + .readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + MatrixBlock parallel = new ReaderDeltaParallel() + .readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", serial.getNumRows(), parallel.getNumRows()); assertEquals("cols", serial.getNumColumns(), parallel.getNumColumns()); @@ -131,8 +135,8 @@ public void parallelReadMatchesSerialMultiFile() throws Exception { public void roundTripDenseSmall() throws Exception { MatrixBlock in = new MatrixBlock(3, 4, false); double v = 1.0; - for(int i = 0; i < 3; i++) - for(int j = 0; j < 4; j++) + for( int i=0; i<3; i++ ) + for( int j=0; j<4; j++ ) in.set(i, j, v++); in.recomputeNonZeros(); @@ -153,7 +157,7 @@ public void roundTripDenseRandom() throws Exception { @Test public void roundTripSparseRandom() throws Exception { - // values written are dense parquet, but exercise a sparse-ish input + //values written are dense parquet, but exercise a sparse-ish input MatrixBlock in = TestUtils.generateTestMatrixBlock(1200, 9, -5, 5, 0.1, 13); in.recomputeNonZeros(); MatrixBlock out = writeThenRead(in); @@ -164,7 +168,7 @@ public void roundTripSparseRandom() throws Exception { @Test public void roundTripMultiBatch() throws Exception { - // more rows than the writer batch size (4096) to exercise chunking + //more rows than the writer batch size (4096) to exercise chunking MatrixBlock in = TestUtils.generateTestMatrixBlock(10000, 5, 0, 100, 1.0, 1); MatrixBlock out = writeThenRead(in); assertEquals("rows", 10000, out.getNumRows()); @@ -178,9 +182,8 @@ public void readDiscoversUnknownDimensions() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, - in.getNonZeros()); - // pass -1 dimensions: the reader must discover them from the table + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + //pass -1 dimensions: the reader must discover them from the table MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 123, out.getNumRows()); assertEquals("cols", 6, out.getNumColumns()); @@ -209,21 +212,22 @@ public void emptyMatrixRoundTrip() throws Exception { @Test public void readNonDoubleNumericColumns() throws Exception { - // tables produced by external tools (or the frame writer) can carry - // long/int/boolean columns; the matrix reader must coerce them to double + //tables produced by external tools (or the frame writer) can carry + //long/int/boolean columns; the matrix reader must coerce them to double Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] longVals = {1, -2, 1_000_000_000L, 0}; - double[] intVals = {7, -8, 123456, 0}; + double[] intVals = {7, -8, 123456, 0}; double[] boolVals = {1, 0, 1, 0}; - writeTypedColumns(tablePath, new DataType[] {LongType.LONG, IntegerType.INTEGER, BooleanType.BOOLEAN}, + writeTypedColumns(tablePath, + new DataType[] {LongType.LONG, IntegerType.INTEGER, BooleanType.BOOLEAN}, new double[][] {longVals, intVals, boolVals}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 3, out.getNumColumns()); - for(int r = 0; r < 4; r++) { + for( int r=0; r<4; r++ ) { assertEquals("long col r" + r, longVals[r], out.get(r, 0), 0.0); assertEquals("int col r" + r, intVals[r], out.get(r, 1), 0.0); assertEquals("bool col r" + r, boolVals[r], out.get(r, 2), 0.0); @@ -236,14 +240,14 @@ public void readNonDoubleNumericColumns() throws Exception { @Test public void rewriteSamePathReplacesData() throws Exception { - // writing to a path that already holds a Delta table must fully replace it + //writing to a path that already holds a Delta table must fully replace it Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { MatrixBlock first = TestUtils.generateTestMatrixBlock(50, 8, 0, 100, 1.0, 1); new WriterDelta().writeMatrixToHDFS(first, tablePath, 50, 8, -1, first.getNonZeros()); - // second write has different dimensions and values + //second write has different dimensions and values MatrixBlock second = TestUtils.generateTestMatrixBlock(20, 3, -5, 5, 1.0, 2); new WriterDelta().writeMatrixToHDFS(second, tablePath, 20, 3, -1, second.getNonZeros()); @@ -259,10 +263,10 @@ public void rewriteSamePathReplacesData() throws Exception { @Test public void parallelBufferedPathMatchesSerial() throws Exception { - // the direct fast path is always taken for SystemDS-written tables (exact - // row stats, no deletion vectors); force the buffered fallback to exercise - // its per-file decode + serial concatenation and assert it matches serial. - // force a multi-file table cheaply via a small writer target file size. + //the direct fast path is always taken for SystemDS-written tables (exact + //row stats, no deletion vectors); force the buffered fallback to exercise + //its per-file decode + serial concatenation and assert it matches serial. + //force a multi-file table cheaply via a small writer target file size. MatrixBlock in = TestUtils.generateTestMatrixBlock(ROWS_MULTI_FILE, 8, -10, 10, 1.0, 23); in.recomputeNonZeros(); @@ -272,22 +276,19 @@ public void parallelBufferedPathMatchesSerial() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_buf_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, - in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); long files; - try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { + try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } assertTrue("expected a multi-file Delta table, got " + files, files > 1); MatrixBlock serial = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); - // subclass that always declines the direct path -> readBuffered() + //subclass that always declines the direct path -> readBuffered() MatrixBlock buffered = new ReaderDeltaParallel() { - @Override - protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { - return false; - } + @Override protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { return false; } }.readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", serial.getNumRows(), buffered.getNumRows()); @@ -303,30 +304,30 @@ protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { @Test public void writerTargetFileSizeConfigProducesMoreFiles() throws Exception { - // a smaller configured target file size must make the writer roll more - // data files for the same matrix (the lever the parallel reader relies on). + //a smaller configured target file size must make the writer roll more + //data files for the same matrix (the lever the parallel reader relies on). MatrixBlock in = TestUtils.generateTestMatrixBlock(400_000, 16, -10, 10, 1.0, 7); in.recomputeNonZeros(); - // isolate the override in a fresh thread-local config (restored in finally) + //isolate the override in a fresh thread-local config (restored in finally) DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(1L * 1024 * 1024)); ConfigurationManager.setLocalConfig(conf); Path dir = Files.createTempDirectory("sysds_delta_cfg_"); try { - assertEquals("config getter reflects the override", 1L * 1024 * 1024, - ConfigurationManager.getDeltaWriterTargetFileSize()); + assertEquals("config getter reflects the override", + 1L * 1024 * 1024, ConfigurationManager.getDeltaWriterTargetFileSize()); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, - in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); long files; - try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { + try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } assertTrue("expected >1 data file with a 1MB target, got " + files, files > 1); - // data still round-trips correctly with the custom layout + //data still round-trips correctly with the custom layout MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); TestUtils.compareMatrices(in, out, 1e-12, "small-target-roundtrip"); } @@ -338,20 +339,21 @@ public void writerTargetFileSizeConfigProducesMoreFiles() throws Exception { @Test public void readerBatchSizeConfigRoundTrips() throws Exception { - // a non-default reader batch size must not change the result (more, smaller - // batches exercise the per-batch extract/concatenate loop more often). + //a non-default reader batch size must not change the result (more, smaller + //batches exercise the per-batch extract/concatenate loop more often). MatrixBlock in = TestUtils.generateTestMatrixBlock(5000, 7, -10, 10, 1.0, 11); - // isolate the override in a fresh thread-local config (restored in finally) + //isolate the override in a fresh thread-local config (restored in finally) DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_READER_BATCH_SIZE, "128"); ConfigurationManager.setLocalConfig(conf); Path dir = Files.createTempDirectory("sysds_delta_bs_"); try { - assertEquals("config getter reflects the override", 128, ConfigurationManager.getDeltaReaderBatchSize()); + assertEquals("config getter reflects the override", + 128, ConfigurationManager.getDeltaReaderBatchSize()); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, - in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); TestUtils.compareMatrices(in, out, 1e-12, "small-batch-roundtrip"); } @@ -363,13 +365,14 @@ public void readerBatchSizeConfigRoundTrips() throws Exception { @Test public void factoryRoutesDeltaToParallelWhenEnabled() { - // the factory must pick the parallel reader iff parallel CP read is enabled + //the factory must pick the parallel reader iff parallel CP read is enabled CompilerConfig cc = ConfigurationManager.getCompilerConfig(); try { cc.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, true); ConfigurationManager.setLocalConfig(cc); MatrixReader par = MatrixReaderFactory.createMatrixReader(FileFormat.DELTA); - assertTrue("expected ReaderDeltaParallel when parallel read enabled", par instanceof ReaderDeltaParallel); + assertTrue("expected ReaderDeltaParallel when parallel read enabled", + par instanceof ReaderDeltaParallel); cc.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, false); ConfigurationManager.setLocalConfig(cc); @@ -384,18 +387,20 @@ public void factoryRoutesDeltaToParallelWhenEnabled() { @Test public void readFloatColumnsCoercedToDouble() throws Exception { - // float columns must be widened to double on read (exact-representable values) + //float columns must be widened to double on read (exact-representable values) Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] f0 = {1.5, -2.25, 0.0, 1024.5}; double[] f1 = {-0.5, 3.75, 100.125, -7.0}; - writeTypedColumns(tablePath, new DataType[] {FloatType.FLOAT, FloatType.FLOAT}, new double[][] {f0, f1}); + writeTypedColumns(tablePath, + new DataType[] {FloatType.FLOAT, FloatType.FLOAT}, + new double[][] {f0, f1}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 2, out.getNumColumns()); - for(int r = 0; r < 4; r++) { + for( int r=0; r<4; r++ ) { assertEquals("f0 r" + r, f0[r], out.get(r, 0), 0.0); assertEquals("f1 r" + r, f1[r], out.get(r, 1), 0.0); } @@ -407,20 +412,21 @@ public void readFloatColumnsCoercedToDouble() throws Exception { @Test public void readShortByteColumnsCoercedToDouble() throws Exception { - // short/byte columns must be coerced to double on read, exercising the - // T_SHORT / T_BYTE branches of ReaderDelta.getDoubleValue. + //short/byte columns must be coerced to double on read, exercising the + //T_SHORT / T_BYTE branches of ReaderDelta.getDoubleValue. Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] shortVals = {1, -2, 30000, 0}; - double[] byteVals = {7, -8, 120, 0}; - writeTypedColumns(tablePath, new DataType[] {ShortType.SHORT, ByteType.BYTE}, + double[] byteVals = {7, -8, 120, 0}; + writeTypedColumns(tablePath, + new DataType[] {ShortType.SHORT, ByteType.BYTE}, new double[][] {shortVals, byteVals}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 2, out.getNumColumns()); - for(int r = 0; r < 4; r++) { + for( int r=0; r<4; r++ ) { assertEquals("short col r" + r, shortVals[r], out.get(r, 0), 0.0); assertEquals("byte col r" + r, byteVals[r], out.get(r, 1), 0.0); } @@ -432,8 +438,8 @@ public void readShortByteColumnsCoercedToDouble() throws Exception { @Test public void writerRejectsDimensionMismatch() throws Exception { - // WriterDelta validates that the passed rlen/clen match the MatrixBlock - // and rejects a mismatch with an IOException. + //WriterDelta validates that the passed rlen/clen match the MatrixBlock + //and rejects a mismatch with an IOException. MatrixBlock in = TestUtils.generateTestMatrixBlock(10, 4, -1, 1, 1.0, 5); in.recomputeNonZeros(); Path dir = Files.createTempDirectory("sysds_delta_"); @@ -453,18 +459,18 @@ public void writerRejectsDimensionMismatch() throws Exception { @Test public void readNullCellsBecomeZero() throws Exception { - // nullable numeric columns with null cells must read back as 0.0 + //nullable numeric columns with null cells must read back as 0.0 Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - double[] vals = {3.0, 7.0, 9.0, 11.0}; - boolean[] nulls = {false, true, false, true}; + double[] vals = {3.0, 7.0, 9.0, 11.0}; + boolean[] nulls = {false, true, false, true}; writeNullableDoubleColumn(tablePath, vals, nulls); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 1, out.getNumColumns()); - for(int r = 0; r < 4; r++) + for( int r=0; r<4; r++ ) assertEquals("r" + r, nulls[r] ? 0.0 : vals[r], out.get(r, 0), 0.0); } finally { @@ -474,7 +480,7 @@ public void readNullCellsBecomeZero() throws Exception { @Test public void readStringColumnRejected() throws Exception { - // string columns cannot back an all-double matrix -> reader must reject them + //string columns cannot back an all-double matrix -> reader must reject them Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { @@ -497,10 +503,9 @@ public void readStringColumnRejected() throws Exception { private static void writeTypedColumns(String tablePath, DataType[] types, double[][] vals) throws Exception { Engine engine = DeltaKernelUtils.createEngine(); StructType schema = new StructType(); - for(int c = 0; c < types.length; c++) + for( int c=0; c singleton(FilteredColumnarBatch fcb) { return new CloseableIterator() { private boolean _done = false; - - @Override - public boolean hasNext() { - return !_done; - } - - @Override - public FilteredColumnarBatch next() { - if(_done) - throw new NoSuchElementException(); + @Override public boolean hasNext() { return !_done; } + @Override public FilteredColumnarBatch next() { + if( _done ) throw new NoSuchElementException(); _done = true; return fcb; } - - @Override - public void close() { - } + @Override public void close() {} }; } - /** - * Minimal in-memory columnar batch backed by per-column double[] values, with an optional per-column null mask - * ({@code nulls==null} => no nulls). - */ + /** Minimal in-memory columnar batch backed by per-column double[] values, with + * an optional per-column null mask ({@code nulls==null} => no nulls). */ private static class TypedBatch implements ColumnarBatch { private final StructType _schema; private final DataType[] _types; private final double[][] _vals; private final boolean[][] _nulls; - TypedBatch(StructType schema, DataType[] types, double[][] vals, boolean[][] nulls) { - _schema = schema; - _types = types; - _vals = vals; - _nulls = nulls; + _schema = schema; _types = types; _vals = vals; _nulls = nulls; } - - @Override - public StructType getSchema() { - return _schema; - } - - @Override - public int getSize() { - return _vals[0].length; - } - - @Override - public ColumnVector getColumnVector(int ordinal) { - return new TypedVector(_types[ordinal], _vals[ordinal], _nulls == null ? null : _nulls[ordinal]); + @Override public StructType getSchema() { return _schema; } + @Override public int getSize() { return _vals[0].length; } + @Override public ColumnVector getColumnVector(int ordinal) { + return new TypedVector(_types[ordinal], _vals[ordinal], + _nulls == null ? null : _nulls[ordinal]); } } @@ -599,98 +568,28 @@ private static class TypedVector implements ColumnVector { private final DataType _type; private final double[] _vals; private final boolean[] _nulls; - - TypedVector(DataType type, double[] vals, boolean[] nulls) { - _type = type; - _vals = vals; - _nulls = nulls; - } - - @Override - public DataType getDataType() { - return _type; - } - - @Override - public int getSize() { - return _vals.length; - } - - @Override - public boolean isNullAt(int rowId) { - return _nulls != null && _nulls[rowId]; - } - - @Override - public double getDouble(int rowId) { - return _vals[rowId]; - } - - @Override - public float getFloat(int rowId) { - return (float) _vals[rowId]; - } - - @Override - public long getLong(int rowId) { - return (long) _vals[rowId]; - } - - @Override - public int getInt(int rowId) { - return (int) _vals[rowId]; - } - - @Override - public short getShort(int rowId) { - return (short) _vals[rowId]; - } - - @Override - public byte getByte(int rowId) { - return (byte) _vals[rowId]; - } - - @Override - public boolean getBoolean(int rowId) { - return _vals[rowId] != 0; - } - - @Override - public void close() { - } + TypedVector(DataType type, double[] vals, boolean[] nulls) { _type = type; _vals = vals; _nulls = nulls; } + @Override public DataType getDataType() { return _type; } + @Override public int getSize() { return _vals.length; } + @Override public boolean isNullAt(int rowId) { return _nulls != null && _nulls[rowId]; } + @Override public double getDouble(int rowId) { return _vals[rowId]; } + @Override public float getFloat(int rowId) { return (float) _vals[rowId]; } + @Override public long getLong(int rowId) { return (long) _vals[rowId]; } + @Override public int getInt(int rowId) { return (int) _vals[rowId]; } + @Override public short getShort(int rowId) { return (short) _vals[rowId]; } + @Override public byte getByte(int rowId) { return (byte) _vals[rowId]; } + @Override public boolean getBoolean(int rowId) { return _vals[rowId] != 0; } + @Override public void close() {} } /** Column view exposing a String[] as a Delta string column. */ private static class StringVector implements ColumnVector { private final String[] _vals; - - StringVector(String[] vals) { - _vals = vals; - } - - @Override - public DataType getDataType() { - return StringType.STRING; - } - - @Override - public int getSize() { - return _vals.length; - } - - @Override - public boolean isNullAt(int rowId) { - return _vals[rowId] == null; - } - - @Override - public String getString(int rowId) { - return _vals[rowId]; - } - - @Override - public void close() { - } + StringVector(String[] vals) { _vals = vals; } + @Override public DataType getDataType() { return StringType.STRING; } + @Override public int getSize() { return _vals.length; } + @Override public boolean isNullAt(int rowId) { return _vals[rowId] == null; } + @Override public String getString(int rowId) { return _vals[rowId]; } + @Override public void close() {} } } diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java index 45194d12b5a..2d79b79f2dd 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java @@ -49,22 +49,24 @@ import org.junit.Test; /** - * Cross-engine interoperability tests for the native (Delta Kernel based) matrix reader/writer against the reference - * Delta implementation (Delta's Spark connector, {@code delta-spark}, pulled in test-only). + * Cross-engine interoperability tests for the native (Delta Kernel based) matrix + * reader/writer against the reference Delta implementation (Delta's Spark + * connector, {@code delta-spark}, pulled in test-only). * - *

- * The other Delta matrix tests round-trip exclusively through SystemDS' own Kernel-based read/write paths, so they - * cannot catch a table that SystemDS writes in a way other Delta engines reject (or vice versa). These tests close that - * gap by routing data through two independent engines: + *

The other Delta matrix tests round-trip exclusively through SystemDS' own + * Kernel-based read/write paths, so they cannot catch a table that SystemDS + * writes in a way other Delta engines reject (or vice versa). These tests close + * that gap by routing data through two independent engines: *

    - *
  • SystemDS writes -> Spark/Delta reads (our output is spec-compliant), and
    • - *
  • Spark/Delta writes -> SystemDS reads, including a multi-file layout and a table with deletion vectors / a - * second commit that the SystemDS writer never produces itself.
    • + *
  • SystemDS writes -> Spark/Delta reads (our output is spec-compliant), and
    • + *
  • Spark/Delta writes -> SystemDS reads, including a multi-file layout and a + * table with deletion vectors / a second commit that the SystemDS writer + * never produces itself.
    • *
* - *

- * Row order is never assumed: every table carries a unique id in column 0 and comparisons are keyed by that id, since - * neither engine guarantees row order across files. + *

Row order is never assumed: every table carries a unique id in column 0 and + * comparisons are keyed by that id, since neither engine guarantees row order + * across files. */ @net.jcip.annotations.NotThreadSafe public class DeltaMatrixSparkInteropTest { @@ -73,19 +75,23 @@ public class DeltaMatrixSparkInteropTest { @BeforeClass public static void startSpark() { - // each test class runs in its own fork (surefire reuseForks=false), so this - // is the only SparkSession in the JVM and gets the Delta extensions injected. + //each test class runs in its own fork (surefire reuseForks=false), so this + //is the only SparkSession in the JVM and gets the Delta extensions injected. SparkSession.clearActiveSession(); SparkSession.clearDefaultSession(); - spark = SparkSession.builder().appName("sysds-delta-interop").master("local[2]") - .config("spark.ui.enabled", "false").config("spark.sql.shuffle.partitions", "2") + spark = SparkSession.builder() + .appName("sysds-delta-interop") + .master("local[2]") + .config("spark.ui.enabled", "false") + .config("spark.sql.shuffle.partitions", "2") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") - .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog").getOrCreate(); + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") + .getOrCreate(); } @AfterClass public static void stopSpark() { - if(spark != null) + if( spark != null ) spark.stop(); SparkSession.clearActiveSession(); SparkSession.clearDefaultSession(); @@ -94,13 +100,13 @@ public static void stopSpark() { @Test public void systemdsWriteSparkReadMultiFile() throws Exception { - // SystemDS writes a (forced) multi-file Delta table; the reference Delta - // engine (Spark) must read every data file back with matching values. + //SystemDS writes a (forced) multi-file Delta table; the reference Delta + //engine (Spark) must read every data file back with matching values. int rows = 500, cols = 5; MatrixBlock in = indexedMatrix(rows, cols); - // small target file size -> multiple parquet data files (exercise that an - // external reader stitches all of our data files, not just the first). + //small target file size -> multiple parquet data files (exercise that an + //external reader stitches all of our data files, not just the first). DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(16L * 1024)); ConfigurationManager.setLocalConfig(conf); @@ -116,10 +122,10 @@ public void systemdsWriteSparkReadMultiFile() throws Exception { List read = df.collectAsList(); assertEquals(rows, read.size()); - for(Row r : read) { + for( Row r : read ) { int id = (int) Math.round(r.getDouble(0)); assertTrue("id in range: " + id, id >= 0 && id < rows); - for(int c = 0; c < cols; c++) + for( int c = 0; c < cols; c++ ) assertEquals("r" + id + " c" + c, in.get(id, c), r.getDouble(c), 1e-9); } } @@ -131,10 +137,10 @@ public void systemdsWriteSparkReadMultiFile() throws Exception { @Test public void sparkWriteSystemdsReadMultiFile() throws Exception { - // the reference Delta engine writes a multi-file table; both the serial and - // parallel SystemDS readers must reconstruct it (coercing long ids to double). + //the reference Delta engine writes a multi-file table; both the serial and + //parallel SystemDS readers must reconstruct it (coercing long ids to double). int rows = 600, cols = 4; - Dataset df = indexedDataFrame(rows, cols).repartition(3); // -> multiple data files + Dataset df = indexedDataFrame(rows, cols).repartition(3); //-> multiple data files Path dir = Files.createTempDirectory("sysds_delta_p2s_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { @@ -142,10 +148,10 @@ public void sparkWriteSystemdsReadMultiFile() throws Exception { assertTrue("spark should have written a multi-file table", countParquet(tablePath) > 1); Map expected = expectedById(rows, cols); - assertMatchesById(new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, - "serial"); - assertMatchesById(new ReaderDeltaParallel().readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, - "parallel"); + assertMatchesById(new ReaderDelta() + .readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, "serial"); + assertMatchesById(new ReaderDeltaParallel() + .readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, "parallel"); } finally { FileUtils.deleteQuietly(dir.toFile()); @@ -154,15 +160,15 @@ public void sparkWriteSystemdsReadMultiFile() throws Exception { @Test public void sparkDeletionVectorsSystemdsRead() throws Exception { - // a Delta table with deletion vectors + a second commit (the DELETE) is a - // layout the SystemDS writer never emits; the readers must honor the DV and - // return only the surviving rows. This exercises the hasDeletionVector path. + //a Delta table with deletion vectors + a second commit (the DELETE) is a + //layout the SystemDS writer never emits; the readers must honor the DV and + //return only the surviving rows. This exercises the hasDeletionVector path. int rows = 400, cols = 3, deleteBelow = 50; Path dir = Files.createTempDirectory("sysds_delta_dv_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - // enable deletion vectors for tables created in this block, then delete a - // row range so Delta records a DV rather than rewriting the data files. + //enable deletion vectors for tables created in this block, then delete a + //row range so Delta records a DV rather than rewriting the data files. spark.conf().set(DV_DEFAULT, "true"); indexedDataFrame(rows, cols).write().format("delta").save(tablePath); spark.sql("DELETE FROM delta.`" + tablePath + "` WHERE c0 < " + deleteBelow); @@ -179,20 +185,21 @@ public void sparkDeletionVectorsSystemdsRead() throws Exception { assertMatchesById(parallel, expected, cols, "parallel-dv"); } finally { - // fresh fork per test class, so simply clearing the override is enough + //fresh fork per test class, so simply clearing the override is enough spark.conf().unset(DV_DEFAULT); FileUtils.deleteQuietly(dir.toFile()); } } - private static final String DV_DEFAULT = "spark.databricks.delta.properties.defaults.enableDeletionVectors"; + private static final String DV_DEFAULT = + "spark.databricks.delta.properties.defaults.enableDeletionVectors"; /** Matrix whose column 0 is the row index and remaining columns are exact doubles. */ private static MatrixBlock indexedMatrix(int rows, int cols) { MatrixBlock mb = new MatrixBlock(rows, cols, false); - for(int r = 0; r < rows; r++) { + for( int r = 0; r < rows; r++ ) { mb.set(r, 0, r); - for(int c = 1; c < cols; c++) + for( int c = 1; c < cols; c++ ) mb.set(r, c, value(r, c)); } mb.recomputeNonZeros(); @@ -202,15 +209,15 @@ private static MatrixBlock indexedMatrix(int rows, int cols) { /** Spark DataFrame mirroring {@link #indexedMatrix} with columns c0..c(cols-1) as doubles. */ private static Dataset indexedDataFrame(int rows, int cols) { StructField[] fields = new StructField[cols]; - for(int c = 0; c < cols; c++) + for( int c = 0; c < cols; c++ ) fields[c] = DataTypes.createStructField("c" + c, DataTypes.DoubleType, false); StructType schema = DataTypes.createStructType(fields); List data = new ArrayList<>(rows); - for(int r = 0; r < rows; r++) { + for( int r = 0; r < rows; r++ ) { Object[] vals = new Object[cols]; vals[0] = (double) r; - for(int c = 1; c < cols; c++) + for( int c = 1; c < cols; c++ ) vals[c] = value(r, c); data.add(RowFactory.create(vals)); } @@ -224,10 +231,10 @@ private static double value(int row, int col) { private static Map expectedById(int rows, int cols) { Map exp = new HashMap<>(rows); - for(int r = 0; r < rows; r++) { + for( int r = 0; r < rows; r++ ) { double[] row = new double[cols]; row[0] = r; - for(int c = 1; c < cols; c++) + for( int c = 1; c < cols; c++ ) row[c] = value(r, c); exp.put(r, row); } @@ -239,25 +246,25 @@ private static void assertMatchesById(MatrixBlock out, Map ex assertEquals(tag + " rows", expected.size(), out.getNumRows()); assertEquals(tag + " cols", cols, out.getNumColumns()); boolean[] seen = new boolean[expected.size() == 0 ? 0 : maxId(expected) + 1]; - for(int r = 0; r < out.getNumRows(); r++) { + for( int r = 0; r < out.getNumRows(); r++ ) { int id = (int) Math.round(out.get(r, 0)); double[] exp = expected.get(id); assertTrue(tag + ": unexpected/duplicate id " + id, exp != null && id < seen.length && !seen[id]); seen[id] = true; - for(int c = 0; c < cols; c++) + for( int c = 0; c < cols; c++ ) assertEquals(tag + " id" + id + " c" + c, exp[c], out.get(r, c), 1e-9); } } private static int maxId(Map expected) { int m = 0; - for(int id : expected.keySet()) + for( int id : expected.keySet() ) m = Math.max(m, id); return m; } private static long countParquet(String tablePath) throws Exception { - try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { + try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { return s.filter(p -> p.toString().endsWith(".parquet")).count(); } } diff --git a/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java b/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java index ae19b291882..472b61d8cd6 100644 --- a/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java +++ b/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -26,10 +26,11 @@ /** * Tests the single-column (unweighted) branch of {@link MatrixBlock#pickValue(double, boolean)} and - * {@link MatrixBlock#median()}. The values are assumed to be sorted in ascending order, mirroring the contract used by - * the quantile pick instructions. The unweighted branch uses the same ceil-based rank as the two-column weighted branch - * (with an implicit weight of 1 per value), so a single column yields the same quantile as the equivalent (value, - * weight) representation. The two-column (weighted) branch is exercised separately through the compressed sort tests. + * {@link MatrixBlock#median()}. The values are assumed to be sorted in ascending order, mirroring the contract used + * by the quantile pick instructions. The unweighted branch uses the same ceil-based rank as the two-column weighted + * branch (with an implicit weight of 1 per value), so a single column yields the same quantile as the equivalent + * (value, weight) representation. The two-column (weighted) branch is exercised separately through the compressed + * sort tests. */ public class QuantilePickTest { diff --git a/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java b/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java index 05aca41ebc7..5c9ed821e78 100644 --- a/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java +++ b/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java @@ -30,7 +30,7 @@ public class TensorToStringTest { @Test public void testDecimalClampsFractionDigits() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 1}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 1}); tb.allocateBlock(); tb.set(0, 0, 5.244058388023880); // decimal=2 must print exactly two fraction digits, not DecimalFormat's default max of 3 @@ -41,10 +41,10 @@ public void testDecimalClampsFractionDigits() { @Test public void testDecimalPadsAndRounds() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 2}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 2}); tb.allocateBlock(); - tb.set(0, 0, 22.0); // integer-valued: padded up to the requested digits - tb.set(0, 1, 5.244058388023880); // rounded at the last requested digit + tb.set(0, 0, 22.0); // integer-valued: padded up to the requested digits + tb.set(0, 1, 5.244058388023880); // rounded at the last requested digit String out = DataConverter.toString(tb, false, " ", "\n", "[", "]", 1, 2, 4); assertTrue("expected 22.0000 padded: " + out, out.contains("22.0000")); assertTrue("expected 5.2441 rounded: " + out, out.contains("5.2441")); @@ -52,10 +52,10 @@ public void testDecimalPadsAndRounds() { @Test public void testNegativeDecimalUsesDefaultFormatting() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 2}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 2}); tb.allocateBlock(); - tb.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained - tb.set(0, 1, 5.244058388023880); // default cap of three fraction digits + tb.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained + tb.set(0, 1, 5.244058388023880); // default cap of three fraction digits // decimal < 0 leaves DecimalFormat unconstrained (no min/max fraction digits set) String out = DataConverter.toString(tb, false, " ", "\n", "[", "]", 1, 2, -1); assertTrue("expected unpadded 22: " + out, out.contains("22")); diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java index 426de81263f..810e2614e7c 100644 --- a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java +++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java @@ -52,12 +52,18 @@ public class QuantileTest extends AutomatedTestBase public void setUp() { TestUtils.clearAssertionInformation(); - addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"})); - addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"})); - addTestConfiguration(TEST_NAME3, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] {"R"})); - addTestConfiguration(TEST_NAME4, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] {"R"})); - addTestConfiguration(TEST_NAME5, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] {"R"})); - addTestConfiguration(TEST_NAME6, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] {"R"})); + addTestConfiguration(TEST_NAME1, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"})); + addTestConfiguration(TEST_NAME2, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"})); + addTestConfiguration(TEST_NAME3, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] {"R"})); + addTestConfiguration(TEST_NAME4, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] {"R"})); + addTestConfiguration(TEST_NAME5, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] {"R"})); + addTestConfiguration(TEST_NAME6, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] {"R"})); } @Test diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java index 35cfb2982fc..e70aedcabe4 100644 --- a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java +++ b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java @@ -59,8 +59,7 @@ private void runSTEPGlmTest(ExecType instType) { // runTest executes the script; fails if the DML script invokes stop() runTest(true, false, null, -1); - } - finally { + } finally { rtplatform = platformOld; } } diff --git a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java index d8cfc4d9fd7..5de429a3c53 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java @@ -91,12 +91,10 @@ public void testBackendPerformance() throws InterruptedException { taskFutures.forEach(res -> { try { Assert.assertEquals("Stats parsed correctly", res.get().statusCode(), 200); - } - catch(InterruptedException e) { + } catch (InterruptedException e) { Thread.currentThread().interrupt(); Assert.fail("Interrupted while fetching statistics: " + e.getMessage()); - } - catch(ExecutionException e) { + } catch (ExecutionException e) { Assert.fail("Failed to fetch statistics: " + e.getMessage()); } }); diff --git a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java index 1cab99ee1ab..f8acdd07930 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java @@ -375,8 +375,9 @@ public void federatedLogicalTest(String testname, Type op_type, ExecMode execMod int port2 = single_fed_worker ? 0 : getRandomAvailablePort(); int port3 = single_fed_worker ? 0 : getRandomAvailablePort(); int port4 = single_fed_worker ? 0 : getRandomAvailablePort(); - Process[] workers = startLocalFedWorkers( - single_fed_worker ? new int[] {port1} : new int[] {port1, port2, port3, port4}); + Process[] workers = startLocalFedWorkers(single_fed_worker + ? new int[] {port1} + : new int[] {port1, port2, port3, port4}); try { if(!isAlive(workers)) diff --git a/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java b/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java index 9d2bb583a4f..dbbf199f8fa 100644 --- a/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java +++ b/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java @@ -72,26 +72,27 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod } if(et == ExecType.SPARK) { - rtplatform = ExecMode.SPARK; + rtplatform = ExecMode.SPARK; } else { // rtplatform = (et==ExecType.MR)? ExecMode.HADOOP : ExecMode.SINGLE_NODE; - rtplatform = ExecMode.HYBRID; + rtplatform = ExecMode.HYBRID; } if( rtplatform == ExecMode.SPARK ) DMLScript.USE_LOCAL_SPARK_CONFIG = true; config.addVariable("rows", rows); - config.addVariable("cols", cols); + config.addVariable("cols", cols); - long rowstart = 816, rowend = 1229, colstart = 967, colend = 1009; - // long rowstart=2, rowend=4, colstart=9, colend=10; + long rowstart=816, rowend=1229, colstart=967, colend=1009; + // long rowstart=2, rowend=4, colstart=9, colend=10; /* - * Random rand=new Random(System.currentTimeMillis()); rowstart=(long)(rand.nextDouble()*((double)rows))+1; - * rowend=(long)(rand.nextDouble()*((double)(rows-rowstart+1)))+rowstart; - * colstart=(long)(rand.nextDouble()*((double)cols))+1; - * colend=(long)(rand.nextDouble()*((double)(cols-colstart+1)))+colstart; - */ + Random rand=new Random(System.currentTimeMillis()); + rowstart=(long)(rand.nextDouble()*((double)rows))+1; + rowend=(long)(rand.nextDouble()*((double)(rows-rowstart+1)))+rowstart; + colstart=(long)(rand.nextDouble()*((double)cols))+1; + colend=(long)(rand.nextDouble()*((double)(cols-colstart+1)))+colstart; + */ config.addVariable("rowstart", rowstart); config.addVariable("rowend", rowend); config.addVariable("colstart", colstart); @@ -118,20 +119,17 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod double sparsity=1.0;//rand.nextDouble(); double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, System.currentTimeMillis()); writeInputMatrix("A", A, true); - - sparsity = 0.1;// rand.nextDouble(); - double[][] B = getRandomMatrix((int) (rowend - rowstart + 1), (int) (colend - colstart + 1), min, max, - sparsity, System.currentTimeMillis()); + + sparsity=0.1;//rand.nextDouble(); + double[][] B = getRandomMatrix((int)(rowend-rowstart+1), (int)(colend-colstart+1), min, max, sparsity, System.currentTimeMillis()); writeInputMatrix("B", B, true); - - sparsity = 0.5;// rand.nextDouble(); - double[][] C = getRandomMatrix((int) (rowend), (int) (cols - colstart + 1), min, max, sparsity, - System.currentTimeMillis()); + + sparsity=0.5;//rand.nextDouble(); + double[][] C = getRandomMatrix((int)(rowend), (int)(cols-colstart+1), min, max, sparsity, System.currentTimeMillis()); writeInputMatrix("C", C, true); - - sparsity = 0.01;// rand.nextDouble(); - double[][] D = getRandomMatrix(rows, (int) (colend - colstart + 1), min, max, sparsity, - System.currentTimeMillis()); + + sparsity=0.01;//rand.nextDouble(); + double[][] D = getRandomMatrix(rows, (int)(colend-colstart+1), min, max, sparsity, System.currentTimeMillis()); writeInputMatrix("D", D, true); /* @@ -140,9 +138,9 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod * While loop iteration - 10 jobs * Final output write - 1 job */ - // boolean exceptionExpected = false; - // int expectedNumberOfJobs = 12; - // runTest(exceptionExpected, null, expectedNumberOfJobs); + //boolean exceptionExpected = false; + //int expectedNumberOfJobs = 12; + //runTest(exceptionExpected, null, expectedNumberOfJobs); boolean exceptionExpected = false; int expectedNumberOfJobs = -1; runTest(true, exceptionExpected, null, expectedNumberOfJobs); diff --git a/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java b/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java index fad1cc123c8..a1b51ee9d81 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java @@ -81,9 +81,8 @@ public void testDoubleScalarWrite() { fullDMLScriptName = HOME + "ScalarComputeWrite.dml"; runTest(true, false, null, -1); - double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)).doubleValue(); - Assert.assertEquals("Computation test for Doubles failed: Values not equal: " + double_scalar - + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar); + double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1,1)).doubleValue(); + Assert.assertEquals("Computation test for Doubles failed: Values not equal: " + double_scalar + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar); } @Test @@ -123,7 +122,8 @@ public void testIntScalarRead() { programArgs = new String[]{"-args", String.valueOf(int_scalar), output("a.scalar")}; runTest(true, false, null, -1); - int int_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)).intValue(); + int int_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)) + .get(new CellIndex(1,1)).intValue(); Assert.assertEquals("Values not equal: " + int_scalar + Opcodes.NOTEQUAL.toString() + int_out_scalar, int_scalar, int_out_scalar); @@ -143,8 +143,8 @@ public void testDoubleScalarRead() { programArgs = new String[]{ "-args", String.valueOf(double_scalar), output("a.scalar") }; runTest(true, false, null, -1); - double double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)) - .doubleValue(); + double double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)) + .get(new CellIndex(1,1)).doubleValue(); Assert.assertEquals("Values not equal: " + double_scalar + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar, 0); diff --git a/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java b/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java index 20ae7aa63ea..a4013c3672d 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java @@ -35,14 +35,14 @@ /** * End-to-end DML test of the native Delta read/write path. * - *

- * The write and the read are run as two separate SystemDS executions on purpose. If they shared a single - * script/process, SystemDS would reuse the still-materialized in-memory matrix for the subsequent read and never invoke - * {@link org.apache.sysds.runtime.io.ReaderDelta} at all (verified: the cache reports 0 HDFS hits in that case). - * Splitting the executions forces a genuine read from disk, and we additionally assert via {@link CacheStatistics} that - * the read run actually performed HDFS reads (the Delta table + the text reference) rather than serving the matrix from - * cache. - *

+ *

The write and the read are run as two separate SystemDS executions + * on purpose. If they shared a single script/process, SystemDS would reuse the + * still-materialized in-memory matrix for the subsequent read and never invoke + * {@link org.apache.sysds.runtime.io.ReaderDelta} at all (verified: the cache + * reports 0 HDFS hits in that case). Splitting the executions forces a genuine + * read from disk, and we additionally assert via {@link CacheStatistics} that + * the read run actually performed HDFS reads (the Delta table + the text + * reference) rather than serving the matrix from cache.

*/ public class DeltaReadWriteTest extends AutomatedTestBase { @@ -54,8 +54,10 @@ public class DeltaReadWriteTest extends AutomatedTestBase { @Override public void setUp() { TestUtils.clearAssertionInformation(); - addTestConfiguration(WRITE_NAME, new TestConfiguration(TEST_CLASS_DIR, WRITE_NAME, new String[] {"ref"})); - addTestConfiguration(READ_NAME, new TestConfiguration(TEST_CLASS_DIR, READ_NAME, new String[] {"R"})); + addTestConfiguration(WRITE_NAME, + new TestConfiguration(TEST_CLASS_DIR, WRITE_NAME, new String[] { "ref" })); + addTestConfiguration(READ_NAME, + new TestConfiguration(TEST_CLASS_DIR, READ_NAME, new String[] { "R" })); } @Test @@ -82,16 +84,17 @@ private void runDeltaRoundTrip(int rows, int cols, double sparsity) { String deltaPath = output("deltaTable"); String refPath = output("ref"); fullDMLScriptName = HOME + WRITE_NAME + ".dml"; - programArgs = new String[] {"-stats", "-args", String.valueOf(rows), String.valueOf(cols), - String.valueOf(sparsity), deltaPath, refPath}; + programArgs = new String[] { "-stats", "-args", + String.valueOf(rows), String.valueOf(cols), String.valueOf(sparsity), + deltaPath, refPath }; runTest(true, false, null, -1); // the write run must have materialized two matrices to disk (the Delta // table under test + the text reference); WriterDelta genuinely hitting // HDFS is what produces these write-side cache statistics. long hdfsWrites = CacheStatistics.getHDFSWrites(); - assertTrue("expected >= 2 HDFS writes in the write run (delta + reference), got " + hdfsWrites, - hdfsWrites >= 2); + assertTrue("expected >= 2 HDFS writes in the write run (delta + reference), got " + + hdfsWrites, hdfsWrites >= 2); // and a real Delta table (transaction log) must have been created assertTrue("missing Delta transaction log under " + deltaPath, new File(deltaPath, "_delta_log").isDirectory()); @@ -99,18 +102,19 @@ private void runDeltaRoundTrip(int rows, int cols, double sparsity) { // ---- phase 2: fresh execution reads the Delta table and compares ---- getAndLoadTestConfiguration(READ_NAME); fullDMLScriptName = HOME + READ_NAME + ".dml"; - programArgs = new String[] {"-stats", "-args", deltaPath, refPath, output("R")}; + programArgs = new String[] { "-stats", "-args", + deltaPath, refPath, output("R") }; runTest(true, false, null, -1); // the read run must have materialized two matrices from disk (the Delta // table under test + the text reference); a cached/short-circuited read // would report fewer HDFS hits and fail here. long hdfsReads = CacheStatistics.getHDFSHits(); - assertTrue("expected >= 2 HDFS reads in the read run (delta + reference), got " + hdfsReads, - hdfsReads >= 2); + assertTrue("expected >= 2 HDFS reads in the read run (delta + reference), got " + + hdfsReads, hdfsReads >= 2); HashMap R = readDMLMatrixFromOutputDir("R"); - // text-cell output omits exact zeros, so a missing cell means 0.0 + //text-cell output omits exact zeros, so a missing cell means 0.0 double diff = R.getOrDefault(new CellIndex(1, 1), 0.0); double nrow = R.getOrDefault(new CellIndex(1, 2), 0.0); double ncol = R.getOrDefault(new CellIndex(1, 3), 0.0); diff --git a/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java index a844321c249..cc1412b1606 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java @@ -49,9 +49,10 @@ public class FrameParquetSchemaTest extends AutomatedTestBase { @Override public void setUp() { - addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"Rout"})); + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[]{"Rout"})); } + /** * Test for sequential writer and reader * diff --git a/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java b/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java index 6e6e4665f5e..dfb3d8a19de 100644 --- a/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java @@ -42,8 +42,12 @@ */ @net.jcip.annotations.NotThreadSafe public class JMLConnectionTest extends AutomatedTestBase { - public static final String META = "{\"data_type\": \"matrix\",\n" + " \"value_type\": \"double\", \n" - + " \"rows\": 1,\n" + " \"cols\": 1,\n" + " \"nnz\": 1,\n" + " \"format\": \"csv\"}"; + public static final String META = "{\"data_type\": \"matrix\",\n" + + " \"value_type\": \"double\", \n" + + " \"rows\": 1,\n" + + " \"cols\": 1,\n" + + " \"nnz\": 1,\n" + + " \"format\": \"csv\"}"; private final static String TEST_NAME = "JMLConnectionTest"; private final static String TEST_DIR = "functions/jmlc/"; @@ -95,14 +99,12 @@ public void testConnectionInvalidInName() throws DMLException { conn.gatherMemStats(false); Assert.assertFalse(DMLScript.STATISTICS); - try(conn) { - conn.prepareScript("printx('hello')", new String[] {"$inScalar1", null}, new String[] {null}); + try (conn) { + conn.prepareScript("printx('hello')", new String[]{"$inScalar1", null}, new String[]{null}); throw new AssertionError("Test should have thrown a LanguageException"); - } - catch(LanguageException e) { + } catch (LanguageException e) { Assert.assertTrue(e.getMessage().startsWith("Invalid variable names")); - } - finally { + } finally { DMLScript.STATISTICS = oldStat; DMLScript.JMLC_MEM_STATISTICS = oldJMLCStat; } @@ -110,24 +112,21 @@ public void testConnectionInvalidInName() throws DMLException { @Test public void testConnectionParseLanguageException() { - try(Connection conn = new Connection()) { - conn.prepareScript("printx('hello')", new String[] {}, new String[] {}); + try (Connection conn = new Connection()) { + conn.prepareScript("printx('hello')", new String[]{}, new String[]{}); throw new AssertionError("Test should have thrown a DMLException"); - } - catch(DMLException e) { + } catch (DMLException e) { Throwable cause = e.getCause(); - Assert.assertTrue(cause.getMessage().startsWith( - "ERROR: [line 1:0] -> printx('hello') -- function printx is undefined in namespace .builtinNS")); + Assert.assertTrue(cause.getMessage().startsWith("ERROR: [line 1:0] -> printx('hello') -- function printx is undefined in namespace .builtinNS")); } } @Test public void testConnectionParseException() { - try(Connection conn = new Connection()) { - conn.prepareScript("print('hello'", new String[] {}, new String[] {}); + try (Connection conn = new Connection()) { + conn.prepareScript("print('hello'", new String[]{}, new String[]{}); throw new AssertionError("Test should have thrown a ParseException"); - } - catch(Exception e) { + } catch (Exception e) { Assert.assertEquals("ParseException", e.getClass().getSimpleName()); } } @@ -145,11 +144,10 @@ public void testConnectionClose() { @Test public void testReadScriptHDFS() { - try(Connection conn = new Connection()) { + try (Connection conn = new Connection()) { conn.readScript("hdfs://localhost:9000/Test"); - } - catch(IOException e) { - Assert.assertEquals("ConnectException", e.getClass().getSimpleName()); + } catch (IOException e) { + Assert.assertEquals("ConnectException",e.getClass().getSimpleName()); } } diff --git a/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java b/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java index 2178884ef5b..4852220861e 100644 --- a/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java @@ -111,16 +111,17 @@ public void federatedReuse(String test) { // Run reference dml script with normal matrix. Reuse of ba+*. fullDMLScriptName = HOME + test + "Reference.dml"; - programArgs = new String[] {"-stats", "-lineage", "reuse_full", "-nvargs", "X1=" + input("X1"), - "X2=" + input("X2"), "Y1=" + input("Y1"), "Y2=" + input("Y2"), "Z=" + expected("Z")}; + programArgs = new String[] {"-stats", "-lineage", "reuse_full", + "-nvargs", "X1=" + input("X1"), "X2=" + input("X2"), "Y1=" + input("Y1"), + "Y2=" + input("Y2"), "Z=" + expected("Z")}; runTest(true, false, null, -1); long mmCount = Statistics.getCPHeavyHitterCount(Opcodes.MMULT.toString()); // Run actual dml script with federated matrix // The fed workers reuse ba+* fullDMLScriptName = HOME + test + ".dml"; - programArgs = new String[] {"-stats", "-lineage", "reuse_full", "-nvargs", - "X1=" + TestUtils.federatedAddress(port1, input("X1")), + programArgs = new String[] {"-stats","-lineage", "reuse_full", + "-nvargs", "X1=" + TestUtils.federatedAddress(port1, input("X1")), "X2=" + TestUtils.federatedAddress(port2, input("X2")), "Y1=" + TestUtils.federatedAddress(port1, input("Y1")), "Y2=" + TestUtils.federatedAddress(port2, input("Y2")), "r=" + rows, "c=" + cols, "Z=" + output("Z")}; @@ -128,12 +129,12 @@ public void federatedReuse(String test) { long mmCount_fed = Statistics.getCPHeavyHitterCount(Opcodes.MMULT.toString()); long fedMMCount = Statistics.getCPHeavyHitterCount("fed_ba+*"); - // compare results + // compare results compareResults(1e-9); // compare matrix multiplication count - // #federated execution of ba+* = #threads times #non-federated execution of ba+* (after reuse) - Assert.assertTrue("Violated reuse count: " + mmCount_fed + " == " + mmCount * 2, - mmCount_fed == mmCount * 2); // #threads = 2 + // #federated execution of ba+* = #threads times #non-federated execution of ba+* (after reuse) + Assert.assertTrue("Violated reuse count: "+mmCount_fed+" == "+mmCount*2, + mmCount_fed == mmCount * 2); // #threads = 2 switch(test) { case TEST_NAME1: // If the o/p is federated, fed_ba+* will be called everytime diff --git a/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java b/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java index c86eb0f4941..eca3628a89b 100644 --- a/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java @@ -121,8 +121,9 @@ private void runTriUDFReuse(ExecMode execMode) { // Run reference dml script with normal matrix fullDMLScriptName = HOME + TEST_NAME + "Reference.dml"; - programArgs = new String[] {"-lineage", "reuse_full", "-stats", "100", "-args", input("X1"), input("X2"), - input("X3"), input("X4"), Boolean.toString(rowPartitioned).toUpperCase(), expected("S")}; + programArgs = new String[] {"-lineage", "reuse_full", "-stats", "100", "-args", + input("X1"), input("X2"), input("X3"), input("X4"), + Boolean.toString(rowPartitioned).toUpperCase(), expected("S")}; runTest(null); // Run actual dml script with federated matrix diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java index 3ffdfe1d30b..18ca2fbc454 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java @@ -272,79 +272,82 @@ protected void toStringTestHelper(ExecMode platform, String testName, String exp } @Test - public void testPrintWithDecimal() { + public void testPrintWithDecimal(){ String testName = "ToString12"; String decimalPoints = "2"; String value = "22"; String expectedOutput = "22.00\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } + @Test - public void testPrintWithDecimal2() { + public void testPrintWithDecimal2(){ String testName = "ToString12"; String decimalPoints = "2"; String value = "5.244058388023880"; String expectedOutput = "5.24\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } + @Test - public void testPrintWithDecimal3() { + public void testPrintWithDecimal3(){ String testName = "ToString12"; String decimalPoints = "10"; String value = "5.244058388023880"; String expectedOutput = "5.2440583880\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } + @Test - public void testPrintWithDecimal4() { + public void testPrintWithDecimal4(){ String testName = "ToString12"; String decimalPoints = "4"; String value = "5.244058388023880"; String expectedOutput = "5.2441\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } + @Test - public void testPrintWithDecimal5() { + public void testPrintWithDecimal5(){ String testName = "ToString12"; String decimalPoints = "10"; String value = "0.000000008023880"; String expectedOutput = "0.0000000080\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - protected void toStringTestHelper2(ExecMode platform, String testName, String expectedOutput, String decimalPoints, - String value) { + protected void toStringTestHelper2(ExecMode platform, String testName, String expectedOutput, String decimalPoints, String value) { ExecMode platformOld = rtplatform; - + rtplatform = platform; boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - if(rtplatform == ExecMode.SPARK) + if (rtplatform == ExecMode.SPARK) DMLScript.USE_LOCAL_SPARK_CONFIG = true; try { // Create and load test configuration getAndLoadTestConfiguration(testName); String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testName + ".dml"; - programArgs = new String[] {"-args", output(OUTPUT_NAME), value, decimalPoints}; + programArgs = new String[]{"-args", output(OUTPUT_NAME), value, decimalPoints}; // Run DML and R scripts runTest(true, false, null, -1); diff --git a/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java index 26143dc16ee..770c5b7c5bf 100644 --- a/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java @@ -76,9 +76,10 @@ public ReshapeTest(int rlen, int clen, int rows, int cols, boolean rowWise) { @Parameterized.Parameters(name = "{0}x{1} {2}x{3} rowWise {4}") public static Iterable getParams() { - int[][][] dims = {{{1000, 1000}, {1, 1000000}}, // single row/col - {{3000, 4000}, {1500, 8000}}, // partialBlocks - {{2400, 1400}, {800, 4200}} // fullBlocks + int[][][] dims = { + {{1000, 1000}, {1, 1000000}}, // single row/col + {{3000, 4000}, {1500, 8000}}, // partialBlocks + {{2400, 1400}, {800, 4200}} // fullBlocks }; ArrayList params = new ArrayList<>(); @@ -116,8 +117,7 @@ public void runTestMatrixReshapeOOC() { double[][] X = getRandomMatrix(rlen, clen, 0, 1, 1, 7); MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(Types.FileFormat.BINARY); - writer.writeMatrixToHDFS(DataConverter.convertToMatrixBlock(X), input(INPUT_NAME), rlen, clen, 1000, - rlen * clen); + writer.writeMatrixToHDFS(DataConverter.convertToMatrixBlock(X), input(INPUT_NAME), rlen, clen, 1000, rlen * clen); HDFSTool.writeMetaDataFile(input(INPUT_NAME + ".mtd"), Types.ValueType.FP64, new MatrixCharacteristics(rlen, clen, blen, rlen * clen), Types.FileFormat.BINARY); @@ -143,8 +143,8 @@ public void runTestMatrixReshapeOOC() { runTest(true, false, null, -1); // compare results - MatrixBlock actual = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME), Types.FileFormat.BINARY, rows, - cols, blen); + MatrixBlock actual = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME), + Types.FileFormat.BINARY, rows, cols, blen); MatrixBlock expected = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME + "_target"), Types.FileFormat.BINARY, rows, cols, blen); diff --git a/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java index 49a52587cde..5f42db7d733 100644 --- a/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java @@ -335,9 +335,9 @@ private void runTestMatrixReshape( ReshapeType type, boolean rowwise, boolean sp String.valueOf(trows), String.valueOf(tcols), output("Y") }; fullRScriptName = HOME + TEST_NAME + ".R"; - rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + trows + " " + tcols + " " - + expectedDir(); - + rCmd = "Rscript" + " " + fullRScriptName + " " + + inputDir() + " " + trows + " " + tcols + " " + expectedDir(); + double[][] X = getRandomMatrix(rows, cols, 0, 1, sparsity, 7); writeInputMatrix("X", X, true); diff --git a/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java index 69d6958f8a6..dcdafddcd47 100644 --- a/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java @@ -94,9 +94,9 @@ private void runVectorReshape(boolean sparse, ExecType et) String.valueOf(rows2), String.valueOf(cols2), output("R") }; fullRScriptName = HOME + TEST_NAME + ".R"; - rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + rows2 + " " + cols2 + " " - + expectedDir(); - + rCmd = "Rscript" + " " + fullRScriptName + " " + + inputDir() + " " + rows2 + " " + cols2 + " " + expectedDir(); + double sparsity = sparse ? sparsitySparse : sparsityDense; double[][] X = getRandomMatrix(rows1, cols1, 0, 1, sparsity, 7); writeInputMatrixWithMTD("X", X, true); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java index b16554045e4..60b491b8141 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java @@ -151,8 +151,8 @@ private void runTestMatrixChainDP(String testName) { overwriteCurrentConfig(dmlConfig); } catch(FileNotFoundException fnfe) { - Assert.fail( - "Could not find DML config file: " + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); + Assert.fail("Could not find DML config file: " + + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); } catch(IOException ioe) { Assert.fail("Could not overwrite the DML configuration file. " + ioe.getMessage()); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java index 96c479e206d..bf9acd9e52a 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java @@ -123,8 +123,8 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { overwriteCurrentConfig(dmlConfig); } catch(FileNotFoundException fnfe) { - Assert.fail( - "Could not find DML config file: " + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); + Assert.fail("Could not find DML config file: " + + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); } catch(IOException ioe) { Assert.fail("Could not overwrite the DML configuration file. " + ioe.getMessage()); @@ -132,7 +132,8 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + TEST_NAME + ".dml"; - programArgs = new String[] {"-explain", "hops", "-stats", "-args", input("X"), input("Y"), output("R")}; + programArgs = new String[] {"-explain", "hops", "-stats", + "-args", input("X"), input("Y"), output("R")}; fullRScriptName = HOME + TEST_NAME + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java index 15b80e49618..e8e885f905f 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java @@ -74,7 +74,7 @@ public void testRewriteQuantizationFusedCompressionNoRewrite() { /** * Unified method to test both scalar and matrix scale factors. - * + * * @param testname Test name * @param rewrites Whether to enable fusion rewrites * @param isScalar Whether the scale factor is a scalar or a matrix diff --git a/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java b/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java index 39266f5f3d3..30681f373e4 100644 --- a/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java +++ b/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -106,8 +106,7 @@ public void testHash2() throws Exception { @Test public void testHash3() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, - new ValueType[] {ValueType.UINT8, ValueType.INT64, ValueType.UINT8}, 32); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.INT64,ValueType.UINT8}, 32); MatrixBlock expected = new MatrixBlock(1, 7, new double[] {1, 1, 1, 0, 1, 1, 1}); String spec = "{\"ids\": true, \"dummycode\": [1,3], \"hash\": [1,3], \"K\": 3}"; @@ -115,11 +114,11 @@ public void testHash3() throws Exception { } + @Test public void testHybrid1() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, - new ValueType[] {ValueType.UINT8, ValueType.INT64, ValueType.UINT8, ValueType.BOOLEAN}, 32); - MatrixBlock expected = new MatrixBlock(1, 9, new double[] {1, 1, 1, 0, 1, 1, 1, 1, 1}); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.INT64,ValueType.UINT8, ValueType.BOOLEAN}, 32); + MatrixBlock expected = new MatrixBlock(1, 9, new double[] {1, 1, 1, 0, 1, 1, 1,1,1}); String spec = "{\"ids\": true, \"dummycode\": [1,3,4], \"hash\": [1,3], \"K\": 3}"; runTransformTest(fb, spec, expected); @@ -128,9 +127,8 @@ public void testHybrid1() throws Exception { @Test public void testHybrid2() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, - new ValueType[] {ValueType.UINT8, ValueType.BOOLEAN, ValueType.UINT8, ValueType.BOOLEAN}, 32); - MatrixBlock expected = new MatrixBlock(1, 10, new double[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.BOOLEAN,ValueType.UINT8, ValueType.BOOLEAN}, 32); + MatrixBlock expected = new MatrixBlock(1, 10, new double[] {1, 1, 1, 1,1, 1, 1, 1,1,1}); String spec = "{\"ids\": true, \"dummycode\": [1,2,3,4], \"hash\": [1,3], \"K\": 3}"; runTransformTest(fb, spec, expected); @@ -141,7 +139,7 @@ private void runTransformTest(FrameBlock fb, String spec, MatrixBlock expected) try { getAndLoadTestConfiguration(TEST_NAME1); - + String inF = input("F-In"); String inS = input("spec"); diff --git a/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java b/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java index cd28649dc42..8c4ba6ae8ad 100644 --- a/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java +++ b/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java @@ -283,8 +283,7 @@ private String[][] readTwoColumnStringCSV(String s) { out[1][i] = in.getString(i, 1); } return out; - } - catch(IOException e) { + } catch (IOException e) { throw new RuntimeException(e); } } diff --git a/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java b/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java index ae13cbd510f..d3d71d820d6 100644 --- a/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java +++ b/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java @@ -92,7 +92,7 @@ private void runVectorizationTest( String testName, boolean rewrites ) runTest(true, false, null, -1); runRScript(true); - // compare results + //compare results HashMap dmlfile = readDMLMatrixFromOutputDir("R"); HashMap rfile = readRMatrixFromExpectedDir("R"); TestUtils.compareMatrices(dmlfile, rfile, 1e-14, "DML", "R"); From 3962ff1698900b7070f658497747993832f78b83 Mon Sep 17 00:00:00 2001 From: bruno Date: Tue, 1 Sep 2026 16:28:54 +0200 Subject: [PATCH 130/132] dev/format-changed.sh --- .../java/org/apache/sysds/api/DMLScript.java | 10 +- .../org/apache/sysds/common/Builtins.java | 376 +++++------------- .../java/org/apache/sysds/hops/BinaryOp.java | 4 +- src/main/java/org/apache/sysds/hops/Hop.java | 14 +- .../java/org/apache/sysds/hops/UnaryOp.java | 9 +- .../sysds/hops/estim/EstimationUtils.java | 12 +- .../sysds/hops/rewrite/ProgramRewriter.java | 6 +- ...riteMatrixMultChainOptimizationSparse.java | 19 +- .../parser/BuiltinFunctionExpression.java | 24 -- .../apache/sysds/parser/DMLTranslator.java | 87 +--- .../apache/sysds/parser/DataExpression.java | 206 +++------- .../compress/CompressedMatrixBlock.java | 2 +- .../runtime/compress/colgroup/AColGroup.java | 17 +- .../compress/colgroup/AColGroupValue.java | 1 - .../runtime/compress/colgroup/ASDCZero.java | 6 +- .../compress/colgroup/ColGroupDDC.java | 4 +- .../compress/colgroup/ColGroupEmpty.java | 7 +- .../colgroup/ColGroupLinearFunctional.java | 2 +- .../compress/colgroup/ColGroupOLE.java | 2 +- .../compress/colgroup/ColGroupRLE.java | 4 +- .../compress/colgroup/ColGroupSDCFOR.java | 4 +- .../colgroup/ColGroupUncompressed.java | 13 +- .../colgroup/ColGroupUncompressedArray.java | 2 +- .../dictionary/AIdentityDictionary.java | 4 +- .../colgroup/dictionary/DeltaDictionary.java | 4 +- .../colgroup/dictionary/IDictionary.java | 6 +- .../dictionary/IdentityDictionary.java | 4 +- .../dictionary/IdentityDictionarySlice.java | 4 +- .../compress/colgroup/mapping/AMapToData.java | 2 +- .../compress/colgroup/offset/AOffset.java | 10 +- .../compress/colgroup/offset/OffsetEmpty.java | 1 + .../compress/lib/CLALibBinaryCellOp.java | 6 +- .../runtime/compress/lib/CLALibMMChain.java | 2 +- .../compress/lib/CLALibRemoveEmpty.java | 15 +- .../runtime/compress/lib/CLALibSort.java | 8 +- .../controlprogram/caching/FrameObject.java | 4 +- .../controlprogram/caching/MatrixObject.java | 2 +- .../context/SparkExecutionContext.java | 34 +- .../federated/FederatedWorker.java | 3 +- .../frame/data/columns/ArrayFactory.java | 22 +- .../frame/data/lib/MatrixBlockFromFrame.java | 4 +- .../runtime/functionobjects/Builtin.java | 139 +++---- .../instructions/cp/BinaryCPInstruction.java | 2 +- .../cp/BinaryFrameScalarCPInstruction.java | 10 +- .../cp/BinaryMatrixMatrixCPInstruction.java | 4 +- .../cp/ParameterizedBuiltinCPInstruction.java | 7 +- .../instructions/ooc/ReorgOOCInstruction.java | 8 +- .../ooc/ReshapeOOCInstruction.java | 69 ++-- .../spark/QuantilePickSPInstruction.java | 3 +- .../spark/data/IndexedMatrixValue.java | 7 +- .../sysds/runtime/io/DeltaKernelUtils.java | 257 ++++++------ .../apache/sysds/runtime/io/ReaderDelta.java | 107 ++--- .../sysds/runtime/io/ReaderDeltaParallel.java | 96 ++--- .../apache/sysds/runtime/io/WriterDelta.java | 74 ++-- .../runtime/matrix/data/LibMatrixReorg.java | 34 +- .../runtime/matrix/data/MatrixBlock.java | 34 +- .../sysds/runtime/ooc/cache/OOCFuture.java | 9 +- .../runtime/ooc/cache/io/CloseableQueue.java | 38 +- .../cache/io/OOCBufferedDataInputStream.java | 12 +- .../cache/io/OOCBufferedDataOutputStream.java | 20 +- .../runtime/ooc/cache/io/OOCIOHandler.java | 25 +- .../ooc/cache/io/OOCMatrixIOHandler.java | 161 ++++---- .../runtime/ooc/cache/io/SpillableObject.java | 6 +- .../ooc/cache/legacy/OOCCacheScheduler.java | 44 +- .../cache/legacy/OOCLRUCacheScheduler.java | 207 +++++----- .../runtime/transform/decode/Decoder.java | 10 +- .../runtime/transform/decode/DecoderBin.java | 4 +- .../transform/decode/DecoderDummycode.java | 2 +- .../transform/decode/DecoderFactory.java | 53 ++- .../transform/decode/DecoderRecode.java | 18 +- .../sysds/runtime/util/CommonThreadPool.java | 8 +- .../sysds/runtime/util/DataConverter.java | 9 +- .../org/apache/sysds/utils/DoubleParser.java | 2 +- .../apache/sysds/utils/SettingsChecker.java | 13 +- .../org/apache/sysds/performance/Main.java | 7 +- .../apache/sysds/test/AutomatedTestBase.java | 23 +- .../java/org/apache/sysds/test/TestUtils.java | 9 +- .../component/compile/CompilerTestBase.java | 21 +- .../SparkTransitiveExecTypeCompileTest.java | 50 +-- .../compress/CompressedSortTest.java | 6 +- .../compress/lib/CLALibMMChainTest.java | 4 +- ...CompressedBinaryMatrixMatrixSolveTest.java | 15 +- .../OffsetClassInitConcurrencyTest.java | 4 +- .../SparkContextReferenceCountTest.java | 32 +- .../component/federated/FedWorkerBase.java | 19 +- .../federated/FedWorkerMatrixCompress.java | 8 +- .../component/frame/FrameToStringTest.java | 14 +- .../frame/MatrixFromFrameSafeCastTest.java | 12 +- .../frame/transform/DecoderCompositeTest.java | 8 +- .../GetCategoricalMaskInstructionTest.java | 21 +- .../TransformDecodeRoundTripTest.java | 39 +- .../frame/transform/TransformDecodeTest.java | 4 +- .../component/io/DeltaMatrixCoverageTest.java | 97 +++-- .../io/DeltaMatrixReadWriteTest.java | 333 ++++++++++------ .../io/DeltaMatrixSparkInteropTest.java | 105 +++-- .../component/matrix/QuantilePickTest.java | 13 +- .../component/tensor/TensorToStringTest.java | 14 +- .../functions/binary/matrix/QuantileTest.java | 18 +- .../builtin/part2/BuiltinSTEPGlmTest.java | 3 +- .../FederatedBackendPerformanceTest.java | 6 +- .../part4/FederatedLogicalTest.java | 5 +- .../functions/indexing/LeftIndexingTest.java | 48 +-- .../sysds/test/functions/io/ScalarIOTest.java | 12 +- .../io/delta/DeltaReadWriteTest.java | 40 +- .../io/parquet/FrameParquetSchemaTest.java | 3 +- .../functions/jmlc/JMLConnectionTest.java | 42 +- .../functions/lineage/FedFullReuseTest.java | 17 +- .../functions/lineage/FedUDFReuseTest.java | 5 +- .../test/functions/misc/ToStringTest.java | 33 +- .../sysds/test/functions/ooc/ReshapeTest.java | 14 +- .../functions/reorg/MatrixReshapeTest.java | 6 +- .../functions/reorg/VectorReshapeTest.java | 6 +- .../rewrite/RewriteMatrixChainDPTest.java | 4 +- .../RewriteMatrixMultChainOptSparseTest.java | 7 +- ...writeQuantizationFusedCompressionTest.java | 2 +- .../transform/GetCategoricalMaskTest.java | 20 +- .../TransformFrameEncodeBagOfWords.java | 3 +- .../vect/LeftIndexingChainUpdateTest.java | 2 +- 118 files changed, 1654 insertions(+), 1958 deletions(-) diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java b/src/main/java/org/apache/sysds/api/DMLScript.java index a7a175bb7b6..0bb1e9b462d 100644 --- a/src/main/java/org/apache/sysds/api/DMLScript.java +++ b/src/main/java/org/apache/sysds/api/DMLScript.java @@ -508,9 +508,9 @@ private static void execute(String dmlScriptStr, String fnameOptConfig, Map inHops1 = new ArrayList<>(); - inHops1.add(expr); - inHops1.add(expr2); - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), inHops1); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - case AVG_POOL: - case MAX_POOL: { - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForPoolingForwardIM2COL(expr, source, 1, hops)); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - case AVG_POOL_BACKWARD: - case MAX_POOL_BACKWARD: { - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForConvOpPoolingCOL2IM(expr, source, 1, hops)); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - case CONV2D: - case CONV2D_BACKWARD_FILTER: - case CONV2D_BACKWARD_DATA: { - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForConvOp(expr, source, 1, hops)); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - - case ROW_COUNT_DISTINCT: - currBuiltinOp = new AggUnaryOp(target.getName(), - DataType.MATRIX, target.getValueType(), AggOp.COUNT_DISTINCT, Direction.Row, expr); - break; - - case COL_COUNT_DISTINCT: - currBuiltinOp = new AggUnaryOp(target.getName(), - DataType.MATRIX, target.getValueType(), AggOp.COUNT_DISTINCT, Direction.Col, expr); - break; - - case GET_CATEGORICAL_MASK: - currBuiltinOp = new BinaryOp(target.getName(), DataType.MATRIX, ValueType.FP64, OpOp2.GET_CATEGORICAL_MASK, expr, expr2); - break; - default: - throw new ParseException("Unsupported builtin function type: "+source.getOpCode()); - } - - boolean isConvolution = source.getOpCode() == Builtins.CONV2D || source.getOpCode() == Builtins.CONV2D_BACKWARD_DATA || - source.getOpCode() == Builtins.CONV2D_BACKWARD_FILTER || - source.getOpCode() == Builtins.MAX_POOL || source.getOpCode() == Builtins.MAX_POOL_BACKWARD || - source.getOpCode() == Builtins.AVG_POOL || source.getOpCode() == Builtins.AVG_POOL_BACKWARD; - if( !isConvolution) { + boolean isConvolution = source.getOpCode() == Builtins.CONV2D || + source.getOpCode() == Builtins.CONV2D_BACKWARD_DATA || + source.getOpCode() == Builtins.CONV2D_BACKWARD_FILTER || source.getOpCode() == Builtins.MAX_POOL || + source.getOpCode() == Builtins.MAX_POOL_BACKWARD || source.getOpCode() == Builtins.AVG_POOL || + source.getOpCode() == Builtins.AVG_POOL_BACKWARD; + if(!isConvolution) { // Since the dimension of output doesnot match that of input variable for these operations setIdentifierParams(currBuiltinOp, source.getOutput()); } diff --git a/src/main/java/org/apache/sysds/parser/DataExpression.java b/src/main/java/org/apache/sysds/parser/DataExpression.java index 68a3d1b7ffe..3d3a90b4f6f 100644 --- a/src/main/java/org/apache/sysds/parser/DataExpression.java +++ b/src/main/java/org/apache/sysds/parser/DataExpression.java @@ -1176,52 +1176,72 @@ else if( getVarParam(READNNZPARAM) != null ) { boolean isHDF5 = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.HDF5.toString())); - boolean isCOG = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.COG.toString())); + // handle all csv default parameters + handleCSVDefaultParam(DELIM_DELIMITER, ValueType.STRING, conditional); + handleCSVDefaultParam(DELIM_FILL_VALUE, ValueType.FP64, conditional); + handleCSVDefaultParam(DELIM_HAS_HEADER_ROW, ValueType.BOOLEAN, conditional); + handleCSVDefaultParam(DELIM_FILL, ValueType.BOOLEAN, conditional); + handleCSVDefaultParam(DELIM_NA_STRINGS, ValueType.STRING, conditional); + } - // Delta tables are self-describing (schema + dimensions discovered from the - // transaction log at read time), so dimensions are optional like CSV. - boolean isDelta = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.DELTA.toString())); + boolean isLIBSVM = false; + isLIBSVM = (formatTypeString != null && + formatTypeString.equalsIgnoreCase(FileFormat.LIBSVM.toString())); + if(isLIBSVM) { + // Handle libsvm file format + shouldReadMTD = true; + + // only allow IO_FILENAME, READROWPARAM, READCOLPARAM + // as valid parameters + if(!inferredFormatType) { + for(String key : _varParams.keySet()) { + if(!(key.equals(IO_FILENAME) || key.equals(FORMAT_TYPE) || key.equals(READROWPARAM) || + key.equals(READCOLPARAM) || key.equals(READNNZPARAM) || key.equals(DATATYPEPARAM) || + key.equals(VALUETYPEPARAM) || key.equals(DELIM_DELIMITER) || + key.equals(LIBSVM_INDEX_DELIM))) { + String msg = "Only parameters allowed are: " + IO_FILENAME + "," + READROWPARAM + "," + + READCOLPARAM + DELIM_DELIMITER + "," + LIBSVM_INDEX_DELIM; + + raiseValidateError( + "Invalid parameter " + key + " in read statement: " + toString() + ". " + msg, + conditional, LanguageErrorCodes.INVALID_PARAMETERS); + } + } + } + // handle all default parameters + handleCSVDefaultParam(DELIM_DELIMITER, ValueType.STRING, conditional); + handleCSVDefaultParam(LIBSVM_INDEX_DELIM, ValueType.STRING, conditional); + } - dataTypeString = (getVarParam(DATATYPEPARAM) == null) ? null : getVarParam(DATATYPEPARAM).toString(); - - if ( dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE) - || dataTypeString.equalsIgnoreCase(Statement.FRAME_DATA_TYPE)) { - - boolean isMatrix = false; - if ( dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE)) + boolean isHDF5 = (formatTypeString != null && + formatTypeString.equalsIgnoreCase(FileFormat.HDF5.toString())); + + boolean isCOG = (formatTypeString != null && + formatTypeString.equalsIgnoreCase(FileFormat.COG.toString())); + + // Delta tables are self-describing (schema + dimensions discovered from the + // transaction log at read time), so dimensions are optional like CSV. + boolean isDelta = (formatTypeString != null && + formatTypeString.equalsIgnoreCase(FileFormat.DELTA.toString())); + + dataTypeString = (getVarParam(DATATYPEPARAM) == null) ? null : getVarParam(DATATYPEPARAM).toString(); + + if(dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE) || + dataTypeString.equalsIgnoreCase(Statement.FRAME_DATA_TYPE)) { + + boolean isMatrix = false; + if(dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE)) isMatrix = true; - - // set data type - getOutput().setDataType(isMatrix ? DataType.MATRIX : DataType.FRAME); - - // set number non-zeros - Expression ennz = getVarParam("nnz"); - long nnz = -1; - if( ennz != null ) { - nnz = Long.valueOf(ennz.toString()); - getOutput().setNnz(nnz); - } - // Following dimension checks must be done when data type = MATRIX_DATA_TYPE - // initialize size of target data identifier to UNKNOWN - getOutput().setDimensions(-1, -1); - - if (!isCSV && !isLIBSVM && !isHDF5 && !isCOG && !isDelta && ConfigurationManager.getCompilerConfig() - .getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) //skip check for csv/libsvm/delta format / jmlc api - && (getVarParam(READROWPARAM) == null || getVarParam(READCOLPARAM) == null) ) { - raiseValidateError("Missing or incomplete dimension information in read statement: " - + mtdFileName, conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - - if (getVarParam(READROWPARAM) instanceof ConstIdentifier - && getVarParam(READCOLPARAM) instanceof ConstIdentifier) - { - // these are strings that are long values - Long dim1 = (getVarParam(READROWPARAM) == null) ? null : Long.valueOf( getVarParam(READROWPARAM).toString()); - Long dim2 = (getVarParam(READCOLPARAM) == null) ? null : Long.valueOf( getVarParam(READCOLPARAM).toString()); - if ( !isCSV && !isDelta && (dim1 < 0 || dim2 < 0) && ConfigurationManager - .getCompilerConfig().getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) ) { - raiseValidateError("Invalid dimension information in read statement", conditional, LanguageErrorCodes.INVALID_PARAMETERS); + // set data type + getOutput().setDataType(isMatrix ? DataType.MATRIX : DataType.FRAME); + + // set number non-zeros + Expression ennz = getVarParam("nnz"); + long nnz = -1; + if(ennz != null) { + nnz = Long.valueOf(ennz.toString()); + getOutput().setNnz(nnz); } // set dim1 and dim2 values @@ -1252,104 +1272,10 @@ && getVarParam(READCOLPARAM) instanceof ConstIdentifier) catch(Exception ex) { raiseValidateError("Invalid format '" + fmt+ "' in statement: " + toString(), conditional); } - - if (getVarParam(ROWBLOCKCOUNTPARAM) instanceof ConstIdentifier && getVarParam(COLUMNBLOCKCOUNTPARAM) instanceof ConstIdentifier) { - Integer rowBlockCount = (getVarParam(ROWBLOCKCOUNTPARAM) == null) ? - null : Integer.valueOf(getVarParam(ROWBLOCKCOUNTPARAM).toString()); - getOutput().setBlocksize(rowBlockCount != null ? rowBlockCount : -1); - } - - // block dimensions must be -1x-1 when format="text" - // NOTE MB: disabled validate of default blocksize for inputs w/ format="binary" - // because we automatically introduce reblocks if blocksizes don't match - if ( (getOutput().getFileFormat().isTextFormat() || !isMatrix) && getOutput().getBlocksize() != -1 ){ - raiseValidateError("Invalid block dimensions (" + getOutput().getBlocksize() + ") when format=" + getVarParam(FORMAT_TYPE) + " in \"" + this.toString() + "\".", conditional); - } - - } - else if ( dataTypeString.equalsIgnoreCase(Statement.SCALAR_DATA_TYPE)) { - getOutput().setDataType(DataType.SCALAR); - getOutput().setNnz(-1L); - } - else if ( dataTypeString.equalsIgnoreCase(DataType.LIST.name())) { - getOutput().setDataType(DataType.LIST); - } - else{ - raiseValidateError("Unknown Data Type " + dataTypeString + ". Valid values: " - + Statement.SCALAR_DATA_TYPE +", " + Statement.MATRIX_DATA_TYPE+", " + Statement.FRAME_DATA_TYPE - +", " + DataType.LIST.name().toLowerCase(), conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - - // handle value type parameter - if (getVarParam(VALUETYPEPARAM) != null && !(getVarParam(VALUETYPEPARAM) instanceof StringIdentifier)){ - raiseValidateError("for read method, parameter " + VALUETYPEPARAM + " can only be a string. " + - "Valid values are: " + Statement.DOUBLE_VALUE_TYPE +", " + Statement.INT_VALUE_TYPE + ", " + Statement.BOOLEAN_VALUE_TYPE + ", " + Statement.STRING_VALUE_TYPE, conditional); - } - // Identify the value type (used only for read method) - String valueTypeString = getVarParam(VALUETYPEPARAM) == null ? null : getVarParam(VALUETYPEPARAM).toString(); - if (valueTypeString != null) { - if (valueTypeString.equalsIgnoreCase(Statement.DOUBLE_VALUE_TYPE)) - getOutput().setValueType(ValueType.FP64); - else if (valueTypeString.equalsIgnoreCase(Statement.STRING_VALUE_TYPE)) - getOutput().setValueType(ValueType.STRING); - else if (valueTypeString.equalsIgnoreCase(Statement.INT_VALUE_TYPE)) - getOutput().setValueType(ValueType.INT64); - else if (valueTypeString.equalsIgnoreCase(Statement.BOOLEAN_VALUE_TYPE)) - getOutput().setValueType(ValueType.BOOLEAN); - else if (valueTypeString.equalsIgnoreCase(ValueType.UNKNOWN.name())) - getOutput().setValueType(ValueType.UNKNOWN); - else { - raiseValidateError("Unknown Value Type " + valueTypeString - + ". Valid values are: " + Statement.DOUBLE_VALUE_TYPE +", " + Statement.INT_VALUE_TYPE + ", " + Statement.BOOLEAN_VALUE_TYPE + ", " + Statement.STRING_VALUE_TYPE, conditional); - } - } else { - getOutput().setValueType(ValueType.FP64); - } - - break; - - case WRITE: - - // for CSV format, if no delimiter specified THEN set default "," - if (getVarParam(FORMAT_TYPE) == null || checkFormatType(FileFormat.CSV) ){ - if (getVarParam(DELIM_DELIMITER) == null) { - addVarParam(DELIM_DELIMITER, new StringIdentifier(DEFAULT_DELIM_DELIMITER, this)); - } - if (getVarParam(DELIM_HAS_HEADER_ROW) == null) { - addVarParam(DELIM_HAS_HEADER_ROW, new BooleanIdentifier(DEFAULT_DELIM_HAS_HEADER_ROW, this)); - } - if (getVarParam(DELIM_SPARSE) == null) { - addVarParam(DELIM_SPARSE, new BooleanIdentifier(DEFAULT_DELIM_SPARSE, this)); - } - } - - // for LIBSVM format, add the default separators if not specified - if (getVarParam(FORMAT_TYPE) == null || checkFormatType(FileFormat.LIBSVM)) { - if(getVarParam(DELIM_DELIMITER) == null) { - addVarParam(DELIM_DELIMITER, new StringIdentifier(DEFAULT_DELIM_DELIMITER, this)); - } - if(getVarParam(LIBSVM_INDEX_DELIM) == null) { - addVarParam(LIBSVM_INDEX_DELIM, new StringIdentifier(DEFAULT_LIBSVM_INDEX_DELIM, this)); - } - if(getVarParam(DELIM_SPARSE) == null) { - addVarParam(DELIM_SPARSE, new BooleanIdentifier(DEFAULT_DELIM_SPARSE, this)); - } - } - - //validate read filename - if (getVarParam(FORMAT_TYPE) == null || FileFormat.isTextFormat(getVarParam(FORMAT_TYPE).toString()) - || checkFormatType(FileFormat.DELTA)) //delta: columnar, no block layout - getOutput().setBlocksize(-1); - else if (checkFormatType(FileFormat.BINARY, FileFormat.COMPRESSED, FileFormat.UNKNOWN)) { - if( getVarParam(ROWBLOCKCOUNTPARAM)!=null ) - getOutput().setBlocksize(Integer.parseInt(getVarParam(ROWBLOCKCOUNTPARAM).toString())); - else - getOutput().setBlocksize(ConfigurationManager.getBlocksize()); - } - else if( getVarParam(FORMAT_TYPE) instanceof StringIdentifier ) //literal format - raiseValidateError("Invalid format " + getVarParam(FORMAT_TYPE) - + " in statement: " + toString(), conditional); - break; + else if(getVarParam(FORMAT_TYPE) instanceof StringIdentifier) // literal format + raiseValidateError("Invalid format " + getVarParam(FORMAT_TYPE) + " in statement: " + toString(), + conditional); + break; case RAND: diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java index d0ba5363939..042e0dc0328 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java @@ -484,7 +484,7 @@ public static CompressedMatrixBlock read(DataInput in) throws IOException { long nonZeros = in.readLong(); boolean overlappingColGroups = in.readBoolean(); List groups = ColGroupIO.readGroups(in, rlen); - CompressedMatrixBlock ret = new CompressedMatrixBlock(rlen, clen, nonZeros, overlappingColGroups, groups); + CompressedMatrixBlock ret = new CompressedMatrixBlock(rlen, clen, nonZeros, overlappingColGroups, groups); LOG.debug("Compressed read serialization time: " + t.stop()); return ret; } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java index 354325e293b..66d4e78cb0f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java @@ -402,7 +402,8 @@ public final AColGroup rightMultByMatrix(MatrixBlock right) { * @param cru The right hand side column upper * @param nRows The number of rows in this column group */ - public void rightDecompressingMult(MatrixBlock right, MatrixBlock ret, int rl, int ru, int nRows, int crl, int cru) { + public void rightDecompressingMult(MatrixBlock right, MatrixBlock ret, int rl, int ru, int nRows, int crl, + int cru) { throw new NotImplementedException( "not supporting right Decompressing Multiply on class: " + this.getClass().getSimpleName()); } @@ -977,9 +978,9 @@ public AColGroup[] splitReshapePushDown(final int multiplier, final int nRow, fi /** * Sort the values of the column group according to double comparison operations and return as another compressed * group. - * + * * This sorting assumes that the column group is sorted independently of everything else. - * + * * @return The sorted group */ public abstract AColGroup sort(); @@ -996,9 +997,9 @@ public String toString() { /** * Return a new column group containing only the selected rows in the given boolean vector. - * + * * Whenever possible only modify the index structure, not the dictionary of the column groups. - * + * * @param selectV The selection vector * @param rOut The number of rows in the output * @return The new column group @@ -1007,9 +1008,9 @@ public String toString() { /** * Return a new column group containing only the selected columns in the given boolean vector. - * + * * Whenever possible only modify the column index, and reduce the dictionaries of the column groups. - * + * * @param selectV The selection vector * @return The new column group, or {@code null} if no column of this group is selected */ @@ -1045,7 +1046,7 @@ public AColGroup removeEmptyCols(boolean[] selectV) { /** * Using the selection of columns, slice out those and return in a new column group with the given column indexes. * Ideally this method should only modify the dictionaries. - * + * * @param newColumnIDs the new column indexes * @param selectedColumns The selected columns of this column group (guaranteed < current number of columns) * @return A new Column group diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java index d825b91f089..d610c1b586c 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java @@ -210,7 +210,6 @@ public void clear() { counts = null; } - @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java index 30de5e120c5..794d90c0d11 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java @@ -212,8 +212,8 @@ public void decompressToSparseBlock(SparseBlock sb, int rl, int ru, int offR, in // TODO make sparse decompression where the iterator is known in argument decompressToSparseBlockSparseDictionary(sb, rl, ru, offR, offC, mb.getSparseBlock()); else - decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, mb.getDenseBlockValues(), - it); + decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, + mb.getDenseBlockValues(), it); } else decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, _dict.getValues(), it); @@ -240,7 +240,7 @@ public void decompressToDenseBlockDenseDictionary(DenseBlock db, int rl, int ru, } public abstract void decompressToSparseBlockDenseDictionaryWithProvidedIterator(SparseBlock db, int rl, int ru, - int offR, int offC, double[] values, AIterator it); + int offR, int offC, double[] values, AIterator it); public abstract void decompressToDenseBlockDenseDictionaryWithProvidedIterator(DenseBlock db, int rl, int ru, int offR, int offC, double[] values, AIterator it); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java index b316e48474a..d643cae440c 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java @@ -674,8 +674,8 @@ private void defaultRightDecompressingMult(MatrixBlock right, MatrixBlock ret, i } } - final void vectMM(double aa, double[] b, double[] c, int endT, int jd, int crl, int cru, int offOut, int k, int vLen, - DoubleVector vVec) { + final void vectMM(double aa, double[] b, double[] c, int endT, int jd, int crl, int cru, int offOut, int k, + int vLen, DoubleVector vVec) { vVec = vVec.broadcast(aa); final int offj = k * jd; final int end = endT + offj; diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java index 64114a054ab..d5ad55772c7 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java @@ -478,14 +478,13 @@ public AColGroup combineWithSameIndex(int nRow, int nCol, List right) return new ColGroupEmpty(combinedIndex); } - @Override - public AColGroup removeEmptyRows(boolean[] selectV, int rOut){ + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { return this; } - @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { return new ColGroupEmpty(newColumnIDs); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java index fa8aa104ffb..e0bea3c3696 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java @@ -747,7 +747,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java index a251d828b5f..b4f0c144a73 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java @@ -738,7 +738,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java index 347cea9c0da..43df7fa3b94 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java @@ -1195,9 +1195,9 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { throw new NotImplementedException("Unimplemented method 'removeEmptyRows'"); } - + @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java index 815ecacf378..4566106a3e2 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java @@ -634,8 +634,8 @@ protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList s for(int i = 0; i < selectedColumns.size(); i++) { ref[i] = _reference[selectedColumns.get(i)]; } - return ColGroupSDCFOR.create(newColumnIDs, _numRows, _dict.sliceColumns(selectedColumns, getNumCols()), _indexes, _data, null, - ref); + return ColGroupSDCFOR.create(newColumnIDs, _numRows, _dict.sliceColumns(selectedColumns, getNumCols()), + _indexes, _data, null, ref); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java index 611add6480f..9797087f8c3 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java @@ -85,7 +85,7 @@ public class ColGroupUncompressed extends AColGroup { /** * Do not use this constructor of column group uncompressed, instead use the create constructor. - * + * * @param mb The contained data. * @param colIndexes Column indexes for this Columngroup */ @@ -96,9 +96,10 @@ protected ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes) { /** * Do not use this constructor of column group quantization-fused uncompressed, instead use the create constructor. - * + * * @param mb The contained data. - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire + * matrix * @param colIndexes Column indexes for this Columngroup */ protected ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes, double[] scaleFactors) { @@ -138,7 +139,8 @@ public static AColGroup create(MatrixBlock mb, IColIndex colIndexes) { * * @param mb The MB / data to contain in the uncompressed column * @param colIndexes The column indexes for the group - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire + * matrix * @return An Uncompressed Column group */ public static AColGroup createQuantized(MatrixBlock mb, IColIndex colIndexes, double[] scaleFactors) { @@ -157,7 +159,8 @@ public static AColGroup createQuantized(MatrixBlock mb, IColIndex colIndexes, do * @param rawBlock The uncompressed block; uncompressed data must be present at the time that the constructor is * called * @param transposed Says if the input matrix raw block have been transposed. - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire + * matrix * @return AColGroup. */ public static AColGroup createQuantized(IColIndex colIndexes, MatrixBlock rawBlock, boolean transposed, diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java index 51e26a3f9d2..de8a740ceb2 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java @@ -290,7 +290,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java index a7e715b59b8..6e66ef6ef9b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java @@ -76,8 +76,8 @@ public double[] productAllRowsToDoubleWithDefault(double[] defaultTuple) { return ret; } - @Override - public int[] sort(){ + @Override + public int[] sort() { throw new NotImplementedException(); } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java index 9a0412145f0..7ebba2f1a76 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java @@ -138,8 +138,8 @@ public IDictionary clone() { throw new NotImplementedException(); } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { throw new NotImplementedException(); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java index c8ddfc4883a..b5e1a99355b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java @@ -1055,7 +1055,7 @@ public IDictionary rightMMPreAggSparse(int numVals, SparseBlock b, IColIndex thi /** * Slice out the selected columns given of this encoded group. - * + * * @param selectedColumns The columns to slice out and return as a new matrix. * @param nCol The number of columns in this dictionary. * @return The returned matrix @@ -1064,9 +1064,9 @@ public IDictionary rightMMPreAggSparse(int numVals, SparseBlock b, IColIndex thi /** * Sort the values of this dictionary via an index of how the values mapped previously. - * + * * In practice this design means we can reuse the previous dictionary for the resulting column group - * + * * @return The sorted index. */ public int[] sort(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java index c2540de959a..4337da7307f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java @@ -541,8 +541,8 @@ public String getString(int colIndexes) { return "IdentityMatrix of size: " + nRowCol + " with empty: " + withEmpty; } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { return getMBDict().sliceColumns(selectedColumns, nCol); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java index c7f642edfd0..47628b43d2a 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java @@ -311,8 +311,8 @@ public String getString(int colIndexes) { return toString(); } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { return getMBDict().sliceColumns(selectedColumns, nCol); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java index 83a74972db7..6d516713689 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java @@ -1064,7 +1064,7 @@ public AMapToData removeEmpty(final boolean[] selectV, final int rOut) { /** * Use the offsets of the select vector to choose which values to keep. - * + * * @param select The row indexes to keep * @return A New MapToData */ diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java index f65876b7f37..bf8ee7f9ee1 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java @@ -56,11 +56,11 @@ public abstract class AOffset implements Serializable { protected static final Log LOG = LogFactory.getLog(AOffset.class.getName()); /** - * Lazy holder for the cached empty slice. The empty slice is built on first use rather than in AOffset's - * static initializer: instantiating the OffsetEmpty subclass from AOffset's {@code } forms a - * superclass/subclass class-initialization cycle that deadlocks when several threads first touch the offset - * classes concurrently (e.g. parallel tests). Deferring it to first use guarantees AOffset is already - * initialized by the time OffsetEmpty is loaded, so no cycle exists. + * Lazy holder for the cached empty slice. The empty slice is built on first use rather than in AOffset's static + * initializer: instantiating the OffsetEmpty subclass from AOffset's {@code } forms a superclass/subclass + * class-initialization cycle that deadlocks when several threads first touch the offset classes concurrently (e.g. + * parallel tests). Deferring it to first use guarantees AOffset is already initialized by the time OffsetEmpty is + * loaded, so no cycle exists. */ private static final class EmptySliceHolder { static final OffsetSliceInfo EMPTY_SLICE = new OffsetSliceInfo(-1, -1, new OffsetEmpty()); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java index 866168ded2f..37ff41cf817 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java @@ -76,6 +76,7 @@ public int getOffsetToLast() { public long getInMemorySize() { return estimateInMemorySize(); } + @Override public boolean equals(AOffset b) { return b instanceof OffsetEmpty; diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java index d981ab87838..7953322350e 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java @@ -139,7 +139,8 @@ private static boolean isDoubleCompressedOpApplicable(CompressedMatrixBlock m1, m1.getColGroups().get(0) instanceof ColGroupDDC && !((CompressedMatrixBlock) that).isOverlapping() && ((CompressedMatrixBlock) that).getColGroups().get(0) instanceof ColGroupDDC && ((IMapToDataGroup) m1.getColGroups().get(0)) - .getMapToData() == ((IMapToDataGroup) ((CompressedMatrixBlock) that).getColGroups().get(0)).getMapToData(); + .getMapToData() == ((IMapToDataGroup) ((CompressedMatrixBlock) that).getColGroups().get(0)) + .getMapToData(); } private static CompressedMatrixBlock doubleCompressedBinaryOp(BinaryOperator op, CompressedMatrixBlock m1, @@ -1062,7 +1063,8 @@ public Long call() { return _ret.recomputeNonZeros(_rl, _ru - 1); } - private final void processBlock(final int rl, final int ru, final List groups, final AIterator[] its) { + private final void processBlock(final int rl, final int ru, final List groups, + final AIterator[] its) { decompressToTmpBlock(rl, ru, tmp.getSparseBlock(), groups, its); // decompressing multiple column groups can leave the temp rows with unsorted column indices, so sort // before reading them in stored order into the (column-sorted) output sparse block. diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java index cc7953f8c5d..a91b75ae73c 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java @@ -96,7 +96,7 @@ public static MatrixBlock mmChain(CompressedMatrixBlock x, MatrixBlock v, Matrix if(x.isEmpty()) return returnEmpty(x, out); - if(ctype == ChainType.XtXv && x.getColGroups().size() < 5 && x.getNumColumns()> 30){ + if(ctype == ChainType.XtXv && x.getColGroups().size() < 5 && x.getNumColumns() > 30) { MatrixBlock tmp = CLALibTSMM.leftMultByTransposeSelf(x, k); return tmp.aggregateBinaryOperations(tmp, v, out, InstructionUtils.getMatMultOperator(k)); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java index 3755e4040e7..802eddffcb8 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java @@ -36,7 +36,7 @@ public class CLALibRemoveEmpty { /** * CP rmempty operation (single input, single output matrix) - * + * * @param in The input matrix * @param ret The output matrix * @param rows If we are removing based on rows, or columns. @@ -66,13 +66,13 @@ private static MatrixBlock rmEmptyCols(CompressedMatrixBlock in, MatrixBlock ret int cOut = (int) select.getNonZeros(); if(cOut == -1) cOut = (int) select.recomputeNonZeros(); - if(cOut == 0){ + if(cOut == 0) { ret.reset(in.getNumRows(), !emptyReturn ? 0 : 1); return ret; } - final boolean[] selectV = DataConverter - .convertToBooleanVector(CompressedMatrixBlock.getUncompressed(select, "decompressing selection in rmempty")); + final boolean[] selectV = DataConverter.convertToBooleanVector( + CompressedMatrixBlock.getUncompressed(select, "decompressing selection in rmempty")); final List inG = in.getColGroups(); final List retG = new ArrayList<>(inG.size()); @@ -102,18 +102,17 @@ private static MatrixBlock rmEmptyRows(CompressedMatrixBlock in, MatrixBlock ret int rOut = (int) select.getNonZeros(); if(rOut == -1) rOut = (int) select.recomputeNonZeros(); - if(rOut == 0){ + if(rOut == 0) { ret.reset(!emptyReturn ? 0 : 1, in.getNumColumns()); return ret; } - // TODO: add optimization to avoid linear scan and make selectV indexes, if selection is small relative to number + // TODO: add optimization to avoid linear scan and make selectV indexes, if selection is small relative to + // number // of rows // TODO: add decompress to boolean vector. final boolean[] selectV = DataConverter.convertToBooleanVector(select); - - final List inG = in.getColGroups(); final List retG = new ArrayList<>(inG.size()); try { diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java index b94f11ae723..5ae7bd5103b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java @@ -40,10 +40,10 @@ private CLALibSort() { /** * Sort (order) a compressed matrix in place of the {@code order} built-in, while keeping the result compressed. * - * The compressed fast-path only supports the case the user can benefit from: a single column held in a single column - * group, sorted ascending and returning the sorted values (not the index permutation). For everything else (multiple - * columns, multiple column groups, descending order, index return, or a column-group encoding without a sort - * implementation) this returns {@code null} so the caller can fall back to a decompressed reorg. + * The compressed fast-path only supports the case the user can benefit from: a single column held in a single + * column group, sorted ascending and returning the sorted values (not the index permutation). For everything else + * (multiple columns, multiple column groups, descending order, index return, or a column-group encoding without a + * sort implementation) this returns {@code null} so the caller can fall back to a decompressed reorg. * * @param mb the compressed matrix to sort * @param fn the sort specification carried by the reorg operator diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java index 87d14dbf87e..9ccaa474f39 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java @@ -208,8 +208,8 @@ protected FrameBlock readBlobFromHDFS(String fname, long[] dims) throws IOExcept if(data == null) throw new IOException("Unable to load frame from file: " + fname); - //Delta and CSV discover dimensions (and Delta also schema) at read time, so - //refresh the cached metadata to reflect the materialized frame block. + // Delta and CSV discover dimensions (and Delta also schema) at read time, so + // refresh the cached metadata to reflect the materialized frame block. if(iimd.getFileFormat() == FileFormat.CSV || iimd.getFileFormat() == FileFormat.DELTA) { _metaData = _metaData instanceof MetaDataFormat ? new MetaDataFormat(data.getDataCharacteristics(), iimd.getFileFormat()) : new MetaData(data.getDataCharacteristics()); diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java index 28fa70f7741..4331da2b426 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java @@ -454,7 +454,7 @@ protected MatrixBlock readBlobFromHDFS(String fname, long[] dims) throws IOExcep rlen, clen, blen, mc.getNonZeros(), getFileFormatProperties()); if(iimd.getFileFormat() == FileFormat.CSV || iimd.getFileFormat() == FileFormat.DELTA) { - //dimensions/nnz are discovered at read time for these self-describing formats + // dimensions/nnz are discovered at read time for these self-describing formats _metaData = _metaData instanceof MetaDataFormat ? new MetaDataFormat(newData.getDataCharacteristics(), iimd.getFileFormat()) : new MetaData(newData.getDataCharacteristics()); } diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java index b52f3777e1f..fbae4925c66 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java @@ -122,9 +122,9 @@ public class SparkExecutionContext extends ExecutionContext //singleton spark context (as there can be only one spark context per JVM) private static JavaSparkContext _spctx = null; - //registered users of the singleton context (guarded by the - //SparkExecutionContext.class monitor); maintained by enterSparkExecution()/ - //exitSparkExecution(), and close() only stops the context once it hits zero + // registered users of the singleton context (guarded by the + // SparkExecutionContext.class monitor); maintained by enterSparkExecution()/ + // exitSparkExecution(), and close() only stops the context once it hits zero private static int _activeExecutions = 0; //registry of parallelized RDDs to enforce that at any time, we spent at most @@ -175,8 +175,8 @@ public synchronized static JavaSparkContext getSparkContextStatic() { initSparkContext(); if(_spctx.sc().isStopped()){ _spctx = null; - //the previous context was stopped; reset the active-execution count so a - //stale registration cannot skip a future legitimate stop of the new one + // the previous context was stopped; reset the active-execution count so a + // stale registration cannot skip a future legitimate stop of the new one _activeExecutions = 0; initSparkContext(); } @@ -196,16 +196,15 @@ public synchronized static boolean isSparkContextCreated() { public static void resetSparkContextStatic() { synchronized(SparkExecutionContext.class) { _spctx = null; - //force-discarding the shared context: drop the active-execution count so - //a stale registration cannot skip a future legitimate stop + // force-discarding the shared context: drop the active-execution count so + // a stale registration cannot skip a future legitimate stop _activeExecutions = 0; } } /** - * Registers an active user of the shared spark context. Must be balanced by a - * later {@link #exitSparkExecution()} so a concurrent execution cannot stop the - * context while this one still has in-flight jobs. + * Registers an active user of the shared spark context. Must be balanced by a later {@link #exitSparkExecution()} + * so a concurrent execution cannot stop the context while this one still has in-flight jobs. */ public static void enterSparkExecution() { synchronized(SparkExecutionContext.class) { @@ -214,9 +213,8 @@ public static void enterSparkExecution() { } /** - * Releases an active user previously registered via {@link #enterSparkExecution()}. - * Only adjusts the count; the actual teardown is left to {@link #close()}, which - * stops the context once no registered execution remains. + * Releases an active user previously registered via {@link #enterSparkExecution()}. Only adjusts the count; the + * actual teardown is left to {@link #close()}, which stops the context once no registered execution remains. */ public static void exitSparkExecution() { synchronized(SparkExecutionContext.class) { @@ -227,13 +225,13 @@ public static void exitSparkExecution() { public void close() { synchronized(SparkExecutionContext.class) { - //keep the shared context alive while a registered execution still uses - //it; close() never changes the count, so an unpaired close() (a caller - //that never entered) cannot stop a context another execution is using + // keep the shared context alive while a registered execution still uses + // it; close() never changes the count, so an unpaired close() (a caller + // that never entered) cannot stop a context another execution is using if(_activeExecutions > 0) { if(LOG.isDebugEnabled()) - LOG.debug("Keeping shared spark context alive; " + _activeExecutions - + " execution(s) still active"); + LOG.debug( + "Keeping shared spark context alive; " + _activeExecutions + " execution(s) still active"); return; } if(_spctx != null) { diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java index 682cc8e3fff..c502817e026 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java @@ -95,8 +95,7 @@ private void run() { int par_conn = ConfigurationManager.getDMLConfig().getIntValue(DMLConfig.FEDERATED_PAR_CONN); final int EVENT_LOOP_THREADS = (par_conn > 0) ? par_conn : InfrastructureAnalyzer.getLocalParallelism(); // Daemon event loops so a leaked in-JVM (test) worker cannot block JVM exit. - NioEventLoopGroup bossGroup = new NioEventLoopGroup(1, - new DefaultThreadFactory("fed-worker-boss", true)); + NioEventLoopGroup bossGroup = new NioEventLoopGroup(1, new DefaultThreadFactory("fed-worker-boss", true)); ThreadPoolExecutor workerTPE = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 10, TimeUnit.SECONDS, new SynchronousQueue(true), new DefaultThreadFactory("fed-worker-pool", true)); NioEventLoopGroup workerGroup = new NioEventLoopGroup(EVENT_LOOP_THREADS, workerTPE); diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java index 80a5d699dfa..ebf05972b87 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java @@ -125,13 +125,15 @@ public static RaggedArray create(T[] col, int m) { /** * Wrap a fully populated raw typed column array into an {@link Array} of the given value type. The runtime type of - * {@code col} must match the primitive backing type of {@code vt} (e.g. {@code double[]} for {@link ValueType#FP64}, - * {@code String[]} for {@link ValueType#STRING}). + * {@code col} must match the primitive backing type of {@code vt} (e.g. {@code double[]} for + * {@link ValueType#FP64}, {@code String[]} for {@link ValueType#STRING}). * - *

For {@link ValueType#BOOLEAN} this mirrors {@link #allocateBoolean(int)}: a {@code boolean[]} longer than - * {@link #bitSetSwitchPoint} is bit-packed into a compact {@link BitSetArray} (so a bulk decoder that fills a - * plain {@code boolean[]} still ends up with the same representation as every other frame allocation path), - * while shorter columns stay a plain {@link BooleanArray}.

+ *

+ * For {@link ValueType#BOOLEAN} this mirrors {@link #allocateBoolean(int)}: a {@code boolean[]} longer than + * {@link #bitSetSwitchPoint} is bit-packed into a compact {@link BitSetArray} (so a bulk decoder that fills a plain + * {@code boolean[]} still ends up with the same representation as every other frame allocation path), while shorter + * columns stay a plain {@link BooleanArray}. + *

* * @param vt the value type of the column * @param col the backing array to wrap @@ -168,10 +170,10 @@ public static Array create(ValueType vt, Object col) { /** * Allocate the raw backing array for a column of the given value type: the inverse of - * {@link #create(ValueType, Object)}. Returns {@code double[]} for {@link ValueType#FP64}, - * {@code int[]} for INT32/UINT/HASH32, {@code long[]} for INT64/HASH64, {@code String[]} for STRING, etc. The - * runtime array type matches what {@link #create(ValueType, Object)} expects, so a bulk decoder can fill this - * primitive array directly and then wrap it via {@code create(vt, backing)}. + * {@link #create(ValueType, Object)}. Returns {@code double[]} for {@link ValueType#FP64}, {@code int[]} for + * INT32/UINT/HASH32, {@code long[]} for INT64/HASH64, {@code String[]} for STRING, etc. The runtime array type + * matches what {@link #create(ValueType, Object)} expects, so a bulk decoder can fill this primitive array directly + * and then wrap it via {@code create(vt, backing)}. * * @param vt the value type of the column * @param nRow the number of rows to allocate diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java b/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java index 9ff58065d97..95be95117e2 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java @@ -41,7 +41,7 @@ public class MatrixBlockFromFrame { public static Boolean WARNED_FOR_FAILED_CAST = false; - private MatrixBlockFromFrame(){ + private MatrixBlockFromFrame() { // private constructor for code coverage. } @@ -115,7 +115,7 @@ private static long convert(FrameBlock frame, MatrixBlock mb, int n, int rl, int return convertStrict(frame, mb, n, rl, ru); } catch(NumberFormatException | DMLRuntimeException e) { - synchronized(WARNED_FOR_FAILED_CAST){ + synchronized(WARNED_FOR_FAILED_CAST) { if(!WARNED_FOR_FAILED_CAST) { LOG.error( "Failed to convert to Matrix because of number format errors, falling back to NaN on incompatible cells", diff --git a/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java b/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java index eed2c58f78c..c12a187cf17 100644 --- a/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java +++ b/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java @@ -30,32 +30,13 @@ import jdk.incubator.vector.VectorSpecies; -/** - * Class with pre-defined set of objects. This class can not be instantiated elsewhere. - * - * Notes on commons.math FastMath: - * * FastMath uses lookup tables and interpolation instead of native calls. - * * The memory overhead for those tables is roughly 48KB in total (acceptable) - * * Micro and application benchmarks showed significantly (30%-3x) performance improvements - * for most operations; without loss of accuracy. - * * atan / sqrt were 20% slower in FastMath and hence, we use Math there - * * round / abs were equivalent in FastMath and hence, we use Math there - * * Finally, there is just one argument against FastMath - The comparison heavily depends - * on the JVM. For example, currently the IBM JDK JIT compiles to HW instructions for sqrt - * which makes this operation very efficient; as soon as other operations like log/exp are - * similarly compiled, we should rerun the micro benchmarks, and switch back if necessary. - * - */ -public class Builtin extends ValueFunction -{ - private static final long serialVersionUID = 3836744687789840574L; - - public enum BuiltinCode { AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, ATAN, LOG, LOG_NZ, MIN, - MAX, ABS, SIGN, SQRT, EXP, PLOGP, PRINT, PRINTF, NROW, NCOL, LENGTH, LINEAGE, ROUND, MAXINDEX, MININDEX, - STOP, CEIL, FLOOR, CUMSUM, ROWCUMSUM, CUMPROD, CUMMIN, CUMMAX, CUMSUMPROD, INVERSE, SPROP, SIGMOID, EVAL, LIST, - TYPEOF, APPLY_SCHEMA, DETECTSCHEMA, ISNA, ISNAN, ISINF, DROP_INVALID_TYPE, - DROP_INVALID_LENGTH, VALUE_SWAP, FRAME_ROW_REPLICATE, GET_CATEGORICAL_MASK, - MAP, COUNT_DISTINCT, COUNT_DISTINCT_APPROX, UNIQUE} + public enum BuiltinCode { + AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, ATAN, LOG, LOG_NZ, MIN, MAX, ABS, SIGN, SQRT, EXP, PLOGP, + PRINT, PRINTF, NROW, NCOL, LENGTH, LINEAGE, ROUND, MAXINDEX, MININDEX, STOP, CEIL, FLOOR, CUMSUM, ROWCUMSUM, + CUMPROD, CUMMIN, CUMMAX, CUMSUMPROD, INVERSE, SPROP, SIGMOID, EVAL, LIST, TYPEOF, APPLY_SCHEMA, DETECTSCHEMA, + ISNA, ISNAN, ISINF, DROP_INVALID_TYPE, DROP_INVALID_LENGTH, VALUE_SWAP, FRAME_ROW_REPLICATE, + GET_CATEGORICAL_MASK, MAP, COUNT_DISTINCT, COUNT_DISTINCT_APPROX, UNIQUE + } private static final VectorSpecies SPECIES = DoubleVector.SPECIES_PREFERRED; private static final int vLen = SPECIES.length(); @@ -68,59 +49,59 @@ public enum BuiltinCode { AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, static public HashMap String2BuiltinCode; static { String2BuiltinCode = new HashMap<>(); - String2BuiltinCode.put( "autoDiff" , BuiltinCode.AUTODIFF); - String2BuiltinCode.put( "sin" , BuiltinCode.SIN); - String2BuiltinCode.put( "cos" , BuiltinCode.COS); - String2BuiltinCode.put( "tan" , BuiltinCode.TAN); - String2BuiltinCode.put( "sinh" , BuiltinCode.SINH); - String2BuiltinCode.put( "cosh" , BuiltinCode.COSH); - String2BuiltinCode.put( "tanh" , BuiltinCode.TANH); - String2BuiltinCode.put( "asin" , BuiltinCode.ASIN); - String2BuiltinCode.put( "acos" , BuiltinCode.ACOS); - String2BuiltinCode.put( "atan" , BuiltinCode.ATAN); - String2BuiltinCode.put( "log" , BuiltinCode.LOG); - String2BuiltinCode.put( "log_nz" , BuiltinCode.LOG_NZ); - String2BuiltinCode.put( "min" , BuiltinCode.MIN); - String2BuiltinCode.put( "max" , BuiltinCode.MAX); - String2BuiltinCode.put( "maxindex", BuiltinCode.MAXINDEX); - String2BuiltinCode.put( "minindex", BuiltinCode.MININDEX); - String2BuiltinCode.put( "abs" , BuiltinCode.ABS); - String2BuiltinCode.put( "sign" , BuiltinCode.SIGN); - String2BuiltinCode.put( "sqrt" , BuiltinCode.SQRT); - String2BuiltinCode.put( "exp" , BuiltinCode.EXP); - String2BuiltinCode.put( "plogp" , BuiltinCode.PLOGP); - String2BuiltinCode.put( "print" , BuiltinCode.PRINT); - String2BuiltinCode.put( "printf" , BuiltinCode.PRINTF); - String2BuiltinCode.put( "eval" , BuiltinCode.EVAL); - String2BuiltinCode.put( "list" , BuiltinCode.LIST); - String2BuiltinCode.put( "nrow" , BuiltinCode.NROW); - String2BuiltinCode.put( "ncol" , BuiltinCode.NCOL); - String2BuiltinCode.put( "length" , BuiltinCode.LENGTH); - String2BuiltinCode.put( "round" , BuiltinCode.ROUND); - String2BuiltinCode.put( "stop" , BuiltinCode.STOP); - String2BuiltinCode.put( "ceil" , BuiltinCode.CEIL); - String2BuiltinCode.put( "floor" , BuiltinCode.FLOOR); - String2BuiltinCode.put( "ucumk+" , BuiltinCode.CUMSUM); - String2BuiltinCode.put( "urowcumk+" , BuiltinCode.ROWCUMSUM); - String2BuiltinCode.put( "ucum*" , BuiltinCode.CUMPROD); - String2BuiltinCode.put( "ucumk+*", BuiltinCode.CUMSUMPROD); - String2BuiltinCode.put( "ucummin", BuiltinCode.CUMMIN); - String2BuiltinCode.put( "ucummax", BuiltinCode.CUMMAX); - String2BuiltinCode.put( "inverse", BuiltinCode.INVERSE); - String2BuiltinCode.put( "sprop", BuiltinCode.SPROP); - String2BuiltinCode.put( "sigmoid", BuiltinCode.SIGMOID); - String2BuiltinCode.put( "typeOf", BuiltinCode.TYPEOF); - String2BuiltinCode.put( "detectSchema", BuiltinCode.DETECTSCHEMA); - String2BuiltinCode.put( "isna", BuiltinCode.ISNA); - String2BuiltinCode.put( "isnan", BuiltinCode.ISNAN); - String2BuiltinCode.put( "isinf", BuiltinCode.ISINF); - String2BuiltinCode.put( "dropInvalidType", BuiltinCode.DROP_INVALID_TYPE); - String2BuiltinCode.put( "freplicate", BuiltinCode.FRAME_ROW_REPLICATE); - String2BuiltinCode.put( "dropInvalidLength", BuiltinCode.DROP_INVALID_LENGTH); - String2BuiltinCode.put( "_map", BuiltinCode.MAP); - String2BuiltinCode.put( "valueSwap", BuiltinCode.VALUE_SWAP); - String2BuiltinCode.put( "applySchema", BuiltinCode.APPLY_SCHEMA); - String2BuiltinCode.put( "get_categorical_mask", BuiltinCode.GET_CATEGORICAL_MASK); + String2BuiltinCode.put("autoDiff", BuiltinCode.AUTODIFF); + String2BuiltinCode.put("sin", BuiltinCode.SIN); + String2BuiltinCode.put("cos", BuiltinCode.COS); + String2BuiltinCode.put("tan", BuiltinCode.TAN); + String2BuiltinCode.put("sinh", BuiltinCode.SINH); + String2BuiltinCode.put("cosh", BuiltinCode.COSH); + String2BuiltinCode.put("tanh", BuiltinCode.TANH); + String2BuiltinCode.put("asin", BuiltinCode.ASIN); + String2BuiltinCode.put("acos", BuiltinCode.ACOS); + String2BuiltinCode.put("atan", BuiltinCode.ATAN); + String2BuiltinCode.put("log", BuiltinCode.LOG); + String2BuiltinCode.put("log_nz", BuiltinCode.LOG_NZ); + String2BuiltinCode.put("min", BuiltinCode.MIN); + String2BuiltinCode.put("max", BuiltinCode.MAX); + String2BuiltinCode.put("maxindex", BuiltinCode.MAXINDEX); + String2BuiltinCode.put("minindex", BuiltinCode.MININDEX); + String2BuiltinCode.put("abs", BuiltinCode.ABS); + String2BuiltinCode.put("sign", BuiltinCode.SIGN); + String2BuiltinCode.put("sqrt", BuiltinCode.SQRT); + String2BuiltinCode.put("exp", BuiltinCode.EXP); + String2BuiltinCode.put("plogp", BuiltinCode.PLOGP); + String2BuiltinCode.put("print", BuiltinCode.PRINT); + String2BuiltinCode.put("printf", BuiltinCode.PRINTF); + String2BuiltinCode.put("eval", BuiltinCode.EVAL); + String2BuiltinCode.put("list", BuiltinCode.LIST); + String2BuiltinCode.put("nrow", BuiltinCode.NROW); + String2BuiltinCode.put("ncol", BuiltinCode.NCOL); + String2BuiltinCode.put("length", BuiltinCode.LENGTH); + String2BuiltinCode.put("round", BuiltinCode.ROUND); + String2BuiltinCode.put("stop", BuiltinCode.STOP); + String2BuiltinCode.put("ceil", BuiltinCode.CEIL); + String2BuiltinCode.put("floor", BuiltinCode.FLOOR); + String2BuiltinCode.put("ucumk+", BuiltinCode.CUMSUM); + String2BuiltinCode.put("urowcumk+", BuiltinCode.ROWCUMSUM); + String2BuiltinCode.put("ucum*", BuiltinCode.CUMPROD); + String2BuiltinCode.put("ucumk+*", BuiltinCode.CUMSUMPROD); + String2BuiltinCode.put("ucummin", BuiltinCode.CUMMIN); + String2BuiltinCode.put("ucummax", BuiltinCode.CUMMAX); + String2BuiltinCode.put("inverse", BuiltinCode.INVERSE); + String2BuiltinCode.put("sprop", BuiltinCode.SPROP); + String2BuiltinCode.put("sigmoid", BuiltinCode.SIGMOID); + String2BuiltinCode.put("typeOf", BuiltinCode.TYPEOF); + String2BuiltinCode.put("detectSchema", BuiltinCode.DETECTSCHEMA); + String2BuiltinCode.put("isna", BuiltinCode.ISNA); + String2BuiltinCode.put("isnan", BuiltinCode.ISNAN); + String2BuiltinCode.put("isinf", BuiltinCode.ISINF); + String2BuiltinCode.put("dropInvalidType", BuiltinCode.DROP_INVALID_TYPE); + String2BuiltinCode.put("freplicate", BuiltinCode.FRAME_ROW_REPLICATE); + String2BuiltinCode.put("dropInvalidLength", BuiltinCode.DROP_INVALID_LENGTH); + String2BuiltinCode.put("_map", BuiltinCode.MAP); + String2BuiltinCode.put("valueSwap", BuiltinCode.VALUE_SWAP); + String2BuiltinCode.put("applySchema", BuiltinCode.APPLY_SCHEMA); + String2BuiltinCode.put("get_categorical_mask", BuiltinCode.GET_CATEGORICAL_MASK); } protected Builtin(BuiltinCode bf) { diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java index 86184f47be6..08d28512d5c 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java @@ -59,7 +59,7 @@ else if (in1.getDataType() == DataType.TENSOR && in2.getDataType() == DataType.T return new BinaryTensorTensorCPInstruction(operator, in1, in2, out, opcode, str); else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.FRAME) return new BinaryFrameFrameCPInstruction(operator, in1, in2, out, opcode, str); - else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.SCALAR) + else if(in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.SCALAR) return new BinaryFrameScalarCPInstruction(operator, in1, in2, out, opcode, str); else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.MATRIX) return new BinaryFrameMatrixCPInstruction(operator, in1, in2, out, opcode, str); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java index 193894fd9bc..de76fca18b8 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java @@ -37,8 +37,8 @@ public class BinaryFrameScalarCPInstruction extends BinaryCPInstruction { // private static final Log LOG = LogFactory.getLog(BinaryFrameFrameCPInstruction.class.getName()); - private static final TfMethod[] UNSUPPORTED_MASK_METHODS = new TfMethod[] {TfMethod.BIN, - TfMethod.WORD_EMBEDDING, TfMethod.BAG_OF_WORDS, TfMethod.UDF}; + private static final TfMethod[] UNSUPPORTED_MASK_METHODS = new TfMethod[] {TfMethod.BIN, TfMethod.WORD_EMBEDDING, + TfMethod.BAG_OF_WORDS, TfMethod.UDF}; protected BinaryFrameScalarCPInstruction(MultiThreadedOperator op, CPOperand in1, CPOperand in2, CPOperand out, String opcode, String istr) { @@ -96,9 +96,9 @@ public void processGetCategorical(ExecutionContext ec, FrameBlock f, ScalarObjec } /** - * Accumulates, per input column, how many output columns it expands to (lengths) and whether those - * output columns are categorical (categorical). The arrays are allocated lazily: a column that no - * method touches keeps the implicit default of a single, non-categorical output column. + * Accumulates, per input column, how many output columns it expands to (lengths) and whether those output columns + * are categorical (categorical). The arrays are allocated lazily: a column that no method touches keeps the + * implicit default of a single, non-categorical output column. */ private static final class CategoricalMask { private final FrameBlock f; diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java index d76dbe0d45e..2c8093c3717 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java @@ -80,10 +80,10 @@ public void processInstruction(ExecutionContext ec) { retBlock = inBlock1; } else { - if(LibCommonsMath.isSupportedMatrixMatrixOperation(getOpcode()) ){ + if(LibCommonsMath.isSupportedMatrixMatrixOperation(getOpcode())) { if(compressedLeft) inBlock1 = CompressedMatrixBlock.getUncompressed(inBlock1, getOpcode()); - + if(compressedRight) inBlock2 = CompressedMatrixBlock.getUncompressed(inBlock2, getOpcode()); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java index e53958ac4b8..97ae151ecc0 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java @@ -350,9 +350,10 @@ else if(opcode.equalsIgnoreCase(Opcodes.TRANSFORMDECODE.toString())) { String[] colnames = meta.getColumnNames(); // compute transformdecode - Decoder decoder = DecoderFactory - .createDecoder(getParameterMap().get("spec"), colnames, null, meta, data.getNumColumns()); - FrameBlock fbout = decoder.decode(data, new FrameBlock(decoder.getSchema()), InfrastructureAnalyzer.getLocalParallelism()); + Decoder decoder = DecoderFactory.createDecoder(getParameterMap().get("spec"), colnames, null, meta, + data.getNumColumns()); + FrameBlock fbout = decoder.decode(data, new FrameBlock(decoder.getSchema()), + InfrastructureAnalyzer.getLocalParallelism()); fbout.setColumnNames(Arrays.copyOfRange(colnames, 0, fbout.getNumColumns())); // release locks diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java index 40a677e5d71..04353806ca3 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java @@ -45,8 +45,8 @@ protected ReorgOOCInstruction(ReorgOperator op, CPOperand in1, CPOperand out, St this(op, in1, out, null, null, null, opcode, istr); } - private ReorgOOCInstruction(Operator op, CPOperand in, CPOperand out, CPOperand col, CPOperand desc, CPOperand ixret, - String opcode, String istr) { + private ReorgOOCInstruction(Operator op, CPOperand in, CPOperand out, CPOperand col, CPOperand desc, + CPOperand ixret, String opcode, String istr) { super(OOCType.Reorg, op, in, out, opcode, istr); _col = col; _desc = desc; @@ -76,8 +76,8 @@ else if(opcode.equalsIgnoreCase(Opcodes.SORT.toString())) { CPOperand desc = new CPOperand(parts[3]); CPOperand ixret = new CPOperand(parts[4]); int k = Integer.parseInt(parts[6]); - return new ReorgOOCInstruction(new ReorgOperator(new SortIndex(1, false, false), k), - in, out, col, desc, ixret, opcode, str); + return new ReorgOOCInstruction(new ReorgOperator(new SortIndex(1, false, false), k), in, out, col, desc, + ixret, opcode, str); } else throw new NotImplementedException(); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java index 7590438b949..091c6785b3c 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java @@ -128,17 +128,20 @@ public void processInstruction(ExecutionContext ec) { long r = tmp.getIndexes().getRowIndex(); long c = tmp.getIndexes().getColumnIndex(); // adapt index to new position in row - return new IndexedMatrixValue(new MatrixIndexes(1, (r - 1) * numBlocksPerRowIn + c), tmp.getValue()); + return new IndexedMatrixValue(new MatrixIndexes(1, (r - 1) * numBlocksPerRowIn + c), + tmp.getValue()); }); } else { f.join(); - reshapeFullColBlocks(rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, singleRowBlocks, qOut); + reshapeFullColBlocks(rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, + singleRowBlocks, qOut); } } else { f.join(); - reshapePartialColBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, singleRowBlocks, qOut); + reshapePartialColBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, + numBlocksPerColOut, singleRowBlocks, qOut); } } else { @@ -166,17 +169,20 @@ public void processInstruction(ExecutionContext ec) { long r = tmp.getIndexes().getRowIndex(); long c = tmp.getIndexes().getColumnIndex(); // adapt index to new position in col - return new IndexedMatrixValue(new MatrixIndexes((c - 1) * numBlocksPerColIn + r, 1), tmp.getValue()); + return new IndexedMatrixValue(new MatrixIndexes((c - 1) * numBlocksPerColIn + r, 1), + tmp.getValue()); }); } else { f.join(); - reshapeFullRowBlocks(rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, singleColBlocks, qOut); + reshapeFullRowBlocks(rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, + singleColBlocks, qOut); } } else { f.join(); - reshapePartialRowBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, singleColBlocks, qOut); + reshapePartialRowBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, + numBlocksPerColOut, singleColBlocks, qOut); } } } @@ -226,7 +232,8 @@ private void reshapeFullColBlocks(long rows, long cols, int blen, int numBlocksP } private void reshapePartialColBlocks(long rlen, long clen, long rows, long cols, int blen, int numBlocksPerRowIn, - int numBlocksPerRowOut, int numBlocksPerColOut, OOCStream singleRowBlocks, OOCStream qOut) { + int numBlocksPerRowOut, int numBlocksPerColOut, OOCStream singleRowBlocks, + OOCStream qOut) { // use cache for accessing input rows by index CachingStream singleRowBlockCache = new CachingStream(singleRowBlocks); singleRowBlockCache.incrSubscriberCount(1); @@ -237,14 +244,16 @@ private void reshapePartialColBlocks(long rlen, long clen, long rows, long cols, int r = 0; // allocate row of output blocks - MatrixBlock[] outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut,true); + MatrixBlock[] outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, + true); int offsetOut = 0; int localColsOut = (cols > blen) ? blen : (int) cols; // iterate through input rows and add to row of output blocks for(int i = 1; i <= rlen; i++) { for(int j = 1; j <= numBlocksPerRowIn; j++) { - try(OOCStream.QueueCallback qcb = singleRowBlockCache.findCached(new MatrixIndexes(i, j))) { + try(OOCStream.QueueCallback qcb = singleRowBlockCache + .findCached(new MatrixIndexes(i, j))) { MatrixBlock blk = (MatrixBlock) qcb.get().getValue(); int offsetIn = 0; @@ -278,10 +287,12 @@ else if(remIn == remOut) { if(r == outputBlockRow[0].getNumRows()) { // enqueue filled output blocks and allocate new ones for(int b = 0; b < outputBlockRow.length; b++) - qOut.enqueue(new IndexedMatrixValue(new MatrixIndexes(br + 1, b + 1), outputBlockRow[b])); + qOut.enqueue( + new IndexedMatrixValue(new MatrixIndexes(br + 1, b + 1), outputBlockRow[b])); br++; // allocate new block row - outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, true); + outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, + numBlocksPerColOut, true); r = 0; } bc = 0; @@ -341,7 +352,8 @@ private void reshapeFullRowBlocks(long rows, long cols, int blen, int numBlocksP } private void reshapePartialRowBlocks(long rlen, long clen, long rows, long cols, int blen, int numBlocksPerRowOut, - int numBlocksPerColIn, int numBlocksPerColOut, OOCStream singleColBlocks, OOCStream qOut) { + int numBlocksPerColIn, int numBlocksPerColOut, OOCStream singleColBlocks, + OOCStream qOut) { // use cache for accessing input cols by index CachingStream singleRowBlockCache = new CachingStream(singleColBlocks); singleRowBlockCache.incrSubscriberCount(1); @@ -352,14 +364,16 @@ private void reshapePartialRowBlocks(long rlen, long clen, long rows, long cols, int c = 0; // allocate col of output blocks - MatrixBlock[] outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, false); + MatrixBlock[] outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, + false); int offsetOut = 0; int localRowsOut = (rows > blen) ? blen : (int) rows; // iterate through input cols and add to col of output blocks for(int j = 1; j <= clen; j++) { for(int i = 1; i <= numBlocksPerColIn; i++) { - try(OOCStream.QueueCallback qcb = singleRowBlockCache.findCached(new MatrixIndexes(i, j))) { + try(OOCStream.QueueCallback qcb = singleRowBlockCache + .findCached(new MatrixIndexes(i, j))) { MatrixBlock blk = (MatrixBlock) qcb.get().getValue(); int offsetIn = 0; @@ -394,11 +408,13 @@ else if(remIn == remOut) { // enqueue filled output blocks and allocate new ones for(int b = 0; b < outputBlockCol.length; b++) { outputBlockCol[b].recomputeNonZeros(); - qOut.enqueue(new IndexedMatrixValue(new MatrixIndexes(b + 1, bc + 1), outputBlockCol[b])); + qOut.enqueue( + new IndexedMatrixValue(new MatrixIndexes(b + 1, bc + 1), outputBlockCol[b])); } bc++; // allocate new block col - outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, false); + outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, + numBlocksPerColOut, false); c = 0; } br = 0; @@ -411,16 +427,20 @@ else if(remIn == remOut) { qOut.closeInput(); } - private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int blen, int numBlocksPerRow, int numBlocksPerCol, boolean isBlockRowSlice) { + private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int blen, int numBlocksPerRow, + int numBlocksPerCol, boolean isBlockRowSlice) { int num = isBlockRowSlice ? numBlocksPerRow : numBlocksPerCol; MatrixBlock[] res = new MatrixBlock[num]; // full inner blocks, adjust for outer blocks - int localRows = ((!isBlockRowSlice || idx == numBlocksPerCol - 1) && rows % blen != 0) ? (int) rows % blen : blen; - int localCols = ((isBlockRowSlice || idx == numBlocksPerRow - 1) && cols % blen != 0) ? (int) cols % blen : blen; + int localRows = ((!isBlockRowSlice || idx == numBlocksPerCol - 1) && rows % blen != 0) ? (int) rows % + blen : blen; + int localCols = ((isBlockRowSlice || idx == numBlocksPerRow - 1) && cols % blen != 0) ? (int) cols % + blen : blen; for(int k = 0; k < num - 1; k++) { - res[k] = isBlockRowSlice ? new MatrixBlock(localRows, blen, false) : new MatrixBlock(blen, localCols, false); + res[k] = isBlockRowSlice ? new MatrixBlock(localRows, blen, false) : new MatrixBlock(blen, localCols, + false); res[k].allocateDenseBlock(); } res[num - 1] = new MatrixBlock(localRows, localCols, false); @@ -428,10 +448,13 @@ private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int ble return res; } - private void setOutputEntries(MatrixBlock src, MatrixBlock dest, int idx, int srcOffset, int destOffset, int length, boolean rowWise) { + private void setOutputEntries(MatrixBlock src, MatrixBlock dest, int idx, int srcOffset, int destOffset, int length, + boolean rowWise) { if(rowWise) - ((DenseBlockFP64) dest.getDenseBlock()).setPartialRow(src.getDenseBlock(), idx, srcOffset, destOffset, length); + ((DenseBlockFP64) dest.getDenseBlock()).setPartialRow(src.getDenseBlock(), idx, srcOffset, destOffset, + length); else - ((DenseBlockFP64) dest.getDenseBlock()).setPartialCol(src.getDenseBlock(), idx, srcOffset, destOffset, length); + ((DenseBlockFP64) dest.getDenseBlock()).setPartialCol(src.getDenseBlock(), idx, srcOffset, destOffset, + length); } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java index 75f84882478..e25219b80ba 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java @@ -114,8 +114,7 @@ public void processInstruction(ExecutionContext ec) { case VALUEPICK: { if( input2.isScalar() ) { ScalarObject quantile = ec.getScalarInput(input2); - double[] wt = getWeightedQuantileSummary(in, mc, - new double[] {quantile.getDoubleValue()}, true); + double[] wt = getWeightedQuantileSummary(in, mc, new double[] {quantile.getDoubleValue()}, true); ec.setScalarOutput(output.getName(), new DoubleObject(wt[3])); } else { diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java index 2f83caa5526..f007558ebdc 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java @@ -30,8 +30,7 @@ import org.apache.sysds.runtime.matrix.data.MatrixValue; import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; -public class IndexedMatrixValue implements SpillableObject, Serializable -{ +public class IndexedMatrixValue implements SpillableObject, Serializable { private static final long serialVersionUID = 6723389820806752110L; private MatrixIndexes _indexes = null; @@ -110,8 +109,8 @@ public void discard() { @Override public void read(DataInput dataInput) throws IOException { - _indexes = new MatrixIndexes(); - _value = new MatrixBlock(); + _indexes = new MatrixIndexes(); + _value = new MatrixBlock(); _indexes.readFields(dataInput); _value.readFields(dataInput); } diff --git a/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java index c3b9351d3d3..cc8491d4515 100644 --- a/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java +++ b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java @@ -87,11 +87,10 @@ import io.delta.kernel.utils.FileStatus; /** - * Shared helpers for the native (Spark-free) Delta Lake read/write paths used - * by both the matrix and frame readers/writers. Centralizes engine creation, - * path qualification, the scan loop (snapshot -> data files -> logical - * columnar batches, honoring deletion vectors), and the write transaction - * (logical data -> parquet -> commit). + * Shared helpers for the native (Spark-free) Delta Lake read/write paths used by both the matrix and frame + * readers/writers. Centralizes engine creation, path qualification, the scan loop (snapshot -> data files -> + * logical columnar batches, honoring deletion vectors), and the write transaction (logical data -> parquet -> + * commit). */ public class DeltaKernelUtils { @@ -102,23 +101,29 @@ public class DeltaKernelUtils { /** Reused thread-safe JSON reader for the per-file Delta stats (numRecords). */ private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); - /** Delta Kernel config key: number of rows per parquet read batch, overridable via - * {@link org.apache.sysds.conf.DMLConfig#DELTA_READER_BATCH_SIZE}. */ + /** + * Delta Kernel config key: number of rows per parquet read batch, overridable via + * {@link org.apache.sysds.conf.DMLConfig#DELTA_READER_BATCH_SIZE}. + */ private static final String CONF_READER_BATCH_SIZE = "delta.kernel.default.parquet.reader.batch-size"; - /** Delta Kernel config key: target size (bytes) at which the writer rolls a new data file, overridable via - * {@link org.apache.sysds.conf.DMLConfig#DELTA_WRITER_TARGET_FILE_SIZE}. */ + /** + * Delta Kernel config key: target size (bytes) at which the writer rolls a new data file, overridable via + * {@link org.apache.sysds.conf.DMLConfig#DELTA_WRITER_TARGET_FILE_SIZE}. + */ private static final String CONF_WRITER_TARGET_FILE_SIZE = "delta.kernel.default.parquet.writer.targetMaxFileSize"; - /** Internal Delta column type codes shared by the matrix and frame readers to - * dispatch boxing-free primitive column access. */ - public static final int T_DOUBLE = 0; - public static final int T_FLOAT = 1; - public static final int T_LONG = 2; - public static final int T_INT = 3; - public static final int T_SHORT = 4; - public static final int T_BYTE = 5; + /** + * Internal Delta column type codes shared by the matrix and frame readers to dispatch boxing-free primitive column + * access. + */ + public static final int T_DOUBLE = 0; + public static final int T_FLOAT = 1; + public static final int T_LONG = 2; + public static final int T_INT = 3; + public static final int T_SHORT = 4; + public static final int T_BYTE = 5; public static final int T_BOOLEAN = 6; - public static final int T_STRING = 7; + public static final int T_STRING = 7; /** * Parquet physical type each {@code T_*} column is stored as, indexed by type code (delta int/short/byte columns @@ -128,22 +133,22 @@ public class DeltaKernelUtils { PrimitiveTypeName.INT64, PrimitiveTypeName.INT32, PrimitiveTypeName.INT32, PrimitiveTypeName.INT32, PrimitiveTypeName.BOOLEAN, PrimitiveTypeName.BINARY}; - //derived configuration cached to avoid copying the (large) base conf on every - //engine creation (createEngine is called once per data file in parallel reads); - //rebuilt whenever the base conf or the relevant SystemDS settings change. + // derived configuration cached to avoid copying the (large) base conf on every + // engine creation (createEngine is called once per data file in parallel reads); + // rebuilt whenever the base conf or the relevant SystemDS settings change. private static Configuration cachedConf; private static Configuration cachedConfBase; private static int cachedBatchSize; private static long cachedTargetFileSize; - private DeltaKernelUtils() {} + private DeltaKernelUtils() { + } /** - * Consumes a whole columnar batch. {@code selected} is {@code null} when all - * {@code size} rows are live; otherwise {@code selected[r]} indicates whether - * row {@code r} survived the deletion/selection vector. Batch-level consumption - * lets callers extract data column-at-a-time (cache friendly, boxing free) - * instead of paying a per-row callback. + * Consumes a whole columnar batch. {@code selected} is {@code null} when all {@code size} rows are live; otherwise + * {@code selected[r]} indicates whether row {@code r} survived the deletion/selection vector. Batch-level + * consumption lets callers extract data column-at-a-time (cache friendly, boxing free) instead of paying a per-row + * callback. */ @FunctionalInterface public interface BatchConsumer { @@ -151,22 +156,29 @@ public interface BatchConsumer { } /** - * Map a Delta Kernel {@link DataType} to an internal type code (see the - * {@code T_*} constants). Returned once per column so the per-cell read loop - * can switch on a primitive int instead of repeating {@code instanceof} checks. + * Map a Delta Kernel {@link DataType} to an internal type code (see the {@code T_*} constants). Returned once per + * column so the per-cell read loop can switch on a primitive int instead of repeating {@code instanceof} checks. * * @param dt the Delta column data type * @return the matching {@code T_*} code, or {@code -1} if the type is not supported */ public static int typeCode(DataType dt) { - if( dt instanceof DoubleType ) return T_DOUBLE; - if( dt instanceof FloatType ) return T_FLOAT; - if( dt instanceof LongType ) return T_LONG; - if( dt instanceof IntegerType ) return T_INT; - if( dt instanceof ShortType ) return T_SHORT; - if( dt instanceof ByteType ) return T_BYTE; - if( dt instanceof BooleanType ) return T_BOOLEAN; - if( dt instanceof StringType ) return T_STRING; + if(dt instanceof DoubleType) + return T_DOUBLE; + if(dt instanceof FloatType) + return T_FLOAT; + if(dt instanceof LongType) + return T_LONG; + if(dt instanceof IntegerType) + return T_INT; + if(dt instanceof ShortType) + return T_SHORT; + if(dt instanceof ByteType) + return T_BYTE; + if(dt instanceof BooleanType) + return T_BOOLEAN; + if(dt instanceof StringType) + return T_STRING; return -1; } @@ -429,8 +441,10 @@ private static void decodeColumnInto(ColumnReader creader, int maxDef, int nrow, } } - /** Floor on the adaptive writer target file size. Below this the per-file metadata/open - * overhead (and tiny-file proliferation) outweighs the extra read parallelism. */ + /** + * Floor on the adaptive writer target file size. Below this the per-file metadata/open overhead (and tiny-file + * proliferation) outweighs the extra read parallelism. + */ public static final long ADAPTIVE_WRITER_MIN_FILE_SIZE = 4L * 1024 * 1024; private static Configuration buildConf(Configuration base, int batchSize, long targetFileSize) { @@ -444,9 +458,8 @@ private static synchronized Configuration deltaConf() { Configuration base = ConfigurationManager.getCachedJobConf(); int batchSize = ConfigurationManager.getDeltaReaderBatchSize(); long targetFileSize = ConfigurationManager.getDeltaWriterTargetFileSize(); - if(cachedConf == null || cachedConfBase != base - || cachedBatchSize != batchSize || cachedTargetFileSize != targetFileSize) - { + if(cachedConf == null || cachedConfBase != base || cachedBatchSize != batchSize || + cachedTargetFileSize != targetFileSize) { cachedConf = buildConf(base, batchSize, targetFileSize); cachedConfBase = base; cachedBatchSize = batchSize; @@ -460,12 +473,11 @@ public static Engine createEngine() { } /** - * Compute the parquet target data-file size (bytes) for writing a table of the given - * estimated size. With adaptive sizing enabled the writer aims for roughly one data - * file per expected parallel reader (so the native per-file parallel read can use all - * threads): never above the configured target, and never below - * {@code ADAPTIVE_WRITER_MIN_FILE_SIZE} unless the configured target is itself smaller - * than that floor (in which case the configured target wins). + * Compute the parquet target data-file size (bytes) for writing a table of the given estimated size. With adaptive + * sizing enabled the writer aims for roughly one data file per expected parallel reader (so the native per-file + * parallel read can use all threads): never above the configured target, and never below + * {@code ADAPTIVE_WRITER_MIN_FILE_SIZE} unless the configured target is itself smaller than that floor (in which + * case the configured target wins). * * @param estimatedBytes estimate of the table's size (the block in-memory size is a fine proxy) * @return the target max parquet data-file size in bytes @@ -476,7 +488,7 @@ public static long adaptiveWriterTargetFileSize(long estimatedBytes) { return configured; int par = Math.max(1, OptimizerUtils.getParallelBinaryReadParallelism()); long perReader = Math.max(1, estimatedBytes / par); - //never above the configured cap, never below the floor (unless the cap itself is lower) + // never above the configured cap, never below the floor (unless the cap itself is lower) long target = Math.min(configured, Math.max(ADAPTIVE_WRITER_MIN_FILE_SIZE, perReader)); if(LOG.isDebugEnabled()) LOG.debug("Delta adaptive file size: est=" + estimatedBytes + "B par=" + par + " -> target=" + target @@ -485,24 +497,24 @@ public static long adaptiveWriterTargetFileSize(long estimatedBytes) { } /** - * Create an engine for writing a table of the given estimated size, configured with an - * adaptive target data-file size (see {@link #adaptiveWriterTargetFileSize(long)}). A fresh - * (uncached) configuration is built since writes happen once per table, not per data file. + * Create an engine for writing a table of the given estimated size, configured with an adaptive target data-file + * size (see {@link #adaptiveWriterTargetFileSize(long)}). A fresh (uncached) configuration is built since writes + * happen once per table, not per data file. * * @param estimatedBytes estimate of the table's size (the block in-memory size is a fine proxy) * @return a Delta Kernel engine for the write */ public static Engine createWriteEngine(long estimatedBytes) { - //the reader batch size is irrelevant on the write path but is set to keep the - //conf shape identical to deltaConf(); only the target file size matters here. + // the reader batch size is irrelevant on the write path but is set to keep the + // conf shape identical to deltaConf(); only the target file size matters here. Configuration c = buildConf(ConfigurationManager.getCachedJobConf(), ConfigurationManager.getDeltaReaderBatchSize(), adaptiveWriterTargetFileSize(estimatedBytes)); return DefaultEngine.create(c); } /** - * Resolve a (possibly relative) path to a fully-qualified URI so the - * kernel's default engine can locate the table on the right filesystem. + * Resolve a (possibly relative) path to a fully-qualified URI so the kernel's default engine can locate the table + * on the right filesystem. * * @param fname input path * @return fully-qualified table path @@ -519,11 +531,9 @@ public static String qualify(String fname) { } /** - * Opened latest snapshot of a Delta table: the logical schema plus everything - * needed to (re)read its data files, including the list of per-data-file scan - * rows. Delta Kernel scan-file rows are self-contained (the kernel's - * distributed design serializes them to workers), so they can be retained and - * read independently / in parallel. + * Opened latest snapshot of a Delta table: the logical schema plus everything needed to (re)read its data files, + * including the list of per-data-file scan rows. Delta Kernel scan-file rows are self-contained (the kernel's + * distributed design serializes them to workers), so they can be retained and read independently / in parallel. */ public static final class ScanHandle { public final StructType schema; @@ -531,19 +541,18 @@ public static final class ScanHandle { public final StructType physicalReadSchema; public final List scanFiles; /** - * Per-file record counts taken from the Delta {@code numRecords} statistic, - * aligned with {@link #scanFiles}; {@code -1} where the statistic is absent. + * Per-file record counts taken from the Delta {@code numRecords} statistic, aligned with {@link #scanFiles}; + * {@code -1} where the statistic is absent. */ public final long[] numRecords; /** - * Per-file flag indicating a deletion vector is present (so the live row - * count differs from {@link #numRecords}), aligned with {@link #scanFiles}. + * Per-file flag indicating a deletion vector is present (so the live row count differs from + * {@link #numRecords}), aligned with {@link #scanFiles}. */ public final boolean[] hasDeletionVector; - private ScanHandle(StructType schema, Row scanState, StructType physicalReadSchema, - List scanFiles, long[] numRecords, boolean[] hasDeletionVector) - { + private ScanHandle(StructType schema, Row scanState, StructType physicalReadSchema, List scanFiles, + long[] numRecords, boolean[] hasDeletionVector) { this.schema = schema; this.scanState = scanState; this.physicalReadSchema = physicalReadSchema; @@ -553,13 +562,12 @@ private ScanHandle(StructType schema, Row scanState, StructType physicalReadSche } /** - * @return true iff every data file carries a {@code numRecords} statistic - * and none has a deletion vector, i.e. exact per-file row offsets - * can be derived from metadata without reading the data. + * @return true iff every data file carries a {@code numRecords} statistic and none has a deletion vector, i.e. + * exact per-file row offsets can be derived from metadata without reading the data. */ public boolean hasExactRowCounts() { - for( int i=0; i scanFileIter = (scan instanceof ScanImpl) - ? ((ScanImpl) scan).getScanFiles(engine, true) - : scan.getScanFiles(engine); + // request the scan files WITH per-file statistics (numRecords) so callers can + // pre-size output and place rows without reading the data; harmless extra + // column for the data-read path. Fall back to the stats-less iterator if the + // concrete scan does not support it. + CloseableIterator scanFileIter = (scan instanceof ScanImpl) ? ((ScanImpl) scan) + .getScanFiles(engine, true) : scan.getScanFiles(engine); List files = new ArrayList<>(); List recs = new ArrayList<>(); List dvs = new ArrayList<>(); - try( CloseableIterator scanFiles = scanFileIter ) { - while( scanFiles.hasNext() ) { + try(CloseableIterator scanFiles = scanFileIter) { + while(scanFiles.hasNext()) { FilteredColumnarBatch scanFileBatch = scanFiles.next(); - try( CloseableIterator scanFileRows = scanFileBatch.getRows() ) { - while( scanFileRows.hasNext() ) { + try(CloseableIterator scanFileRows = scanFileBatch.getRows()) { + while(scanFileRows.hasNext()) { Row scanFileRow = scanFileRows.next(); files.add(scanFileRow); recs.add(numRecords(scanFileRow)); @@ -609,7 +615,7 @@ public static ScanHandle openScan(Engine engine, String tablePath) throws IOExce } long[] numRecords = new long[recs.size()]; boolean[] hasDv = new boolean[dvs.size()]; - for( int i=0; i physicalData = engine.getParquetHandler() .readParquetFiles(Utils.singletonCloseableIterator(dataFile), physicalReadSchema, Optional.empty()); - try( CloseableIterator logicalData = - Scan.transformPhysicalData(engine, scanState, scanFileRow, physicalData) ) - { - while( logicalData.hasNext() ) + try(CloseableIterator logicalData = Scan.transformPhysicalData(engine, scanState, + scanFileRow, physicalData)) { + while(logicalData.hasNext()) consumeBatch(logicalData.next(), consumer); } } /** - * Scan the latest snapshot of a Delta table sequentially, invoking the batch - * consumer for every data batch. The consumer is created lazily from the table - * schema (so callers can size buffers / derive per-column types up front). + * Scan the latest snapshot of a Delta table sequentially, invoking the batch consumer for every data batch. The + * consumer is created lazily from the table schema (so callers can size buffers / derive per-column types up + * front). * * @param engine delta kernel engine * @param tablePath fully-qualified table path @@ -677,11 +679,10 @@ public static void readScanFile(Engine engine, Row scanState, StructType physica * @throws IOException on read failure */ public static StructType scan(Engine engine, String tablePath, Function consumerFactory) - throws IOException - { + throws IOException { ScanHandle h = openScan(engine, tablePath); BatchConsumer consumer = consumerFactory.apply(h.schema); - for( Row scanFileRow : h.scanFiles ) + for(Row scanFileRow : h.scanFiles) readScanFile(engine, h.scanState, h.physicalReadSchema, scanFileRow, consumer); return h.schema; } @@ -690,28 +691,27 @@ private static void consumeBatch(FilteredColumnarBatch fcb, BatchConsumer consum ColumnarBatch batch = fcb.getData(); int ncol = batch.getSchema().length(); ColumnVector[] cols = new ColumnVector[ncol]; - for( int c=0; c all rows live) + // materialize the deletion/selection mask once (null => all rows live) Optional selVector = fcb.getSelectionVector(); boolean[] selected = null; - if( selVector.isPresent() ) { + if(selVector.isPresent()) { ColumnVector sv = selVector.get(); selected = new boolean[size]; - for( int r=0; r logicalData) throws IOException - { - //replace any existing table at the path (the other SystemDS writers delete - //the output first; the caching layer does not do it on our behalf) + CloseableIterator logicalData) throws IOException { + // replace any existing table at the path (the other SystemDS writers delete + // the output first; the caching layer does not do it on our behalf) HDFSTool.deleteFileIfExistOnHDFS(tablePath); Table table = Table.forPath(engine, tablePath); - TransactionBuilder txnBuilder = table - .createTransactionBuilder(engine, ENGINE_INFO, Operation.CREATE_TABLE) + TransactionBuilder txnBuilder = table.createTransactionBuilder(engine, ENGINE_INFO, Operation.CREATE_TABLE) .withSchema(engine, schema); Transaction txn = txnBuilder.build(engine); Row txnState = txn.getTransactionState(engine); - CloseableIterator physicalData = - Transaction.transformLogicalData(engine, txnState, logicalData, Collections.emptyMap()); - DataWriteContext writeContext = - Transaction.getWriteContext(engine, txnState, Collections.emptyMap()); + CloseableIterator physicalData = Transaction.transformLogicalData(engine, txnState, + logicalData, Collections.emptyMap()); + DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, Collections.emptyMap()); CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles(writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns()); - CloseableIterator appendActions = - Transaction.generateAppendActions(engine, txnState, dataFiles, writeContext); + CloseableIterator appendActions = Transaction.generateAppendActions(engine, txnState, dataFiles, + writeContext); txn.commit(engine, CloseableIterable.inMemoryIterable(appendActions)); } } diff --git a/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java b/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java index 58a98741975..55d8f8f7c2d 100644 --- a/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java +++ b/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java @@ -33,29 +33,27 @@ import io.delta.kernel.types.StructType; /** - * Single-threaded native Delta Lake reader for matrices, built on the - * Spark-free Delta Kernel library. It opens the latest snapshot of a Delta - * table directory, reads its parquet data files through the kernel's default - * engine (honoring deletion vectors), and materializes the numeric columns - * into a dense {@link MatrixBlock}. + * Single-threaded native Delta Lake reader for matrices, built on the Spark-free Delta Kernel library. It opens the + * latest snapshot of a Delta table directory, reads its parquet data files through the kernel's default engine + * (honoring deletion vectors), and materializes the numeric columns into a dense {@link MatrixBlock}. * - *

Only numeric columns (double/float/long/int/short/byte/boolean) are - * supported, matching the all-double nature of a SystemDS matrix. Dimensions - * do not need to be known up front: the row count is discovered while scanning - * and the column count is taken from the table schema.

+ *

+ * Only numeric columns (double/float/long/int/short/byte/boolean) are supported, matching the all-double nature of a + * SystemDS matrix. Dimensions do not need to be known up front: the row count is discovered while scanning and the + * column count is taken from the table schema. + *

*/ public class ReaderDelta extends MatrixReader { @Override public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int blen, long estnnz) - throws IOException, DMLRuntimeException - { + throws IOException, DMLRuntimeException { Engine engine = DeltaKernelUtils.createEngine(); String tablePath = DeltaKernelUtils.qualify(fname); - //Scan column-at-a-time into one row-major buffer per batch (no per-row - //allocation, no boxing, no per-cell set()). Buffers are concatenated into - //the dense output via bulk array copies below. + // Scan column-at-a-time into one row-major buffer per batch (no per-row + // allocation, no boxing, no per-cell set()). Buffers are concatenated into + // the dense output via bulk array copies below. ArrayList batches = new ArrayList<>(); int[] nrowH = new int[1]; StructType schema = DeltaKernelUtils.scan(engine, tablePath, sch -> { @@ -72,7 +70,7 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl long lestnnz = (estnnz >= 0) ? estnnz : (long) nrow * ncol; MatrixBlock ret = createOutputMatrixBlock(nrow, ncol, Math.max(nrow, 1), lestnnz, true, false); - if( nrow > 0 && ncol > 0 ) + if(nrow > 0 && ncol > 0) fillDense(ret, batches); ret.recomputeNonZeros(); ret.examSparsity(); @@ -83,14 +81,14 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl static int[] columnTypes(StructType schema) { int ncol = schema.length(); int[] types = new int[ncol]; - for( int c=0; c batches) { DenseBlock db = ret.getDenseBlock(); - if( db.isContiguous() ) { + if(db.isContiguous()) { double[] dv = db.valuesAt(0); int off = 0; - for( double[] buf : batches ) { + for(double[] buf : batches) { System.arraycopy(buf, 0, dv, off, buf.length); off += buf.length; } } else { - //rare large multi-block fallback: route each row through the block API + // rare large multi-block fallback: route each row through the block API int ncol = ret.getNumColumns(); int r = 0; - for( double[] buf : batches ) { + for(double[] buf : batches) { int rowsInBuf = buf.length / ncol; - for( int i=0; iThe expensive part of a Delta read is the parquet decode, which the kernel - * performs per data file; parallelizing across files is therefore the natural - * way to bridge the gap to the (near-raw) binary reader. A table backed by a - * single data file (the default for tables <= the parquet target file size) - * cannot be split this way, so the reader transparently falls back to the - * sequential {@link ReaderDelta} path in that case.

+ *

+ * The expensive part of a Delta read is the parquet decode, which the kernel performs per data file; parallelizing + * across files is therefore the natural way to bridge the gap to the (near-raw) binary reader. A table backed by a + * single data file (the default for tables <= the parquet target file size) cannot be split this way, so the reader + * transparently falls back to the sequential {@link ReaderDelta} path in that case. + *

*/ public class ReaderDeltaParallel extends ReaderDelta { @@ -57,28 +56,27 @@ public ReaderDeltaParallel() { @Override public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int blen, long estnnz) - throws IOException, DMLRuntimeException - { + throws IOException, DMLRuntimeException { Engine engine = DeltaKernelUtils.createEngine(); String tablePath = DeltaKernelUtils.qualify(fname); DeltaKernelUtils.ScanHandle handle = DeltaKernelUtils.openScan(engine, tablePath); final int nfiles = handle.scanFiles.size(); - //nothing to gain from parallelism for single-file (or empty) tables - if( _numThreads <= 1 || nfiles <= 1 ) + // nothing to gain from parallelism for single-file (or empty) tables + if(_numThreads <= 1 || nfiles <= 1) return super.readMatrixFromHDFS(fname, rlen, clen, blen, estnnz); final int ncol = handle.schema.length(); final int[] types = columnTypes(handle.schema); - //fast path: exact per-file row counts are known from metadata and the dense - //output fits a single contiguous array -> pre-size once and let each thread - //decode directly into its slice (no intermediate buffers, no serial copy). - if( useDirectPath(handle) ) { + // fast path: exact per-file row counts are known from metadata and the dense + // output fits a single contiguous array -> pre-size once and let each thread + // decode directly into its slice (no intermediate buffers, no serial copy). + if(useDirectPath(handle)) { long total = 0; - for( long r : handle.numRecords ) + for(long r : handle.numRecords) total += r; - if( total > 0 && (long) total * ncol <= Integer.MAX_VALUE ) + if(total > 0 && (long) total * ncol <= Integer.MAX_VALUE) return readDirect(fname, handle, ncol, types, (int) total, estnnz); } @@ -86,11 +84,9 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl } /** - * Whether the metadata-driven direct-write fast path can be used for this - * table (exact per-file row counts and no deletion vectors). Visible for - * testing: the buffered fallback is otherwise only reachable for tables - * lacking row statistics or carrying deletion vectors, which the SystemDS - * Delta writer never produces. + * Whether the metadata-driven direct-write fast path can be used for this table (exact per-file row counts and no + * deletion vectors). Visible for testing: the buffered fallback is otherwise only reachable for tables lacking row + * statistics or carrying deletion vectors, which the SystemDS Delta writer never produces. * * @param handle the opened scan handle * @return true if the direct path is applicable @@ -100,38 +96,37 @@ protected boolean useDirectPath(DeltaKernelUtils.ScanHandle handle) { } /** - * Fast path: each thread decodes one data file straight into the final dense - * array at a metadata-derived row offset. Single allocation, fully parallel. + * Fast path: each thread decodes one data file straight into the final dense array at a metadata-derived row + * offset. Single allocation, fully parallel. */ - private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, - int ncol, int[] types, int nrow, long estnnz) throws IOException - { + private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, int ncol, int[] types, int nrow, + long estnnz) throws IOException { final int nfiles = handle.scanFiles.size(); final int[] rowOffset = new int[nfiles]; int acc = 0; - for( int i=0; i> tasks = new ArrayList<>(nfiles); - for( int i=0; i { int[] cur = new int[] {base}; Engine eng = DeltaKernelUtils.createEngine(); DeltaKernelUtils.readScanFile(eng, handle.scanState, handle.physicalReadSchema, scanFileRow, (cols, size, selected) -> { - if( cur[0] + DeltaKernelUtils.countSelected(size, selected) > limit ) + if(cur[0] + DeltaKernelUtils.countSelected(size, selected) > limit) throw new DMLRuntimeException("Delta file produced more rows than its " + "numRecords statistic; refusing parallel direct read of " + fname); cur[0] += extractBatchInto(cols, size, selected, types, ncol, dv, cur[0]); @@ -147,19 +142,17 @@ private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, } /** - * Fallback path: decode each file in parallel into per-file buffers (used when - * row counts are unknown, deletion vectors are present, or the matrix exceeds a - * single contiguous array), then concatenate in file order. + * Fallback path: decode each file in parallel into per-file buffers (used when row counts are unknown, deletion + * vectors are present, or the matrix exceeds a single contiguous array), then concatenate in file order. */ - private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handle, - int ncol, int[] types, long estnnz) throws IOException - { + private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handle, int ncol, int[] types, + long estnnz) throws IOException { final int nfiles = handle.scanFiles.size(); @SuppressWarnings("unchecked") final ArrayList[] fileBufs = new ArrayList[nfiles]; final int[] fileRows = new int[nfiles]; ArrayList> tasks = new ArrayList<>(nfiles); - for( int i=0; i { @@ -179,15 +172,15 @@ private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handl awaitFileTasks(tasks, fname); int nrow = 0; - for( int i=0; i ordered = new ArrayList<>(); - for( int i=0; i= 0) ? estnnz : (long) nrow * ncol; MatrixBlock ret = createOutputMatrixBlock(nrow, ncol, Math.max(nrow, 1), lestnnz, true, false); - if( nrow > 0 && ncol > 0 ) + if(nrow > 0 && ncol > 0) fillDense(ret, ordered); ret.recomputeNonZeros(_numThreads); ret.examSparsity(); @@ -195,16 +188,15 @@ private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handl } /** - * Run one decode task per data file on the shared common thread pool and await - * completion. Full parallelism is requested (the task count, one per data file, - * naturally caps concurrency); this avoids the per-thread pool-size caching in - * {@code CommonThreadPool.get(k)} that could otherwise throttle this reader to a - * smaller pool created earlier on the same thread. + * Run one decode task per data file on the shared common thread pool and await completion. Full parallelism is + * requested (the task count, one per data file, naturally caps concurrency); this avoids the per-thread pool-size + * caching in {@code CommonThreadPool.get(k)} that could otherwise throttle this reader to a smaller pool created + * earlier on the same thread. */ private void awaitFileTasks(List> tasks, String fname) throws IOException { ExecutorService pool = CommonThreadPool.get(_numThreads); try { - for( Future f : pool.invokeAll(tasks) ) + for(Future f : pool.invokeAll(tasks)) f.get(); } catch(Exception ex) { diff --git a/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java b/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java index 55ea8a54297..602b540c407 100644 --- a/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java +++ b/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java @@ -39,60 +39,54 @@ import io.delta.kernel.utils.CloseableIterator; /** - * Single-threaded native Delta Lake writer for matrices, built on the - * Spark-free Delta Kernel library. It creates a Delta table at the target - * directory with an all-double schema {@code c0..c(n-1)} (replacing any existing - * table at that path), streams the {@link MatrixBlock} rows as columnar batches - * into parquet data files via the kernel's default engine, and commits the - * corresponding add-file actions to the transaction log. + * Single-threaded native Delta Lake writer for matrices, built on the Spark-free Delta Kernel library. It creates a + * Delta table at the target directory with an all-double schema {@code c0..c(n-1)} (replacing any existing table at + * that path), streams the {@link MatrixBlock} rows as columnar batches into parquet data files via the kernel's default + * engine, and commits the corresponding add-file actions to the transaction log. */ public class WriterDelta extends MatrixWriter { @Override public void writeMatrixToHDFS(MatrixBlock src, String fname, long rlen, long clen, int blen, long nnz, boolean diag) - throws IOException - { - if( src.getNumRows() != rlen || src.getNumColumns() != clen ) - throw new IOException("Matrix dimensions mismatch with metadata: (" - + src.getNumRows() + "x" + src.getNumColumns() + ") vs (" + rlen + "x" + clen + ")."); + throws IOException { + if(src.getNumRows() != rlen || src.getNumColumns() != clen) + throw new IOException("Matrix dimensions mismatch with metadata: (" + src.getNumRows() + "x" + + src.getNumColumns() + ") vs (" + rlen + "x" + clen + ")."); int ncol = (int) clen; int nrow = (int) rlen; int batchRows = ConfigurationManager.getDeltaWriterBatchSize(); - //fast path: a contiguous dense block lets the column views read straight - //from the backing double[] (avoids per-cell MatrixBlock.get dispatch). - double[] dense = (!src.isInSparseFormat() && src.getDenseBlock() != null - && src.getDenseBlock().isContiguous()) ? src.getDenseBlockValues() : null; - //size data files adaptively (toward one file per parallel reader) for faster parallel reads. - //Delta writes every cell as a double, so size by the dense footprint rather than the (possibly - //sparse) in-memory size, which would understate the on-disk table for sparse inputs. + // fast path: a contiguous dense block lets the column views read straight + // from the backing double[] (avoids per-cell MatrixBlock.get dispatch). + double[] dense = (!src.isInSparseFormat() && src.getDenseBlock() != null && + src.getDenseBlock().isContiguous()) ? src.getDenseBlockValues() : null; + // size data files adaptively (toward one file per parallel reader) for faster parallel reads. + // Delta writes every cell as a double, so size by the dense footprint rather than the (possibly + // sparse) in-memory size, which would understate the on-disk table for sparse inputs. long estimatedBytes = (long) nrow * ncol * 8L; Engine engine = DeltaKernelUtils.createWriteEngine(estimatedBytes); - DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), - buildSchema(ncol), new MatrixBatchIterator(src, dense, nrow, ncol, batchRows)); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), buildSchema(ncol), + new MatrixBatchIterator(src, dense, nrow, ncol, batchRows)); } @Override - public void writeEmptyMatrixToHDFS(String fname, long rlen, long clen, int blen) - throws IOException - { - //empty table: create with schema but no data files + public void writeEmptyMatrixToHDFS(String fname, long rlen, long clen, int blen) throws IOException { + // empty table: create with schema but no data files Engine engine = DeltaKernelUtils.createEngine(); - DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), - buildSchema((int) clen), CloseableIterable.emptyIterable().iterator()); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), buildSchema((int) clen), + CloseableIterable.emptyIterable().iterator()); } private static StructType buildSchema(int ncol) { StructType schema = new StructType(); - for( int c=0; c stream, long rlen, long clen, int blen) - throws IOException - { + public long writeMatrixFromStream(String fname, OOCStream stream, long rlen, long clen, + int blen) throws IOException { throw new UnsupportedOperationException("Out-of-core stream write is not supported for the Delta format."); } @@ -122,18 +116,18 @@ public boolean hasNext() { @Override public FilteredColumnarBatch next() { - if( !hasNext() ) + if(!hasNext()) throw new NoSuchElementException(); int size = Math.min(_batchRows, _nrow - _pos); ColumnarBatch batch = new MatrixColumnarBatch(_mb, _dense, _schema, _pos, size, _ncol); _pos += size; - //no selection vector: all rows in the batch are written + // no selection vector: all rows in the batch are written return new FilteredColumnarBatch(batch, Optional.empty()); } @Override public void close() { - //nothing to release + // nothing to release } } @@ -162,7 +156,7 @@ public StructType getSchema() { @Override public ColumnVector getColumnVector(int ordinal) { - if( ordinal < 0 || ordinal >= _ncol ) + if(ordinal < 0 || ordinal >= _ncol) throw new IndexOutOfBoundsException("column ordinal " + ordinal); return new MatrixColumnVector(_mb, _dense, _rowStart, _size, _ncol, ordinal); } @@ -208,16 +202,14 @@ public boolean isNullAt(int rowId) { @Override public double getDouble(int rowId) { - //dense contiguous single block => index fits in int (getDenseBlockValues - //is only handed over for single-block dense matrices) - return (_dense != null) - ? _dense[(_rowStart + rowId) * _ncol + _col] - : _mb.get(_rowStart + rowId, _col); + // dense contiguous single block => index fits in int (getDenseBlockValues + // is only handed over for single-block dense matrices) + return (_dense != null) ? _dense[(_rowStart + rowId) * _ncol + _col] : _mb.get(_rowStart + rowId, _col); } @Override public void close() { - //nothing to release + // nothing to release } } } diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java index 5f478979104..0bf9f7d87ba 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java @@ -977,7 +977,7 @@ public static MatrixBlock rmempty(MatrixBlock in, MatrixBlock ret, boolean rows, if(ret == null) ret = new MatrixBlock(); MatrixBlock ret2 = rmemptyEarlyAbort(in, ret, rows, emptyReturn, select); - if(ret2 != null ) + if(ret2 != null) return ret2; // core removeEmpty return rmemptyUnsafe(in, ret, rows, emptyReturn, select); @@ -985,7 +985,7 @@ public static MatrixBlock rmempty(MatrixBlock in, MatrixBlock ret, boolean rows, public static MatrixBlock rmemptyUnsafe(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, MatrixBlock select) { - if( rows ) + if(rows) return removeEmptyRows(in, ret, select, emptyReturn); else // cols return removeEmptyColumns(in, ret, select, emptyReturn); @@ -999,14 +999,15 @@ public static MatrixBlock rmemptyUnsafe(MatrixBlock in, MatrixBlock ret, boolean * @param rows If removing based on rows, or columns * @param emptyReturn Return a row/column of zeros for empty input * @param select An optional selection vector - * @return The early-abort result, or {@code null} if no early termination applies and the caller must continue. - * For the select-all case the returned block is the input {@code in} itself (a shallow alias, not a copy). + * @return The early-abort result, or {@code null} if no early termination applies and the caller must continue. For + * the select-all case the returned block is the input {@code in} itself (a shallow alias, not a copy). */ - public static MatrixBlock rmemptyEarlyAbort(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, MatrixBlock select){ - //check for empty inputs - //(the semantics of removeEmpty are that for an empty m-by-n matrix, the output - //is an empty 1-by-n or m-by-1 matrix because we don't allow matrices with dims 0) - if( in.isEmptyBlock(false) && select == null ) { + public static MatrixBlock rmemptyEarlyAbort(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, + MatrixBlock select) { + // check for empty inputs + // (the semantics of removeEmpty are that for an empty m-by-n matrix, the output + // is an empty 1-by-n or m-by-1 matrix because we don't allow matrices with dims 0) + if(in.isEmptyBlock(false) && select == null) { int n = emptyReturn ? 1 : 0; if( rows ) ret.reset(n, in.clen, in.sparse); @@ -3651,7 +3652,7 @@ private static MatrixBlock removeEmptyRows(MatrixBlock in, MatrixBlock ret, Matr /** * Remove selected rows, based on the boolean array given. Note this function is internal use only, and require a * boolean vector to be constructed first. - * + * * @param in Input to remove rows from * @param ret Output to assign the result into * @param emptyReturn If the output is allowed to be empty. @@ -3672,9 +3673,9 @@ public static MatrixBlock removeEmptyRows(MatrixBlock in, MatrixBlock ret, boole ret.reset(rlen2, n, sp); if( in.isEmptyBlock(false) ) return ret; - - if( SHALLOW_COPY_REORG && m == rlen2 && selectNull ) { - // the condition m==rlen2 is not enough with non-empty 1-row input but empty + + if(SHALLOW_COPY_REORG && m == rlen2 && selectNull) { + // the condition m==rlen2 is not enough with non-empty 1-row input but empty // 1-row select vector because if emptyReturn should output a single empty row ret.sparse = in.sparse; if( ret.sparse ) @@ -3714,10 +3715,9 @@ else if( !in.sparse && !ret.sparse ) //DENSE <- DENSE ci++; } } - - //check sparsity - ret.nonZeros = (selectNull) ? - in.nonZeros : ret.recomputeNonZeros(); + + // check sparsity + ret.nonZeros = (selectNull) ? in.nonZeros : ret.recomputeNonZeros(); ret.examSparsity(); return ret; diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java index 7525dab2f7f..c450ecbe1f5 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java @@ -4756,13 +4756,13 @@ public static double computeIQMCorrection(double sum, double sum_wt, } /** - * Pick the quantiles out of this matrix. If this matrix contains two columns it is weighted quantile picking. - * If a single column it is unweighted. - * + * Pick the quantiles out of this matrix. If this matrix contains two columns it is weighted quantile picking. If a + * single column it is unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantiles The quantiles to pick - * @param ret The result matrix + * @param ret The result matrix * @return The result matrix */ public final MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret) { @@ -4792,11 +4792,11 @@ public MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret, boolean av } /** - * Pick the median value from this matrix. If this matrix has two columns it is weighted picking using the - * weight column, otherwise it is unweighted over the single column. - * + * Pick the median value from this matrix. If this matrix has two columns it is weighted picking using the weight + * column, otherwise it is unweighted over the single column. + * * Note the values are assumed to be sorted. - * + * * @return The median value */ public double median() { @@ -4807,10 +4807,11 @@ public double median() { } /** - * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is unweighted. - * + * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is + * unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantile The quantile to pick * @return The quantile */ @@ -4819,12 +4820,13 @@ public final double pickValue(double quantile){ } /** - * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is unweighted. - * + * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is + * unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantile The quantile to pick - * @param average If the quantile is averaged. + * @param average If the quantile is averaged. * @return The quantile */ public final double pickValue(double quantile, boolean average) { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java index 9e3494a7d48..9ea39bd1e2f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java @@ -34,8 +34,8 @@ import java.util.function.Function; /** - * Small future implementation for OOC hot paths. It supports multiple synchronous subscribers without - * the completion-stage support of {@link java.util.concurrent.CompletableFuture}. + * Small future implementation for OOC hot paths. It supports multiple synchronous subscribers without the + * completion-stage support of {@link java.util.concurrent.CompletableFuture}. */ public class OOCFuture { private Subscriber _subscribers; @@ -240,7 +240,7 @@ private static void accept(Function mapper, Consu if(resultError == null) { try { @SuppressWarnings("unchecked") - R mapped = mapper == null ? (R)value : mapper.apply(value); + R mapped = mapper == null ? (R) value : mapper.apply(value); result = mapped; } catch(Throwable t) { @@ -345,8 +345,7 @@ public T get() throws InterruptedException, ExecutionException { } @Override - public T get(long timeout, TimeUnit unit) - throws InterruptedException, ExecutionException, TimeoutException { + public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { try { return mapper.apply(source.get(timeout, unit)); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java index 94411242df1..df981f40223 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java @@ -25,20 +25,22 @@ public class CloseableQueue { private final BlockingQueue queue = new LinkedBlockingQueue<>(); - private final Object POISON = new Object(); // sentinel + private final Object POISON = new Object(); // sentinel private volatile boolean closed = false; - public CloseableQueue() { } + public CloseableQueue() { + } /** * Enqueue if the queue is not closed. + * * @return false if already closed */ public boolean enqueueIfOpen(T task) throws InterruptedException { - if (task == null) + if(task == null) throw new IllegalArgumentException("null tasks not allowed"); - synchronized (this) { - if (closed) + synchronized(this) { + if(closed) return false; queue.put(task); } @@ -47,12 +49,12 @@ public boolean enqueueIfOpen(T task) throws InterruptedException { @SuppressWarnings("unchecked") public T take() throws InterruptedException { - if (closed && queue.isEmpty()) + if(closed && queue.isEmpty()) return null; Object x = queue.take(); - if (x == POISON) + if(x == POISON) return null; return (T) x; @@ -60,33 +62,31 @@ public T take() throws InterruptedException { /** * Poll with max timeout. - * @return item, or null if: - * - timeout, or - * - queue has been closed and this consumer reached its poison pill + * + * @return item, or null if: - timeout, or - queue has been closed and this consumer reached its poison pill */ @SuppressWarnings("unchecked") public T poll(long timeout, TimeUnit unit) throws InterruptedException { - if (closed && queue.isEmpty()) + if(closed && queue.isEmpty()) return null; Object x = queue.poll(timeout, unit); - if (x == null) - return null; // timeout + if(x == null) + return null; // timeout - if (x == POISON) + if(x == POISON) return null; return (T) x; } /** - * Close queue for N consumers. - * Each consumer will receive exactly one poison pill and then should stop. + * Close queue for N consumers. Each consumer will receive exactly one poison pill and then should stop. */ public boolean close() throws InterruptedException { - synchronized (this) { - if (closed) - return false; // idempotent + synchronized(this) { + if(closed) + return false; // idempotent closed = true; } queue.put(POISON); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java index ee02b28404c..ccc73ff6160 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java @@ -78,7 +78,7 @@ public void readFully(byte[] b) throws IOException { } @Override - public void readFully(byte [] b, int off, int len) throws IOException { + public void readFully(byte[] b, int off, int len) throws IOException { if(len < 0) throw new IndexOutOfBoundsException(); @@ -127,12 +127,12 @@ public int readUnsignedByte() throws IOException { @Override public short readShort() throws IOException { if(_count - _pos >= 2) { - short ret = (short)baToShort(_buff, _pos); + short ret = (short) baToShort(_buff, _pos); _pos += 2; return ret; } readFully(_tmp, 0, 2); - return (short)baToShort(_tmp, 0); + return (short) baToShort(_tmp, 0); } @Override @@ -142,7 +142,7 @@ public int readUnsignedShort() throws IOException { @Override public char readChar() throws IOException { - return (char)readUnsignedShort(); + return (char) readUnsignedShort(); } @Override @@ -222,7 +222,7 @@ public long readDoubleArray(int len, double[] varr) throws IOException { @Override public long readSparseRows(int rlen, long nnz, SparseBlock rows) throws IOException { if(rows instanceof SparseBlockCSR) { - ((SparseBlockCSR)rows).initSparse(rlen, (int)nnz, this); + ((SparseBlockCSR) rows).initSparse(rlen, (int) nnz, this); return nnz; } @@ -256,7 +256,7 @@ private void refill() throws IOException { } private int getRefillLength() { - int pageOffset = (int)(_filePos & PAGE_MASK); + int pageOffset = (int) (_filePos & PAGE_MASK); if(pageOffset == 0) return _bufflen; return Math.min(_bufflen, PAGE_SIZE - pageOffset); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java index 9854a94586b..22b7b23706c 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java @@ -65,7 +65,7 @@ long getFlushedPosition() { public void write(int b) throws IOException { if(_count >= _bufflen) flushBuffer(); - _buff[_count++] = (byte)b; + _buff[_count++] = (byte) b; _position++; } @@ -109,7 +109,7 @@ public void close() throws IOException { public void writeBoolean(boolean v) throws IOException { if(_count >= _bufflen) flushBuffer(); - _buff[_count++] = (byte)(v ? 1 : 0); + _buff[_count++] = (byte) (v ? 1 : 0); _position++; } @@ -153,7 +153,7 @@ public void writeFloat(float v) throws IOException { public void writeByte(int v) throws IOException { if(_count + 1 > _bufflen) flushBuffer(); - _buff[_count++] = (byte)v; + _buff[_count++] = (byte) v; _position++; } @@ -194,18 +194,18 @@ public void writeUTF(String s) throws IOException { flushBuffer(); final char c = s.charAt(i); if(c >= 0x0001 && c <= 0x007F) { - _buff[_count++] = (byte)c; + _buff[_count++] = (byte) c; _position++; } else if(c >= 0x0800) { - _buff[_count++] = (byte)(0xE0 | ((c >> 12) & 0x0F)); - _buff[_count++] = (byte)(0x80 | ((c >> 6) & 0x3F)); - _buff[_count++] = (byte)(0x80 | (c & 0x3F)); + _buff[_count++] = (byte) (0xE0 | ((c >> 12) & 0x0F)); + _buff[_count++] = (byte) (0x80 | ((c >> 6) & 0x3F)); + _buff[_count++] = (byte) (0x80 | (c & 0x3F)); _position += 3; } else { - _buff[_count++] = (byte)(0xC0 | ((c >> 6) & 0x1F)); - _buff[_count++] = (byte)(0x80 | (c & 0x3F)); + _buff[_count++] = (byte) (0xC0 | ((c >> 6) & 0x1F)); + _buff[_count++] = (byte) (0x80 | (c & 0x3F)); _position += 2; } } @@ -213,7 +213,7 @@ else if(c >= 0x0800) { @Override public void writeDoubleArray(int len, double[] varr) throws IOException { - for(int i = 0; i < len; ) { + for(int i = 0; i < len;) { if(_count >= _bufflen) flushBuffer(); int lblen = Math.min(len - i, (_bufflen - _count) / 8); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java index ab28df0ef0f..6c13a062561 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java @@ -50,10 +50,10 @@ public interface OOCIOHandler { void registerSourceLocation(BlockKey key, SourceBlockDescriptor descriptor); /** - * Schedule an asynchronous read from an external source into the provided target stream. - * The returned future completes when either EOF is reached or the requested byte budget - * is exhausted. When the budget is reached and keepOpenOnLimit is true, the target stream - * is kept open and a continuation token is provided so the caller can resume. + * Schedule an asynchronous read from an external source into the provided target stream. The returned future + * completes when either EOF is reached or the requested byte budget is exhausted. When the budget is reached and + * keepOpenOnLimit is true, the target stream is kept open and a continuation token is provided so the caller can + * resume. */ CompletableFuture scheduleSourceRead(SourceReadRequest request); @@ -62,7 +62,8 @@ public interface OOCIOHandler { */ CompletableFuture continueSourceRead(SourceReadContinuation continuation, long maxBytesInFlight); - interface SourceReadContinuation {} + interface SourceReadContinuation { + } class SourceReadRequest { public final String path; @@ -75,9 +76,8 @@ class SourceReadRequest { public final boolean keepOpenOnLimit; public final OOCStream target; - public SourceReadRequest(String path, Types.FileFormat format, long rows, long cols, - int blen, long estNnz, long maxBytesInFlight, boolean keepOpenOnLimit, - OOCStream target) { + public SourceReadRequest(String path, Types.FileFormat format, long rows, long cols, int blen, long estNnz, + long maxBytesInFlight, boolean keepOpenOnLimit, OOCStream target) { this.path = path; this.format = format; this.rows = rows; @@ -113,9 +113,8 @@ class SourceBlockDescriptor { public final int recordLength; public final long serializedSize; - public SourceBlockDescriptor(String path, Types.FileFormat format, - MatrixIndexes indexes, long offset, int recordLength, - long serializedSize) { + public SourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, + int recordLength, long serializedSize) { this.path = path; this.format = format; this.indexes = indexes; @@ -129,8 +128,8 @@ class GroupSourceBlockDescriptor extends SourceBlockDescriptor { public final List blocks; public final int count; - public GroupSourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, int recordLength, - long serializedSize, List blocks) { + public GroupSourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, + int recordLength, long serializedSize, List blocks) { super(path, format, indexes, offset, recordLength, serializedSize); this.blocks = blocks; this.count = blocks.size(); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java index e9487f7bd45..55ac038205f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java @@ -81,7 +81,7 @@ public class OOCMatrixIOHandler implements OOCIOHandler { private final AtomicLong _readSeq = new AtomicLong(0); // Spill related structures - private final ConcurrentHashMap _spillLocations = new ConcurrentHashMap<>(); + private final ConcurrentHashMap _spillLocations = new ConcurrentHashMap<>(); private final ConcurrentHashMap _partitions = new ConcurrentHashMap<>(); private final ConcurrentHashMap _sourceLocations = new ConcurrentHashMap<>(); private final AtomicInteger _partitionCounter = new AtomicInteger(0); @@ -97,38 +97,21 @@ public class OOCMatrixIOHandler implements OOCIOHandler { @SuppressWarnings("unchecked") public OOCMatrixIOHandler() { this._spillDir = LocalFileUtils.getUniqueWorkingDir("ooc_stream"); - _writeExec = new ThreadPoolExecutor( - WRITER_SIZE, - WRITER_SIZE, - 0L, - TimeUnit.MILLISECONDS, + _writeExec = new ThreadPoolExecutor(WRITER_SIZE, WRITER_SIZE, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); - _readExec = new ThreadPoolExecutor( - READER_SIZE, - READER_SIZE, - 0L, - TimeUnit.MILLISECONDS, + _readExec = new ThreadPoolExecutor(READER_SIZE, READER_SIZE, 0L, TimeUnit.MILLISECONDS, new PriorityBlockingQueue<>()); - _srcReadExec = new ThreadPoolExecutor( - READER_SIZE, - READER_SIZE, - 0L, - TimeUnit.MILLISECONDS, - new ArrayBlockingQueue<>(100000)); - _deleteExec = new ThreadPoolExecutor( - 1, - 1, - 0L, - TimeUnit.MILLISECONDS, + _srcReadExec = new ThreadPoolExecutor(READER_SIZE, READER_SIZE, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); + _deleteExec = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); _q = new CloseableQueue[WRITER_SIZE]; _wCtr = new AtomicLong(0); - _started = new AtomicBoolean(false); + _started = new AtomicBoolean(false); } private synchronized void start() { - if (_started.compareAndSet(false, true)) { - for (int i = 0; i < WRITER_SIZE; i++) { + if(_started.compareAndSet(false, true)) { + for(int i = 0; i < WRITER_SIZE; i++) { final int finalIdx = i; _q[i] = new CloseableQueue<>(); _writeExec.submit(() -> evictTask(_q[finalIdx])); @@ -139,7 +122,7 @@ private synchronized void start() { @Override public void shutdown() { boolean started = _started.get(); - if (started) { + if(started) { try { for(int i = 0; i < WRITER_SIZE; i++) { if(_q[i] != null) @@ -159,7 +142,7 @@ public void shutdown() { _deleteExec.shutdownNow(); _spillLocations.clear(); _partitions.clear(); - if (started) + if(started) LocalFileUtils.deleteFileIfExists(_spillDir); } @@ -169,7 +152,7 @@ public CompletableFuture scheduleEviction(BlockEntry block) { CompletableFuture future = new CompletableFuture<>(); try { long q = _wCtr.getAndAdd(block.getSize()) / OVERFLOW; - int i = (int)(q % WRITER_SIZE); + int i = (int) (q % WRITER_SIZE); if(!_q[i].enqueueIfOpen(new Tuple2<>(block, future))) future.completeExceptionally(new DMLRuntimeException("OOC writer queue is closed")); } @@ -189,7 +172,8 @@ public OOCFuture scheduleRead(final BlockEntry block) { ReadTask task = new ReadTask(block, future, _readSeq.getAndIncrement(), pinnedPartitionId); _pendingReads.put(block.getKey(), task); _readExec.execute(task); - } catch (RejectedExecutionException e) { + } + catch(RejectedExecutionException e) { unpinPartitionForRead(pinnedPartitionId); _pendingReads.remove(block.getKey()); future.completeExceptionally(e); @@ -199,12 +183,12 @@ public OOCFuture scheduleRead(final BlockEntry block) { @Override public void prioritizeRead(BlockKey key, double priority) { - if (priority == 0) + if(priority == 0) return; ReadTask task = _pendingReads.get(key); - if (task == null) + if(task == null) return; - if (_readExec.getQueue().remove(task)) { + if(_readExec.getQueue().remove(task)) { task.addPriority(priority); _readExec.getQueue().offer(task); } @@ -228,8 +212,9 @@ public CompletableFuture scheduleSourceRead(SourceReadRequest } @Override - public CompletableFuture continueSourceRead(SourceReadContinuation continuation, long maxBytesInFlight) { - if (!(continuation instanceof SourceReadState state)) { + public CompletableFuture continueSourceRead(SourceReadContinuation continuation, + long maxBytesInFlight) { + if(!(continuation instanceof SourceReadState state)) { CompletableFuture failed = new CompletableFuture<>(); failed.completeExceptionally(new DMLRuntimeException("Unsupported continuation type: " + continuation)); return failed; @@ -240,8 +225,8 @@ public CompletableFuture continueSourceRead(SourceReadContinua private CompletableFuture submitSourceRead(SourceReadRequest request, SourceReadState state, long maxBytesInFlight) { if(request.format != Types.FileFormat.BINARY) - return CompletableFuture.failedFuture( - new DMLRuntimeException("Unsupported format for source read: " + request.format)); + return CompletableFuture + .failedFuture(new DMLRuntimeException("Unsupported format for source read: " + request.format)); return readBinarySourceParallel(request, state, maxBytesInFlight); } @@ -335,17 +320,18 @@ private CompletableFuture readBinarySourceParallel(SourceReadR return result; } - private void completeResult(CompletableFuture future, AtomicLong bytesRead, AtomicBoolean budgetHit, - AtomicReference error, SourceReadRequest request, Path[] files, AtomicLongArray filePositions, - AtomicIntegerArray completed, ConcurrentLinkedDeque descriptors) { + private void completeResult(CompletableFuture future, AtomicLong bytesRead, + AtomicBoolean budgetHit, AtomicReference error, SourceReadRequest request, Path[] files, + AtomicLongArray filePositions, AtomicIntegerArray completed, + ConcurrentLinkedDeque descriptors) { Throwable err = error.get(); - if (err != null) { + if(err != null) { future.completeExceptionally(err instanceof Exception ? err : new Exception(err)); return; } try { - if (budgetHit.get()) { + if(budgetHit.get()) { if(!request.keepOpenOnLimit) { closeTarget(request.target, false); } @@ -365,29 +351,29 @@ private void completeResult(CompletableFuture future, AtomicLo private void readSequenceFile(JobConf job, Path path, SourceReadRequest request, int fileIdx, AtomicLongArray filePositions, AtomicIntegerArray completed, AtomicBoolean stop, AtomicBoolean budgetHit, - AtomicLong bytesRead, long byteLimit, Object budgetLock, ConcurrentLinkedDeque descriptors) - throws IOException { + AtomicLong bytesRead, long byteLimit, Object budgetLock, + ConcurrentLinkedDeque descriptors) throws IOException { MatrixIndexes key = new MatrixIndexes(); try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { long pos = filePositions.get(fileIdx); - if (pos > 0) + if(pos > 0) reader.seek(pos); long ioStart = DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; while(!stop.get()) { long recordStart = reader.getPosition(); MatrixBlock value = new MatrixBlock(); - if (!reader.next(key, value)) + if(!reader.next(key, value)) break; long recordEnd = reader.getPosition(); long blockSize = value.getExactSerializedSize(); boolean shouldBreak = false; synchronized(budgetLock) { - if (stop.get()) + if(stop.get()) shouldBreak = true; - else if (bytesRead.get() + blockSize > byteLimit) { + else if(bytesRead.get() + blockSize > byteLimit) { stop.set(true); budgetHit.set(true); shouldBreak = true; @@ -398,7 +384,7 @@ else if (bytesRead.get() + blockSize > byteLimit) { MatrixIndexes outIdx = new MatrixIndexes(key); IndexedMatrixValue imv = new IndexedMatrixValue(outIdx, value); SourceBlockDescriptor descriptor = new SourceBlockDescriptor(path.toString(), request.format, outIdx, - recordStart, (int)(recordEnd - recordStart), blockSize); + recordStart, (int) (recordEnd - recordStart), blockSize); if(request.target instanceof SourceOOCStream src) src.enqueue(imv, descriptor); @@ -407,22 +393,23 @@ else if (bytesRead.get() + blockSize > byteLimit) { descriptors.add(descriptor); filePositions.set(fileIdx, reader.getPosition()); - if (DMLScript.OOC_LOG_EVENTS) { + if(DMLScript.OOC_LOG_EVENTS) { long currTime = System.nanoTime(); OOCEventLog.onDiskReadEvent(_srcReadCallerId, ioStart, currTime, blockSize); ioStart = currTime; } - if (shouldBreak) + if(shouldBreak) break; // Note that we knowingly go over limit, which could result in READER_SIZE*8MB overshoot } - if (!stop.get()) + if(!stop.get()) completed.set(fileIdx, 1); } } - private void closeTarget(org.apache.sysds.runtime.instructions.ooc.OOCStream target, boolean close) { + private void closeTarget(org.apache.sysds.runtime.instructions.ooc.OOCStream target, + boolean close) { if(close) { try { target.closeInput(); @@ -437,10 +424,10 @@ private void loadFromDisk(BlockEntry block) { String key = block.getKey().toFileKey(); SourceBlockDescriptor src = _sourceLocations.get(block.getKey()); - if (src != null) { + if(src != null) { long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; loadFromSource(block, src); - if (DMLScript.OOC_STATISTICS) { + if(DMLScript.OOC_STATISTICS) { Statistics.incrementOOCLoadFromDisk(); Statistics.accumulateOOCLoadFromDiskTime(System.nanoTime() - ioStart); Statistics.accumulateOOCLoadFromDiskBytes(block.getSize()); @@ -451,34 +438,36 @@ private void loadFromDisk(BlockEntry block) { long ioDuration = 0; // 1. find the blocks address (spill location) SpillLocation sloc = _spillLocations.get(key); - if (sloc == null) + if(sloc == null) throw new DMLRuntimeException("Failed to load spill location for: " + key); PartitionFile partFile = _partitions.get(sloc.partitionId); - if (partFile == null) + if(partFile == null) throw new DMLRuntimeException("Failed to load partition for: " + sloc.partitionId); String filename = partFile.filePath; SpillableObject obj; - try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { + try(RandomAccessFile raf = new RandomAccessFile(filename, "r")) { raf.seek(sloc.offset); DataInput dis = new OOCBufferedDataInputStream(raf); long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; obj = SpillableObjectRegistry.read(dis); - if (DMLScript.OOC_STATISTICS) + if(DMLScript.OOC_STATISTICS) ioDuration = System.nanoTime() - ioStart; - } catch (ClosedByInterruptException ignored) { + } + catch(ClosedByInterruptException ignored) { return; - } catch (IOException e) { + } + catch(IOException e) { throw new RuntimeException(e); } block.setDataUnsafe(obj); - if (DMLScript.OOC_STATISTICS) { + if(DMLScript.OOC_STATISTICS) { Statistics.incrementOOCLoadFromDisk(); Statistics.accumulateOOCLoadFromDiskTime(ioDuration); Statistics.accumulateOOCLoadFromDiskBytes(block.getSize()); @@ -486,22 +475,23 @@ private void loadFromDisk(BlockEntry block) { } private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { - if (src.format != Types.FileFormat.BINARY) + if(src.format != Types.FileFormat.BINARY) throw new DMLRuntimeException("Unsupported format for source read: " + src.format); JobConf job = new JobConf(ConfigurationManager.getCachedJobConf()); Path path = new Path(src.path); - if (src instanceof OOCIOHandler.GroupSourceBlockDescriptor gsrc) { + if(src instanceof OOCIOHandler.GroupSourceBlockDescriptor gsrc) { List values = new ArrayList<>(gsrc.count); try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { reader.seek(gsrc.offset); - for (int i = 0; i < gsrc.blocks.size(); i++) { + for(int i = 0; i < gsrc.blocks.size(); i++) { SourceBlockDescriptor d = gsrc.blocks.get(i); MatrixIndexes ix = new MatrixIndexes(); MatrixBlock mb = new MatrixBlock(); - if (!reader.next(ix, mb)) - throw new DMLRuntimeException("Failed to read source block at offset " + d.offset + " in " + d.path); + if(!reader.next(ix, mb)) + throw new DMLRuntimeException( + "Failed to read source block at offset " + d.offset + " in " + d.path); values.add(new IndexedMatrixValue(ix, mb)); } } @@ -516,8 +506,9 @@ private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { reader.seek(src.offset); - if (!reader.next(ix, mb)) - throw new DMLRuntimeException("Failed to read source block at offset " + src.offset + " in " + src.path); + if(!reader.next(ix, mb)) + throw new DMLRuntimeException( + "Failed to read source block at offset " + src.offset + " in " + src.path); } catch(IOException e) { throw new DMLRuntimeException(e); @@ -530,7 +521,7 @@ private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { private void evictTask(CloseableQueue>> q) { long byteCtr = 0; - while (!q.isFinished()) { + while(!q.isFinished()) { // --- 1. WRITE PHASE --- int partitionId = _partitionCounter.getAndIncrement(); @@ -572,17 +563,17 @@ private void evictTask(CloseableQueue } byteCtr += wrote; - if (byteCtr >= MAX_PARTITION_SIZE) { + if(byteCtr >= MAX_PARTITION_SIZE) { closePartition = true; byteCtr = 0; break; } - if (DMLScript.OOC_LOG_EVENTS) + if(DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskWriteEvent(_evictCallerId, ioStart, System.nanoTime(), wrote); } - if (!closePartition && q.close()) { + if(!closePartition && q.close()) { while((tpl = q.take()) != null) { long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; BlockEntry entry = tpl._1(); @@ -595,7 +586,7 @@ private void evictTask(CloseableQueue Statistics.accumulateOOCEvictionWriteTime(System.nanoTime() - ioStart); } - if (DMLScript.OOC_LOG_EVENTS) + if(DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskWriteEvent(_evictCallerId, ioStart, System.nanoTime(), wrote); } } @@ -617,13 +608,13 @@ private void evictTask(CloseableQueue } private long writeOut(int partitionId, BlockEntry entry, CompletableFuture future, - OOCBufferedDataOutputStream dos, - ConcurrentLinkedDeque>> flushQueue) throws IOException { + OOCBufferedDataOutputStream dos, ConcurrentLinkedDeque>> flushQueue) + throws IOException { String key = entry.getKey().toFileKey(); boolean alreadySpilled = _spillLocations.containsKey(key); - if (!alreadySpilled) { + if(!alreadySpilled) { long offsetBefore = dos.getPosition(); if(future.isCancelled()) @@ -656,9 +647,10 @@ private long writeOut(int partitionId, BlockEntry entry, CompletableFuture return 0; } - private void flushQueue(long offset, ConcurrentLinkedDeque>> flushQueue) { + private void flushQueue(long offset, + ConcurrentLinkedDeque>> flushQueue) { Tuple3> tmp; - while ((tmp = flushQueue.peek()) != null && tmp._2() <= offset) { + while((tmp = flushQueue.peek()) != null && tmp._2() <= offset) { flushQueue.poll(); tmp._3().complete(null); } @@ -777,12 +769,14 @@ public void run() { try { long ioStart = DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; loadFromDisk(_block); - if (DMLScript.OOC_LOG_EVENTS) + if(DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskReadEvent(_readCallerId, ioStart, System.nanoTime(), _block.getSize()); _future.complete(_block); - } catch (Throwable e) { + } + catch(Throwable e) { _future.completeExceptionally(e); - } finally { + } + finally { unpinPartitionForRead(_pinnedPartitionId); } } @@ -790,15 +784,12 @@ public void run() { @Override public int compareTo(ReadTask other) { int byPriority = Double.compare(other._priority, _priority); - if (byPriority != 0) + if(byPriority != 0) return byPriority; return Long.compare(_sequence, other._sequence); } } - - - private static class SpillLocation { // structure of spillLocation: file, offset final int partitionId; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java index a93b1d18f2a..3458838c071 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java @@ -24,8 +24,10 @@ import java.io.IOException; public interface SpillableObject { - boolean tryWrite(DataOutput out) throws IOException; - void read(DataInput in) throws IOException; + boolean tryWrite(DataOutput out) throws IOException; + + void read(DataInput in) throws IOException; + long size(); default void discard() { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java index d8ff79dd797..0d318d83d94 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java @@ -34,14 +34,16 @@ public interface OOCCacheScheduler { /** * Requests a single block from the cache. + * * @param key the requested key associated to the block * @return the available BlockEntry */ CompletableFuture request(BlockKey key); /** - * Tries to request a single block from the cache. - * Immediately returns the entry if present, otherwise null without scheduling reads. + * Tries to request a single block from the cache. Immediately returns the entry if present, otherwise null without + * scheduling reads. + * * @param key the requested key associated to the block * @return the available BlockEntry or null */ @@ -52,14 +54,16 @@ default BlockEntry tryRequest(BlockKey key) { /** * Requests a list of blocks from the cache that must be available at the same time. + * * @param keys the requested keys associated to the block * @return the list of available BlockEntries */ CompletableFuture> request(List keys); /** - * Tries to request a list of blocks from the cache that must be available at the same time. - * Immediately returns the list of entries if present, otherwise null without scheduling reads. + * Tries to request a list of blocks from the cache that must be available at the same time. Immediately returns the + * list of entries if present, otherwise null without scheduling reads. + * * @param keys the requested keys associated to the block * @return the list of available BlockEntries */ @@ -76,24 +80,25 @@ default BlockEntry tryRequest(BlockKey key) { List tryRequestAnyOf(List keys, int n, List selectionOut); /** - * Adds the given priority to any pending request accessing the key. - * Multi-requests are prioritized partially. + * Adds the given priority to any pending request accessing the key. Multi-requests are prioritized partially. */ void prioritize(BlockKey key, double priority); /** - * Places a new block in the cache. Note that objects are immutable and cannot be overwritten. - * The object data should now only be accessed via cache, as ownership has been transferred. - * @param key the associated key of the block + * Places a new block in the cache. Note that objects are immutable and cannot be overwritten. The object data + * should now only be accessed via cache, as ownership has been transferred. + * + * @param key the associated key of the block * @param data the block data * @param size the size of the data */ BlockKey put(BlockKey key, Object data, long size); /** - * Places a new block in the cache and returns a pinned handle. - * Note that objects are immutable and cannot be overwritten. - * @param key the associated key of the block + * Places a new block in the cache and returns a pinned handle. Note that objects are immutable and cannot be + * overwritten. + * + * @param key the associated key of the block * @param data the block data * @param size the size of the data */ @@ -101,8 +106,11 @@ default BlockEntry tryRequest(BlockKey key) { interface HandoverHandle { BlockKey getKey(); + boolean isCommitted(); + CompletableFuture getCompletionFuture(); + OOCStream.QueueCallback reclaim(); } @@ -131,27 +139,30 @@ BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.SourceBlockDescriptor descriptor); /** - * Notifies the cache that there is another reference to the same block key. - * This will prevent forget(key) from removing the block from cache. - * A block will only be forgotten after all referencing instances called forget(key). + * Notifies the cache that there is another reference to the same block key. This will prevent forget(key) from + * removing the block from cache. A block will only be forgotten after all referencing instances called forget(key). + * * @param key */ void addReference(BlockKey key); /** * Forgets a block from the cache. + * * @param key the associated key of the block */ void forget(BlockKey key); /** * Pins a BlockEntry in cache to prevent eviction. + * * @param entry the entry to be pinned */ void pin(BlockEntry entry); /** * Unpins a pinned block. + * * @param entry the entry to be unpinned */ void unpin(BlockEntry entry); @@ -192,8 +203,7 @@ BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, void updateLimits(long evictionLimit, long hardLimit); /** - * Creates a snapshot of the cache. - * Should only be used for debugging or diagnoses. + * Creates a snapshot of the cache. Should only be used for debugging or diagnoses. */ Collection snapshot(); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java index 96de368ccc9..9b0acc97b35 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java @@ -80,7 +80,7 @@ public class OOCLRUCacheScheduler implements OOCCacheScheduler { public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long hardLimit, long readBuffer) { this._ioHandler = ioHandler; this._cache = new LinkedHashMap<>(1024, 0.75f, true); - this._evictionCache = new HashMap<>(); + this._evictionCache = new HashMap<>(); this._deferredReadRequests = new DeferredReadQueue(); this._processingReadRequests = new ArrayDeque<>(); this._pendingHandovers = new ArrayDeque<>(); @@ -103,7 +103,7 @@ public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long har this._maintenanceNeedsIncr = new AtomicBoolean(false); this._callerId = DMLScript.OOC_LOG_EVENTS ? OOCEventLog.registerCaller("LRUCacheScheduler") : 0; - if (DMLScript.OOC_LOG_EVENTS) { + if(DMLScript.OOC_LOG_EVENTS) { OOCEventLog.putRunSetting("CacheEvictionLimit", _evictionLimit); OOCEventLog.putRunSetting("CacheHardLimit", _hardLimit); } @@ -111,7 +111,7 @@ public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long har @Override public CompletableFuture request(BlockKey key) { - if (!this._running) + if(!this._running) throw new IllegalStateException("Cache scheduler has been shut down."); Statistics.incrementOOCEvictionGet(); @@ -120,21 +120,21 @@ public CompletableFuture request(BlockKey key) { boolean couldPin = false; synchronized(this) { entry = _cache.get(key); - if (entry == null) + if(entry == null) entry = _evictionCache.get(key); - if (entry == null) + if(entry == null) throw new IllegalArgumentException("Could not find requested block with key " + key); synchronized(entry) { - if (entry.getState().isAvailable()) { - if (pinEntryWithAccounting(entry) == 0) + if(entry.getState().isAvailable()) { + if(pinEntryWithAccounting(entry) == 0) throw new IllegalStateException(); couldPin = true; } } } - if (couldPin) { + if(couldPin) { // Then we could pin the required entry and can terminate return CompletableFuture.completedFuture(entry); } @@ -184,7 +184,7 @@ public CompletableFuture> request(List keys) { } public CompletableFuture> request(List keys, boolean onlyIfAvailable) { - if (!this._running) + if(!this._running) throw new IllegalStateException("Cache scheduler has been shut down."); Statistics.incrementOOCEvictionGet(keys.size()); @@ -193,11 +193,11 @@ public CompletableFuture> request(List keys, boolean boolean allAvailable = true; synchronized(this) { - for (BlockKey key : keys) { + for(BlockKey key : keys) { BlockEntry entry = _cache.get(key); - if (entry == null) + if(entry == null) entry = _evictionCache.get(key); - if (entry == null) + if(entry == null) throw new IllegalArgumentException("Could not find requested block with key " + key); synchronized(entry) { @@ -217,7 +217,7 @@ public CompletableFuture> request(List keys, boolean } } - if (allAvailable) { + if(allAvailable) { // Then we could pin all entries return CompletableFuture.completedFuture(entries); } @@ -226,12 +226,12 @@ public CompletableFuture> request(List keys, boolean return null; // Schedule deferred read otherwise - final CompletableFuture> future = new CompletableFuture<>(); + final CompletableFuture> future = new CompletableFuture<>(); DeferredReadRequest request = new DeferredReadRequest(future, entries); - for (int i = 0; i < entries.size(); i++) { + for(int i = 0; i < entries.size(); i++) { BlockEntry entry = entries.get(i); synchronized(entry) { - if (entry.getState().isAvailable()) { + if(entry.getState().isAvailable()) { entry.addRetainHint(); request.markRetainHinted(i); } @@ -243,9 +243,9 @@ public CompletableFuture> request(List keys, boolean @Override public void prioritize(BlockKey key, double priority) { - if (!this._running) + if(!this._running) return; - if (priority == 0) + if(priority == 0) return; synchronized(this) { @@ -267,18 +267,18 @@ private void scheduleDeferredRead(DeferredReadRequest deferredReadRequest) { if(entry.getState().isAvailable()) readyCount++; BlockReadState state = _blockReads.get(entry.getKey()); - if (state != null) + if(state != null) score += state.priority; } - if (!deferredReadRequest.getEntries().isEmpty()) + if(!deferredReadRequest.getEntries().isEmpty()) score /= deferredReadRequest.getEntries().size(); - if (!deferredReadRequest.getEntries().isEmpty()) + if(!deferredReadRequest.getEntries().isEmpty()) score += ((double) readyCount) / deferredReadRequest.getEntries().size(); deferredReadRequest.setPriorityScore(score); _deferredReadRequests.add(deferredReadRequest); _deferredReadCountHint = _deferredReadRequests.size(); } - onCacheSizeChanged(true); // Apply pressure from deferred read demand. + onCacheSizeChanged(true); // Apply pressure from deferred read demand. onCacheSizeChanged(false); // Attempt to schedule deferred reads. } @@ -317,7 +317,8 @@ public void putSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.S } @Override - public BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.SourceBlockDescriptor descriptor) { + public BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, + OOCIOHandler.SourceBlockDescriptor descriptor) { return put(key, data, size, true, descriptor); } @@ -333,23 +334,24 @@ public void addReference(BlockKey key) { } } - private BlockEntry put(BlockKey key, Object data, long size, boolean pin, OOCIOHandler.SourceBlockDescriptor descriptor) { - if (!this._running) + private BlockEntry put(BlockKey key, Object data, long size, boolean pin, + OOCIOHandler.SourceBlockDescriptor descriptor) { + if(!this._running) throw new IllegalStateException(); - if (data == null) + if(data == null) throw new IllegalArgumentException(); - if (descriptor != null) + if(descriptor != null) _ioHandler.registerSourceLocation(key, descriptor); Statistics.incrementOOCEvictionPut(); BlockEntry entry = new BlockEntry(key, size, data); - if (descriptor != null) + if(descriptor != null) entry.setState(BlockState.WARM); - if (pin) + if(pin) entry.pin(); synchronized(this) { BlockEntry avail = _cache.putIfAbsent(key, entry); - if (avail != null || _evictionCache.containsKey(key)) + if(avail != null || _evictionCache.containsKey(key)) throw new IllegalStateException("Cannot overwrite existing entries: " + key); _cacheSize += size; if(pin) { @@ -364,7 +366,7 @@ private BlockEntry put(BlockKey key, Object data, long size, boolean pin, OOCIOH @Override public void forget(BlockKey key) { - if (!this._running) + if(!this._running) return; final MutableObject mEntry = new MutableObject<>(); BlockEntry entry; @@ -381,7 +383,7 @@ public void forget(BlockKey key) { return e; }); - if (mEntry.getValue() == null) { + if(mEntry.getValue() == null) { _evictionCache.compute(key, (k, e) -> { if(e == null) return null; @@ -395,10 +397,10 @@ public void forget(BlockKey key) { entry = mEntry.getValue(); - if (entry != null) { + if(entry != null) { synchronized(entry) { - shouldScheduleDeletion = entry.getState().isBackedByDisk() - || entry.getState() == BlockState.EVICTING; + shouldScheduleDeletion = entry.getState().isBackedByDisk() || + entry.getState() == BlockState.EVICTING; cacheSizeDelta = transitionMemState(entry, BlockState.REMOVED); if(entry.isPinned() && entry.getDataUnsafe() != null) _pinnedBytes -= entry.getSize(); @@ -408,9 +410,9 @@ public void forget(BlockKey key) { } } } - if (cacheSizeDelta != 0) + if(cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); - if (shouldScheduleDeletion) + if(shouldScheduleDeletion) _ioHandler.scheduleDeletion(entry); } @@ -424,11 +426,11 @@ public void pin(BlockEntry entry) { synchronized(this) { synchronized(entry) { int pinCount = pinEntryWithAccounting(entry); - if (pinCount == 0) + if(pinCount == 0) throw new IllegalStateException("Could not pin the requested entry: " + entry.getKey()); } // Access element in cache for Lru - //_cache.get(entry.getKey()); + // _cache.get(entry.getKey()); } } @@ -442,26 +444,26 @@ public void unpin(BlockEntry entry) { synchronized(entry) { if(!unpinEntryWithAccounting(entry)) return; - if (_cacheSize <= _evictionLimit) + if(_cacheSize <= _evictionLimit) return; // Nothing to do - if (entry.isPinned()) + if(entry.isPinned()) return; // Pin state changed so we cannot evict - if (entry.getState().isAvailable() && entry.getState().isBackedByDisk()) { - if (entry.getRetainHintCount() > 0) { + if(entry.getState().isAvailable() && entry.getState().isBackedByDisk()) { + if(entry.getRetainHintCount() > 0) { shouldCheckEviction = true; } else { - cacheSizeDelta = transitionMemState(entry, BlockState.COLD); + cacheSizeDelta = transitionMemState(entry, BlockState.COLD); long cleared = entry.clear(); - if (cleared != entry.getSize()) + if(cleared != entry.getSize()) throw new IllegalStateException(); _cache.remove(entry.getKey()); _evictionCache.put(entry.getKey(), entry); } } - else if (entry.getState() == BlockState.HOT) { - if (entry.getRetainHintCount() > 0) { + else if(entry.getState() == BlockState.HOT) { + if(entry.getRetainHintCount() > 0) { shouldCheckEviction = true; } else { @@ -470,9 +472,9 @@ else if (entry.getState() == BlockState.HOT) { } } } - if (cacheSizeDelta != 0) + if(cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); - else if (shouldCheckEviction) + else if(shouldCheckEviction) onCacheSizeChanged(true); } @@ -508,10 +510,11 @@ public synchronized void shutdown() { System.out.println("[WARN] Cache still holds " + _cache.size() + " / " + _evictionCache.size() + " blocks"); Set cachedStreams = _cache.keySet().stream().map(BlockKey::getStreamId).collect(Collectors.toSet()); - Set evictedStreams = _evictionCache.keySet().stream().map(BlockKey::getStreamId).collect(Collectors.toSet()); + Set evictedStreams = _evictionCache.keySet().stream().map(BlockKey::getStreamId) + .collect(Collectors.toSet()); cachedStreams.addAll(evictedStreams); - System.out.println("[WARN] Affected stream IDs: " + cachedStreams + ", Pinned: " + _cache.values().stream().mapToInt( - e -> e.isPinned() ? 1 : 0).sum()); + System.out.println("[WARN] Affected stream IDs: " + cachedStreams + ", Pinned: " + + _cache.values().stream().mapToInt(e -> e.isPinned() ? 1 : 0).sum()); } _cache.clear(); _evictionCache.clear(); @@ -575,7 +578,8 @@ private void runMaintenanceLoop() { do { _maintenanceRequested.set(false); onCacheSizeChangedInternal(_maintenanceNeedsIncr.getAndSet(false)); - } while(_maintenanceRequested.get()); + } + while(_maintenanceRequested.get()); } finally { _maintenanceRunning.set(false); @@ -591,7 +595,8 @@ private void onCacheSizeChangedInternal(boolean incr) { if(incr) onCacheSizeIncremented(); else - while(onCacheSizeDecremented()) {} + while(onCacheSizeDecremented()) { + } while(processPendingHandovers()) { onCacheSizeIncremented(); } @@ -601,18 +606,23 @@ private void onCacheSizeChangedInternal(boolean incr) { } private synchronized void sanityCheck() { - if (_cacheSize > _hardLimit * 1.1) { - if (!_warnThrottling) { + if(_cacheSize > _hardLimit * 1.1) { + if(!_warnThrottling) { _warnThrottling = true; - System.out.println("[WARN] Cache hard limit exceeded by over 10%: " + String.format("%.2f", _cacheSize/1000000.0) + "MB (-" + String.format("%.2f", _bytesUpForEviction/1000000.0) + "MB) > " + String.format("%.2f", _hardLimit/1000000.0) + "MB"); + System.out.println( + "[WARN] Cache hard limit exceeded by over 10%: " + String.format("%.2f", _cacheSize / 1000000.0) + + "MB (-" + String.format("%.2f", _bytesUpForEviction / 1000000.0) + "MB) > " + + String.format("%.2f", _hardLimit / 1000000.0) + "MB"); } } - else if (_warnThrottling && _cacheSize < _hardLimit) { + else if(_warnThrottling && _cacheSize < _hardLimit) { _warnThrottling = false; - System.out.println("[INFO] Cache within limit: " + String.format("%.2f", _cacheSize/1000000.0) + "MB (-" + String.format("%.2f", _bytesUpForEviction/1000000.0) + "MB) <= " + String.format("%.2f", _hardLimit/1000000.0) + "MB"); + System.out.println("[INFO] Cache within limit: " + String.format("%.2f", _cacheSize / 1000000.0) + "MB (-" + + String.format("%.2f", _bytesUpForEviction / 1000000.0) + "MB) <= " + + String.format("%.2f", _hardLimit / 1000000.0) + "MB"); } - if (!SANITY_CHECKS) + if(!SANITY_CHECKS) return; int pinned = 0; @@ -625,16 +635,16 @@ else if (_warnThrottling && _cacheSize < _hardLimit) { long actualPinnedEvictingBytes = 0; long actualWarmPinnedBytes = 0; long actualReadingReservedBytes = 0; - for (BlockEntry entry : _cache.values()) { - if (entry.isPinned()) { + for(BlockEntry entry : _cache.values()) { + if(entry.isPinned()) { pinned++; actualPinnedBytes += entry.getSize(); if(entry.getState() == BlockState.WARM) actualWarmPinnedBytes += entry.getSize(); } - if (entry.getState().isBackedByDisk()) + if(entry.getState().isBackedByDisk()) backedByDisk++; - if (entry.getState() == BlockState.EVICTING) { + if(entry.getState() == BlockState.EVICTING) { evicting++; upForEviction += entry.getSize(); if(entry.isPinned()) @@ -642,43 +652,44 @@ else if (_warnThrottling && _cacheSize < _hardLimit) { } if(entry.getState() == BlockState.READING) actualReadingReservedBytes += entry.getSize(); - if (!entry.getState().isAvailable()) + if(!entry.getState().isAvailable()) throw new IllegalStateException(); total++; actualCacheSize += entry.getSize(); } - for (BlockEntry entry : _evictionCache.values()) { - if (entry.getState().isAvailable()) + for(BlockEntry entry : _evictionCache.values()) { + if(entry.getState().isAvailable()) throw new IllegalStateException("Invalid eviction state: " + entry.getState()); - if (entry.getState() == BlockState.EVICTING && entry.isPinned()) + if(entry.getState() == BlockState.EVICTING && entry.isPinned()) actualPinnedEvictingBytes += entry.getSize(); - if (entry.getState() == BlockState.READING) + if(entry.getState() == BlockState.READING) actualCacheSize += entry.getSize(); - if (entry.getState() == BlockState.READING) + if(entry.getState() == BlockState.READING) actualReadingReservedBytes += entry.getSize(); - if (entry.isPinned()) { + if(entry.isPinned()) { actualPinnedBytes += entry.getSize(); if(entry.getState() == BlockState.WARM) actualWarmPinnedBytes += entry.getSize(); } } - if (actualCacheSize != _cacheSize) + if(actualCacheSize != _cacheSize) throw new IllegalStateException(actualCacheSize + " != " + _cacheSize); - if (upForEviction != _bytesUpForEviction) + if(upForEviction != _bytesUpForEviction) throw new IllegalStateException(upForEviction + " != " + _bytesUpForEviction); - if (actualPinnedBytes != _pinnedBytes) + if(actualPinnedBytes != _pinnedBytes) throw new IllegalStateException(actualPinnedBytes + " != " + _pinnedBytes); - if (actualPinnedEvictingBytes != _pinnedEvictingBytes) + if(actualPinnedEvictingBytes != _pinnedEvictingBytes) throw new IllegalStateException(actualPinnedEvictingBytes + " != " + _pinnedEvictingBytes); - if (_pinnedEvictingBytes > _bytesUpForEviction) + if(_pinnedEvictingBytes > _bytesUpForEviction) throw new IllegalStateException(_pinnedEvictingBytes + " > " + _bytesUpForEviction); if(actualWarmPinnedBytes != _warmPinnedBytes) throw new IllegalStateException(actualWarmPinnedBytes + " != " + _warmPinnedBytes); - if (actualReadingReservedBytes != _readingReservedBytes) + if(actualReadingReservedBytes != _readingReservedBytes) throw new IllegalStateException(actualReadingReservedBytes + " != " + _readingReservedBytes); System.out.println("=========="); - System.out.println("Limit: " + _evictionLimit/1000 + "KB"); - System.out.println("Memory: (" + _cacheSize/1000 + "KB - " + _bytesUpForEviction/1000 + "KB) / " + _hardLimit/1000 + "KB"); + System.out.println("Limit: " + _evictionLimit / 1000 + "KB"); + System.out.println("Memory: (" + _cacheSize / 1000 + "KB - " + _bytesUpForEviction / 1000 + "KB) / " + + _hardLimit / 1000 + "KB"); System.out.println("Pinned: " + pinned + " / " + total); System.out.println("Disk backed: " + backedByDisk + " / " + total); System.out.println("Evicting: " + evicting + " / " + total); @@ -695,10 +706,11 @@ private void onCacheSizeIncremented() { if(pressure <= _evictionLimit) return; // Nothing to do - long overshoot = Math.max((long)(0.1 * _evictionLimit), 10000000); + long overshoot = Math.max((long) (0.1 * _evictionLimit), 10000000); long lowLimit = _evictionLimit - _readBuffer - overshoot; - //System.out.println("[CACHE] Claiming " + (pressure + overshoot - _evictionLimit)/1000 + "kB (last claim was " + (System.currentTimeMillis() - _lastEvictRun) + "ms ago)"); + // System.out.println("[CACHE] Claiming " + (pressure + overshoot - _evictionLimit)/1000 + "kB (last claim + // was " + (System.currentTimeMillis() - _lastEvictRun) + "ms ago)"); // Scan for values that can be evicted Collection entries = _cache.values(); @@ -713,8 +725,8 @@ private void onCacheSizeIncremented() { break; synchronized(entry) { - //if(entry.isPinned()) - // continue; + // if(entry.isPinned()) + // continue; if(!allowRetainHint && entry.getRetainHintCount() > 0) continue; if(entry.getState() == BlockState.COLD || entry.getState() == BlockState.EVICTING) @@ -748,12 +760,12 @@ private void onCacheSizeIncremented() { _lastEvictRun = System.currentTimeMillis(); } - for (BlockEntry entry : upForEvictionNeedsWrite) + for(BlockEntry entry : upForEvictionNeedsWrite) evict(entry, true); - for (BlockEntry entry : upForEvictionNoWrite) + for(BlockEntry entry : upForEvictionNoWrite) evict(entry, false); - if (cacheSizeDelta != 0) + if(cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); } @@ -813,7 +825,7 @@ private boolean onCacheSizeDecremented() { throw new IllegalStateException(); req.setPinned(idx); } - else if (entry.getState() == BlockState.READING) { + else if(entry.getState() == BlockState.READING) { req.schedule(idx); registerWaiter(entry.getKey(), req, idx); reading = true; @@ -836,7 +848,7 @@ else if (entry.getState() == BlockState.READING) { if(allReserved) { _deferredReadRequests.poll(); _deferredReadCountHint = _deferredReadRequests.size(); - if (!toRead.isEmpty()) + if(!toRead.isEmpty()) _processingReadRequests.add(req); } @@ -943,17 +955,17 @@ private void onEvicted(final BlockEntry entry) { if(tmp != null && tmp != entry) throw new IllegalStateException(); tmp = _evictionCache.put(entry.getKey(), entry); - if (tmp != null) + if(tmp != null) throw new IllegalStateException(); sanityCheck(); } - if (cacheSizeDelta != 0) + if(cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); } private void clearRetainHints(DeferredReadRequest request) { - for (int i = 0; i < request.getEntries().size(); i++) { - if (!request.isRetainHinted(i)) + for(int i = 0; i < request.getEntries().size(); i++) { + if(!request.isRetainHinted(i)) continue; BlockEntry entry = request.getEntries().get(i); synchronized(entry) { @@ -963,12 +975,12 @@ private void clearRetainHints(DeferredReadRequest request) { } /** - * Cleanly transitions state of a BlockEntry and handles accounting. - * Requires both the scheduler object and the entry to be locked: + * Cleanly transitions state of a BlockEntry and handles accounting. Requires both the scheduler object and the + * entry to be locked: */ private long transitionMemState(BlockEntry entry, BlockState newState) { BlockState oldState = entry.getState(); - if (oldState == newState) + if(oldState == newState) return 0; long sz = entry.getSize(); @@ -976,7 +988,7 @@ private long transitionMemState(BlockEntry entry, BlockState newState) { boolean pinned = entry.isPinned(); // Remove old contribution - switch (oldState) { + switch(oldState) { case REMOVED: throw new IllegalStateException(); case HOT: @@ -1000,7 +1012,7 @@ private long transitionMemState(BlockEntry entry, BlockState newState) { } // Add new contribution - switch (newState) { + switch(newState) { case REMOVED: case COLD: break; @@ -1056,6 +1068,7 @@ private int pinEntryWithAccounting(BlockEntry entry) { /** * Requires scheduler lock and entry lock. + * * @return true if this call transitioned pin count to zero. */ private boolean unpinEntryWithAccounting(BlockEntry entry) { diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java b/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java index 1f731fc3aa5..f04534481c9 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java @@ -66,8 +66,8 @@ protected boolean isHashCol(int colID) { } /** - * Domain size of a dummycoded source column: the hash domain K from the meta cell for - * feature-hashed columns, otherwise the column's {@code numDistinct} (0 when unset). + * Domain size of a dummycoded source column: the hash domain K from the meta cell for feature-hashed columns, + * otherwise the column's {@code numDistinct} (0 when unset). * * @param meta transform meta frame * @param colID 1-based column id of the dummycoded source column @@ -144,13 +144,13 @@ public FrameBlock decode(final MatrixBlock in, final FrameBlock out, final int k try { final List> tasks = new ArrayList<>(); int blz = Math.max((in.getNumRows() + k) / k, 1000); - - for(int i = 0; i < in.getNumRows(); i += blz){ + + for(int i = 0; i < in.getNumRows(); i += blz) { final int start = i; final int end = Math.min(in.getNumRows(), i + blz); tasks.add(pool.submit(() -> decode(in, out, start, end))); } - + for(Future f : tasks) f.get(); return out; diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java index a286c03dce8..01a502154cd 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java @@ -72,10 +72,10 @@ public void decode(MatrixBlock in, FrameBlock out, int rl, int ru) { final double val = in.get(i, _srcCols[j] - 1); if(!Double.isNaN(val)){ final int key = (int) Math.round(val); - if(key == 0){ + if(key == 0) { a.set(i, _binMins[j][key]); } - else{ + else { double bmin = _binMins[j][key - 1]; double bmax = _binMaxs[j][key - 1]; double oval = bmin + (bmax - bmin) / 2 // bin center diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java index ee1a33c49fd..8aa0b60d990 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java @@ -34,7 +34,7 @@ /** * Simple atomic decoder for dummycoded columns. This decoder builds internally inverted column mappings from the given * frame meta data. - * + * */ public class DecoderDummycode extends Decoder { private static final long serialVersionUID = 4758831042891032129L; diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java index 8f6c45d63e8..fc123657032 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java @@ -64,15 +64,15 @@ public static Decoder createDecoder(String spec, String[] colnames, ValueType[] try { //parse transform specification JSONObject jSpec = new JSONObject(spec); - - //create decoders 'bin', 'recode', 'hash', 'dummy', and 'pass-through' + + // create decoders 'bin', 'recode', 'hash', 'dummy', and 'pass-through' List binIDs = TfMetaUtils.parseBinningColIDs(jSpec, colnames, minCol, maxCol); - List rcIDs = Arrays.asList(ArrayUtils.toObject( - TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.RECODE.toString(), minCol, maxCol))); - List hcIDs = Arrays.asList(ArrayUtils.toObject( - TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.HASH.toString(), minCol, maxCol))); - List dcIDs = Arrays.asList(ArrayUtils.toObject( - TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.DUMMYCODE.toString(), minCol, maxCol))); + List rcIDs = Arrays.asList(ArrayUtils + .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.RECODE.toString(), minCol, maxCol))); + List hcIDs = Arrays.asList(ArrayUtils + .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.HASH.toString(), minCol, maxCol))); + List dcIDs = Arrays.asList(ArrayUtils + .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.DUMMYCODE.toString(), minCol, maxCol))); // only specially treat the columns with both recode and dictionary rcIDs = unionDistinct(rcIDs, dcIDs); // hashing is a lossy, one-way transform with no inverse recode map, so hash columns @@ -106,29 +106,18 @@ public static Decoder createDecoder(String spec, String[] colnames, ValueType[] // collect all the decoders in one list. List ldecoders = new ArrayList<>(); - - if( !binIDs.isEmpty() ) { - ldecoders.add(new DecoderBin(schema, - ArrayUtils.toPrimitive(binIDs.toArray(new Integer[0])), + + if(!binIDs.isEmpty()) { + ldecoders.add(new DecoderBin(schema, ArrayUtils.toPrimitive(binIDs.toArray(new Integer[0])), ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); } if( !dcIDs.isEmpty() ) { ldecoders.add(new DecoderDummycode(schema, ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); } - if( !rcIDs.isEmpty() ) { - // recode on output (after dummycode rebuilds the categorical columns) when dummycoding is present - ldecoders.add(new DecoderRecode(schema, !dcIDs.isEmpty(), - ArrayUtils.toPrimitive(rcIDs.toArray(new Integer[0])))); - } - if( !ptIDs.isEmpty() ) { - ldecoders.add(new DecoderPassThrough(schema, - ArrayUtils.toPrimitive(ptIDs.toArray(new Integer[0])), - ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); - } - - //create composite decoder of all created decoders - //and initialize with given meta data (recode, dummy, bin) + + // create composite decoder of all created decoders + // and initialize with given meta data (recode, dummy, bin) decoder = new DecoderComposite(schema, ldecoders); decoder.setColnames(colnames); decoder.initMetaData(meta); @@ -147,7 +136,7 @@ else if( decoder instanceof DecoderRecode ) return DecoderType.Recode.ordinal(); else if( decoder instanceof DecoderPassThrough ) return DecoderType.PassThrough.ordinal(); - else if( decoder instanceof DecoderBin ) + else if(decoder instanceof DecoderBin) return DecoderType.Bin.ordinal(); throw new DMLRuntimeException("Unsupported decoder type: " + decoder.getClass().getCanonicalName()); @@ -158,10 +147,14 @@ public static Decoder createInstance(int type) { // create instance switch(dtype) { - case Bin: return new DecoderBin(); - case Dummycode: return new DecoderDummycode(null, null); - case PassThrough: return new DecoderPassThrough(null, null, null); - case Recode: return new DecoderRecode(null, false, null); + case Bin: + return new DecoderBin(); + case Dummycode: + return new DecoderDummycode(null, null); + case PassThrough: + return new DecoderPassThrough(null, null, null); + case Recode: + return new DecoderRecode(null, false, null); default: throw new DMLRuntimeException("Unsupported Encoder Type used: " + dtype); } diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java index 11dd2c7faa5..d8d624122a0 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java @@ -126,20 +126,20 @@ public void initMetaData(FrameBlock meta) { _rcMaps = new HashMap[_colList.length]; for( int j=0; j<_colList.length; j++ ) { HashMap map = new HashMap<>(); - for( int i=0; i= 0} - * both the minimum and maximum fraction digits are pinned to {@code decimal}, so values are - * printed with exactly that many decimals; otherwise the {@link DecimalFormat} defaults apply. + * Creates a non-grouping {@link DecimalFormat} for printing values. When {@code decimal >= 0} both the minimum and + * maximum fraction digits are pinned to {@code decimal}, so values are printed with exactly that many decimals; + * otherwise the {@link DecimalFormat} defaults apply. + * * @param decimal number of decimal places to print, -1 for default * @return a configured {@link DecimalFormat} */ private static DecimalFormat createDecimalFormat(int decimal) { DecimalFormat df = new DecimalFormat(); df.setGroupingUsed(false); - if (decimal >= 0) { + if(decimal >= 0) { df.setMinimumFractionDigits(decimal); df.setMaximumFractionDigits(decimal); } diff --git a/src/main/java/org/apache/sysds/utils/DoubleParser.java b/src/main/java/org/apache/sysds/utils/DoubleParser.java index c0122f8061f..2252b7270c8 100644 --- a/src/main/java/org/apache/sysds/utils/DoubleParser.java +++ b/src/main/java/org/apache/sysds/utils/DoubleParser.java @@ -200,7 +200,7 @@ public static double parseFloatingPointLiteral(String str, int offset, int endIn // : is the first character after numbers. // 0 is the first number. // we use the last position, since this is not allowed to be other values than a number. - if(str.charAt(endIndex - 1) > '9' || str.charAt(endIndex - 1) < '0') + if(str.charAt(endIndex - 1) > '9' || str.charAt(endIndex - 1) < '0') return Double.parseDouble(str); final double val = parseDecFloatLiteral(str, index, offset, endIndex); diff --git a/src/main/java/org/apache/sysds/utils/SettingsChecker.java b/src/main/java/org/apache/sysds/utils/SettingsChecker.java index c5f14a7043b..10d1a42b246 100644 --- a/src/main/java/org/apache/sysds/utils/SettingsChecker.java +++ b/src/main/java/org/apache/sysds/utils/SettingsChecker.java @@ -106,25 +106,24 @@ private static long maxMemMachineOSX() { } private static long maxMemMachineWin() { - //try modern powershell, otherwise wmic, log errors as warning but avoid crashes + // try modern powershell, otherwise wmic, log errors as warning but avoid crashes long tmp = maxMemMachineWin(true); - if( tmp < 0 ) + if(tmp < 0) tmp = maxMemMachineWin(false); return tmp; } - + private static long maxMemMachineWin(boolean modern) { int startIx = modern ? 3 : 1; - String command = modern ? - "powershell Get-CimInstance -ClassName Win32_PhysicalMemory | Select-Object Capacity" : - "wmic memorychip get capacity"; //in bytes + String command = modern ? "powershell Get-CimInstance -ClassName Win32_PhysicalMemory | Select-Object Capacity" : "wmic memorychip get capacity"; // in + // bytes try { Runtime rt = Runtime.getRuntime(); Process pr = rt.exec(command); String[] memStr = new String(pr.getInputStream().readAllBytes(), StandardCharsets.UTF_8).split("\n"); //skip header, and aggregate DIMM capacities long capacity = 0; - for( int i=startIx; i 0 ) capacity += Long.parseLong(tmp); diff --git a/src/test/java/org/apache/sysds/performance/Main.java b/src/test/java/org/apache/sysds/performance/Main.java index 0622e789baa..a941e90f724 100644 --- a/src/test/java/org/apache/sysds/performance/Main.java +++ b/src/test/java/org/apache/sysds/performance/Main.java @@ -243,10 +243,9 @@ private static void run17(String[] args) throws Exception { } /** - * Repeatedly read the same on-disk Delta frame table (written once as setup). - * Args: {@code 18 [mode] [targetFileSizeMB]} - * where mode is one of serial|parallel|both (default parallel) and an omitted - * target file size uses the adaptive default sizing. + * Repeatedly read the same on-disk Delta frame table (written once as setup). Args: + * {@code 18 [mode] [targetFileSizeMB]} where mode is one of serial|parallel|both (default parallel) + * and an omitted target file size uses the adaptive default sizing. */ private static void run18(String[] args) throws Exception { int rows = Integer.parseInt(args[1]); diff --git a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java index 36ea11b3e2f..4aac0685698 100644 --- a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java @@ -117,8 +117,8 @@ public abstract class AutomatedTestBase { public static final double GPU_TOLERANCE = 1e-9; /** - * Default deadline (ms) for federated worker/monitoring readiness waits and a few legacy - * {@code sleep()} calls. {@link FederatedWorkerUtils} enforces its own minimum floor. + * Default deadline (ms) for federated worker/monitoring readiness waits and a few legacy {@code sleep()} calls. + * {@link FederatedWorkerUtils} enforces its own minimum floor. */ public static final int FED_WORKER_WAIT = 3000; @@ -1761,9 +1761,9 @@ private static Process spawnLocalFedWorker(int port, String[] addArgs) { /** * Start a new JVM for a federated monitoring backend at the port. * - *

Returns once the backend's TCP port accepts connections (Netty's bind has completed), or - * throws a {@link RuntimeException} once the {@link FederatedWorkerUtils} readiness floor - * elapses. + *

+ * Returns once the backend's TCP port accepts connections (Netty's bind has completed), or throws a + * {@link RuntimeException} once the {@link FederatedWorkerUtils} readiness floor elapses. * * @param port Port to use for the JVM * @param addArgs Extra CLI args to append, or null @@ -1776,10 +1776,10 @@ protected Process startLocalFedMonitoring(int port, String[] addArgs) { /** * Start a new JVM for a federated monitoring backend at the port. * - *

Returns once the backend's TCP port accepts connections, or throws a - * {@link RuntimeException} after {@code timeoutMs} elapses. The monitoring server opens the - * port after Netty's {@code bind().sync()} returns; a successful TCP connect therefore signals - * that the HTTP listener is ready to accept requests. + *

+ * Returns once the backend's TCP port accepts connections, or throws a {@link RuntimeException} after + * {@code timeoutMs} elapses. The monitoring server opens the port after Netty's {@code bind().sync()} returns; a + * successful TCP connect therefore signals that the HTTP listener is ready to accept requests. * * @param port Port to use for the JVM * @param addArgs Extra CLI args to append, or null @@ -1798,8 +1798,9 @@ private static Process spawnLocalFedMonitoring(int port, String[] addArgs) { String separator = System.getProperty("file.separator"); String classpath = System.getProperty("java.class.path"); String path = System.getProperty("java.home") + separator + "bin" + separator + "java"; - String[] args = ArrayUtils.addAll(new String[] {path, "-cp", classpath, DMLScript.class.getName(), - "-fedMonitoring", Integer.toString(port)}, addArgs); + String[] args = ArrayUtils.addAll( + new String[] {path, "-cp", classpath, DMLScript.class.getName(), "-fedMonitoring", Integer.toString(port)}, + addArgs); try { return new ProcessBuilder(args).start(); } diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java index f14a614b583..0c7cd2046e5 100644 --- a/src/test/java/org/apache/sysds/test/TestUtils.java +++ b/src/test/java/org/apache/sysds/test/TestUtils.java @@ -2941,10 +2941,9 @@ public static void writeTestScalar(String file, double value) { } } - /** * Write scalar to file - * + * * @param file File to write to * @param value Value to write */ @@ -3519,9 +3518,9 @@ public static void shutdownThread(Thread t) { // Bounded join: workers are daemon threads, so even if one ignores the interrupt // we must not block cleanup (and the JVM) indefinitely waiting for it. t.join(THREAD_SHUTDOWN_JOIN_MS); - if( t.isAlive() ) - LOG.warn("Federated worker thread " + t.getName() - + " did not stop within " + THREAD_SHUTDOWN_JOIN_MS + "ms; leaving it as a daemon."); + if(t.isAlive()) + LOG.warn("Federated worker thread " + t.getName() + " did not stop within " + + THREAD_SHUTDOWN_JOIN_MS + "ms; leaving it as a daemon."); } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java b/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java index 07ec9752928..d76c741d870 100644 --- a/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java +++ b/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java @@ -67,10 +67,10 @@ public void setUp() { /** * Compile a DML script string into a runtime {@link Program} without executing it. * - * @param dmlScript the DML source - * @param args named command-line arguments ($name -> value), may be null - * @param mode the global execution mode (e.g. {@link ExecMode#HYBRID}) - * @param localMaxMem the local memory budget in bytes used for memory-based exec-type decisions + * @param dmlScript the DML source + * @param args named command-line arguments ($name -> value), may be null + * @param mode the global execution mode (e.g. {@link ExecMode#HYBRID}) + * @param localMaxMem the local memory budget in bytes used for memory-based exec-type decisions * @return the compiled runtime program */ protected Program compile(String dmlScript, Map args, ExecMode mode, long localMaxMem) { @@ -145,8 +145,7 @@ else if(pb instanceof FunctionProgramBlock) { /** All instructions whose opcode equals {@code opcode} (exact match). */ protected List getByOpcode(Program prog, String opcode) { - return getInstructions(prog).stream().filter(i -> opcode.equals(i.getOpcode())) - .collect(Collectors.toList()); + return getInstructions(prog).stream().filter(i -> opcode.equals(i.getOpcode())).collect(Collectors.toList()); } protected static boolean isSpark(Instruction inst) { @@ -169,13 +168,13 @@ protected void assertCP(Program prog, String opcode) { private void assertExecType(Program prog, String opcode, boolean expectSpark) { List matches = getByOpcode(prog, opcode); - Assert.assertFalse("Expected at least one '" + opcode + "' instruction but found none.\n" - + Explain.explain(prog), matches.isEmpty()); + Assert.assertFalse( + "Expected at least one '" + opcode + "' instruction but found none.\n" + Explain.explain(prog), + matches.isEmpty()); for(Instruction inst : matches) { boolean spark = isSpark(inst); - Assert.assertEquals("Instruction '" + opcode + "' expected exec type " - + (expectSpark ? "SPARK" : "CP") + " but was " + (spark ? "SPARK" : "CP") + ".\n" - + Explain.explain(prog), expectSpark, spark); + Assert.assertEquals("Instruction '" + opcode + "' expected exec type " + (expectSpark ? "SPARK" : "CP") + + " but was " + (spark ? "SPARK" : "CP") + ".\n" + Explain.explain(prog), expectSpark, spark); } } diff --git a/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java b/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java index 0b3889db908..7b45120d8f1 100644 --- a/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java +++ b/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java @@ -33,14 +33,13 @@ */ public class SparkTransitiveExecTypeCompileTest extends CompilerTestBase { - private static final String DML_HEADER = - "X = rand(rows=20000000, cols=8, seed=1);\n" + // ~1.2GB -> rand and colSums run on Spark - "v = colSums(X);\n"; // 1x8 Spark-resident vector (opcode uack+) + private static final String DML_HEADER = "X = rand(rows=20000000, cols=8, seed=1);\n" + // ~1.2GB -> rand and + // colSums run on Spark + "v = colSums(X);\n"; // 1x8 Spark-resident vector (opcode uack+) @Test public void singleConsumerUnaryPulledIntoSpark() { - String dml = DML_HEADER + - "r = round(v);\n" + // sole consumer of the Spark-resident vector -> pulled into Spark + String dml = DML_HEADER + "r = round(v);\n" + // sole consumer of the Spark-resident vector -> pulled into Spark "print(sum(r));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); @@ -50,62 +49,58 @@ public void singleConsumerUnaryPulledIntoSpark() { @Test public void multiConsumerUnaryStaysCP() { - String dml = DML_HEADER + - "a = round(v);\n" + // v now has two consumers (round + abs) ... - "b = abs(v);\n" + - "print(sum(a) + sum(b));\n"; + String dml = DML_HEADER + "a = round(v);\n" + // v now has two consumers (round + abs) ... + "b = abs(v);\n" + "print(sum(a) + sum(b));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uack+"); // input still has a Spark output ... - assertCP(prog, "round"); // ... but the multi-parent guard keeps both unaries in CP + assertCP(prog, "round"); // ... but the multi-parent guard keeps both unaries in CP assertCP(prog, "abs"); } // A tall, Spark-resident column vector that is still small enough (40 KB) to be CP by memory // estimate: rowSums over a very wide matrix runs on Spark, but its 1-column result fits in CP. - private static final String TALL_VECTOR_HEADER = - "X = rand(rows=5000, cols=200000, seed=1);\n" + // ~8GB -> rand and rowSums run on Spark - "c = rowSums(X);\n"; // 5000x1 Spark-resident vector (opcode uark+) + private static final String TALL_VECTOR_HEADER = "X = rand(rows=5000, cols=200000, seed=1);\n" + // ~8GB -> rand and + // rowSums run + // on Spark + "c = rowSums(X);\n"; // 5000x1 Spark-resident vector (opcode uark+) @Test public void cumulativeUnaryStaysCP() { - String dml = TALL_VECTOR_HEADER + - "r = cumsum(c);\n" + // sole consumer of the Spark-resident vector, CP by estimate ... + String dml = TALL_VECTOR_HEADER + "r = cumsum(c);\n" + // sole consumer of the Spark-resident vector, CP by + // estimate ... "print(as.scalar(r[2500,1]));\n"; // ... consume via indexing (avoids the sum(cumsum) rewrite) Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); - assertSpark(prog, "uark+"); // input genuinely has a Spark output + assertSpark(prog, "uark+"); // input genuinely has a Spark output assertCP(prog, "ucumk+"); // ... but cumulative ops are excluded from the transitive pull } @Test public void singleConsumerBinaryPulledIntoSpark() { - String dml = TALL_VECTOR_HEADER + - "r = c + 2.0;\n" + // matrix-scalar on the Spark-resident vector, sole consumer -> pulled into Spark + String dml = TALL_VECTOR_HEADER + "r = c + 2.0;\n" + // matrix-scalar on the Spark-resident vector, sole + // consumer -> pulled into Spark "print(as.scalar(r[2500,1]));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uark+"); // input genuinely has a Spark output (multi-block column vector) - assertSpark(prog, "+"); // matrix-scalar binary pulled into Spark (CP by estimate, single consumer) + assertSpark(prog, "+"); // matrix-scalar binary pulled into Spark (CP by estimate, single consumer) } @Test public void multiConsumerBinaryStaysCP() { - String dml = TALL_VECTOR_HEADER + - "a = c + 2.0;\n" + // c now has two consumers (+ and *) ... - "b = c * 3.0;\n" + - "print(as.scalar(a[2500,1]) + as.scalar(b[2500,1]));\n"; + String dml = TALL_VECTOR_HEADER + "a = c + 2.0;\n" + // c now has two consumers (+ and *) ... + "b = c * 3.0;\n" + "print(as.scalar(a[2500,1]) + as.scalar(b[2500,1]));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uark+"); // input still has a Spark output ... - assertCP(prog, "+"); // ... but the multi-parent guard keeps both binaries in CP + assertCP(prog, "+"); // ... but the multi-parent guard keeps both binaries in CP assertCP(prog, "*"); } @Test public void transitiveDisabledUnaryStaysCP() { - String dml = DML_HEADER + - "r = round(v);\n" + // pullable unary, but flag is off + String dml = DML_HEADER + "r = round(v);\n" + // pullable unary, but flag is off "print(sum(r));\n"; Program prog = compileWithTransitive(dml, false); @@ -115,8 +110,7 @@ public void transitiveDisabledUnaryStaysCP() { @Test public void transitiveDisabledBinaryStaysCP() { - String dml = TALL_VECTOR_HEADER + - "r = c + 2.0;\n" + // pullable matrix-scalar, but flag is off + String dml = TALL_VECTOR_HEADER + "r = c + 2.0;\n" + // pullable matrix-scalar, but flag is off "print(as.scalar(r[2500,1]));\n"; Program prog = compileWithTransitive(dml, false); diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java index 083a29f965b..22f867819c3 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java @@ -43,7 +43,8 @@ /** * Tests the {@code order} (sort) reorg operation on compressed matrices. A single column held in a single column group * is sorted ascending while staying compressed (via {@link org.apache.sysds.runtime.compress.lib.CLALibSort}); every - * other configuration falls back to a decompressed reorg. In all cases the result must match the uncompressed reference. + * other configuration falls back to a decompressed reorg. In all cases the result must match the uncompressed + * reference. */ public class CompressedSortTest { @@ -263,8 +264,7 @@ private void runCompressed(MatrixBlock mb, CompressionType ct) { assertEquals("Expected a single column group", 1, cmb.getColGroups().size()); MatrixBlock actual = cmb.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); - assertTrue("Expected the sorted result to stay compressed for " + ct, - actual instanceof CompressedMatrixBlock); + assertTrue("Expected the sorted result to stay compressed for " + ct, actual instanceof CompressedMatrixBlock); MatrixBlock expected = mb.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); TestUtils.compareMatrices(expected, CompressedMatrixBlock.getUncompressed(actual, "sort"), 0.0, "sort " + ct); diff --git a/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java index 833128ad9f0..49cbbdd248d 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java @@ -62,8 +62,8 @@ public static void setup() { } /** - * Build a compressed matrix backed by a single DDC column group spanning all {@code nCol} columns. This guarantees a - * single (non-uncompressed) column group, which is what triggers the mm-chain fast path for wide enough matrices. + * Build a compressed matrix backed by a single DDC column group spanning all {@code nCol} columns. This guarantees + * a single (non-uncompressed) column group, which is what triggers the mm-chain fast path for wide enough matrices. */ private static CompressedMatrixBlock singleDDC(int nRow, int nCol, int nVal, int seed) { Random r = new Random(seed); diff --git a/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java b/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java index 549010a78cb..c6afd5a3543 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java @@ -43,8 +43,8 @@ /** * Drive the solve opcode through {@link BinaryMatrixMatrixCPInstruction} with compressed inputs to cover the - * commons-math matrix-matrix branch that decompresses compressed left/right operands before solving. The - * script-level solve tests only ever see uncompressed inputs, so this branch is otherwise unreached. + * commons-math matrix-matrix branch that decompresses compressed left/right operands before solving. The script-level + * solve tests only ever see uncompressed inputs, so this branch is otherwise unreached. */ public class CompressedBinaryMatrixMatrixSolveTest { @@ -72,8 +72,8 @@ public void solveCompressedLeftCompressedRight() { assertTrue("A must compress to exercise the compressed-left path", aC instanceof CompressedMatrixBlock); CompressedMatrixBlock bC = CompressedMatrixBlockFactory.createConstant(n, 2, 1.0); - MatrixBlock expected = LibCommonsMath.matrixMatrixOperations( - CompressedMatrixBlock.getUncompressed(aC), CompressedMatrixBlock.getUncompressed(bC), SOLVE); + MatrixBlock expected = LibCommonsMath.matrixMatrixOperations(CompressedMatrixBlock.getUncompressed(aC), + CompressedMatrixBlock.getUncompressed(bC), SOLVE); MatrixBlock actual = runSolve(aC, bC); TestUtils.compareMatricesBitAvgDistance(expected, actual, 0, 0, SOLVE); @@ -94,8 +94,8 @@ public void solveCompressedLeftDenseRight() { assertTrue("A must compress to exercise the compressed-left path", aC instanceof CompressedMatrixBlock); MatrixBlock b = TestUtils.round(TestUtils.generateTestMatrixBlock(n, 1, -5, 5, 1.0, 7)); - MatrixBlock expected = LibCommonsMath.matrixMatrixOperations( - CompressedMatrixBlock.getUncompressed(aC), b, SOLVE); + MatrixBlock expected = LibCommonsMath.matrixMatrixOperations(CompressedMatrixBlock.getUncompressed(aC), b, + SOLVE); MatrixBlock actual = runSolve(aC, b); TestUtils.compareMatricesBitAvgDistance(expected, actual, 0, 0, SOLVE); @@ -119,7 +119,8 @@ private static BinaryMatrixMatrixCPInstruction solveInstruction() { } private static MatrixObject matrixObject(String name, MatrixBlock mb) { - MatrixCharacteristics mc = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), 1000, mb.getNonZeros()); + MatrixCharacteristics mc = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), 1000, + mb.getNonZeros()); MatrixObject mo = new MatrixObject(ValueType.FP64, "/dev/null/" + name, new MetaDataFormat(mc, FileFormat.BINARY), mb); return mo; diff --git a/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java b/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java index 8907c82f1d1..0d2f37e319c 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java @@ -50,7 +50,9 @@ public class OffsetClassInitConcurrencyTest { private static final String[] INIT_TARGETS = {PKG + "AOffset", PKG + "OffsetEmpty", PKG + "OffsetChar", PKG + "OffsetByte", PKG + "OffsetSingle", PKG + "OffsetTwo"}; - /** Whether a class-init cycle deadlocks depends on thread timing, so repeat to make a regression reliable to catch. */ + /** + * Whether a class-init cycle deadlocks depends on thread timing, so repeat to make a regression reliable to catch. + */ private static final int ROUNDS = 20; /** A real init deadlock never resolves; a healthy round finishes in milliseconds, so this bound is generous. */ diff --git a/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java b/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java index 3493da7d2b1..f7dffa9021c 100644 --- a/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java +++ b/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java @@ -35,12 +35,10 @@ public class SparkContextReferenceCountTest { /** - * Two DML executions sharing the JVM-wide singleton spark context (as happens - * with surefire parallel tests, threadCount>1). When the first execution - * finishes and calls close(), the shared context must stay alive because the - * second execution still has in-flight work. Before reference counting, - * close() stopped the context unconditionally, which cancelled the second - * execution's spark job and wedged it until the test watchdog. + * Two DML executions sharing the JVM-wide singleton spark context (as happens with surefire parallel tests, + * threadCount>1). When the first execution finishes and calls close(), the shared context must stay alive + * because the second execution still has in-flight work. Before reference counting, close() stopped the context + * unconditionally, which cancelled the second execution's spark job and wedged it until the test watchdog. */ @Test public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { @@ -64,16 +62,13 @@ public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { // context that B still uses SparkExecutionContext.exitSparkExecution(); ecA.close(); - assertFalse("shared context must stay alive while another execution is active", - sc.sc().isStopped()); - assertEquals("B's job must still run on the live context", - 10L, rdd.reduce(Integer::sum).longValue()); + assertFalse("shared context must stay alive while another execution is active", sc.sc().isStopped()); + assertEquals("B's job must still run on the live context", 10L, rdd.reduce(Integer::sum).longValue()); // B finishes last: releasing the final registration lets close() stop it SparkExecutionContext.exitSparkExecution(); ecB.close(); - assertTrue("shared context must be stopped once the last execution closes", - sc.sc().isStopped()); + assertTrue("shared context must be stopped once the last execution closes", sc.sc().isStopped()); } finally { // drain any remaining registrations and stop the context so a failed @@ -87,10 +82,9 @@ public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { } /** - * An unpaired close() (a caller that borrows the shared context but never - * registered via enterSparkExecution()) must not stop a context another - * execution still uses. This fails on the old unconditional-stop code, which - * tore the context down out from under the active execution. + * An unpaired close() (a caller that borrows the shared context but never registered via enterSparkExecution()) + * must not stop a context another execution still uses. This fails on the old unconditional-stop code, which tore + * the context down out from under the active execution. */ @Test public void unpairedCloseDoesNotStopAContextStillInUse() { @@ -106,14 +100,12 @@ public void unpairedCloseDoesNotStopAContextStillInUse() { // borrows the shared context): close() must not stop a context in use unregistered = ExecutionContextFactory.createSparkExecutionContext(); unregistered.close(); - assertFalse("unpaired close() must not stop a context still in use", - sc.sc().isStopped()); + assertFalse("unpaired close() must not stop a context still in use", sc.sc().isStopped()); // the registered execution finishing stops the context as the last user SparkExecutionContext.exitSparkExecution(); active.close(); - assertTrue("context must stop once the last registered execution closes", - sc.sc().isStopped()); + assertTrue("context must stop once the last registered execution closes", sc.sc().isStopped()); } finally { SparkExecutionContext.exitSparkExecution(); diff --git a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java index 2c854b4a81b..459936fa24c 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java @@ -78,17 +78,18 @@ public MatrixBlock getMatrixBlock(long id) { } /** - * Poll the federated worker until the matrix at {@code id} is observed as a - * {@link CompressedMatrixBlock}, or {@link #COMPRESS_TIMEOUT_MS} elapses. + * Poll the federated worker until the matrix at {@code id} is observed as a {@link CompressedMatrixBlock}, or + * {@link #COMPRESS_TIMEOUT_MS} elapses. * - *

Federated workers compress asynchronously after a PUT/READ_VAR (see - * {@code CompressedMatrixBlockFactory.compressAsync}), so a {@code getMatrixBlock} fired right - * after the operation can race against the in-flight compression and return the uncompressed - * block. Tests that need to observe the compressed form should poll instead of sleeping a fixed - * amount. + *

+ * Federated workers compress asynchronously after a PUT/READ_VAR (see + * {@code CompressedMatrixBlockFactory.compressAsync}), so a {@code getMatrixBlock} fired right after the operation + * can race against the in-flight compression and return the uncompressed block. Tests that need to observe the + * compressed form should poll instead of sleeping a fixed amount. * - *

On timeout this returns the most recent (uncompressed) read so the caller can produce a - * meaningful assertion failure naming the variable. + *

+ * On timeout this returns the most recent (uncompressed) read so the caller can produce a meaningful assertion + * failure naming the variable. * * @param id federated variable id * @return the matrix block, compressed if compression finished in time, otherwise the latest read diff --git a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java index 2b5ff327ef3..d3e4485bdf4 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java @@ -68,13 +68,11 @@ public void verifySameOrAlsoCompressedAsLocalCompress() { // federated. Compression on the worker is async; poll only when we expect compression to // match the local result, otherwise a single read is enough. final long id = putMatrixBlock(mb); - final MatrixBlock mbr = (mbcLocal instanceof CompressedMatrixBlock) - ? awaitCompressed(id) - : getMatrixBlock(id); + final MatrixBlock mbr = (mbcLocal instanceof CompressedMatrixBlock) ? awaitCompressed(id) : getMatrixBlock(id); if(mbcLocal instanceof CompressedMatrixBlock && !(mbr instanceof CompressedMatrixBlock)) - fail("Invalid result, the federated site did not compress the matrix block within " - + COMPRESS_TIMEOUT_MS + "ms"); + fail("Invalid result, the federated site did not compress the matrix block within " + COMPRESS_TIMEOUT_MS + + "ms"); TestUtils.compareMatricesBitAvgDistance(mbcLocal, mbr, 0, 0, "Not equivalent matrix block returned from federated site"); diff --git a/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java b/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java index 60587bf51a2..0de3b6db640 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java @@ -42,7 +42,7 @@ public void test100x100() { @Test public void testDecimalClampsFractionDigits() { - FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); + FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); f.ensureAllocatedColumns(1); f.set(0, 0, 5.244058388023880); // decimal=2 must print exactly two fraction digits, not DecimalFormat's default max of 3 @@ -53,10 +53,10 @@ public void testDecimalClampsFractionDigits() { @Test public void testDecimalPadsAndRounds() { - FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); + FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); f.ensureAllocatedColumns(2); - f.set(0, 0, 22.0); // integer-valued: padded up to the requested digits - f.set(1, 0, 5.244058388023880); // rounded at the last requested digit + f.set(0, 0, 22.0); // integer-valued: padded up to the requested digits + f.set(1, 0, 5.244058388023880); // rounded at the last requested digit String out = DataConverter.toString(f, false, " ", "\n", 2, 1, 4); assertTrue("expected 22.0000 padded: " + out, out.contains("22.0000\n")); assertTrue("expected 5.2441 rounded: " + out, out.contains("5.2441\n")); @@ -64,10 +64,10 @@ public void testDecimalPadsAndRounds() { @Test public void testNegativeDecimalUsesDefaultFormatting() { - FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); + FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); f.ensureAllocatedColumns(2); - f.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained - f.set(1, 0, 5.244058388023880); // default cap of three fraction digits + f.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained + f.set(1, 0, 5.244058388023880); // default cap of three fraction digits // decimal < 0 leaves DecimalFormat unconstrained (no min/max fraction digits set) String out = DataConverter.toString(f, false, " ", "\n", 2, 1, -1); assertTrue("expected unpadded 22: " + out, out.contains("22\n")); diff --git a/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java b/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java index 43a53879f17..7d67c04983b 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java @@ -142,8 +142,7 @@ public void safeCastWarnsOnlyOnce() { // the fallback warning must be logged exactly once across both conversions final List log = LoggingUtils.reinsert(appender); long warnings = log.stream() - .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")) - .count(); + .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")).count(); assertEquals(1, warnings); } @@ -153,8 +152,7 @@ public void strictThrowsWhenWarnCastDisabled() { setWarnCast(false); FrameBlock fb = mixedFrame(); - Exception e = assertThrows(DMLRuntimeException.class, - () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 1)); + Exception e = assertThrows(DMLRuntimeException.class, () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 1)); assertTrue(e.getMessage().contains("Failed to convert FrameBlock to MatrixBlock")); } @@ -164,8 +162,7 @@ public void strictThrowsParallelWhenWarnCastDisabled() { setWarnCast(false); FrameBlock fb = mixedFrame(); - Exception e = assertThrows(DMLRuntimeException.class, - () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 4)); + Exception e = assertThrows(DMLRuntimeException.class, () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 4)); assertTrue(e.getMessage().contains("Failed to convert FrameBlock to MatrixBlock")); } @@ -185,8 +182,7 @@ public void warnCastValidFrameConvertsWithoutFallback() { final List log = LoggingUtils.reinsert(appender); long warnings = log.stream() - .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")) - .count(); + .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")).count(); assertEquals(0, warnings); } diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java index ccba674707b..982941bb190 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java @@ -74,8 +74,8 @@ private void runDecode(String spec, int nCol, int nCat) { try { FrameBlock data = categoricalFrame(ROWS, nCol, nCat, 17); - MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), - data.getNumColumns(), null); + MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), data.getNumColumns(), + null); MatrixBlock encoded = encoder.encode(data, 1); Decoder decoder = buildDecoder(data, spec, encoder); @@ -122,8 +122,8 @@ public void singleThreadEqualsParallelManyCategories() { public void decoderIsComposite() { FrameBlock data = categoricalFrame(100, 2, 3, 1); String spec = "{recode:[C1], dummycode:[C2]}"; - MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), - data.getNumColumns(), null); + MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), data.getNumColumns(), + null); encoder.encode(data, 1); Decoder decoder = buildDecoder(data, spec, encoder); if(!(decoder instanceof DecoderComposite)) diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java index d9c540f54c5..412686c1e83 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java @@ -47,9 +47,9 @@ import org.junit.Test; /** - * Unit tests that drive the get_categorical_mask instruction directly to exercise the defensive code - * paths (distinct-count prefix in the metadata frame, default column metadata, non id-based specs and - * the unsupported opcode guard) that the script-level transform tests cannot reach. + * Unit tests that drive the get_categorical_mask instruction directly to exercise the defensive code paths + * (distinct-count prefix in the metadata frame, default column metadata, non id-based specs and the unsupported opcode + * guard) that the script-level transform tests cannot reach. */ public class GetCategoricalMaskInstructionTest { protected static final Log LOG = LogFactory.getLog(GetCategoricalMaskInstructionTest.class.getName()); @@ -210,7 +210,8 @@ public void imputeAndOmitAreAccepted() { // impute and omit do not change the output column count or categorical flag, so a spec that // only adds them on top of a recoded column must still succeed and mark that column categorical FrameBlock meta = new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}); - MatrixBlock res = run(meta, "{\"ids\": true, \"recode\": [1], \"impute\": [{\"id\": 1, \"method\": \"global_mode\"}], \"omit\": [1]}"); + MatrixBlock res = run(meta, + "{\"ids\": true, \"recode\": [1], \"impute\": [{\"id\": 1, \"method\": \"global_mode\"}], \"omit\": [1]}"); assertEquals(1, res.getNumRows()); assertEquals(1, res.getNumColumns()); @@ -230,8 +231,7 @@ public void unsupportedOpcodeThrows() { // any frame-scalar binary opcode other than get_categorical_mask must be rejected ExecutionContext ec = ExecutionContextFactory.createContext(); ec.setAutoCreateVars(true); - ec.setVariable("F", frameObject(new FrameBlock(new ValueType[] {ValueType.STRING}, - new String[][] {{"a"}}))); + ec.setVariable("F", frameObject(new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}))); assertThrowsMessage("Unsupported operation", () -> maskInstruction("+").processInstruction(ec)); } @@ -263,9 +263,9 @@ private static void assertMask(MatrixBlock res, double[] expected) { } /** - * Build a single-row metadata frame of nCol string columns. A positive distinct[i] is written to - * that column's metadata as the recode distinct count (the path real transformencode uses), while - * a zero leaves the column with default metadata (a continuous / non-dummycoded column). + * Build a single-row metadata frame of nCol string columns. A positive distinct[i] is written to that column's + * metadata as the recode distinct count (the path real transformencode uses), while a zero leaves the column with + * default metadata (a continuous / non-dummycoded column). */ private static FrameBlock metaWithDistinct(int nCol, int[] distinct) { ValueType[] schema = new ValueType[nCol]; @@ -290,7 +290,8 @@ private static MatrixBlock run(FrameBlock meta, String spec) { private static BinaryFrameScalarCPInstruction maskInstruction(String opcode) { String in1 = InstructionUtils.concatOperandParts("F", DataType.FRAME.name(), ValueType.STRING.name(), "false"); - String in2 = InstructionUtils.concatOperandParts("spec", DataType.SCALAR.name(), ValueType.STRING.name(), "true"); + String in2 = InstructionUtils.concatOperandParts("spec", DataType.SCALAR.name(), ValueType.STRING.name(), + "true"); String out = InstructionUtils.concatOperandParts("out", DataType.MATRIX.name(), ValueType.FP64.name(), "false"); String str = InstructionUtils.concatOperands("CP", opcode, in1, in2, out); return (BinaryFrameScalarCPInstruction) BinaryCPInstruction.parseInstruction(str); diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java index b2d31f43b83..645c4dd09bd 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -44,10 +44,10 @@ import org.junit.Test; /** - * Exact inverse correctness tests for the transform decoders. Recode and dummycode are lossless category encodings, so a - * decode of the encoded matrix must reconstruct the original categorical frame. These tests assert exact reconstruction - * for the dense path, the sparse path, and the parallel path so that the dummycode sparse binary search and the parallel - * block split are validated against ground truth rather than only against each other. + * Exact inverse correctness tests for the transform decoders. Recode and dummycode are lossless category encodings, so + * a decode of the encoded matrix must reconstruct the original categorical frame. These tests assert exact + * reconstruction for the dense path, the sparse path, and the parallel path so that the dummycode sparse binary search + * and the parallel block split are validated against ground truth rather than only against each other. */ public class TransformDecodeRoundTripTest { protected static final Log LOG = LogFactory.getLog(TransformDecodeRoundTripTest.class.getName()); @@ -59,9 +59,9 @@ public void setUp() { } private static FrameBlock categoricalFrame() { - final String[] values = new String[] { - "apple", "banana", "apple", "cherry", "banana", "date", "apple", "cherry", "date", "banana", "elderberry", - "apple", "fig", "banana", "cherry", "apple", "date", "fig", "elderberry", "banana"}; + final String[] values = new String[] {"apple", "banana", "apple", "cherry", "banana", "date", "apple", "cherry", + "date", "banana", "elderberry", "apple", "fig", "banana", "cherry", "apple", "date", "fig", "elderberry", + "banana"}; final FrameBlock f = new FrameBlock(new ValueType[] {ValueType.STRING}); f.ensureAllocatedColumns(values.length); for(int i = 0; i < values.length; i++) @@ -117,8 +117,8 @@ public void binWithDummycodeOnOtherColumnConsistency() { /** * Dummycode on an earlier column (1) shifts the bin column (2) to the right in the encoded matrix. The bin decoder - * must walk the dummycode domain sizes to recover the bin column's true source position. This drives the - * non-magic offset branch of the bin source-column mapping. + * must walk the dummycode domain sizes to recover the bin column's true source position. This drives the non-magic + * offset branch of the bin source-column mapping. */ @Test public void binAfterDummycodeOnEarlierColumnConsistency() { @@ -128,9 +128,9 @@ public void binAfterDummycodeOnEarlierColumnConsistency() { } /** - * Same right-shift as above, but the earlier column is feature-hashed before being dummycoded. The hash domain - * size K is stored as a plain integer in the single meta cell, so the bin source-column mapping reads it (instead - * of numDistinct) to compute the offset. + * Same right-shift as above, but the earlier column is feature-hashed before being dummycoded. The hash domain size + * K is stored as a plain integer in the single meta cell, so the bin source-column mapping reads it (instead of + * numDistinct) to compute the offset. */ @Test public void binAfterHashDummycodeOnEarlierColumnConsistency() { @@ -211,10 +211,10 @@ public void binDecodeZeroCodeUsesFirstBinBoundary() { } /** - * Spark broadcasts the decoder to executors via Java serialization without re-running initMetaData, so the - * decoder must round-trip all of its decode state through writeExternal/readExternal. Decode with a freshly - * deserialized decoder and assert it matches the in-memory decode. Covers plain bin and bin-with-dummycode - * (the latter exercises the serialized _srcCols/_dcCols source-column mapping). + * Spark broadcasts the decoder to executors via Java serialization without re-running initMetaData, so the decoder + * must round-trip all of its decode state through writeExternal/readExternal. Decode with a freshly deserialized + * decoder and assert it matches the in-memory decode. Covers plain bin and bin-with-dummycode (the latter exercises + * the serialized _srcCols/_dcCols source-column mapping). */ @Test public void binDecoderSurvivesSerialization() { @@ -481,8 +481,7 @@ public void parallelDecodeWrapsWorkerException() { fail("expected the parallel decode wrapper to propagate the worker failure"); } catch(DMLRuntimeException expected) { - assertNotNull("parallel decode wrapper must retain the worker exception as cause", - expected.getCause()); + assertNotNull("parallel decode wrapper must retain the worker exception as cause", expected.getCause()); } } catch(Exception e) { diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java index 54bd1679716..5745a35d506 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java index 8d7ad14539a..64012c06bbf 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java @@ -57,17 +57,15 @@ import io.delta.kernel.types.TimestampType; /** - * Targeted tests for the error/defensive branches of the native Delta matrix - * read/write code that the round-trip and interop tests do not reach: malformed - * per-file statistics, unsupported column types, unsupported stream operations, + * Targeted tests for the error/defensive branches of the native Delta matrix read/write code that the round-trip and + * interop tests do not reach: malformed per-file statistics, unsupported column types, unsupported stream operations, * bad table paths, and the non-dense writer input path. * - *

A few of these branches guard against inputs that the SystemDS writer and - * the Delta Kernel scan API never produce in a normal round trip (e.g. a - * statistics JSON without {@code numRecords}, or a column type code outside the - * supported set). They are exercised here by mocking the Delta Kernel data - * objects and invoking the (package-private) helpers reflectively, rather than - * widening their production visibility purely for testing. + *

+ * A few of these branches guard against inputs that the SystemDS writer and the Delta Kernel scan API never produce in + * a normal round trip (e.g. a statistics JSON without {@code numRecords}, or a column type code outside the supported + * set). They are exercised here by mocking the Delta Kernel data objects and invoking the (package-private) helpers + * reflectively, rather than widening their production visibility purely for testing. */ public class DeltaMatrixCoverageTest { @@ -89,8 +87,8 @@ public void qualifyRejectsUnknownFilesystemScheme() { @Test public void typeCodeReturnsNegativeForUnsupportedTypes() { - //non-numeric / unsupported Delta types must map to the sentinel -1 so the - //reader can reject them with a clear message rather than mis-decoding. + // non-numeric / unsupported Delta types must map to the sentinel -1 so the + // reader can reject them with a clear message rather than mis-decoding. assertEquals(-1, DeltaKernelUtils.typeCode(DateType.DATE)); assertEquals(-1, DeltaKernelUtils.typeCode(TimestampType.TIMESTAMP)); assertEquals(-1, DeltaKernelUtils.typeCode(BinaryType.BINARY)); @@ -112,17 +110,17 @@ public void writerRejectsStreamWrite() throws Exception { @Test public void numRecordsHandlesAbsentNullAndMalformedStats() throws Exception { - //no "stats" field at all -> -1 + // no "stats" field at all -> -1 assertEquals(-1, numRecords(addFileRow(new StructType().add("path", StringType.STRING), false, null))); - //stats column present but null-at -> -1 + // stats column present but null-at -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), true, null))); - //stats string explicitly null -> -1 + // stats string explicitly null -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, null))); - //malformed JSON -> JsonProcessingException -> -1 + // malformed JSON -> JsonProcessingException -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, "{not valid json"))); - //valid JSON but no numRecords field -> -1 + // valid JSON but no numRecords field -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, "{\"minValues\":{}}"))); - //well-formed stats -> the parsed count + // well-formed stats -> the parsed count assertEquals(1234L, numRecords(addFileRow(statsSchema(), false, "{\"numRecords\":1234}"))); } @@ -131,8 +129,8 @@ public void getDoubleValueRejectsUnknownTypeCode() throws Exception { Method m = ReaderDelta.class.getDeclaredMethod("getDoubleValue", ColumnVector.class, int.class, int.class); m.setAccessible(true); try { - //type code outside the supported T_* set; the switch default must throw - //before touching the (null) vector. + // type code outside the supported T_* set; the switch default must throw + // before touching the (null) vector. m.invoke(null, (ColumnVector) null, 0, 999); fail("expected a DMLRuntimeException for an unsupported type code"); } @@ -156,9 +154,9 @@ public void numericTypeCodeRejectsNonNumericType() throws Exception { @Test public void parallelReadWrapsFileFailure() throws Exception { - //a per-file decode failure in the parallel reader must surface as a single - //clear IOException (the awaitFileTasks catch), not a raw executor error. - //Provoke it by deleting one data file after the table (and its log) exist. + // a per-file decode failure in the parallel reader must surface as a single + // clear IOException (the awaitFileTasks catch), not a raw executor error. + // Provoke it by deleting one data file after the table (and its log) exist. MatrixBlock in = TestUtils.generateTestMatrixBlock(100_000, 8, -10, 10, 1.0, 13); in.recomputeNonZeros(); DMLConfig conf = new DMLConfig(); @@ -167,15 +165,14 @@ public void parallelReadWrapsFileFailure() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_fail_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); - //delete one parquet data file; the transaction log still references it, - //so the scan enumerates it but the decode task fails. + // delete one parquet data file; the transaction log still references it, + // so the scan enumerates it but the decode task fails. File victim; - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { - victim = s.filter(p -> p.toString().endsWith(".parquet")) - .findFirst().map(Path::toFile).orElse(null); + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { + victim = s.filter(p -> p.toString().endsWith(".parquet")).findFirst().map(Path::toFile).orElse(null); } assertTrue("expected at least one data file to delete", victim != null && victim.delete()); @@ -200,8 +197,8 @@ public void parallelReadWrapsFileFailure() throws Exception { @Test public void sparseFormatMatrixRoundTrips() throws Exception { - //a sparse-backed MatrixBlock takes the writer's non-contiguous path (no - //direct double[] view), exercising MatrixColumnVector.get via MatrixBlock. + // a sparse-backed MatrixBlock takes the writer's non-contiguous path (no + // direct double[] view), exercising MatrixColumnVector.get via MatrixBlock. MatrixBlock in = TestUtils.generateTestMatrixBlock(2000, 7, -5, 5, 0.05, 13); in.recomputeNonZeros(); in.examSparsity(); @@ -210,8 +207,8 @@ public void sparseFormatMatrixRoundTrips() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_sparse_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", in.getNumRows(), out.getNumRows()); assertEquals("cols", in.getNumColumns(), out.getNumColumns()); @@ -224,15 +221,15 @@ public void sparseFormatMatrixRoundTrips() throws Exception { @Test public void fillDenseHandlesNonContiguousBlock() throws Exception { - //the dense fill normally hits the contiguous fast path; force a multi-block - //(non-contiguous) dense block so the row-by-row fallback is exercised. Such - //blocks only arise for matrices beyond a single contiguous array, so we - //shrink the per-block allocation cap to provoke it on a tiny matrix. + // the dense fill normally hits the contiguous fast path; force a multi-block + // (non-contiguous) dense block so the row-by-row fallback is exercised. Such + // blocks only arise for matrices beyond a single contiguous array, so we + // shrink the per-block allocation cap to provoke it on a tiny matrix. int rows = 5, cols = 4; int savedMaxAlloc = DenseBlockLDRB.MAX_ALLOC; DenseBlock db; try { - DenseBlockLDRB.MAX_ALLOC = 2 * cols; //~2 rows per block -> multiple blocks + DenseBlockLDRB.MAX_ALLOC = 2 * cols; // ~2 rows per block -> multiple blocks db = new DenseBlockLFP64(new int[] {rows, cols}); } finally { @@ -240,14 +237,14 @@ public void fillDenseHandlesNonContiguousBlock() throws Exception { } assertTrue("expected a non-contiguous (multi-block) dense block", !db.isContiguous()); - //two row-major batches (3 rows + 2 rows) covering all 5 rows + // two row-major batches (3 rows + 2 rows) covering all 5 rows double[] b0 = new double[3 * cols]; double[] b1 = new double[2 * cols]; - for( int r = 0; r < 3; r++ ) - for( int c = 0; c < cols; c++ ) + for(int r = 0; r < 3; r++) + for(int c = 0; c < cols; c++) b0[r * cols + c] = cell(r, c); - for( int r = 0; r < 2; r++ ) - for( int c = 0; c < cols; c++ ) + for(int r = 0; r < 2; r++) + for(int c = 0; c < cols; c++) b1[r * cols + c] = cell(3 + r, c); java.util.ArrayList batches = new java.util.ArrayList<>(); batches.add(b0); @@ -258,8 +255,8 @@ public void fillDenseHandlesNonContiguousBlock() throws Exception { m.setAccessible(true); m.invoke(null, ret, batches); - for( int r = 0; r < rows; r++ ) - for( int c = 0; c < cols; c++ ) + for(int r = 0; r < rows; r++) + for(int c = 0; c < cols; c++) assertEquals("r" + r + " c" + c, cell(r, c), ret.getDenseBlock().get(r, c), 0.0); } @@ -276,8 +273,8 @@ private static StructType statsSchema() { } /** - * Build a mocked scan-file row whose AddFile child has the given schema, null - * flag and (when not null) stats string, matching what {@code numRecords} reads. + * Build a mocked scan-file row whose AddFile child has the given schema, null flag and (when not null) stats + * string, matching what {@code numRecords} reads. */ private static Row addFileRow(StructType addSchema, boolean statsNull, String statsValue) { Row outer = mock(Row.class); @@ -285,12 +282,12 @@ private static Row addFileRow(StructType addSchema, boolean statsNull, String st when(outer.getStruct(InternalScanFileUtils.ADD_FILE_ORDINAL)).thenReturn(add); when(add.getSchema()).thenReturn(addSchema); int statsOrd = addSchema.fieldNames().indexOf("stats"); - if( statsOrd >= 0 ) { + if(statsOrd >= 0) { when(add.isNullAt(statsOrd)).thenReturn(statsNull); - if( !statsNull ) + if(!statsNull) when(add.getString(statsOrd)).thenReturn(statsValue); } - return outer; //the scan-file row numRecords consumes (its AddFile child is 'add') + return outer; // the scan-file row numRecords consumes (its AddFile child is 'add') } private static long numRecords(Row scanFileRow) throws Exception { diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java index 54a3bcf6334..65c4ec21c0f 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java @@ -63,20 +63,19 @@ import io.delta.kernel.utils.CloseableIterator; /** - * Direct (no DML) round-trip tests for the native Delta Kernel based matrix - * reader/writer. Each test writes a MatrixBlock to a fresh local Delta table - * directory and reads it back, asserting dimensions and values match. + * Direct (no DML) round-trip tests for the native Delta Kernel based matrix reader/writer. Each test writes a + * MatrixBlock to a fresh local Delta table directory and reads it back, asserting dimensions and values match. */ public class DeltaMatrixReadWriteTest { - //small writer target file size (bytes) used to force a multi-file table - //layout cheaply, instead of brute-forcing huge row counts. + // small writer target file size (bytes) used to force a multi-file table + // layout cheaply, instead of brute-forcing huge row counts. private static final long SMALL_TARGET_FILE_SIZE = 256L * 1024; private static final int ROWS_MULTI_FILE = 100_000; private static MatrixBlock writeThenRead(MatrixBlock in) throws Exception { Path dir = Files.createTempDirectory("sysds_delta_"); - //WriterDelta creates the table at the given (empty) directory + // WriterDelta creates the table at the given (empty) directory String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { WriterDelta writer = new WriterDelta(); @@ -92,9 +91,9 @@ private static MatrixBlock writeThenRead(MatrixBlock in) throws Exception { @Test public void parallelReadMatchesSerialMultiFile() throws Exception { - //force a multi-file table cheaply via a small writer target file size - //(rather than a huge row count), so the parallel per-file path is - //actually exercised rather than falling back to serial. + // force a multi-file table cheaply via a small writer target file size + // (rather than a huge row count), so the parallel per-file path is + // actually exercised rather than falling back to serial. MatrixBlock in = TestUtils.generateTestMatrixBlock(ROWS_MULTI_FILE, 8, -10, 10, 1.0, 13); in.recomputeNonZeros(); @@ -104,21 +103,18 @@ public void parallelReadMatchesSerialMultiFile() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_par_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); - //sanity: confirm the table really is split across multiple files + // sanity: confirm the table really is split across multiple files long files; - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } - assertTrue("expected a multi-file Delta table to exercise the parallel path, got " + files, - files > 1); + assertTrue("expected a multi-file Delta table to exercise the parallel path, got " + files, files > 1); - MatrixBlock serial = new ReaderDelta() - .readMatrixFromHDFS(tablePath, -1, -1, -1, -1); - MatrixBlock parallel = new ReaderDeltaParallel() - .readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + MatrixBlock serial = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + MatrixBlock parallel = new ReaderDeltaParallel().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", serial.getNumRows(), parallel.getNumRows()); assertEquals("cols", serial.getNumColumns(), parallel.getNumColumns()); @@ -135,8 +131,8 @@ public void parallelReadMatchesSerialMultiFile() throws Exception { public void roundTripDenseSmall() throws Exception { MatrixBlock in = new MatrixBlock(3, 4, false); double v = 1.0; - for( int i=0; i<3; i++ ) - for( int j=0; j<4; j++ ) + for(int i = 0; i < 3; i++) + for(int j = 0; j < 4; j++) in.set(i, j, v++); in.recomputeNonZeros(); @@ -157,7 +153,7 @@ public void roundTripDenseRandom() throws Exception { @Test public void roundTripSparseRandom() throws Exception { - //values written are dense parquet, but exercise a sparse-ish input + // values written are dense parquet, but exercise a sparse-ish input MatrixBlock in = TestUtils.generateTestMatrixBlock(1200, 9, -5, 5, 0.1, 13); in.recomputeNonZeros(); MatrixBlock out = writeThenRead(in); @@ -168,7 +164,7 @@ public void roundTripSparseRandom() throws Exception { @Test public void roundTripMultiBatch() throws Exception { - //more rows than the writer batch size (4096) to exercise chunking + // more rows than the writer batch size (4096) to exercise chunking MatrixBlock in = TestUtils.generateTestMatrixBlock(10000, 5, 0, 100, 1.0, 1); MatrixBlock out = writeThenRead(in); assertEquals("rows", 10000, out.getNumRows()); @@ -182,8 +178,9 @@ public void readDiscoversUnknownDimensions() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); - //pass -1 dimensions: the reader must discover them from the table + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); + // pass -1 dimensions: the reader must discover them from the table MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 123, out.getNumRows()); assertEquals("cols", 6, out.getNumColumns()); @@ -212,22 +209,21 @@ public void emptyMatrixRoundTrip() throws Exception { @Test public void readNonDoubleNumericColumns() throws Exception { - //tables produced by external tools (or the frame writer) can carry - //long/int/boolean columns; the matrix reader must coerce them to double + // tables produced by external tools (or the frame writer) can carry + // long/int/boolean columns; the matrix reader must coerce them to double Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] longVals = {1, -2, 1_000_000_000L, 0}; - double[] intVals = {7, -8, 123456, 0}; + double[] intVals = {7, -8, 123456, 0}; double[] boolVals = {1, 0, 1, 0}; - writeTypedColumns(tablePath, - new DataType[] {LongType.LONG, IntegerType.INTEGER, BooleanType.BOOLEAN}, + writeTypedColumns(tablePath, new DataType[] {LongType.LONG, IntegerType.INTEGER, BooleanType.BOOLEAN}, new double[][] {longVals, intVals, boolVals}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 3, out.getNumColumns()); - for( int r=0; r<4; r++ ) { + for(int r = 0; r < 4; r++) { assertEquals("long col r" + r, longVals[r], out.get(r, 0), 0.0); assertEquals("int col r" + r, intVals[r], out.get(r, 1), 0.0); assertEquals("bool col r" + r, boolVals[r], out.get(r, 2), 0.0); @@ -240,14 +236,14 @@ public void readNonDoubleNumericColumns() throws Exception { @Test public void rewriteSamePathReplacesData() throws Exception { - //writing to a path that already holds a Delta table must fully replace it + // writing to a path that already holds a Delta table must fully replace it Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { MatrixBlock first = TestUtils.generateTestMatrixBlock(50, 8, 0, 100, 1.0, 1); new WriterDelta().writeMatrixToHDFS(first, tablePath, 50, 8, -1, first.getNonZeros()); - //second write has different dimensions and values + // second write has different dimensions and values MatrixBlock second = TestUtils.generateTestMatrixBlock(20, 3, -5, 5, 1.0, 2); new WriterDelta().writeMatrixToHDFS(second, tablePath, 20, 3, -1, second.getNonZeros()); @@ -263,10 +259,10 @@ public void rewriteSamePathReplacesData() throws Exception { @Test public void parallelBufferedPathMatchesSerial() throws Exception { - //the direct fast path is always taken for SystemDS-written tables (exact - //row stats, no deletion vectors); force the buffered fallback to exercise - //its per-file decode + serial concatenation and assert it matches serial. - //force a multi-file table cheaply via a small writer target file size. + // the direct fast path is always taken for SystemDS-written tables (exact + // row stats, no deletion vectors); force the buffered fallback to exercise + // its per-file decode + serial concatenation and assert it matches serial. + // force a multi-file table cheaply via a small writer target file size. MatrixBlock in = TestUtils.generateTestMatrixBlock(ROWS_MULTI_FILE, 8, -10, 10, 1.0, 23); in.recomputeNonZeros(); @@ -276,19 +272,22 @@ public void parallelBufferedPathMatchesSerial() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_buf_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); long files; - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } assertTrue("expected a multi-file Delta table, got " + files, files > 1); MatrixBlock serial = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); - //subclass that always declines the direct path -> readBuffered() + // subclass that always declines the direct path -> readBuffered() MatrixBlock buffered = new ReaderDeltaParallel() { - @Override protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { return false; } + @Override + protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { + return false; + } }.readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", serial.getNumRows(), buffered.getNumRows()); @@ -304,30 +303,30 @@ public void parallelBufferedPathMatchesSerial() throws Exception { @Test public void writerTargetFileSizeConfigProducesMoreFiles() throws Exception { - //a smaller configured target file size must make the writer roll more - //data files for the same matrix (the lever the parallel reader relies on). + // a smaller configured target file size must make the writer roll more + // data files for the same matrix (the lever the parallel reader relies on). MatrixBlock in = TestUtils.generateTestMatrixBlock(400_000, 16, -10, 10, 1.0, 7); in.recomputeNonZeros(); - //isolate the override in a fresh thread-local config (restored in finally) + // isolate the override in a fresh thread-local config (restored in finally) DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(1L * 1024 * 1024)); ConfigurationManager.setLocalConfig(conf); Path dir = Files.createTempDirectory("sysds_delta_cfg_"); try { - assertEquals("config getter reflects the override", - 1L * 1024 * 1024, ConfigurationManager.getDeltaWriterTargetFileSize()); + assertEquals("config getter reflects the override", 1L * 1024 * 1024, + ConfigurationManager.getDeltaWriterTargetFileSize()); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); long files; - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } assertTrue("expected >1 data file with a 1MB target, got " + files, files > 1); - //data still round-trips correctly with the custom layout + // data still round-trips correctly with the custom layout MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); TestUtils.compareMatrices(in, out, 1e-12, "small-target-roundtrip"); } @@ -339,21 +338,20 @@ public void writerTargetFileSizeConfigProducesMoreFiles() throws Exception { @Test public void readerBatchSizeConfigRoundTrips() throws Exception { - //a non-default reader batch size must not change the result (more, smaller - //batches exercise the per-batch extract/concatenate loop more often). + // a non-default reader batch size must not change the result (more, smaller + // batches exercise the per-batch extract/concatenate loop more often). MatrixBlock in = TestUtils.generateTestMatrixBlock(5000, 7, -10, 10, 1.0, 11); - //isolate the override in a fresh thread-local config (restored in finally) + // isolate the override in a fresh thread-local config (restored in finally) DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_READER_BATCH_SIZE, "128"); ConfigurationManager.setLocalConfig(conf); Path dir = Files.createTempDirectory("sysds_delta_bs_"); try { - assertEquals("config getter reflects the override", - 128, ConfigurationManager.getDeltaReaderBatchSize()); + assertEquals("config getter reflects the override", 128, ConfigurationManager.getDeltaReaderBatchSize()); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); TestUtils.compareMatrices(in, out, 1e-12, "small-batch-roundtrip"); } @@ -365,14 +363,13 @@ public void readerBatchSizeConfigRoundTrips() throws Exception { @Test public void factoryRoutesDeltaToParallelWhenEnabled() { - //the factory must pick the parallel reader iff parallel CP read is enabled + // the factory must pick the parallel reader iff parallel CP read is enabled CompilerConfig cc = ConfigurationManager.getCompilerConfig(); try { cc.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, true); ConfigurationManager.setLocalConfig(cc); MatrixReader par = MatrixReaderFactory.createMatrixReader(FileFormat.DELTA); - assertTrue("expected ReaderDeltaParallel when parallel read enabled", - par instanceof ReaderDeltaParallel); + assertTrue("expected ReaderDeltaParallel when parallel read enabled", par instanceof ReaderDeltaParallel); cc.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, false); ConfigurationManager.setLocalConfig(cc); @@ -387,20 +384,18 @@ public void factoryRoutesDeltaToParallelWhenEnabled() { @Test public void readFloatColumnsCoercedToDouble() throws Exception { - //float columns must be widened to double on read (exact-representable values) + // float columns must be widened to double on read (exact-representable values) Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] f0 = {1.5, -2.25, 0.0, 1024.5}; double[] f1 = {-0.5, 3.75, 100.125, -7.0}; - writeTypedColumns(tablePath, - new DataType[] {FloatType.FLOAT, FloatType.FLOAT}, - new double[][] {f0, f1}); + writeTypedColumns(tablePath, new DataType[] {FloatType.FLOAT, FloatType.FLOAT}, new double[][] {f0, f1}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 2, out.getNumColumns()); - for( int r=0; r<4; r++ ) { + for(int r = 0; r < 4; r++) { assertEquals("f0 r" + r, f0[r], out.get(r, 0), 0.0); assertEquals("f1 r" + r, f1[r], out.get(r, 1), 0.0); } @@ -412,21 +407,20 @@ public void readFloatColumnsCoercedToDouble() throws Exception { @Test public void readShortByteColumnsCoercedToDouble() throws Exception { - //short/byte columns must be coerced to double on read, exercising the - //T_SHORT / T_BYTE branches of ReaderDelta.getDoubleValue. + // short/byte columns must be coerced to double on read, exercising the + // T_SHORT / T_BYTE branches of ReaderDelta.getDoubleValue. Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] shortVals = {1, -2, 30000, 0}; - double[] byteVals = {7, -8, 120, 0}; - writeTypedColumns(tablePath, - new DataType[] {ShortType.SHORT, ByteType.BYTE}, + double[] byteVals = {7, -8, 120, 0}; + writeTypedColumns(tablePath, new DataType[] {ShortType.SHORT, ByteType.BYTE}, new double[][] {shortVals, byteVals}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 2, out.getNumColumns()); - for( int r=0; r<4; r++ ) { + for(int r = 0; r < 4; r++) { assertEquals("short col r" + r, shortVals[r], out.get(r, 0), 0.0); assertEquals("byte col r" + r, byteVals[r], out.get(r, 1), 0.0); } @@ -438,8 +432,8 @@ public void readShortByteColumnsCoercedToDouble() throws Exception { @Test public void writerRejectsDimensionMismatch() throws Exception { - //WriterDelta validates that the passed rlen/clen match the MatrixBlock - //and rejects a mismatch with an IOException. + // WriterDelta validates that the passed rlen/clen match the MatrixBlock + // and rejects a mismatch with an IOException. MatrixBlock in = TestUtils.generateTestMatrixBlock(10, 4, -1, 1, 1.0, 5); in.recomputeNonZeros(); Path dir = Files.createTempDirectory("sysds_delta_"); @@ -459,18 +453,18 @@ public void writerRejectsDimensionMismatch() throws Exception { @Test public void readNullCellsBecomeZero() throws Exception { - //nullable numeric columns with null cells must read back as 0.0 + // nullable numeric columns with null cells must read back as 0.0 Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - double[] vals = {3.0, 7.0, 9.0, 11.0}; - boolean[] nulls = {false, true, false, true}; + double[] vals = {3.0, 7.0, 9.0, 11.0}; + boolean[] nulls = {false, true, false, true}; writeNullableDoubleColumn(tablePath, vals, nulls); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 1, out.getNumColumns()); - for( int r=0; r<4; r++ ) + for(int r = 0; r < 4; r++) assertEquals("r" + r, nulls[r] ? 0.0 : vals[r], out.get(r, 0), 0.0); } finally { @@ -480,7 +474,7 @@ public void readNullCellsBecomeZero() throws Exception { @Test public void readStringColumnRejected() throws Exception { - //string columns cannot back an all-double matrix -> reader must reject them + // string columns cannot back an all-double matrix -> reader must reject them Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { @@ -503,9 +497,10 @@ public void readStringColumnRejected() throws Exception { private static void writeTypedColumns(String tablePath, DataType[] types, double[][] vals) throws Exception { Engine engine = DeltaKernelUtils.createEngine(); StructType schema = new StructType(); - for( int c=0; c singleton(FilteredColumnarBatch fcb) { return new CloseableIterator() { private boolean _done = false; - @Override public boolean hasNext() { return !_done; } - @Override public FilteredColumnarBatch next() { - if( _done ) throw new NoSuchElementException(); + + @Override + public boolean hasNext() { + return !_done; + } + + @Override + public FilteredColumnarBatch next() { + if(_done) + throw new NoSuchElementException(); _done = true; return fcb; } - @Override public void close() {} + + @Override + public void close() { + } }; } - /** Minimal in-memory columnar batch backed by per-column double[] values, with - * an optional per-column null mask ({@code nulls==null} => no nulls). */ + /** + * Minimal in-memory columnar batch backed by per-column double[] values, with an optional per-column null mask + * ({@code nulls==null} => no nulls). + */ private static class TypedBatch implements ColumnarBatch { private final StructType _schema; private final DataType[] _types; private final double[][] _vals; private final boolean[][] _nulls; + TypedBatch(StructType schema, DataType[] types, double[][] vals, boolean[][] nulls) { - _schema = schema; _types = types; _vals = vals; _nulls = nulls; + _schema = schema; + _types = types; + _vals = vals; + _nulls = nulls; } - @Override public StructType getSchema() { return _schema; } - @Override public int getSize() { return _vals[0].length; } - @Override public ColumnVector getColumnVector(int ordinal) { - return new TypedVector(_types[ordinal], _vals[ordinal], - _nulls == null ? null : _nulls[ordinal]); + + @Override + public StructType getSchema() { + return _schema; + } + + @Override + public int getSize() { + return _vals[0].length; + } + + @Override + public ColumnVector getColumnVector(int ordinal) { + return new TypedVector(_types[ordinal], _vals[ordinal], _nulls == null ? null : _nulls[ordinal]); } } @@ -568,28 +599,98 @@ private static class TypedVector implements ColumnVector { private final DataType _type; private final double[] _vals; private final boolean[] _nulls; - TypedVector(DataType type, double[] vals, boolean[] nulls) { _type = type; _vals = vals; _nulls = nulls; } - @Override public DataType getDataType() { return _type; } - @Override public int getSize() { return _vals.length; } - @Override public boolean isNullAt(int rowId) { return _nulls != null && _nulls[rowId]; } - @Override public double getDouble(int rowId) { return _vals[rowId]; } - @Override public float getFloat(int rowId) { return (float) _vals[rowId]; } - @Override public long getLong(int rowId) { return (long) _vals[rowId]; } - @Override public int getInt(int rowId) { return (int) _vals[rowId]; } - @Override public short getShort(int rowId) { return (short) _vals[rowId]; } - @Override public byte getByte(int rowId) { return (byte) _vals[rowId]; } - @Override public boolean getBoolean(int rowId) { return _vals[rowId] != 0; } - @Override public void close() {} + + TypedVector(DataType type, double[] vals, boolean[] nulls) { + _type = type; + _vals = vals; + _nulls = nulls; + } + + @Override + public DataType getDataType() { + return _type; + } + + @Override + public int getSize() { + return _vals.length; + } + + @Override + public boolean isNullAt(int rowId) { + return _nulls != null && _nulls[rowId]; + } + + @Override + public double getDouble(int rowId) { + return _vals[rowId]; + } + + @Override + public float getFloat(int rowId) { + return (float) _vals[rowId]; + } + + @Override + public long getLong(int rowId) { + return (long) _vals[rowId]; + } + + @Override + public int getInt(int rowId) { + return (int) _vals[rowId]; + } + + @Override + public short getShort(int rowId) { + return (short) _vals[rowId]; + } + + @Override + public byte getByte(int rowId) { + return (byte) _vals[rowId]; + } + + @Override + public boolean getBoolean(int rowId) { + return _vals[rowId] != 0; + } + + @Override + public void close() { + } } /** Column view exposing a String[] as a Delta string column. */ private static class StringVector implements ColumnVector { private final String[] _vals; - StringVector(String[] vals) { _vals = vals; } - @Override public DataType getDataType() { return StringType.STRING; } - @Override public int getSize() { return _vals.length; } - @Override public boolean isNullAt(int rowId) { return _vals[rowId] == null; } - @Override public String getString(int rowId) { return _vals[rowId]; } - @Override public void close() {} + + StringVector(String[] vals) { + _vals = vals; + } + + @Override + public DataType getDataType() { + return StringType.STRING; + } + + @Override + public int getSize() { + return _vals.length; + } + + @Override + public boolean isNullAt(int rowId) { + return _vals[rowId] == null; + } + + @Override + public String getString(int rowId) { + return _vals[rowId]; + } + + @Override + public void close() { + } } } diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java index 2d79b79f2dd..45194d12b5a 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java @@ -49,24 +49,22 @@ import org.junit.Test; /** - * Cross-engine interoperability tests for the native (Delta Kernel based) matrix - * reader/writer against the reference Delta implementation (Delta's Spark - * connector, {@code delta-spark}, pulled in test-only). + * Cross-engine interoperability tests for the native (Delta Kernel based) matrix reader/writer against the reference + * Delta implementation (Delta's Spark connector, {@code delta-spark}, pulled in test-only). * - *

The other Delta matrix tests round-trip exclusively through SystemDS' own - * Kernel-based read/write paths, so they cannot catch a table that SystemDS - * writes in a way other Delta engines reject (or vice versa). These tests close - * that gap by routing data through two independent engines: + *

+ * The other Delta matrix tests round-trip exclusively through SystemDS' own Kernel-based read/write paths, so they + * cannot catch a table that SystemDS writes in a way other Delta engines reject (or vice versa). These tests close that + * gap by routing data through two independent engines: *

    - *
  • SystemDS writes -> Spark/Delta reads (our output is spec-compliant), and
    • - *
  • Spark/Delta writes -> SystemDS reads, including a multi-file layout and a - * table with deletion vectors / a second commit that the SystemDS writer - * never produces itself.
    • + *
  • SystemDS writes -> Spark/Delta reads (our output is spec-compliant), and
    • + *
  • Spark/Delta writes -> SystemDS reads, including a multi-file layout and a table with deletion vectors / a + * second commit that the SystemDS writer never produces itself.
    • *
* - *

Row order is never assumed: every table carries a unique id in column 0 and - * comparisons are keyed by that id, since neither engine guarantees row order - * across files. + *

+ * Row order is never assumed: every table carries a unique id in column 0 and comparisons are keyed by that id, since + * neither engine guarantees row order across files. */ @net.jcip.annotations.NotThreadSafe public class DeltaMatrixSparkInteropTest { @@ -75,23 +73,19 @@ public class DeltaMatrixSparkInteropTest { @BeforeClass public static void startSpark() { - //each test class runs in its own fork (surefire reuseForks=false), so this - //is the only SparkSession in the JVM and gets the Delta extensions injected. + // each test class runs in its own fork (surefire reuseForks=false), so this + // is the only SparkSession in the JVM and gets the Delta extensions injected. SparkSession.clearActiveSession(); SparkSession.clearDefaultSession(); - spark = SparkSession.builder() - .appName("sysds-delta-interop") - .master("local[2]") - .config("spark.ui.enabled", "false") - .config("spark.sql.shuffle.partitions", "2") + spark = SparkSession.builder().appName("sysds-delta-interop").master("local[2]") + .config("spark.ui.enabled", "false").config("spark.sql.shuffle.partitions", "2") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") - .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") - .getOrCreate(); + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog").getOrCreate(); } @AfterClass public static void stopSpark() { - if( spark != null ) + if(spark != null) spark.stop(); SparkSession.clearActiveSession(); SparkSession.clearDefaultSession(); @@ -100,13 +94,13 @@ public static void stopSpark() { @Test public void systemdsWriteSparkReadMultiFile() throws Exception { - //SystemDS writes a (forced) multi-file Delta table; the reference Delta - //engine (Spark) must read every data file back with matching values. + // SystemDS writes a (forced) multi-file Delta table; the reference Delta + // engine (Spark) must read every data file back with matching values. int rows = 500, cols = 5; MatrixBlock in = indexedMatrix(rows, cols); - //small target file size -> multiple parquet data files (exercise that an - //external reader stitches all of our data files, not just the first). + // small target file size -> multiple parquet data files (exercise that an + // external reader stitches all of our data files, not just the first). DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(16L * 1024)); ConfigurationManager.setLocalConfig(conf); @@ -122,10 +116,10 @@ public void systemdsWriteSparkReadMultiFile() throws Exception { List read = df.collectAsList(); assertEquals(rows, read.size()); - for( Row r : read ) { + for(Row r : read) { int id = (int) Math.round(r.getDouble(0)); assertTrue("id in range: " + id, id >= 0 && id < rows); - for( int c = 0; c < cols; c++ ) + for(int c = 0; c < cols; c++) assertEquals("r" + id + " c" + c, in.get(id, c), r.getDouble(c), 1e-9); } } @@ -137,10 +131,10 @@ public void systemdsWriteSparkReadMultiFile() throws Exception { @Test public void sparkWriteSystemdsReadMultiFile() throws Exception { - //the reference Delta engine writes a multi-file table; both the serial and - //parallel SystemDS readers must reconstruct it (coercing long ids to double). + // the reference Delta engine writes a multi-file table; both the serial and + // parallel SystemDS readers must reconstruct it (coercing long ids to double). int rows = 600, cols = 4; - Dataset df = indexedDataFrame(rows, cols).repartition(3); //-> multiple data files + Dataset df = indexedDataFrame(rows, cols).repartition(3); // -> multiple data files Path dir = Files.createTempDirectory("sysds_delta_p2s_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { @@ -148,10 +142,10 @@ public void sparkWriteSystemdsReadMultiFile() throws Exception { assertTrue("spark should have written a multi-file table", countParquet(tablePath) > 1); Map expected = expectedById(rows, cols); - assertMatchesById(new ReaderDelta() - .readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, "serial"); - assertMatchesById(new ReaderDeltaParallel() - .readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, "parallel"); + assertMatchesById(new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, + "serial"); + assertMatchesById(new ReaderDeltaParallel().readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, + "parallel"); } finally { FileUtils.deleteQuietly(dir.toFile()); @@ -160,15 +154,15 @@ public void sparkWriteSystemdsReadMultiFile() throws Exception { @Test public void sparkDeletionVectorsSystemdsRead() throws Exception { - //a Delta table with deletion vectors + a second commit (the DELETE) is a - //layout the SystemDS writer never emits; the readers must honor the DV and - //return only the surviving rows. This exercises the hasDeletionVector path. + // a Delta table with deletion vectors + a second commit (the DELETE) is a + // layout the SystemDS writer never emits; the readers must honor the DV and + // return only the surviving rows. This exercises the hasDeletionVector path. int rows = 400, cols = 3, deleteBelow = 50; Path dir = Files.createTempDirectory("sysds_delta_dv_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - //enable deletion vectors for tables created in this block, then delete a - //row range so Delta records a DV rather than rewriting the data files. + // enable deletion vectors for tables created in this block, then delete a + // row range so Delta records a DV rather than rewriting the data files. spark.conf().set(DV_DEFAULT, "true"); indexedDataFrame(rows, cols).write().format("delta").save(tablePath); spark.sql("DELETE FROM delta.`" + tablePath + "` WHERE c0 < " + deleteBelow); @@ -185,21 +179,20 @@ public void sparkDeletionVectorsSystemdsRead() throws Exception { assertMatchesById(parallel, expected, cols, "parallel-dv"); } finally { - //fresh fork per test class, so simply clearing the override is enough + // fresh fork per test class, so simply clearing the override is enough spark.conf().unset(DV_DEFAULT); FileUtils.deleteQuietly(dir.toFile()); } } - private static final String DV_DEFAULT = - "spark.databricks.delta.properties.defaults.enableDeletionVectors"; + private static final String DV_DEFAULT = "spark.databricks.delta.properties.defaults.enableDeletionVectors"; /** Matrix whose column 0 is the row index and remaining columns are exact doubles. */ private static MatrixBlock indexedMatrix(int rows, int cols) { MatrixBlock mb = new MatrixBlock(rows, cols, false); - for( int r = 0; r < rows; r++ ) { + for(int r = 0; r < rows; r++) { mb.set(r, 0, r); - for( int c = 1; c < cols; c++ ) + for(int c = 1; c < cols; c++) mb.set(r, c, value(r, c)); } mb.recomputeNonZeros(); @@ -209,15 +202,15 @@ private static MatrixBlock indexedMatrix(int rows, int cols) { /** Spark DataFrame mirroring {@link #indexedMatrix} with columns c0..c(cols-1) as doubles. */ private static Dataset indexedDataFrame(int rows, int cols) { StructField[] fields = new StructField[cols]; - for( int c = 0; c < cols; c++ ) + for(int c = 0; c < cols; c++) fields[c] = DataTypes.createStructField("c" + c, DataTypes.DoubleType, false); StructType schema = DataTypes.createStructType(fields); List data = new ArrayList<>(rows); - for( int r = 0; r < rows; r++ ) { + for(int r = 0; r < rows; r++) { Object[] vals = new Object[cols]; vals[0] = (double) r; - for( int c = 1; c < cols; c++ ) + for(int c = 1; c < cols; c++) vals[c] = value(r, c); data.add(RowFactory.create(vals)); } @@ -231,10 +224,10 @@ private static double value(int row, int col) { private static Map expectedById(int rows, int cols) { Map exp = new HashMap<>(rows); - for( int r = 0; r < rows; r++ ) { + for(int r = 0; r < rows; r++) { double[] row = new double[cols]; row[0] = r; - for( int c = 1; c < cols; c++ ) + for(int c = 1; c < cols; c++) row[c] = value(r, c); exp.put(r, row); } @@ -246,25 +239,25 @@ private static void assertMatchesById(MatrixBlock out, Map ex assertEquals(tag + " rows", expected.size(), out.getNumRows()); assertEquals(tag + " cols", cols, out.getNumColumns()); boolean[] seen = new boolean[expected.size() == 0 ? 0 : maxId(expected) + 1]; - for( int r = 0; r < out.getNumRows(); r++ ) { + for(int r = 0; r < out.getNumRows(); r++) { int id = (int) Math.round(out.get(r, 0)); double[] exp = expected.get(id); assertTrue(tag + ": unexpected/duplicate id " + id, exp != null && id < seen.length && !seen[id]); seen[id] = true; - for( int c = 0; c < cols; c++ ) + for(int c = 0; c < cols; c++) assertEquals(tag + " id" + id + " c" + c, exp[c], out.get(r, c), 1e-9); } } private static int maxId(Map expected) { int m = 0; - for( int id : expected.keySet() ) + for(int id : expected.keySet()) m = Math.max(m, id); return m; } private static long countParquet(String tablePath) throws Exception { - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { return s.filter(p -> p.toString().endsWith(".parquet")).count(); } } diff --git a/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java b/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java index 472b61d8cd6..ae19b291882 100644 --- a/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java +++ b/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -26,11 +26,10 @@ /** * Tests the single-column (unweighted) branch of {@link MatrixBlock#pickValue(double, boolean)} and - * {@link MatrixBlock#median()}. The values are assumed to be sorted in ascending order, mirroring the contract used - * by the quantile pick instructions. The unweighted branch uses the same ceil-based rank as the two-column weighted - * branch (with an implicit weight of 1 per value), so a single column yields the same quantile as the equivalent - * (value, weight) representation. The two-column (weighted) branch is exercised separately through the compressed - * sort tests. + * {@link MatrixBlock#median()}. The values are assumed to be sorted in ascending order, mirroring the contract used by + * the quantile pick instructions. The unweighted branch uses the same ceil-based rank as the two-column weighted branch + * (with an implicit weight of 1 per value), so a single column yields the same quantile as the equivalent (value, + * weight) representation. The two-column (weighted) branch is exercised separately through the compressed sort tests. */ public class QuantilePickTest { diff --git a/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java b/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java index 5c9ed821e78..05aca41ebc7 100644 --- a/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java +++ b/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java @@ -30,7 +30,7 @@ public class TensorToStringTest { @Test public void testDecimalClampsFractionDigits() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 1}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 1}); tb.allocateBlock(); tb.set(0, 0, 5.244058388023880); // decimal=2 must print exactly two fraction digits, not DecimalFormat's default max of 3 @@ -41,10 +41,10 @@ public void testDecimalClampsFractionDigits() { @Test public void testDecimalPadsAndRounds() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 2}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 2}); tb.allocateBlock(); - tb.set(0, 0, 22.0); // integer-valued: padded up to the requested digits - tb.set(0, 1, 5.244058388023880); // rounded at the last requested digit + tb.set(0, 0, 22.0); // integer-valued: padded up to the requested digits + tb.set(0, 1, 5.244058388023880); // rounded at the last requested digit String out = DataConverter.toString(tb, false, " ", "\n", "[", "]", 1, 2, 4); assertTrue("expected 22.0000 padded: " + out, out.contains("22.0000")); assertTrue("expected 5.2441 rounded: " + out, out.contains("5.2441")); @@ -52,10 +52,10 @@ public void testDecimalPadsAndRounds() { @Test public void testNegativeDecimalUsesDefaultFormatting() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 2}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 2}); tb.allocateBlock(); - tb.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained - tb.set(0, 1, 5.244058388023880); // default cap of three fraction digits + tb.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained + tb.set(0, 1, 5.244058388023880); // default cap of three fraction digits // decimal < 0 leaves DecimalFormat unconstrained (no min/max fraction digits set) String out = DataConverter.toString(tb, false, " ", "\n", "[", "]", 1, 2, -1); assertTrue("expected unpadded 22: " + out, out.contains("22")); diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java index 810e2614e7c..426de81263f 100644 --- a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java +++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java @@ -52,18 +52,12 @@ public class QuantileTest extends AutomatedTestBase public void setUp() { TestUtils.clearAssertionInformation(); - addTestConfiguration(TEST_NAME1, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"})); - addTestConfiguration(TEST_NAME2, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"})); - addTestConfiguration(TEST_NAME3, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] {"R"})); - addTestConfiguration(TEST_NAME4, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] {"R"})); - addTestConfiguration(TEST_NAME5, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] {"R"})); - addTestConfiguration(TEST_NAME6, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] {"R"})); + addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"})); + addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"})); + addTestConfiguration(TEST_NAME3, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] {"R"})); + addTestConfiguration(TEST_NAME4, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] {"R"})); + addTestConfiguration(TEST_NAME5, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] {"R"})); + addTestConfiguration(TEST_NAME6, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] {"R"})); } @Test diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java index e70aedcabe4..35cfb2982fc 100644 --- a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java +++ b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java @@ -59,7 +59,8 @@ private void runSTEPGlmTest(ExecType instType) { // runTest executes the script; fails if the DML script invokes stop() runTest(true, false, null, -1); - } finally { + } + finally { rtplatform = platformOld; } } diff --git a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java index 5de429a3c53..d8cfc4d9fd7 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java @@ -91,10 +91,12 @@ public void testBackendPerformance() throws InterruptedException { taskFutures.forEach(res -> { try { Assert.assertEquals("Stats parsed correctly", res.get().statusCode(), 200); - } catch (InterruptedException e) { + } + catch(InterruptedException e) { Thread.currentThread().interrupt(); Assert.fail("Interrupted while fetching statistics: " + e.getMessage()); - } catch (ExecutionException e) { + } + catch(ExecutionException e) { Assert.fail("Failed to fetch statistics: " + e.getMessage()); } }); diff --git a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java index f8acdd07930..1cab99ee1ab 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java @@ -375,9 +375,8 @@ public void federatedLogicalTest(String testname, Type op_type, ExecMode execMod int port2 = single_fed_worker ? 0 : getRandomAvailablePort(); int port3 = single_fed_worker ? 0 : getRandomAvailablePort(); int port4 = single_fed_worker ? 0 : getRandomAvailablePort(); - Process[] workers = startLocalFedWorkers(single_fed_worker - ? new int[] {port1} - : new int[] {port1, port2, port3, port4}); + Process[] workers = startLocalFedWorkers( + single_fed_worker ? new int[] {port1} : new int[] {port1, port2, port3, port4}); try { if(!isAlive(workers)) diff --git a/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java b/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java index dbbf199f8fa..9d2bb583a4f 100644 --- a/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java +++ b/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java @@ -72,27 +72,26 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod } if(et == ExecType.SPARK) { - rtplatform = ExecMode.SPARK; + rtplatform = ExecMode.SPARK; } else { // rtplatform = (et==ExecType.MR)? ExecMode.HADOOP : ExecMode.SINGLE_NODE; - rtplatform = ExecMode.HYBRID; + rtplatform = ExecMode.HYBRID; } if( rtplatform == ExecMode.SPARK ) DMLScript.USE_LOCAL_SPARK_CONFIG = true; config.addVariable("rows", rows); - config.addVariable("cols", cols); + config.addVariable("cols", cols); - long rowstart=816, rowend=1229, colstart=967, colend=1009; - // long rowstart=2, rowend=4, colstart=9, colend=10; + long rowstart = 816, rowend = 1229, colstart = 967, colend = 1009; + // long rowstart=2, rowend=4, colstart=9, colend=10; /* - Random rand=new Random(System.currentTimeMillis()); - rowstart=(long)(rand.nextDouble()*((double)rows))+1; - rowend=(long)(rand.nextDouble()*((double)(rows-rowstart+1)))+rowstart; - colstart=(long)(rand.nextDouble()*((double)cols))+1; - colend=(long)(rand.nextDouble()*((double)(cols-colstart+1)))+colstart; - */ + * Random rand=new Random(System.currentTimeMillis()); rowstart=(long)(rand.nextDouble()*((double)rows))+1; + * rowend=(long)(rand.nextDouble()*((double)(rows-rowstart+1)))+rowstart; + * colstart=(long)(rand.nextDouble()*((double)cols))+1; + * colend=(long)(rand.nextDouble()*((double)(cols-colstart+1)))+colstart; + */ config.addVariable("rowstart", rowstart); config.addVariable("rowend", rowend); config.addVariable("colstart", colstart); @@ -119,17 +118,20 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod double sparsity=1.0;//rand.nextDouble(); double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, System.currentTimeMillis()); writeInputMatrix("A", A, true); - - sparsity=0.1;//rand.nextDouble(); - double[][] B = getRandomMatrix((int)(rowend-rowstart+1), (int)(colend-colstart+1), min, max, sparsity, System.currentTimeMillis()); + + sparsity = 0.1;// rand.nextDouble(); + double[][] B = getRandomMatrix((int) (rowend - rowstart + 1), (int) (colend - colstart + 1), min, max, + sparsity, System.currentTimeMillis()); writeInputMatrix("B", B, true); - - sparsity=0.5;//rand.nextDouble(); - double[][] C = getRandomMatrix((int)(rowend), (int)(cols-colstart+1), min, max, sparsity, System.currentTimeMillis()); + + sparsity = 0.5;// rand.nextDouble(); + double[][] C = getRandomMatrix((int) (rowend), (int) (cols - colstart + 1), min, max, sparsity, + System.currentTimeMillis()); writeInputMatrix("C", C, true); - - sparsity=0.01;//rand.nextDouble(); - double[][] D = getRandomMatrix(rows, (int)(colend-colstart+1), min, max, sparsity, System.currentTimeMillis()); + + sparsity = 0.01;// rand.nextDouble(); + double[][] D = getRandomMatrix(rows, (int) (colend - colstart + 1), min, max, sparsity, + System.currentTimeMillis()); writeInputMatrix("D", D, true); /* @@ -138,9 +140,9 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod * While loop iteration - 10 jobs * Final output write - 1 job */ - //boolean exceptionExpected = false; - //int expectedNumberOfJobs = 12; - //runTest(exceptionExpected, null, expectedNumberOfJobs); + // boolean exceptionExpected = false; + // int expectedNumberOfJobs = 12; + // runTest(exceptionExpected, null, expectedNumberOfJobs); boolean exceptionExpected = false; int expectedNumberOfJobs = -1; runTest(true, exceptionExpected, null, expectedNumberOfJobs); diff --git a/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java b/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java index a1b51ee9d81..fad1cc123c8 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java @@ -81,8 +81,9 @@ public void testDoubleScalarWrite() { fullDMLScriptName = HOME + "ScalarComputeWrite.dml"; runTest(true, false, null, -1); - double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1,1)).doubleValue(); - Assert.assertEquals("Computation test for Doubles failed: Values not equal: " + double_scalar + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar); + double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)).doubleValue(); + Assert.assertEquals("Computation test for Doubles failed: Values not equal: " + double_scalar + + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar); } @Test @@ -122,8 +123,7 @@ public void testIntScalarRead() { programArgs = new String[]{"-args", String.valueOf(int_scalar), output("a.scalar")}; runTest(true, false, null, -1); - int int_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)) - .get(new CellIndex(1,1)).intValue(); + int int_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)).intValue(); Assert.assertEquals("Values not equal: " + int_scalar + Opcodes.NOTEQUAL.toString() + int_out_scalar, int_scalar, int_out_scalar); @@ -143,8 +143,8 @@ public void testDoubleScalarRead() { programArgs = new String[]{ "-args", String.valueOf(double_scalar), output("a.scalar") }; runTest(true, false, null, -1); - double double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)) - .get(new CellIndex(1,1)).doubleValue(); + double double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)) + .doubleValue(); Assert.assertEquals("Values not equal: " + double_scalar + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar, 0); diff --git a/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java b/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java index a4013c3672d..20ae7aa63ea 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java @@ -35,14 +35,14 @@ /** * End-to-end DML test of the native Delta read/write path. * - *

The write and the read are run as two separate SystemDS executions - * on purpose. If they shared a single script/process, SystemDS would reuse the - * still-materialized in-memory matrix for the subsequent read and never invoke - * {@link org.apache.sysds.runtime.io.ReaderDelta} at all (verified: the cache - * reports 0 HDFS hits in that case). Splitting the executions forces a genuine - * read from disk, and we additionally assert via {@link CacheStatistics} that - * the read run actually performed HDFS reads (the Delta table + the text - * reference) rather than serving the matrix from cache.

+ *

+ * The write and the read are run as two separate SystemDS executions on purpose. If they shared a single + * script/process, SystemDS would reuse the still-materialized in-memory matrix for the subsequent read and never invoke + * {@link org.apache.sysds.runtime.io.ReaderDelta} at all (verified: the cache reports 0 HDFS hits in that case). + * Splitting the executions forces a genuine read from disk, and we additionally assert via {@link CacheStatistics} that + * the read run actually performed HDFS reads (the Delta table + the text reference) rather than serving the matrix from + * cache. + *

*/ public class DeltaReadWriteTest extends AutomatedTestBase { @@ -54,10 +54,8 @@ public class DeltaReadWriteTest extends AutomatedTestBase { @Override public void setUp() { TestUtils.clearAssertionInformation(); - addTestConfiguration(WRITE_NAME, - new TestConfiguration(TEST_CLASS_DIR, WRITE_NAME, new String[] { "ref" })); - addTestConfiguration(READ_NAME, - new TestConfiguration(TEST_CLASS_DIR, READ_NAME, new String[] { "R" })); + addTestConfiguration(WRITE_NAME, new TestConfiguration(TEST_CLASS_DIR, WRITE_NAME, new String[] {"ref"})); + addTestConfiguration(READ_NAME, new TestConfiguration(TEST_CLASS_DIR, READ_NAME, new String[] {"R"})); } @Test @@ -84,17 +82,16 @@ private void runDeltaRoundTrip(int rows, int cols, double sparsity) { String deltaPath = output("deltaTable"); String refPath = output("ref"); fullDMLScriptName = HOME + WRITE_NAME + ".dml"; - programArgs = new String[] { "-stats", "-args", - String.valueOf(rows), String.valueOf(cols), String.valueOf(sparsity), - deltaPath, refPath }; + programArgs = new String[] {"-stats", "-args", String.valueOf(rows), String.valueOf(cols), + String.valueOf(sparsity), deltaPath, refPath}; runTest(true, false, null, -1); // the write run must have materialized two matrices to disk (the Delta // table under test + the text reference); WriterDelta genuinely hitting // HDFS is what produces these write-side cache statistics. long hdfsWrites = CacheStatistics.getHDFSWrites(); - assertTrue("expected >= 2 HDFS writes in the write run (delta + reference), got " - + hdfsWrites, hdfsWrites >= 2); + assertTrue("expected >= 2 HDFS writes in the write run (delta + reference), got " + hdfsWrites, + hdfsWrites >= 2); // and a real Delta table (transaction log) must have been created assertTrue("missing Delta transaction log under " + deltaPath, new File(deltaPath, "_delta_log").isDirectory()); @@ -102,19 +99,18 @@ private void runDeltaRoundTrip(int rows, int cols, double sparsity) { // ---- phase 2: fresh execution reads the Delta table and compares ---- getAndLoadTestConfiguration(READ_NAME); fullDMLScriptName = HOME + READ_NAME + ".dml"; - programArgs = new String[] { "-stats", "-args", - deltaPath, refPath, output("R") }; + programArgs = new String[] {"-stats", "-args", deltaPath, refPath, output("R")}; runTest(true, false, null, -1); // the read run must have materialized two matrices from disk (the Delta // table under test + the text reference); a cached/short-circuited read // would report fewer HDFS hits and fail here. long hdfsReads = CacheStatistics.getHDFSHits(); - assertTrue("expected >= 2 HDFS reads in the read run (delta + reference), got " - + hdfsReads, hdfsReads >= 2); + assertTrue("expected >= 2 HDFS reads in the read run (delta + reference), got " + hdfsReads, + hdfsReads >= 2); HashMap R = readDMLMatrixFromOutputDir("R"); - //text-cell output omits exact zeros, so a missing cell means 0.0 + // text-cell output omits exact zeros, so a missing cell means 0.0 double diff = R.getOrDefault(new CellIndex(1, 1), 0.0); double nrow = R.getOrDefault(new CellIndex(1, 2), 0.0); double ncol = R.getOrDefault(new CellIndex(1, 3), 0.0); diff --git a/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java index cc1412b1606..a844321c249 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java @@ -49,10 +49,9 @@ public class FrameParquetSchemaTest extends AutomatedTestBase { @Override public void setUp() { - addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[]{"Rout"})); + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"Rout"})); } - /** * Test for sequential writer and reader * diff --git a/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java b/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java index dfb3d8a19de..6e6e4665f5e 100644 --- a/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java @@ -42,12 +42,8 @@ */ @net.jcip.annotations.NotThreadSafe public class JMLConnectionTest extends AutomatedTestBase { - public static final String META = "{\"data_type\": \"matrix\",\n" + - " \"value_type\": \"double\", \n" + - " \"rows\": 1,\n" + - " \"cols\": 1,\n" + - " \"nnz\": 1,\n" + - " \"format\": \"csv\"}"; + public static final String META = "{\"data_type\": \"matrix\",\n" + " \"value_type\": \"double\", \n" + + " \"rows\": 1,\n" + " \"cols\": 1,\n" + " \"nnz\": 1,\n" + " \"format\": \"csv\"}"; private final static String TEST_NAME = "JMLConnectionTest"; private final static String TEST_DIR = "functions/jmlc/"; @@ -99,12 +95,14 @@ public void testConnectionInvalidInName() throws DMLException { conn.gatherMemStats(false); Assert.assertFalse(DMLScript.STATISTICS); - try (conn) { - conn.prepareScript("printx('hello')", new String[]{"$inScalar1", null}, new String[]{null}); + try(conn) { + conn.prepareScript("printx('hello')", new String[] {"$inScalar1", null}, new String[] {null}); throw new AssertionError("Test should have thrown a LanguageException"); - } catch (LanguageException e) { + } + catch(LanguageException e) { Assert.assertTrue(e.getMessage().startsWith("Invalid variable names")); - } finally { + } + finally { DMLScript.STATISTICS = oldStat; DMLScript.JMLC_MEM_STATISTICS = oldJMLCStat; } @@ -112,21 +110,24 @@ public void testConnectionInvalidInName() throws DMLException { @Test public void testConnectionParseLanguageException() { - try (Connection conn = new Connection()) { - conn.prepareScript("printx('hello')", new String[]{}, new String[]{}); + try(Connection conn = new Connection()) { + conn.prepareScript("printx('hello')", new String[] {}, new String[] {}); throw new AssertionError("Test should have thrown a DMLException"); - } catch (DMLException e) { + } + catch(DMLException e) { Throwable cause = e.getCause(); - Assert.assertTrue(cause.getMessage().startsWith("ERROR: [line 1:0] -> printx('hello') -- function printx is undefined in namespace .builtinNS")); + Assert.assertTrue(cause.getMessage().startsWith( + "ERROR: [line 1:0] -> printx('hello') -- function printx is undefined in namespace .builtinNS")); } } @Test public void testConnectionParseException() { - try (Connection conn = new Connection()) { - conn.prepareScript("print('hello'", new String[]{}, new String[]{}); + try(Connection conn = new Connection()) { + conn.prepareScript("print('hello'", new String[] {}, new String[] {}); throw new AssertionError("Test should have thrown a ParseException"); - } catch (Exception e) { + } + catch(Exception e) { Assert.assertEquals("ParseException", e.getClass().getSimpleName()); } } @@ -144,10 +145,11 @@ public void testConnectionClose() { @Test public void testReadScriptHDFS() { - try (Connection conn = new Connection()) { + try(Connection conn = new Connection()) { conn.readScript("hdfs://localhost:9000/Test"); - } catch (IOException e) { - Assert.assertEquals("ConnectException",e.getClass().getSimpleName()); + } + catch(IOException e) { + Assert.assertEquals("ConnectException", e.getClass().getSimpleName()); } } diff --git a/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java b/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java index 4852220861e..2178884ef5b 100644 --- a/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java @@ -111,17 +111,16 @@ public void federatedReuse(String test) { // Run reference dml script with normal matrix. Reuse of ba+*. fullDMLScriptName = HOME + test + "Reference.dml"; - programArgs = new String[] {"-stats", "-lineage", "reuse_full", - "-nvargs", "X1=" + input("X1"), "X2=" + input("X2"), "Y1=" + input("Y1"), - "Y2=" + input("Y2"), "Z=" + expected("Z")}; + programArgs = new String[] {"-stats", "-lineage", "reuse_full", "-nvargs", "X1=" + input("X1"), + "X2=" + input("X2"), "Y1=" + input("Y1"), "Y2=" + input("Y2"), "Z=" + expected("Z")}; runTest(true, false, null, -1); long mmCount = Statistics.getCPHeavyHitterCount(Opcodes.MMULT.toString()); // Run actual dml script with federated matrix // The fed workers reuse ba+* fullDMLScriptName = HOME + test + ".dml"; - programArgs = new String[] {"-stats","-lineage", "reuse_full", - "-nvargs", "X1=" + TestUtils.federatedAddress(port1, input("X1")), + programArgs = new String[] {"-stats", "-lineage", "reuse_full", "-nvargs", + "X1=" + TestUtils.federatedAddress(port1, input("X1")), "X2=" + TestUtils.federatedAddress(port2, input("X2")), "Y1=" + TestUtils.federatedAddress(port1, input("Y1")), "Y2=" + TestUtils.federatedAddress(port2, input("Y2")), "r=" + rows, "c=" + cols, "Z=" + output("Z")}; @@ -129,12 +128,12 @@ public void federatedReuse(String test) { long mmCount_fed = Statistics.getCPHeavyHitterCount(Opcodes.MMULT.toString()); long fedMMCount = Statistics.getCPHeavyHitterCount("fed_ba+*"); - // compare results + // compare results compareResults(1e-9); // compare matrix multiplication count - // #federated execution of ba+* = #threads times #non-federated execution of ba+* (after reuse) - Assert.assertTrue("Violated reuse count: "+mmCount_fed+" == "+mmCount*2, - mmCount_fed == mmCount * 2); // #threads = 2 + // #federated execution of ba+* = #threads times #non-federated execution of ba+* (after reuse) + Assert.assertTrue("Violated reuse count: " + mmCount_fed + " == " + mmCount * 2, + mmCount_fed == mmCount * 2); // #threads = 2 switch(test) { case TEST_NAME1: // If the o/p is federated, fed_ba+* will be called everytime diff --git a/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java b/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java index eca3628a89b..c86eb0f4941 100644 --- a/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java @@ -121,9 +121,8 @@ private void runTriUDFReuse(ExecMode execMode) { // Run reference dml script with normal matrix fullDMLScriptName = HOME + TEST_NAME + "Reference.dml"; - programArgs = new String[] {"-lineage", "reuse_full", "-stats", "100", "-args", - input("X1"), input("X2"), input("X3"), input("X4"), - Boolean.toString(rowPartitioned).toUpperCase(), expected("S")}; + programArgs = new String[] {"-lineage", "reuse_full", "-stats", "100", "-args", input("X1"), input("X2"), + input("X3"), input("X4"), Boolean.toString(rowPartitioned).toUpperCase(), expected("S")}; runTest(null); // Run actual dml script with federated matrix diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java index 18ca2fbc454..3ffdfe1d30b 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java @@ -272,82 +272,79 @@ protected void toStringTestHelper(ExecMode platform, String testName, String exp } @Test - public void testPrintWithDecimal(){ + public void testPrintWithDecimal() { String testName = "ToString12"; String decimalPoints = "2"; String value = "22"; String expectedOutput = "22.00\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - @Test - public void testPrintWithDecimal2(){ + public void testPrintWithDecimal2() { String testName = "ToString12"; String decimalPoints = "2"; String value = "5.244058388023880"; String expectedOutput = "5.24\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - @Test - public void testPrintWithDecimal3(){ + public void testPrintWithDecimal3() { String testName = "ToString12"; String decimalPoints = "10"; String value = "5.244058388023880"; String expectedOutput = "5.2440583880\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - @Test - public void testPrintWithDecimal4(){ + public void testPrintWithDecimal4() { String testName = "ToString12"; String decimalPoints = "4"; String value = "5.244058388023880"; String expectedOutput = "5.2441\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - @Test - public void testPrintWithDecimal5(){ + public void testPrintWithDecimal5() { String testName = "ToString12"; String decimalPoints = "10"; String value = "0.000000008023880"; String expectedOutput = "0.0000000080\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - protected void toStringTestHelper2(ExecMode platform, String testName, String expectedOutput, String decimalPoints, String value) { + protected void toStringTestHelper2(ExecMode platform, String testName, String expectedOutput, String decimalPoints, + String value) { ExecMode platformOld = rtplatform; - + rtplatform = platform; boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - if (rtplatform == ExecMode.SPARK) + if(rtplatform == ExecMode.SPARK) DMLScript.USE_LOCAL_SPARK_CONFIG = true; try { // Create and load test configuration getAndLoadTestConfiguration(testName); String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testName + ".dml"; - programArgs = new String[]{"-args", output(OUTPUT_NAME), value, decimalPoints}; + programArgs = new String[] {"-args", output(OUTPUT_NAME), value, decimalPoints}; // Run DML and R scripts runTest(true, false, null, -1); diff --git a/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java index 770c5b7c5bf..26143dc16ee 100644 --- a/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java @@ -76,10 +76,9 @@ public ReshapeTest(int rlen, int clen, int rows, int cols, boolean rowWise) { @Parameterized.Parameters(name = "{0}x{1} {2}x{3} rowWise {4}") public static Iterable getParams() { - int[][][] dims = { - {{1000, 1000}, {1, 1000000}}, // single row/col - {{3000, 4000}, {1500, 8000}}, // partialBlocks - {{2400, 1400}, {800, 4200}} // fullBlocks + int[][][] dims = {{{1000, 1000}, {1, 1000000}}, // single row/col + {{3000, 4000}, {1500, 8000}}, // partialBlocks + {{2400, 1400}, {800, 4200}} // fullBlocks }; ArrayList params = new ArrayList<>(); @@ -117,7 +116,8 @@ public void runTestMatrixReshapeOOC() { double[][] X = getRandomMatrix(rlen, clen, 0, 1, 1, 7); MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(Types.FileFormat.BINARY); - writer.writeMatrixToHDFS(DataConverter.convertToMatrixBlock(X), input(INPUT_NAME), rlen, clen, 1000, rlen * clen); + writer.writeMatrixToHDFS(DataConverter.convertToMatrixBlock(X), input(INPUT_NAME), rlen, clen, 1000, + rlen * clen); HDFSTool.writeMetaDataFile(input(INPUT_NAME + ".mtd"), Types.ValueType.FP64, new MatrixCharacteristics(rlen, clen, blen, rlen * clen), Types.FileFormat.BINARY); @@ -143,8 +143,8 @@ public void runTestMatrixReshapeOOC() { runTest(true, false, null, -1); // compare results - MatrixBlock actual = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME), - Types.FileFormat.BINARY, rows, cols, blen); + MatrixBlock actual = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME), Types.FileFormat.BINARY, rows, + cols, blen); MatrixBlock expected = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME + "_target"), Types.FileFormat.BINARY, rows, cols, blen); diff --git a/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java index 5f42db7d733..49a52587cde 100644 --- a/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java @@ -335,9 +335,9 @@ private void runTestMatrixReshape( ReshapeType type, boolean rowwise, boolean sp String.valueOf(trows), String.valueOf(tcols), output("Y") }; fullRScriptName = HOME + TEST_NAME + ".R"; - rCmd = "Rscript" + " " + fullRScriptName + " " + - inputDir() + " " + trows + " " + tcols + " " + expectedDir(); - + rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + trows + " " + tcols + " " + + expectedDir(); + double[][] X = getRandomMatrix(rows, cols, 0, 1, sparsity, 7); writeInputMatrix("X", X, true); diff --git a/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java index dcdafddcd47..69d6958f8a6 100644 --- a/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java @@ -94,9 +94,9 @@ private void runVectorReshape(boolean sparse, ExecType et) String.valueOf(rows2), String.valueOf(cols2), output("R") }; fullRScriptName = HOME + TEST_NAME + ".R"; - rCmd = "Rscript" + " " + fullRScriptName + " " + - inputDir() + " " + rows2 + " " + cols2 + " " + expectedDir(); - + rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + rows2 + " " + cols2 + " " + + expectedDir(); + double sparsity = sparse ? sparsitySparse : sparsityDense; double[][] X = getRandomMatrix(rows1, cols1, 0, 1, sparsity, 7); writeInputMatrixWithMTD("X", X, true); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java index 60b491b8141..b16554045e4 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java @@ -151,8 +151,8 @@ private void runTestMatrixChainDP(String testName) { overwriteCurrentConfig(dmlConfig); } catch(FileNotFoundException fnfe) { - Assert.fail("Could not find DML config file: " + - getCurConfigFile().getPath() + " . " + fnfe.getMessage()); + Assert.fail( + "Could not find DML config file: " + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); } catch(IOException ioe) { Assert.fail("Could not overwrite the DML configuration file. " + ioe.getMessage()); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java index bf9acd9e52a..96c479e206d 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java @@ -123,8 +123,8 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { overwriteCurrentConfig(dmlConfig); } catch(FileNotFoundException fnfe) { - Assert.fail("Could not find DML config file: " + - getCurConfigFile().getPath() + " . " + fnfe.getMessage()); + Assert.fail( + "Could not find DML config file: " + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); } catch(IOException ioe) { Assert.fail("Could not overwrite the DML configuration file. " + ioe.getMessage()); @@ -132,8 +132,7 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + TEST_NAME + ".dml"; - programArgs = new String[] {"-explain", "hops", "-stats", - "-args", input("X"), input("Y"), output("R")}; + programArgs = new String[] {"-explain", "hops", "-stats", "-args", input("X"), input("Y"), output("R")}; fullRScriptName = HOME + TEST_NAME + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java index e8e885f905f..15b80e49618 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java @@ -74,7 +74,7 @@ public void testRewriteQuantizationFusedCompressionNoRewrite() { /** * Unified method to test both scalar and matrix scale factors. - * + * * @param testname Test name * @param rewrites Whether to enable fusion rewrites * @param isScalar Whether the scale factor is a scalar or a matrix diff --git a/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java b/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java index 30681f373e4..39266f5f3d3 100644 --- a/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java +++ b/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -106,7 +106,8 @@ public void testHash2() throws Exception { @Test public void testHash3() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.INT64,ValueType.UINT8}, 32); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, + new ValueType[] {ValueType.UINT8, ValueType.INT64, ValueType.UINT8}, 32); MatrixBlock expected = new MatrixBlock(1, 7, new double[] {1, 1, 1, 0, 1, 1, 1}); String spec = "{\"ids\": true, \"dummycode\": [1,3], \"hash\": [1,3], \"K\": 3}"; @@ -114,11 +115,11 @@ public void testHash3() throws Exception { } - @Test public void testHybrid1() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.INT64,ValueType.UINT8, ValueType.BOOLEAN}, 32); - MatrixBlock expected = new MatrixBlock(1, 9, new double[] {1, 1, 1, 0, 1, 1, 1,1,1}); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, + new ValueType[] {ValueType.UINT8, ValueType.INT64, ValueType.UINT8, ValueType.BOOLEAN}, 32); + MatrixBlock expected = new MatrixBlock(1, 9, new double[] {1, 1, 1, 0, 1, 1, 1, 1, 1}); String spec = "{\"ids\": true, \"dummycode\": [1,3,4], \"hash\": [1,3], \"K\": 3}"; runTransformTest(fb, spec, expected); @@ -127,8 +128,9 @@ public void testHybrid1() throws Exception { @Test public void testHybrid2() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.BOOLEAN,ValueType.UINT8, ValueType.BOOLEAN}, 32); - MatrixBlock expected = new MatrixBlock(1, 10, new double[] {1, 1, 1, 1,1, 1, 1, 1,1,1}); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, + new ValueType[] {ValueType.UINT8, ValueType.BOOLEAN, ValueType.UINT8, ValueType.BOOLEAN}, 32); + MatrixBlock expected = new MatrixBlock(1, 10, new double[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}); String spec = "{\"ids\": true, \"dummycode\": [1,2,3,4], \"hash\": [1,3], \"K\": 3}"; runTransformTest(fb, spec, expected); @@ -139,7 +141,7 @@ private void runTransformTest(FrameBlock fb, String spec, MatrixBlock expected) try { getAndLoadTestConfiguration(TEST_NAME1); - + String inF = input("F-In"); String inS = input("spec"); diff --git a/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java b/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java index 8c4ba6ae8ad..cd28649dc42 100644 --- a/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java +++ b/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java @@ -283,7 +283,8 @@ private String[][] readTwoColumnStringCSV(String s) { out[1][i] = in.getString(i, 1); } return out; - } catch (IOException e) { + } + catch(IOException e) { throw new RuntimeException(e); } } diff --git a/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java b/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java index d3d71d820d6..ae13cbd510f 100644 --- a/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java +++ b/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java @@ -92,7 +92,7 @@ private void runVectorizationTest( String testName, boolean rewrites ) runTest(true, false, null, -1); runRScript(true); - //compare results + // compare results HashMap dmlfile = readDMLMatrixFromOutputDir("R"); HashMap rfile = readRMatrixFromExpectedDir("R"); TestUtils.compareMatrices(dmlfile, rfile, 1e-14, "DML", "R"); From 325d24f99055137d090678be6459986bb96cedca Mon Sep 17 00:00:00 2001 From: bruno Date: Tue, 1 Sep 2026 16:33:44 +0200 Subject: [PATCH 131/132] Revert "dev/format-changed.sh" This reverts commit 3962ff1698900b7070f658497747993832f78b83. --- .../java/org/apache/sysds/api/DMLScript.java | 10 +- .../org/apache/sysds/common/Builtins.java | 376 +++++++++++++----- .../java/org/apache/sysds/hops/BinaryOp.java | 4 +- src/main/java/org/apache/sysds/hops/Hop.java | 14 +- .../java/org/apache/sysds/hops/UnaryOp.java | 9 +- .../sysds/hops/estim/EstimationUtils.java | 12 +- .../sysds/hops/rewrite/ProgramRewriter.java | 6 +- ...riteMatrixMultChainOptimizationSparse.java | 19 +- .../parser/BuiltinFunctionExpression.java | 24 ++ .../apache/sysds/parser/DMLTranslator.java | 87 +++- .../apache/sysds/parser/DataExpression.java | 206 +++++++--- .../compress/CompressedMatrixBlock.java | 2 +- .../runtime/compress/colgroup/AColGroup.java | 17 +- .../compress/colgroup/AColGroupValue.java | 1 + .../runtime/compress/colgroup/ASDCZero.java | 6 +- .../compress/colgroup/ColGroupDDC.java | 4 +- .../compress/colgroup/ColGroupEmpty.java | 7 +- .../colgroup/ColGroupLinearFunctional.java | 2 +- .../compress/colgroup/ColGroupOLE.java | 2 +- .../compress/colgroup/ColGroupRLE.java | 4 +- .../compress/colgroup/ColGroupSDCFOR.java | 4 +- .../colgroup/ColGroupUncompressed.java | 13 +- .../colgroup/ColGroupUncompressedArray.java | 2 +- .../dictionary/AIdentityDictionary.java | 4 +- .../colgroup/dictionary/DeltaDictionary.java | 4 +- .../colgroup/dictionary/IDictionary.java | 6 +- .../dictionary/IdentityDictionary.java | 4 +- .../dictionary/IdentityDictionarySlice.java | 4 +- .../compress/colgroup/mapping/AMapToData.java | 2 +- .../compress/colgroup/offset/AOffset.java | 10 +- .../compress/colgroup/offset/OffsetEmpty.java | 1 - .../compress/lib/CLALibBinaryCellOp.java | 6 +- .../runtime/compress/lib/CLALibMMChain.java | 2 +- .../compress/lib/CLALibRemoveEmpty.java | 15 +- .../runtime/compress/lib/CLALibSort.java | 8 +- .../controlprogram/caching/FrameObject.java | 4 +- .../controlprogram/caching/MatrixObject.java | 2 +- .../context/SparkExecutionContext.java | 34 +- .../federated/FederatedWorker.java | 3 +- .../frame/data/columns/ArrayFactory.java | 22 +- .../frame/data/lib/MatrixBlockFromFrame.java | 4 +- .../runtime/functionobjects/Builtin.java | 139 ++++--- .../instructions/cp/BinaryCPInstruction.java | 2 +- .../cp/BinaryFrameScalarCPInstruction.java | 10 +- .../cp/BinaryMatrixMatrixCPInstruction.java | 4 +- .../cp/ParameterizedBuiltinCPInstruction.java | 7 +- .../instructions/ooc/ReorgOOCInstruction.java | 8 +- .../ooc/ReshapeOOCInstruction.java | 69 ++-- .../spark/QuantilePickSPInstruction.java | 3 +- .../spark/data/IndexedMatrixValue.java | 7 +- .../sysds/runtime/io/DeltaKernelUtils.java | 257 ++++++------ .../apache/sysds/runtime/io/ReaderDelta.java | 107 +++-- .../sysds/runtime/io/ReaderDeltaParallel.java | 96 +++-- .../apache/sysds/runtime/io/WriterDelta.java | 74 ++-- .../runtime/matrix/data/LibMatrixReorg.java | 34 +- .../runtime/matrix/data/MatrixBlock.java | 34 +- .../sysds/runtime/ooc/cache/OOCFuture.java | 9 +- .../runtime/ooc/cache/io/CloseableQueue.java | 38 +- .../cache/io/OOCBufferedDataInputStream.java | 12 +- .../cache/io/OOCBufferedDataOutputStream.java | 20 +- .../runtime/ooc/cache/io/OOCIOHandler.java | 25 +- .../ooc/cache/io/OOCMatrixIOHandler.java | 161 ++++---- .../runtime/ooc/cache/io/SpillableObject.java | 6 +- .../ooc/cache/legacy/OOCCacheScheduler.java | 44 +- .../cache/legacy/OOCLRUCacheScheduler.java | 207 +++++----- .../runtime/transform/decode/Decoder.java | 10 +- .../runtime/transform/decode/DecoderBin.java | 4 +- .../transform/decode/DecoderDummycode.java | 2 +- .../transform/decode/DecoderFactory.java | 53 +-- .../transform/decode/DecoderRecode.java | 18 +- .../sysds/runtime/util/CommonThreadPool.java | 8 +- .../sysds/runtime/util/DataConverter.java | 9 +- .../org/apache/sysds/utils/DoubleParser.java | 2 +- .../apache/sysds/utils/SettingsChecker.java | 13 +- .../org/apache/sysds/performance/Main.java | 7 +- .../apache/sysds/test/AutomatedTestBase.java | 23 +- .../java/org/apache/sysds/test/TestUtils.java | 9 +- .../component/compile/CompilerTestBase.java | 21 +- .../SparkTransitiveExecTypeCompileTest.java | 50 ++- .../compress/CompressedSortTest.java | 6 +- .../compress/lib/CLALibMMChainTest.java | 4 +- ...CompressedBinaryMatrixMatrixSolveTest.java | 15 +- .../OffsetClassInitConcurrencyTest.java | 4 +- .../SparkContextReferenceCountTest.java | 32 +- .../component/federated/FedWorkerBase.java | 19 +- .../federated/FedWorkerMatrixCompress.java | 8 +- .../component/frame/FrameToStringTest.java | 14 +- .../frame/MatrixFromFrameSafeCastTest.java | 12 +- .../frame/transform/DecoderCompositeTest.java | 8 +- .../GetCategoricalMaskInstructionTest.java | 21 +- .../TransformDecodeRoundTripTest.java | 39 +- .../frame/transform/TransformDecodeTest.java | 4 +- .../component/io/DeltaMatrixCoverageTest.java | 97 ++--- .../io/DeltaMatrixReadWriteTest.java | 333 ++++++---------- .../io/DeltaMatrixSparkInteropTest.java | 105 ++--- .../component/matrix/QuantilePickTest.java | 13 +- .../component/tensor/TensorToStringTest.java | 14 +- .../functions/binary/matrix/QuantileTest.java | 18 +- .../builtin/part2/BuiltinSTEPGlmTest.java | 3 +- .../FederatedBackendPerformanceTest.java | 6 +- .../part4/FederatedLogicalTest.java | 5 +- .../functions/indexing/LeftIndexingTest.java | 48 ++- .../sysds/test/functions/io/ScalarIOTest.java | 12 +- .../io/delta/DeltaReadWriteTest.java | 40 +- .../io/parquet/FrameParquetSchemaTest.java | 3 +- .../functions/jmlc/JMLConnectionTest.java | 42 +- .../functions/lineage/FedFullReuseTest.java | 17 +- .../functions/lineage/FedUDFReuseTest.java | 5 +- .../test/functions/misc/ToStringTest.java | 33 +- .../sysds/test/functions/ooc/ReshapeTest.java | 14 +- .../functions/reorg/MatrixReshapeTest.java | 6 +- .../functions/reorg/VectorReshapeTest.java | 6 +- .../rewrite/RewriteMatrixChainDPTest.java | 4 +- .../RewriteMatrixMultChainOptSparseTest.java | 7 +- ...writeQuantizationFusedCompressionTest.java | 2 +- .../transform/GetCategoricalMaskTest.java | 20 +- .../TransformFrameEncodeBagOfWords.java | 3 +- .../vect/LeftIndexingChainUpdateTest.java | 2 +- 118 files changed, 1958 insertions(+), 1654 deletions(-) diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java b/src/main/java/org/apache/sysds/api/DMLScript.java index 0bb1e9b462d..a7a175bb7b6 100644 --- a/src/main/java/org/apache/sysds/api/DMLScript.java +++ b/src/main/java/org/apache/sysds/api/DMLScript.java @@ -508,9 +508,9 @@ private static void execute(String dmlScriptStr, String fnameOptConfig, Map inHops1 = new ArrayList<>(); + inHops1.add(expr); + inHops1.add(expr2); + currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), + OpOpDnn.valueOf(source.getOpCode().name()), inHops1); + setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); + break; + } + case AVG_POOL: + case MAX_POOL: { + currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), + OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForPoolingForwardIM2COL(expr, source, 1, hops)); + setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); + break; + } + case AVG_POOL_BACKWARD: + case MAX_POOL_BACKWARD: { + currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), + OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForConvOpPoolingCOL2IM(expr, source, 1, hops)); + setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); + break; + } + case CONV2D: + case CONV2D_BACKWARD_FILTER: + case CONV2D_BACKWARD_DATA: { + currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), + OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForConvOp(expr, source, 1, hops)); + setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); + break; + } + + case ROW_COUNT_DISTINCT: + currBuiltinOp = new AggUnaryOp(target.getName(), + DataType.MATRIX, target.getValueType(), AggOp.COUNT_DISTINCT, Direction.Row, expr); + break; + + case COL_COUNT_DISTINCT: + currBuiltinOp = new AggUnaryOp(target.getName(), + DataType.MATRIX, target.getValueType(), AggOp.COUNT_DISTINCT, Direction.Col, expr); + break; + + case GET_CATEGORICAL_MASK: + currBuiltinOp = new BinaryOp(target.getName(), DataType.MATRIX, ValueType.FP64, OpOp2.GET_CATEGORICAL_MASK, expr, expr2); + break; + default: + throw new ParseException("Unsupported builtin function type: "+source.getOpCode()); + } + + boolean isConvolution = source.getOpCode() == Builtins.CONV2D || source.getOpCode() == Builtins.CONV2D_BACKWARD_DATA || + source.getOpCode() == Builtins.CONV2D_BACKWARD_FILTER || + source.getOpCode() == Builtins.MAX_POOL || source.getOpCode() == Builtins.MAX_POOL_BACKWARD || + source.getOpCode() == Builtins.AVG_POOL || source.getOpCode() == Builtins.AVG_POOL_BACKWARD; + if( !isConvolution) { // Since the dimension of output doesnot match that of input variable for these operations setIdentifierParams(currBuiltinOp, source.getOutput()); } diff --git a/src/main/java/org/apache/sysds/parser/DataExpression.java b/src/main/java/org/apache/sysds/parser/DataExpression.java index 3d3a90b4f6f..68a3d1b7ffe 100644 --- a/src/main/java/org/apache/sysds/parser/DataExpression.java +++ b/src/main/java/org/apache/sysds/parser/DataExpression.java @@ -1176,72 +1176,52 @@ else if( getVarParam(READNNZPARAM) != null ) { boolean isHDF5 = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.HDF5.toString())); - // handle all csv default parameters - handleCSVDefaultParam(DELIM_DELIMITER, ValueType.STRING, conditional); - handleCSVDefaultParam(DELIM_FILL_VALUE, ValueType.FP64, conditional); - handleCSVDefaultParam(DELIM_HAS_HEADER_ROW, ValueType.BOOLEAN, conditional); - handleCSVDefaultParam(DELIM_FILL, ValueType.BOOLEAN, conditional); - handleCSVDefaultParam(DELIM_NA_STRINGS, ValueType.STRING, conditional); - } - - boolean isLIBSVM = false; - isLIBSVM = (formatTypeString != null && - formatTypeString.equalsIgnoreCase(FileFormat.LIBSVM.toString())); - if(isLIBSVM) { - // Handle libsvm file format - shouldReadMTD = true; - - // only allow IO_FILENAME, READROWPARAM, READCOLPARAM - // as valid parameters - if(!inferredFormatType) { - for(String key : _varParams.keySet()) { - if(!(key.equals(IO_FILENAME) || key.equals(FORMAT_TYPE) || key.equals(READROWPARAM) || - key.equals(READCOLPARAM) || key.equals(READNNZPARAM) || key.equals(DATATYPEPARAM) || - key.equals(VALUETYPEPARAM) || key.equals(DELIM_DELIMITER) || - key.equals(LIBSVM_INDEX_DELIM))) { - String msg = "Only parameters allowed are: " + IO_FILENAME + "," + READROWPARAM + "," - + READCOLPARAM + DELIM_DELIMITER + "," + LIBSVM_INDEX_DELIM; - - raiseValidateError( - "Invalid parameter " + key + " in read statement: " + toString() + ". " + msg, - conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - } - } - // handle all default parameters - handleCSVDefaultParam(DELIM_DELIMITER, ValueType.STRING, conditional); - handleCSVDefaultParam(LIBSVM_INDEX_DELIM, ValueType.STRING, conditional); - } + boolean isCOG = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.COG.toString())); - boolean isHDF5 = (formatTypeString != null && - formatTypeString.equalsIgnoreCase(FileFormat.HDF5.toString())); + // Delta tables are self-describing (schema + dimensions discovered from the + // transaction log at read time), so dimensions are optional like CSV. + boolean isDelta = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.DELTA.toString())); - boolean isCOG = (formatTypeString != null && - formatTypeString.equalsIgnoreCase(FileFormat.COG.toString())); - - // Delta tables are self-describing (schema + dimensions discovered from the - // transaction log at read time), so dimensions are optional like CSV. - boolean isDelta = (formatTypeString != null && - formatTypeString.equalsIgnoreCase(FileFormat.DELTA.toString())); - - dataTypeString = (getVarParam(DATATYPEPARAM) == null) ? null : getVarParam(DATATYPEPARAM).toString(); - - if(dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE) || - dataTypeString.equalsIgnoreCase(Statement.FRAME_DATA_TYPE)) { - - boolean isMatrix = false; - if(dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE)) + dataTypeString = (getVarParam(DATATYPEPARAM) == null) ? null : getVarParam(DATATYPEPARAM).toString(); + + if ( dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE) + || dataTypeString.equalsIgnoreCase(Statement.FRAME_DATA_TYPE)) { + + boolean isMatrix = false; + if ( dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE)) isMatrix = true; + + // set data type + getOutput().setDataType(isMatrix ? DataType.MATRIX : DataType.FRAME); + + // set number non-zeros + Expression ennz = getVarParam("nnz"); + long nnz = -1; + if( ennz != null ) { + nnz = Long.valueOf(ennz.toString()); + getOutput().setNnz(nnz); + } - // set data type - getOutput().setDataType(isMatrix ? DataType.MATRIX : DataType.FRAME); - - // set number non-zeros - Expression ennz = getVarParam("nnz"); - long nnz = -1; - if(ennz != null) { - nnz = Long.valueOf(ennz.toString()); - getOutput().setNnz(nnz); + // Following dimension checks must be done when data type = MATRIX_DATA_TYPE + // initialize size of target data identifier to UNKNOWN + getOutput().setDimensions(-1, -1); + + if (!isCSV && !isLIBSVM && !isHDF5 && !isCOG && !isDelta && ConfigurationManager.getCompilerConfig() + .getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) //skip check for csv/libsvm/delta format / jmlc api + && (getVarParam(READROWPARAM) == null || getVarParam(READCOLPARAM) == null) ) { + raiseValidateError("Missing or incomplete dimension information in read statement: " + + mtdFileName, conditional, LanguageErrorCodes.INVALID_PARAMETERS); + } + + if (getVarParam(READROWPARAM) instanceof ConstIdentifier + && getVarParam(READCOLPARAM) instanceof ConstIdentifier) + { + // these are strings that are long values + Long dim1 = (getVarParam(READROWPARAM) == null) ? null : Long.valueOf( getVarParam(READROWPARAM).toString()); + Long dim2 = (getVarParam(READCOLPARAM) == null) ? null : Long.valueOf( getVarParam(READCOLPARAM).toString()); + if ( !isCSV && !isDelta && (dim1 < 0 || dim2 < 0) && ConfigurationManager + .getCompilerConfig().getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) ) { + raiseValidateError("Invalid dimension information in read statement", conditional, LanguageErrorCodes.INVALID_PARAMETERS); } // set dim1 and dim2 values @@ -1272,10 +1252,104 @@ else if( getVarParam(READNNZPARAM) != null ) { catch(Exception ex) { raiseValidateError("Invalid format '" + fmt+ "' in statement: " + toString(), conditional); } - else if(getVarParam(FORMAT_TYPE) instanceof StringIdentifier) // literal format - raiseValidateError("Invalid format " + getVarParam(FORMAT_TYPE) + " in statement: " + toString(), - conditional); - break; + + if (getVarParam(ROWBLOCKCOUNTPARAM) instanceof ConstIdentifier && getVarParam(COLUMNBLOCKCOUNTPARAM) instanceof ConstIdentifier) { + Integer rowBlockCount = (getVarParam(ROWBLOCKCOUNTPARAM) == null) ? + null : Integer.valueOf(getVarParam(ROWBLOCKCOUNTPARAM).toString()); + getOutput().setBlocksize(rowBlockCount != null ? rowBlockCount : -1); + } + + // block dimensions must be -1x-1 when format="text" + // NOTE MB: disabled validate of default blocksize for inputs w/ format="binary" + // because we automatically introduce reblocks if blocksizes don't match + if ( (getOutput().getFileFormat().isTextFormat() || !isMatrix) && getOutput().getBlocksize() != -1 ){ + raiseValidateError("Invalid block dimensions (" + getOutput().getBlocksize() + ") when format=" + getVarParam(FORMAT_TYPE) + " in \"" + this.toString() + "\".", conditional); + } + + } + else if ( dataTypeString.equalsIgnoreCase(Statement.SCALAR_DATA_TYPE)) { + getOutput().setDataType(DataType.SCALAR); + getOutput().setNnz(-1L); + } + else if ( dataTypeString.equalsIgnoreCase(DataType.LIST.name())) { + getOutput().setDataType(DataType.LIST); + } + else{ + raiseValidateError("Unknown Data Type " + dataTypeString + ". Valid values: " + + Statement.SCALAR_DATA_TYPE +", " + Statement.MATRIX_DATA_TYPE+", " + Statement.FRAME_DATA_TYPE + +", " + DataType.LIST.name().toLowerCase(), conditional, LanguageErrorCodes.INVALID_PARAMETERS); + } + + // handle value type parameter + if (getVarParam(VALUETYPEPARAM) != null && !(getVarParam(VALUETYPEPARAM) instanceof StringIdentifier)){ + raiseValidateError("for read method, parameter " + VALUETYPEPARAM + " can only be a string. " + + "Valid values are: " + Statement.DOUBLE_VALUE_TYPE +", " + Statement.INT_VALUE_TYPE + ", " + Statement.BOOLEAN_VALUE_TYPE + ", " + Statement.STRING_VALUE_TYPE, conditional); + } + // Identify the value type (used only for read method) + String valueTypeString = getVarParam(VALUETYPEPARAM) == null ? null : getVarParam(VALUETYPEPARAM).toString(); + if (valueTypeString != null) { + if (valueTypeString.equalsIgnoreCase(Statement.DOUBLE_VALUE_TYPE)) + getOutput().setValueType(ValueType.FP64); + else if (valueTypeString.equalsIgnoreCase(Statement.STRING_VALUE_TYPE)) + getOutput().setValueType(ValueType.STRING); + else if (valueTypeString.equalsIgnoreCase(Statement.INT_VALUE_TYPE)) + getOutput().setValueType(ValueType.INT64); + else if (valueTypeString.equalsIgnoreCase(Statement.BOOLEAN_VALUE_TYPE)) + getOutput().setValueType(ValueType.BOOLEAN); + else if (valueTypeString.equalsIgnoreCase(ValueType.UNKNOWN.name())) + getOutput().setValueType(ValueType.UNKNOWN); + else { + raiseValidateError("Unknown Value Type " + valueTypeString + + ". Valid values are: " + Statement.DOUBLE_VALUE_TYPE +", " + Statement.INT_VALUE_TYPE + ", " + Statement.BOOLEAN_VALUE_TYPE + ", " + Statement.STRING_VALUE_TYPE, conditional); + } + } else { + getOutput().setValueType(ValueType.FP64); + } + + break; + + case WRITE: + + // for CSV format, if no delimiter specified THEN set default "," + if (getVarParam(FORMAT_TYPE) == null || checkFormatType(FileFormat.CSV) ){ + if (getVarParam(DELIM_DELIMITER) == null) { + addVarParam(DELIM_DELIMITER, new StringIdentifier(DEFAULT_DELIM_DELIMITER, this)); + } + if (getVarParam(DELIM_HAS_HEADER_ROW) == null) { + addVarParam(DELIM_HAS_HEADER_ROW, new BooleanIdentifier(DEFAULT_DELIM_HAS_HEADER_ROW, this)); + } + if (getVarParam(DELIM_SPARSE) == null) { + addVarParam(DELIM_SPARSE, new BooleanIdentifier(DEFAULT_DELIM_SPARSE, this)); + } + } + + // for LIBSVM format, add the default separators if not specified + if (getVarParam(FORMAT_TYPE) == null || checkFormatType(FileFormat.LIBSVM)) { + if(getVarParam(DELIM_DELIMITER) == null) { + addVarParam(DELIM_DELIMITER, new StringIdentifier(DEFAULT_DELIM_DELIMITER, this)); + } + if(getVarParam(LIBSVM_INDEX_DELIM) == null) { + addVarParam(LIBSVM_INDEX_DELIM, new StringIdentifier(DEFAULT_LIBSVM_INDEX_DELIM, this)); + } + if(getVarParam(DELIM_SPARSE) == null) { + addVarParam(DELIM_SPARSE, new BooleanIdentifier(DEFAULT_DELIM_SPARSE, this)); + } + } + + //validate read filename + if (getVarParam(FORMAT_TYPE) == null || FileFormat.isTextFormat(getVarParam(FORMAT_TYPE).toString()) + || checkFormatType(FileFormat.DELTA)) //delta: columnar, no block layout + getOutput().setBlocksize(-1); + else if (checkFormatType(FileFormat.BINARY, FileFormat.COMPRESSED, FileFormat.UNKNOWN)) { + if( getVarParam(ROWBLOCKCOUNTPARAM)!=null ) + getOutput().setBlocksize(Integer.parseInt(getVarParam(ROWBLOCKCOUNTPARAM).toString())); + else + getOutput().setBlocksize(ConfigurationManager.getBlocksize()); + } + else if( getVarParam(FORMAT_TYPE) instanceof StringIdentifier ) //literal format + raiseValidateError("Invalid format " + getVarParam(FORMAT_TYPE) + + " in statement: " + toString(), conditional); + break; case RAND: diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java index 042e0dc0328..d0ba5363939 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java @@ -484,7 +484,7 @@ public static CompressedMatrixBlock read(DataInput in) throws IOException { long nonZeros = in.readLong(); boolean overlappingColGroups = in.readBoolean(); List groups = ColGroupIO.readGroups(in, rlen); - CompressedMatrixBlock ret = new CompressedMatrixBlock(rlen, clen, nonZeros, overlappingColGroups, groups); + CompressedMatrixBlock ret = new CompressedMatrixBlock(rlen, clen, nonZeros, overlappingColGroups, groups); LOG.debug("Compressed read serialization time: " + t.stop()); return ret; } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java index 66d4e78cb0f..354325e293b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java @@ -402,8 +402,7 @@ public final AColGroup rightMultByMatrix(MatrixBlock right) { * @param cru The right hand side column upper * @param nRows The number of rows in this column group */ - public void rightDecompressingMult(MatrixBlock right, MatrixBlock ret, int rl, int ru, int nRows, int crl, - int cru) { + public void rightDecompressingMult(MatrixBlock right, MatrixBlock ret, int rl, int ru, int nRows, int crl, int cru) { throw new NotImplementedException( "not supporting right Decompressing Multiply on class: " + this.getClass().getSimpleName()); } @@ -978,9 +977,9 @@ public AColGroup[] splitReshapePushDown(final int multiplier, final int nRow, fi /** * Sort the values of the column group according to double comparison operations and return as another compressed * group. - * + * * This sorting assumes that the column group is sorted independently of everything else. - * + * * @return The sorted group */ public abstract AColGroup sort(); @@ -997,9 +996,9 @@ public String toString() { /** * Return a new column group containing only the selected rows in the given boolean vector. - * + * * Whenever possible only modify the index structure, not the dictionary of the column groups. - * + * * @param selectV The selection vector * @param rOut The number of rows in the output * @return The new column group @@ -1008,9 +1007,9 @@ public String toString() { /** * Return a new column group containing only the selected columns in the given boolean vector. - * + * * Whenever possible only modify the column index, and reduce the dictionaries of the column groups. - * + * * @param selectV The selection vector * @return The new column group, or {@code null} if no column of this group is selected */ @@ -1046,7 +1045,7 @@ public AColGroup removeEmptyCols(boolean[] selectV) { /** * Using the selection of columns, slice out those and return in a new column group with the given column indexes. * Ideally this method should only modify the dictionaries. - * + * * @param newColumnIDs the new column indexes * @param selectedColumns The selected columns of this column group (guaranteed < current number of columns) * @return A new Column group diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java index d610c1b586c..d825b91f089 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java @@ -210,6 +210,7 @@ public void clear() { counts = null; } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java index 794d90c0d11..30de5e120c5 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java @@ -212,8 +212,8 @@ public void decompressToSparseBlock(SparseBlock sb, int rl, int ru, int offR, in // TODO make sparse decompression where the iterator is known in argument decompressToSparseBlockSparseDictionary(sb, rl, ru, offR, offC, mb.getSparseBlock()); else - decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, - mb.getDenseBlockValues(), it); + decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, mb.getDenseBlockValues(), + it); } else decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, _dict.getValues(), it); @@ -240,7 +240,7 @@ public void decompressToDenseBlockDenseDictionary(DenseBlock db, int rl, int ru, } public abstract void decompressToSparseBlockDenseDictionaryWithProvidedIterator(SparseBlock db, int rl, int ru, - int offR, int offC, double[] values, AIterator it); + int offR, int offC, double[] values, AIterator it); public abstract void decompressToDenseBlockDenseDictionaryWithProvidedIterator(DenseBlock db, int rl, int ru, int offR, int offC, double[] values, AIterator it); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java index d643cae440c..b316e48474a 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java @@ -674,8 +674,8 @@ private void defaultRightDecompressingMult(MatrixBlock right, MatrixBlock ret, i } } - final void vectMM(double aa, double[] b, double[] c, int endT, int jd, int crl, int cru, int offOut, int k, - int vLen, DoubleVector vVec) { + final void vectMM(double aa, double[] b, double[] c, int endT, int jd, int crl, int cru, int offOut, int k, int vLen, + DoubleVector vVec) { vVec = vVec.broadcast(aa); final int offj = k * jd; final int end = endT + offj; diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java index d5ad55772c7..64114a054ab 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java @@ -478,13 +478,14 @@ public AColGroup combineWithSameIndex(int nRow, int nCol, List right) return new ColGroupEmpty(combinedIndex); } - @Override - public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut){ return this; } + @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ return new ColGroupEmpty(newColumnIDs); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java index e0bea3c3696..fa8aa104ffb 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java @@ -747,7 +747,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java index b4f0c144a73..a251d828b5f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java @@ -738,7 +738,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java index 43df7fa3b94..347cea9c0da 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java @@ -1195,9 +1195,9 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { throw new NotImplementedException("Unimplemented method 'removeEmptyRows'"); } - + @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java index 4566106a3e2..815ecacf378 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java @@ -634,8 +634,8 @@ protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList s for(int i = 0; i < selectedColumns.size(); i++) { ref[i] = _reference[selectedColumns.get(i)]; } - return ColGroupSDCFOR.create(newColumnIDs, _numRows, _dict.sliceColumns(selectedColumns, getNumCols()), - _indexes, _data, null, ref); + return ColGroupSDCFOR.create(newColumnIDs, _numRows, _dict.sliceColumns(selectedColumns, getNumCols()), _indexes, _data, null, + ref); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java index 9797087f8c3..611add6480f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java @@ -85,7 +85,7 @@ public class ColGroupUncompressed extends AColGroup { /** * Do not use this constructor of column group uncompressed, instead use the create constructor. - * + * * @param mb The contained data. * @param colIndexes Column indexes for this Columngroup */ @@ -96,10 +96,9 @@ protected ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes) { /** * Do not use this constructor of column group quantization-fused uncompressed, instead use the create constructor. - * + * * @param mb The contained data. - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire - * matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix * @param colIndexes Column indexes for this Columngroup */ protected ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes, double[] scaleFactors) { @@ -139,8 +138,7 @@ public static AColGroup create(MatrixBlock mb, IColIndex colIndexes) { * * @param mb The MB / data to contain in the uncompressed column * @param colIndexes The column indexes for the group - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire - * matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix * @return An Uncompressed Column group */ public static AColGroup createQuantized(MatrixBlock mb, IColIndex colIndexes, double[] scaleFactors) { @@ -159,8 +157,7 @@ public static AColGroup createQuantized(MatrixBlock mb, IColIndex colIndexes, do * @param rawBlock The uncompressed block; uncompressed data must be present at the time that the constructor is * called * @param transposed Says if the input matrix raw block have been transposed. - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire - * matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix * @return AColGroup. */ public static AColGroup createQuantized(IColIndex colIndexes, MatrixBlock rawBlock, boolean transposed, diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java index de8a740ceb2..51e26a3f9d2 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java @@ -290,7 +290,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java index 6e66ef6ef9b..a7e715b59b8 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java @@ -76,8 +76,8 @@ public double[] productAllRowsToDoubleWithDefault(double[] defaultTuple) { return ret; } - @Override - public int[] sort() { + @Override + public int[] sort(){ throw new NotImplementedException(); } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java index 7ebba2f1a76..9a0412145f0 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java @@ -138,8 +138,8 @@ public IDictionary clone() { throw new NotImplementedException(); } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ throw new NotImplementedException(); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java index b5e1a99355b..c8ddfc4883a 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java @@ -1055,7 +1055,7 @@ public IDictionary rightMMPreAggSparse(int numVals, SparseBlock b, IColIndex thi /** * Slice out the selected columns given of this encoded group. - * + * * @param selectedColumns The columns to slice out and return as a new matrix. * @param nCol The number of columns in this dictionary. * @return The returned matrix @@ -1064,9 +1064,9 @@ public IDictionary rightMMPreAggSparse(int numVals, SparseBlock b, IColIndex thi /** * Sort the values of this dictionary via an index of how the values mapped previously. - * + * * In practice this design means we can reuse the previous dictionary for the resulting column group - * + * * @return The sorted index. */ public int[] sort(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java index 4337da7307f..c2540de959a 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java @@ -541,8 +541,8 @@ public String getString(int colIndexes) { return "IdentityMatrix of size: " + nRowCol + " with empty: " + withEmpty; } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ return getMBDict().sliceColumns(selectedColumns, nCol); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java index 47628b43d2a..c7f642edfd0 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java @@ -311,8 +311,8 @@ public String getString(int colIndexes) { return toString(); } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ return getMBDict().sliceColumns(selectedColumns, nCol); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java index 6d516713689..83a74972db7 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java @@ -1064,7 +1064,7 @@ public AMapToData removeEmpty(final boolean[] selectV, final int rOut) { /** * Use the offsets of the select vector to choose which values to keep. - * + * * @param select The row indexes to keep * @return A New MapToData */ diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java index bf8ee7f9ee1..f65876b7f37 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java @@ -56,11 +56,11 @@ public abstract class AOffset implements Serializable { protected static final Log LOG = LogFactory.getLog(AOffset.class.getName()); /** - * Lazy holder for the cached empty slice. The empty slice is built on first use rather than in AOffset's static - * initializer: instantiating the OffsetEmpty subclass from AOffset's {@code } forms a superclass/subclass - * class-initialization cycle that deadlocks when several threads first touch the offset classes concurrently (e.g. - * parallel tests). Deferring it to first use guarantees AOffset is already initialized by the time OffsetEmpty is - * loaded, so no cycle exists. + * Lazy holder for the cached empty slice. The empty slice is built on first use rather than in AOffset's + * static initializer: instantiating the OffsetEmpty subclass from AOffset's {@code } forms a + * superclass/subclass class-initialization cycle that deadlocks when several threads first touch the offset + * classes concurrently (e.g. parallel tests). Deferring it to first use guarantees AOffset is already + * initialized by the time OffsetEmpty is loaded, so no cycle exists. */ private static final class EmptySliceHolder { static final OffsetSliceInfo EMPTY_SLICE = new OffsetSliceInfo(-1, -1, new OffsetEmpty()); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java index 37ff41cf817..866168ded2f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java @@ -76,7 +76,6 @@ public int getOffsetToLast() { public long getInMemorySize() { return estimateInMemorySize(); } - @Override public boolean equals(AOffset b) { return b instanceof OffsetEmpty; diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java index 7953322350e..d981ab87838 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java @@ -139,8 +139,7 @@ private static boolean isDoubleCompressedOpApplicable(CompressedMatrixBlock m1, m1.getColGroups().get(0) instanceof ColGroupDDC && !((CompressedMatrixBlock) that).isOverlapping() && ((CompressedMatrixBlock) that).getColGroups().get(0) instanceof ColGroupDDC && ((IMapToDataGroup) m1.getColGroups().get(0)) - .getMapToData() == ((IMapToDataGroup) ((CompressedMatrixBlock) that).getColGroups().get(0)) - .getMapToData(); + .getMapToData() == ((IMapToDataGroup) ((CompressedMatrixBlock) that).getColGroups().get(0)).getMapToData(); } private static CompressedMatrixBlock doubleCompressedBinaryOp(BinaryOperator op, CompressedMatrixBlock m1, @@ -1063,8 +1062,7 @@ public Long call() { return _ret.recomputeNonZeros(_rl, _ru - 1); } - private final void processBlock(final int rl, final int ru, final List groups, - final AIterator[] its) { + private final void processBlock(final int rl, final int ru, final List groups, final AIterator[] its) { decompressToTmpBlock(rl, ru, tmp.getSparseBlock(), groups, its); // decompressing multiple column groups can leave the temp rows with unsorted column indices, so sort // before reading them in stored order into the (column-sorted) output sparse block. diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java index a91b75ae73c..cc7953f8c5d 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java @@ -96,7 +96,7 @@ public static MatrixBlock mmChain(CompressedMatrixBlock x, MatrixBlock v, Matrix if(x.isEmpty()) return returnEmpty(x, out); - if(ctype == ChainType.XtXv && x.getColGroups().size() < 5 && x.getNumColumns() > 30) { + if(ctype == ChainType.XtXv && x.getColGroups().size() < 5 && x.getNumColumns()> 30){ MatrixBlock tmp = CLALibTSMM.leftMultByTransposeSelf(x, k); return tmp.aggregateBinaryOperations(tmp, v, out, InstructionUtils.getMatMultOperator(k)); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java index 802eddffcb8..3755e4040e7 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java @@ -36,7 +36,7 @@ public class CLALibRemoveEmpty { /** * CP rmempty operation (single input, single output matrix) - * + * * @param in The input matrix * @param ret The output matrix * @param rows If we are removing based on rows, or columns. @@ -66,13 +66,13 @@ private static MatrixBlock rmEmptyCols(CompressedMatrixBlock in, MatrixBlock ret int cOut = (int) select.getNonZeros(); if(cOut == -1) cOut = (int) select.recomputeNonZeros(); - if(cOut == 0) { + if(cOut == 0){ ret.reset(in.getNumRows(), !emptyReturn ? 0 : 1); return ret; } - final boolean[] selectV = DataConverter.convertToBooleanVector( - CompressedMatrixBlock.getUncompressed(select, "decompressing selection in rmempty")); + final boolean[] selectV = DataConverter + .convertToBooleanVector(CompressedMatrixBlock.getUncompressed(select, "decompressing selection in rmempty")); final List inG = in.getColGroups(); final List retG = new ArrayList<>(inG.size()); @@ -102,17 +102,18 @@ private static MatrixBlock rmEmptyRows(CompressedMatrixBlock in, MatrixBlock ret int rOut = (int) select.getNonZeros(); if(rOut == -1) rOut = (int) select.recomputeNonZeros(); - if(rOut == 0) { + if(rOut == 0){ ret.reset(!emptyReturn ? 0 : 1, in.getNumColumns()); return ret; } - // TODO: add optimization to avoid linear scan and make selectV indexes, if selection is small relative to - // number + // TODO: add optimization to avoid linear scan and make selectV indexes, if selection is small relative to number // of rows // TODO: add decompress to boolean vector. final boolean[] selectV = DataConverter.convertToBooleanVector(select); + + final List inG = in.getColGroups(); final List retG = new ArrayList<>(inG.size()); try { diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java index 5ae7bd5103b..b94f11ae723 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java @@ -40,10 +40,10 @@ private CLALibSort() { /** * Sort (order) a compressed matrix in place of the {@code order} built-in, while keeping the result compressed. * - * The compressed fast-path only supports the case the user can benefit from: a single column held in a single - * column group, sorted ascending and returning the sorted values (not the index permutation). For everything else - * (multiple columns, multiple column groups, descending order, index return, or a column-group encoding without a - * sort implementation) this returns {@code null} so the caller can fall back to a decompressed reorg. + * The compressed fast-path only supports the case the user can benefit from: a single column held in a single column + * group, sorted ascending and returning the sorted values (not the index permutation). For everything else (multiple + * columns, multiple column groups, descending order, index return, or a column-group encoding without a sort + * implementation) this returns {@code null} so the caller can fall back to a decompressed reorg. * * @param mb the compressed matrix to sort * @param fn the sort specification carried by the reorg operator diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java index 9ccaa474f39..87d14dbf87e 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java @@ -208,8 +208,8 @@ protected FrameBlock readBlobFromHDFS(String fname, long[] dims) throws IOExcept if(data == null) throw new IOException("Unable to load frame from file: " + fname); - // Delta and CSV discover dimensions (and Delta also schema) at read time, so - // refresh the cached metadata to reflect the materialized frame block. + //Delta and CSV discover dimensions (and Delta also schema) at read time, so + //refresh the cached metadata to reflect the materialized frame block. if(iimd.getFileFormat() == FileFormat.CSV || iimd.getFileFormat() == FileFormat.DELTA) { _metaData = _metaData instanceof MetaDataFormat ? new MetaDataFormat(data.getDataCharacteristics(), iimd.getFileFormat()) : new MetaData(data.getDataCharacteristics()); diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java index 4331da2b426..28fa70f7741 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java @@ -454,7 +454,7 @@ protected MatrixBlock readBlobFromHDFS(String fname, long[] dims) throws IOExcep rlen, clen, blen, mc.getNonZeros(), getFileFormatProperties()); if(iimd.getFileFormat() == FileFormat.CSV || iimd.getFileFormat() == FileFormat.DELTA) { - // dimensions/nnz are discovered at read time for these self-describing formats + //dimensions/nnz are discovered at read time for these self-describing formats _metaData = _metaData instanceof MetaDataFormat ? new MetaDataFormat(newData.getDataCharacteristics(), iimd.getFileFormat()) : new MetaData(newData.getDataCharacteristics()); } diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java index fbae4925c66..b52f3777e1f 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java @@ -122,9 +122,9 @@ public class SparkExecutionContext extends ExecutionContext //singleton spark context (as there can be only one spark context per JVM) private static JavaSparkContext _spctx = null; - // registered users of the singleton context (guarded by the - // SparkExecutionContext.class monitor); maintained by enterSparkExecution()/ - // exitSparkExecution(), and close() only stops the context once it hits zero + //registered users of the singleton context (guarded by the + //SparkExecutionContext.class monitor); maintained by enterSparkExecution()/ + //exitSparkExecution(), and close() only stops the context once it hits zero private static int _activeExecutions = 0; //registry of parallelized RDDs to enforce that at any time, we spent at most @@ -175,8 +175,8 @@ public synchronized static JavaSparkContext getSparkContextStatic() { initSparkContext(); if(_spctx.sc().isStopped()){ _spctx = null; - // the previous context was stopped; reset the active-execution count so a - // stale registration cannot skip a future legitimate stop of the new one + //the previous context was stopped; reset the active-execution count so a + //stale registration cannot skip a future legitimate stop of the new one _activeExecutions = 0; initSparkContext(); } @@ -196,15 +196,16 @@ public synchronized static boolean isSparkContextCreated() { public static void resetSparkContextStatic() { synchronized(SparkExecutionContext.class) { _spctx = null; - // force-discarding the shared context: drop the active-execution count so - // a stale registration cannot skip a future legitimate stop + //force-discarding the shared context: drop the active-execution count so + //a stale registration cannot skip a future legitimate stop _activeExecutions = 0; } } /** - * Registers an active user of the shared spark context. Must be balanced by a later {@link #exitSparkExecution()} - * so a concurrent execution cannot stop the context while this one still has in-flight jobs. + * Registers an active user of the shared spark context. Must be balanced by a + * later {@link #exitSparkExecution()} so a concurrent execution cannot stop the + * context while this one still has in-flight jobs. */ public static void enterSparkExecution() { synchronized(SparkExecutionContext.class) { @@ -213,8 +214,9 @@ public static void enterSparkExecution() { } /** - * Releases an active user previously registered via {@link #enterSparkExecution()}. Only adjusts the count; the - * actual teardown is left to {@link #close()}, which stops the context once no registered execution remains. + * Releases an active user previously registered via {@link #enterSparkExecution()}. + * Only adjusts the count; the actual teardown is left to {@link #close()}, which + * stops the context once no registered execution remains. */ public static void exitSparkExecution() { synchronized(SparkExecutionContext.class) { @@ -225,13 +227,13 @@ public static void exitSparkExecution() { public void close() { synchronized(SparkExecutionContext.class) { - // keep the shared context alive while a registered execution still uses - // it; close() never changes the count, so an unpaired close() (a caller - // that never entered) cannot stop a context another execution is using + //keep the shared context alive while a registered execution still uses + //it; close() never changes the count, so an unpaired close() (a caller + //that never entered) cannot stop a context another execution is using if(_activeExecutions > 0) { if(LOG.isDebugEnabled()) - LOG.debug( - "Keeping shared spark context alive; " + _activeExecutions + " execution(s) still active"); + LOG.debug("Keeping shared spark context alive; " + _activeExecutions + + " execution(s) still active"); return; } if(_spctx != null) { diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java index c502817e026..682cc8e3fff 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java @@ -95,7 +95,8 @@ private void run() { int par_conn = ConfigurationManager.getDMLConfig().getIntValue(DMLConfig.FEDERATED_PAR_CONN); final int EVENT_LOOP_THREADS = (par_conn > 0) ? par_conn : InfrastructureAnalyzer.getLocalParallelism(); // Daemon event loops so a leaked in-JVM (test) worker cannot block JVM exit. - NioEventLoopGroup bossGroup = new NioEventLoopGroup(1, new DefaultThreadFactory("fed-worker-boss", true)); + NioEventLoopGroup bossGroup = new NioEventLoopGroup(1, + new DefaultThreadFactory("fed-worker-boss", true)); ThreadPoolExecutor workerTPE = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 10, TimeUnit.SECONDS, new SynchronousQueue(true), new DefaultThreadFactory("fed-worker-pool", true)); NioEventLoopGroup workerGroup = new NioEventLoopGroup(EVENT_LOOP_THREADS, workerTPE); diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java index ebf05972b87..80a5d699dfa 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java @@ -125,15 +125,13 @@ public static RaggedArray create(T[] col, int m) { /** * Wrap a fully populated raw typed column array into an {@link Array} of the given value type. The runtime type of - * {@code col} must match the primitive backing type of {@code vt} (e.g. {@code double[]} for - * {@link ValueType#FP64}, {@code String[]} for {@link ValueType#STRING}). + * {@code col} must match the primitive backing type of {@code vt} (e.g. {@code double[]} for {@link ValueType#FP64}, + * {@code String[]} for {@link ValueType#STRING}). * - *

- * For {@link ValueType#BOOLEAN} this mirrors {@link #allocateBoolean(int)}: a {@code boolean[]} longer than - * {@link #bitSetSwitchPoint} is bit-packed into a compact {@link BitSetArray} (so a bulk decoder that fills a plain - * {@code boolean[]} still ends up with the same representation as every other frame allocation path), while shorter - * columns stay a plain {@link BooleanArray}. - *

+ *

For {@link ValueType#BOOLEAN} this mirrors {@link #allocateBoolean(int)}: a {@code boolean[]} longer than + * {@link #bitSetSwitchPoint} is bit-packed into a compact {@link BitSetArray} (so a bulk decoder that fills a + * plain {@code boolean[]} still ends up with the same representation as every other frame allocation path), + * while shorter columns stay a plain {@link BooleanArray}.

* * @param vt the value type of the column * @param col the backing array to wrap @@ -170,10 +168,10 @@ public static Array create(ValueType vt, Object col) { /** * Allocate the raw backing array for a column of the given value type: the inverse of - * {@link #create(ValueType, Object)}. Returns {@code double[]} for {@link ValueType#FP64}, {@code int[]} for - * INT32/UINT/HASH32, {@code long[]} for INT64/HASH64, {@code String[]} for STRING, etc. The runtime array type - * matches what {@link #create(ValueType, Object)} expects, so a bulk decoder can fill this primitive array directly - * and then wrap it via {@code create(vt, backing)}. + * {@link #create(ValueType, Object)}. Returns {@code double[]} for {@link ValueType#FP64}, + * {@code int[]} for INT32/UINT/HASH32, {@code long[]} for INT64/HASH64, {@code String[]} for STRING, etc. The + * runtime array type matches what {@link #create(ValueType, Object)} expects, so a bulk decoder can fill this + * primitive array directly and then wrap it via {@code create(vt, backing)}. * * @param vt the value type of the column * @param nRow the number of rows to allocate diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java b/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java index 95be95117e2..9ff58065d97 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java @@ -41,7 +41,7 @@ public class MatrixBlockFromFrame { public static Boolean WARNED_FOR_FAILED_CAST = false; - private MatrixBlockFromFrame() { + private MatrixBlockFromFrame(){ // private constructor for code coverage. } @@ -115,7 +115,7 @@ private static long convert(FrameBlock frame, MatrixBlock mb, int n, int rl, int return convertStrict(frame, mb, n, rl, ru); } catch(NumberFormatException | DMLRuntimeException e) { - synchronized(WARNED_FOR_FAILED_CAST) { + synchronized(WARNED_FOR_FAILED_CAST){ if(!WARNED_FOR_FAILED_CAST) { LOG.error( "Failed to convert to Matrix because of number format errors, falling back to NaN on incompatible cells", diff --git a/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java b/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java index c12a187cf17..eed2c58f78c 100644 --- a/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java +++ b/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java @@ -30,13 +30,32 @@ import jdk.incubator.vector.VectorSpecies; - public enum BuiltinCode { - AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, ATAN, LOG, LOG_NZ, MIN, MAX, ABS, SIGN, SQRT, EXP, PLOGP, - PRINT, PRINTF, NROW, NCOL, LENGTH, LINEAGE, ROUND, MAXINDEX, MININDEX, STOP, CEIL, FLOOR, CUMSUM, ROWCUMSUM, - CUMPROD, CUMMIN, CUMMAX, CUMSUMPROD, INVERSE, SPROP, SIGMOID, EVAL, LIST, TYPEOF, APPLY_SCHEMA, DETECTSCHEMA, - ISNA, ISNAN, ISINF, DROP_INVALID_TYPE, DROP_INVALID_LENGTH, VALUE_SWAP, FRAME_ROW_REPLICATE, - GET_CATEGORICAL_MASK, MAP, COUNT_DISTINCT, COUNT_DISTINCT_APPROX, UNIQUE - } +/** + * Class with pre-defined set of objects. This class can not be instantiated elsewhere. + * + * Notes on commons.math FastMath: + * * FastMath uses lookup tables and interpolation instead of native calls. + * * The memory overhead for those tables is roughly 48KB in total (acceptable) + * * Micro and application benchmarks showed significantly (30%-3x) performance improvements + * for most operations; without loss of accuracy. + * * atan / sqrt were 20% slower in FastMath and hence, we use Math there + * * round / abs were equivalent in FastMath and hence, we use Math there + * * Finally, there is just one argument against FastMath - The comparison heavily depends + * on the JVM. For example, currently the IBM JDK JIT compiles to HW instructions for sqrt + * which makes this operation very efficient; as soon as other operations like log/exp are + * similarly compiled, we should rerun the micro benchmarks, and switch back if necessary. + * + */ +public class Builtin extends ValueFunction +{ + private static final long serialVersionUID = 3836744687789840574L; + + public enum BuiltinCode { AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, ATAN, LOG, LOG_NZ, MIN, + MAX, ABS, SIGN, SQRT, EXP, PLOGP, PRINT, PRINTF, NROW, NCOL, LENGTH, LINEAGE, ROUND, MAXINDEX, MININDEX, + STOP, CEIL, FLOOR, CUMSUM, ROWCUMSUM, CUMPROD, CUMMIN, CUMMAX, CUMSUMPROD, INVERSE, SPROP, SIGMOID, EVAL, LIST, + TYPEOF, APPLY_SCHEMA, DETECTSCHEMA, ISNA, ISNAN, ISINF, DROP_INVALID_TYPE, + DROP_INVALID_LENGTH, VALUE_SWAP, FRAME_ROW_REPLICATE, GET_CATEGORICAL_MASK, + MAP, COUNT_DISTINCT, COUNT_DISTINCT_APPROX, UNIQUE} private static final VectorSpecies SPECIES = DoubleVector.SPECIES_PREFERRED; private static final int vLen = SPECIES.length(); @@ -49,59 +68,59 @@ public enum BuiltinCode { static public HashMap String2BuiltinCode; static { String2BuiltinCode = new HashMap<>(); - String2BuiltinCode.put("autoDiff", BuiltinCode.AUTODIFF); - String2BuiltinCode.put("sin", BuiltinCode.SIN); - String2BuiltinCode.put("cos", BuiltinCode.COS); - String2BuiltinCode.put("tan", BuiltinCode.TAN); - String2BuiltinCode.put("sinh", BuiltinCode.SINH); - String2BuiltinCode.put("cosh", BuiltinCode.COSH); - String2BuiltinCode.put("tanh", BuiltinCode.TANH); - String2BuiltinCode.put("asin", BuiltinCode.ASIN); - String2BuiltinCode.put("acos", BuiltinCode.ACOS); - String2BuiltinCode.put("atan", BuiltinCode.ATAN); - String2BuiltinCode.put("log", BuiltinCode.LOG); - String2BuiltinCode.put("log_nz", BuiltinCode.LOG_NZ); - String2BuiltinCode.put("min", BuiltinCode.MIN); - String2BuiltinCode.put("max", BuiltinCode.MAX); - String2BuiltinCode.put("maxindex", BuiltinCode.MAXINDEX); - String2BuiltinCode.put("minindex", BuiltinCode.MININDEX); - String2BuiltinCode.put("abs", BuiltinCode.ABS); - String2BuiltinCode.put("sign", BuiltinCode.SIGN); - String2BuiltinCode.put("sqrt", BuiltinCode.SQRT); - String2BuiltinCode.put("exp", BuiltinCode.EXP); - String2BuiltinCode.put("plogp", BuiltinCode.PLOGP); - String2BuiltinCode.put("print", BuiltinCode.PRINT); - String2BuiltinCode.put("printf", BuiltinCode.PRINTF); - String2BuiltinCode.put("eval", BuiltinCode.EVAL); - String2BuiltinCode.put("list", BuiltinCode.LIST); - String2BuiltinCode.put("nrow", BuiltinCode.NROW); - String2BuiltinCode.put("ncol", BuiltinCode.NCOL); - String2BuiltinCode.put("length", BuiltinCode.LENGTH); - String2BuiltinCode.put("round", BuiltinCode.ROUND); - String2BuiltinCode.put("stop", BuiltinCode.STOP); - String2BuiltinCode.put("ceil", BuiltinCode.CEIL); - String2BuiltinCode.put("floor", BuiltinCode.FLOOR); - String2BuiltinCode.put("ucumk+", BuiltinCode.CUMSUM); - String2BuiltinCode.put("urowcumk+", BuiltinCode.ROWCUMSUM); - String2BuiltinCode.put("ucum*", BuiltinCode.CUMPROD); - String2BuiltinCode.put("ucumk+*", BuiltinCode.CUMSUMPROD); - String2BuiltinCode.put("ucummin", BuiltinCode.CUMMIN); - String2BuiltinCode.put("ucummax", BuiltinCode.CUMMAX); - String2BuiltinCode.put("inverse", BuiltinCode.INVERSE); - String2BuiltinCode.put("sprop", BuiltinCode.SPROP); - String2BuiltinCode.put("sigmoid", BuiltinCode.SIGMOID); - String2BuiltinCode.put("typeOf", BuiltinCode.TYPEOF); - String2BuiltinCode.put("detectSchema", BuiltinCode.DETECTSCHEMA); - String2BuiltinCode.put("isna", BuiltinCode.ISNA); - String2BuiltinCode.put("isnan", BuiltinCode.ISNAN); - String2BuiltinCode.put("isinf", BuiltinCode.ISINF); - String2BuiltinCode.put("dropInvalidType", BuiltinCode.DROP_INVALID_TYPE); - String2BuiltinCode.put("freplicate", BuiltinCode.FRAME_ROW_REPLICATE); - String2BuiltinCode.put("dropInvalidLength", BuiltinCode.DROP_INVALID_LENGTH); - String2BuiltinCode.put("_map", BuiltinCode.MAP); - String2BuiltinCode.put("valueSwap", BuiltinCode.VALUE_SWAP); - String2BuiltinCode.put("applySchema", BuiltinCode.APPLY_SCHEMA); - String2BuiltinCode.put("get_categorical_mask", BuiltinCode.GET_CATEGORICAL_MASK); + String2BuiltinCode.put( "autoDiff" , BuiltinCode.AUTODIFF); + String2BuiltinCode.put( "sin" , BuiltinCode.SIN); + String2BuiltinCode.put( "cos" , BuiltinCode.COS); + String2BuiltinCode.put( "tan" , BuiltinCode.TAN); + String2BuiltinCode.put( "sinh" , BuiltinCode.SINH); + String2BuiltinCode.put( "cosh" , BuiltinCode.COSH); + String2BuiltinCode.put( "tanh" , BuiltinCode.TANH); + String2BuiltinCode.put( "asin" , BuiltinCode.ASIN); + String2BuiltinCode.put( "acos" , BuiltinCode.ACOS); + String2BuiltinCode.put( "atan" , BuiltinCode.ATAN); + String2BuiltinCode.put( "log" , BuiltinCode.LOG); + String2BuiltinCode.put( "log_nz" , BuiltinCode.LOG_NZ); + String2BuiltinCode.put( "min" , BuiltinCode.MIN); + String2BuiltinCode.put( "max" , BuiltinCode.MAX); + String2BuiltinCode.put( "maxindex", BuiltinCode.MAXINDEX); + String2BuiltinCode.put( "minindex", BuiltinCode.MININDEX); + String2BuiltinCode.put( "abs" , BuiltinCode.ABS); + String2BuiltinCode.put( "sign" , BuiltinCode.SIGN); + String2BuiltinCode.put( "sqrt" , BuiltinCode.SQRT); + String2BuiltinCode.put( "exp" , BuiltinCode.EXP); + String2BuiltinCode.put( "plogp" , BuiltinCode.PLOGP); + String2BuiltinCode.put( "print" , BuiltinCode.PRINT); + String2BuiltinCode.put( "printf" , BuiltinCode.PRINTF); + String2BuiltinCode.put( "eval" , BuiltinCode.EVAL); + String2BuiltinCode.put( "list" , BuiltinCode.LIST); + String2BuiltinCode.put( "nrow" , BuiltinCode.NROW); + String2BuiltinCode.put( "ncol" , BuiltinCode.NCOL); + String2BuiltinCode.put( "length" , BuiltinCode.LENGTH); + String2BuiltinCode.put( "round" , BuiltinCode.ROUND); + String2BuiltinCode.put( "stop" , BuiltinCode.STOP); + String2BuiltinCode.put( "ceil" , BuiltinCode.CEIL); + String2BuiltinCode.put( "floor" , BuiltinCode.FLOOR); + String2BuiltinCode.put( "ucumk+" , BuiltinCode.CUMSUM); + String2BuiltinCode.put( "urowcumk+" , BuiltinCode.ROWCUMSUM); + String2BuiltinCode.put( "ucum*" , BuiltinCode.CUMPROD); + String2BuiltinCode.put( "ucumk+*", BuiltinCode.CUMSUMPROD); + String2BuiltinCode.put( "ucummin", BuiltinCode.CUMMIN); + String2BuiltinCode.put( "ucummax", BuiltinCode.CUMMAX); + String2BuiltinCode.put( "inverse", BuiltinCode.INVERSE); + String2BuiltinCode.put( "sprop", BuiltinCode.SPROP); + String2BuiltinCode.put( "sigmoid", BuiltinCode.SIGMOID); + String2BuiltinCode.put( "typeOf", BuiltinCode.TYPEOF); + String2BuiltinCode.put( "detectSchema", BuiltinCode.DETECTSCHEMA); + String2BuiltinCode.put( "isna", BuiltinCode.ISNA); + String2BuiltinCode.put( "isnan", BuiltinCode.ISNAN); + String2BuiltinCode.put( "isinf", BuiltinCode.ISINF); + String2BuiltinCode.put( "dropInvalidType", BuiltinCode.DROP_INVALID_TYPE); + String2BuiltinCode.put( "freplicate", BuiltinCode.FRAME_ROW_REPLICATE); + String2BuiltinCode.put( "dropInvalidLength", BuiltinCode.DROP_INVALID_LENGTH); + String2BuiltinCode.put( "_map", BuiltinCode.MAP); + String2BuiltinCode.put( "valueSwap", BuiltinCode.VALUE_SWAP); + String2BuiltinCode.put( "applySchema", BuiltinCode.APPLY_SCHEMA); + String2BuiltinCode.put( "get_categorical_mask", BuiltinCode.GET_CATEGORICAL_MASK); } protected Builtin(BuiltinCode bf) { diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java index 08d28512d5c..86184f47be6 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java @@ -59,7 +59,7 @@ else if (in1.getDataType() == DataType.TENSOR && in2.getDataType() == DataType.T return new BinaryTensorTensorCPInstruction(operator, in1, in2, out, opcode, str); else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.FRAME) return new BinaryFrameFrameCPInstruction(operator, in1, in2, out, opcode, str); - else if(in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.SCALAR) + else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.SCALAR) return new BinaryFrameScalarCPInstruction(operator, in1, in2, out, opcode, str); else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.MATRIX) return new BinaryFrameMatrixCPInstruction(operator, in1, in2, out, opcode, str); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java index de76fca18b8..193894fd9bc 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java @@ -37,8 +37,8 @@ public class BinaryFrameScalarCPInstruction extends BinaryCPInstruction { // private static final Log LOG = LogFactory.getLog(BinaryFrameFrameCPInstruction.class.getName()); - private static final TfMethod[] UNSUPPORTED_MASK_METHODS = new TfMethod[] {TfMethod.BIN, TfMethod.WORD_EMBEDDING, - TfMethod.BAG_OF_WORDS, TfMethod.UDF}; + private static final TfMethod[] UNSUPPORTED_MASK_METHODS = new TfMethod[] {TfMethod.BIN, + TfMethod.WORD_EMBEDDING, TfMethod.BAG_OF_WORDS, TfMethod.UDF}; protected BinaryFrameScalarCPInstruction(MultiThreadedOperator op, CPOperand in1, CPOperand in2, CPOperand out, String opcode, String istr) { @@ -96,9 +96,9 @@ public void processGetCategorical(ExecutionContext ec, FrameBlock f, ScalarObjec } /** - * Accumulates, per input column, how many output columns it expands to (lengths) and whether those output columns - * are categorical (categorical). The arrays are allocated lazily: a column that no method touches keeps the - * implicit default of a single, non-categorical output column. + * Accumulates, per input column, how many output columns it expands to (lengths) and whether those + * output columns are categorical (categorical). The arrays are allocated lazily: a column that no + * method touches keeps the implicit default of a single, non-categorical output column. */ private static final class CategoricalMask { private final FrameBlock f; diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java index 2c8093c3717..d76dbe0d45e 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java @@ -80,10 +80,10 @@ public void processInstruction(ExecutionContext ec) { retBlock = inBlock1; } else { - if(LibCommonsMath.isSupportedMatrixMatrixOperation(getOpcode())) { + if(LibCommonsMath.isSupportedMatrixMatrixOperation(getOpcode()) ){ if(compressedLeft) inBlock1 = CompressedMatrixBlock.getUncompressed(inBlock1, getOpcode()); - + if(compressedRight) inBlock2 = CompressedMatrixBlock.getUncompressed(inBlock2, getOpcode()); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java index 97ae151ecc0..e53958ac4b8 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java @@ -350,10 +350,9 @@ else if(opcode.equalsIgnoreCase(Opcodes.TRANSFORMDECODE.toString())) { String[] colnames = meta.getColumnNames(); // compute transformdecode - Decoder decoder = DecoderFactory.createDecoder(getParameterMap().get("spec"), colnames, null, meta, - data.getNumColumns()); - FrameBlock fbout = decoder.decode(data, new FrameBlock(decoder.getSchema()), - InfrastructureAnalyzer.getLocalParallelism()); + Decoder decoder = DecoderFactory + .createDecoder(getParameterMap().get("spec"), colnames, null, meta, data.getNumColumns()); + FrameBlock fbout = decoder.decode(data, new FrameBlock(decoder.getSchema()), InfrastructureAnalyzer.getLocalParallelism()); fbout.setColumnNames(Arrays.copyOfRange(colnames, 0, fbout.getNumColumns())); // release locks diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java index 04353806ca3..40a677e5d71 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java @@ -45,8 +45,8 @@ protected ReorgOOCInstruction(ReorgOperator op, CPOperand in1, CPOperand out, St this(op, in1, out, null, null, null, opcode, istr); } - private ReorgOOCInstruction(Operator op, CPOperand in, CPOperand out, CPOperand col, CPOperand desc, - CPOperand ixret, String opcode, String istr) { + private ReorgOOCInstruction(Operator op, CPOperand in, CPOperand out, CPOperand col, CPOperand desc, CPOperand ixret, + String opcode, String istr) { super(OOCType.Reorg, op, in, out, opcode, istr); _col = col; _desc = desc; @@ -76,8 +76,8 @@ else if(opcode.equalsIgnoreCase(Opcodes.SORT.toString())) { CPOperand desc = new CPOperand(parts[3]); CPOperand ixret = new CPOperand(parts[4]); int k = Integer.parseInt(parts[6]); - return new ReorgOOCInstruction(new ReorgOperator(new SortIndex(1, false, false), k), in, out, col, desc, - ixret, opcode, str); + return new ReorgOOCInstruction(new ReorgOperator(new SortIndex(1, false, false), k), + in, out, col, desc, ixret, opcode, str); } else throw new NotImplementedException(); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java index 091c6785b3c..7590438b949 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java @@ -128,20 +128,17 @@ public void processInstruction(ExecutionContext ec) { long r = tmp.getIndexes().getRowIndex(); long c = tmp.getIndexes().getColumnIndex(); // adapt index to new position in row - return new IndexedMatrixValue(new MatrixIndexes(1, (r - 1) * numBlocksPerRowIn + c), - tmp.getValue()); + return new IndexedMatrixValue(new MatrixIndexes(1, (r - 1) * numBlocksPerRowIn + c), tmp.getValue()); }); } else { f.join(); - reshapeFullColBlocks(rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, - singleRowBlocks, qOut); + reshapeFullColBlocks(rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, singleRowBlocks, qOut); } } else { f.join(); - reshapePartialColBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, - numBlocksPerColOut, singleRowBlocks, qOut); + reshapePartialColBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, singleRowBlocks, qOut); } } else { @@ -169,20 +166,17 @@ public void processInstruction(ExecutionContext ec) { long r = tmp.getIndexes().getRowIndex(); long c = tmp.getIndexes().getColumnIndex(); // adapt index to new position in col - return new IndexedMatrixValue(new MatrixIndexes((c - 1) * numBlocksPerColIn + r, 1), - tmp.getValue()); + return new IndexedMatrixValue(new MatrixIndexes((c - 1) * numBlocksPerColIn + r, 1), tmp.getValue()); }); } else { f.join(); - reshapeFullRowBlocks(rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, - singleColBlocks, qOut); + reshapeFullRowBlocks(rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, singleColBlocks, qOut); } } else { f.join(); - reshapePartialRowBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, - numBlocksPerColOut, singleColBlocks, qOut); + reshapePartialRowBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, singleColBlocks, qOut); } } } @@ -232,8 +226,7 @@ private void reshapeFullColBlocks(long rows, long cols, int blen, int numBlocksP } private void reshapePartialColBlocks(long rlen, long clen, long rows, long cols, int blen, int numBlocksPerRowIn, - int numBlocksPerRowOut, int numBlocksPerColOut, OOCStream singleRowBlocks, - OOCStream qOut) { + int numBlocksPerRowOut, int numBlocksPerColOut, OOCStream singleRowBlocks, OOCStream qOut) { // use cache for accessing input rows by index CachingStream singleRowBlockCache = new CachingStream(singleRowBlocks); singleRowBlockCache.incrSubscriberCount(1); @@ -244,16 +237,14 @@ private void reshapePartialColBlocks(long rlen, long clen, long rows, long cols, int r = 0; // allocate row of output blocks - MatrixBlock[] outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, - true); + MatrixBlock[] outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut,true); int offsetOut = 0; int localColsOut = (cols > blen) ? blen : (int) cols; // iterate through input rows and add to row of output blocks for(int i = 1; i <= rlen; i++) { for(int j = 1; j <= numBlocksPerRowIn; j++) { - try(OOCStream.QueueCallback qcb = singleRowBlockCache - .findCached(new MatrixIndexes(i, j))) { + try(OOCStream.QueueCallback qcb = singleRowBlockCache.findCached(new MatrixIndexes(i, j))) { MatrixBlock blk = (MatrixBlock) qcb.get().getValue(); int offsetIn = 0; @@ -287,12 +278,10 @@ else if(remIn == remOut) { if(r == outputBlockRow[0].getNumRows()) { // enqueue filled output blocks and allocate new ones for(int b = 0; b < outputBlockRow.length; b++) - qOut.enqueue( - new IndexedMatrixValue(new MatrixIndexes(br + 1, b + 1), outputBlockRow[b])); + qOut.enqueue(new IndexedMatrixValue(new MatrixIndexes(br + 1, b + 1), outputBlockRow[b])); br++; // allocate new block row - outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, - numBlocksPerColOut, true); + outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, true); r = 0; } bc = 0; @@ -352,8 +341,7 @@ private void reshapeFullRowBlocks(long rows, long cols, int blen, int numBlocksP } private void reshapePartialRowBlocks(long rlen, long clen, long rows, long cols, int blen, int numBlocksPerRowOut, - int numBlocksPerColIn, int numBlocksPerColOut, OOCStream singleColBlocks, - OOCStream qOut) { + int numBlocksPerColIn, int numBlocksPerColOut, OOCStream singleColBlocks, OOCStream qOut) { // use cache for accessing input cols by index CachingStream singleRowBlockCache = new CachingStream(singleColBlocks); singleRowBlockCache.incrSubscriberCount(1); @@ -364,16 +352,14 @@ private void reshapePartialRowBlocks(long rlen, long clen, long rows, long cols, int c = 0; // allocate col of output blocks - MatrixBlock[] outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, - false); + MatrixBlock[] outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, false); int offsetOut = 0; int localRowsOut = (rows > blen) ? blen : (int) rows; // iterate through input cols and add to col of output blocks for(int j = 1; j <= clen; j++) { for(int i = 1; i <= numBlocksPerColIn; i++) { - try(OOCStream.QueueCallback qcb = singleRowBlockCache - .findCached(new MatrixIndexes(i, j))) { + try(OOCStream.QueueCallback qcb = singleRowBlockCache.findCached(new MatrixIndexes(i, j))) { MatrixBlock blk = (MatrixBlock) qcb.get().getValue(); int offsetIn = 0; @@ -408,13 +394,11 @@ else if(remIn == remOut) { // enqueue filled output blocks and allocate new ones for(int b = 0; b < outputBlockCol.length; b++) { outputBlockCol[b].recomputeNonZeros(); - qOut.enqueue( - new IndexedMatrixValue(new MatrixIndexes(b + 1, bc + 1), outputBlockCol[b])); + qOut.enqueue(new IndexedMatrixValue(new MatrixIndexes(b + 1, bc + 1), outputBlockCol[b])); } bc++; // allocate new block col - outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, - numBlocksPerColOut, false); + outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, false); c = 0; } br = 0; @@ -427,20 +411,16 @@ else if(remIn == remOut) { qOut.closeInput(); } - private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int blen, int numBlocksPerRow, - int numBlocksPerCol, boolean isBlockRowSlice) { + private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int blen, int numBlocksPerRow, int numBlocksPerCol, boolean isBlockRowSlice) { int num = isBlockRowSlice ? numBlocksPerRow : numBlocksPerCol; MatrixBlock[] res = new MatrixBlock[num]; // full inner blocks, adjust for outer blocks - int localRows = ((!isBlockRowSlice || idx == numBlocksPerCol - 1) && rows % blen != 0) ? (int) rows % - blen : blen; - int localCols = ((isBlockRowSlice || idx == numBlocksPerRow - 1) && cols % blen != 0) ? (int) cols % - blen : blen; + int localRows = ((!isBlockRowSlice || idx == numBlocksPerCol - 1) && rows % blen != 0) ? (int) rows % blen : blen; + int localCols = ((isBlockRowSlice || idx == numBlocksPerRow - 1) && cols % blen != 0) ? (int) cols % blen : blen; for(int k = 0; k < num - 1; k++) { - res[k] = isBlockRowSlice ? new MatrixBlock(localRows, blen, false) : new MatrixBlock(blen, localCols, - false); + res[k] = isBlockRowSlice ? new MatrixBlock(localRows, blen, false) : new MatrixBlock(blen, localCols, false); res[k].allocateDenseBlock(); } res[num - 1] = new MatrixBlock(localRows, localCols, false); @@ -448,13 +428,10 @@ private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int ble return res; } - private void setOutputEntries(MatrixBlock src, MatrixBlock dest, int idx, int srcOffset, int destOffset, int length, - boolean rowWise) { + private void setOutputEntries(MatrixBlock src, MatrixBlock dest, int idx, int srcOffset, int destOffset, int length, boolean rowWise) { if(rowWise) - ((DenseBlockFP64) dest.getDenseBlock()).setPartialRow(src.getDenseBlock(), idx, srcOffset, destOffset, - length); + ((DenseBlockFP64) dest.getDenseBlock()).setPartialRow(src.getDenseBlock(), idx, srcOffset, destOffset, length); else - ((DenseBlockFP64) dest.getDenseBlock()).setPartialCol(src.getDenseBlock(), idx, srcOffset, destOffset, - length); + ((DenseBlockFP64) dest.getDenseBlock()).setPartialCol(src.getDenseBlock(), idx, srcOffset, destOffset, length); } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java index e25219b80ba..75f84882478 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java @@ -114,7 +114,8 @@ public void processInstruction(ExecutionContext ec) { case VALUEPICK: { if( input2.isScalar() ) { ScalarObject quantile = ec.getScalarInput(input2); - double[] wt = getWeightedQuantileSummary(in, mc, new double[] {quantile.getDoubleValue()}, true); + double[] wt = getWeightedQuantileSummary(in, mc, + new double[] {quantile.getDoubleValue()}, true); ec.setScalarOutput(output.getName(), new DoubleObject(wt[3])); } else { diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java index f007558ebdc..2f83caa5526 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java @@ -30,7 +30,8 @@ import org.apache.sysds.runtime.matrix.data.MatrixValue; import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; -public class IndexedMatrixValue implements SpillableObject, Serializable { +public class IndexedMatrixValue implements SpillableObject, Serializable +{ private static final long serialVersionUID = 6723389820806752110L; private MatrixIndexes _indexes = null; @@ -109,8 +110,8 @@ public void discard() { @Override public void read(DataInput dataInput) throws IOException { - _indexes = new MatrixIndexes(); - _value = new MatrixBlock(); + _indexes = new MatrixIndexes(); + _value = new MatrixBlock(); _indexes.readFields(dataInput); _value.readFields(dataInput); } diff --git a/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java index cc8491d4515..c3b9351d3d3 100644 --- a/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java +++ b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java @@ -87,10 +87,11 @@ import io.delta.kernel.utils.FileStatus; /** - * Shared helpers for the native (Spark-free) Delta Lake read/write paths used by both the matrix and frame - * readers/writers. Centralizes engine creation, path qualification, the scan loop (snapshot -> data files -> - * logical columnar batches, honoring deletion vectors), and the write transaction (logical data -> parquet -> - * commit). + * Shared helpers for the native (Spark-free) Delta Lake read/write paths used + * by both the matrix and frame readers/writers. Centralizes engine creation, + * path qualification, the scan loop (snapshot -> data files -> logical + * columnar batches, honoring deletion vectors), and the write transaction + * (logical data -> parquet -> commit). */ public class DeltaKernelUtils { @@ -101,29 +102,23 @@ public class DeltaKernelUtils { /** Reused thread-safe JSON reader for the per-file Delta stats (numRecords). */ private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); - /** - * Delta Kernel config key: number of rows per parquet read batch, overridable via - * {@link org.apache.sysds.conf.DMLConfig#DELTA_READER_BATCH_SIZE}. - */ + /** Delta Kernel config key: number of rows per parquet read batch, overridable via + * {@link org.apache.sysds.conf.DMLConfig#DELTA_READER_BATCH_SIZE}. */ private static final String CONF_READER_BATCH_SIZE = "delta.kernel.default.parquet.reader.batch-size"; - /** - * Delta Kernel config key: target size (bytes) at which the writer rolls a new data file, overridable via - * {@link org.apache.sysds.conf.DMLConfig#DELTA_WRITER_TARGET_FILE_SIZE}. - */ + /** Delta Kernel config key: target size (bytes) at which the writer rolls a new data file, overridable via + * {@link org.apache.sysds.conf.DMLConfig#DELTA_WRITER_TARGET_FILE_SIZE}. */ private static final String CONF_WRITER_TARGET_FILE_SIZE = "delta.kernel.default.parquet.writer.targetMaxFileSize"; - /** - * Internal Delta column type codes shared by the matrix and frame readers to dispatch boxing-free primitive column - * access. - */ - public static final int T_DOUBLE = 0; - public static final int T_FLOAT = 1; - public static final int T_LONG = 2; - public static final int T_INT = 3; - public static final int T_SHORT = 4; - public static final int T_BYTE = 5; + /** Internal Delta column type codes shared by the matrix and frame readers to + * dispatch boxing-free primitive column access. */ + public static final int T_DOUBLE = 0; + public static final int T_FLOAT = 1; + public static final int T_LONG = 2; + public static final int T_INT = 3; + public static final int T_SHORT = 4; + public static final int T_BYTE = 5; public static final int T_BOOLEAN = 6; - public static final int T_STRING = 7; + public static final int T_STRING = 7; /** * Parquet physical type each {@code T_*} column is stored as, indexed by type code (delta int/short/byte columns @@ -133,22 +128,22 @@ public class DeltaKernelUtils { PrimitiveTypeName.INT64, PrimitiveTypeName.INT32, PrimitiveTypeName.INT32, PrimitiveTypeName.INT32, PrimitiveTypeName.BOOLEAN, PrimitiveTypeName.BINARY}; - // derived configuration cached to avoid copying the (large) base conf on every - // engine creation (createEngine is called once per data file in parallel reads); - // rebuilt whenever the base conf or the relevant SystemDS settings change. + //derived configuration cached to avoid copying the (large) base conf on every + //engine creation (createEngine is called once per data file in parallel reads); + //rebuilt whenever the base conf or the relevant SystemDS settings change. private static Configuration cachedConf; private static Configuration cachedConfBase; private static int cachedBatchSize; private static long cachedTargetFileSize; - private DeltaKernelUtils() { - } + private DeltaKernelUtils() {} /** - * Consumes a whole columnar batch. {@code selected} is {@code null} when all {@code size} rows are live; otherwise - * {@code selected[r]} indicates whether row {@code r} survived the deletion/selection vector. Batch-level - * consumption lets callers extract data column-at-a-time (cache friendly, boxing free) instead of paying a per-row - * callback. + * Consumes a whole columnar batch. {@code selected} is {@code null} when all + * {@code size} rows are live; otherwise {@code selected[r]} indicates whether + * row {@code r} survived the deletion/selection vector. Batch-level consumption + * lets callers extract data column-at-a-time (cache friendly, boxing free) + * instead of paying a per-row callback. */ @FunctionalInterface public interface BatchConsumer { @@ -156,29 +151,22 @@ public interface BatchConsumer { } /** - * Map a Delta Kernel {@link DataType} to an internal type code (see the {@code T_*} constants). Returned once per - * column so the per-cell read loop can switch on a primitive int instead of repeating {@code instanceof} checks. + * Map a Delta Kernel {@link DataType} to an internal type code (see the + * {@code T_*} constants). Returned once per column so the per-cell read loop + * can switch on a primitive int instead of repeating {@code instanceof} checks. * * @param dt the Delta column data type * @return the matching {@code T_*} code, or {@code -1} if the type is not supported */ public static int typeCode(DataType dt) { - if(dt instanceof DoubleType) - return T_DOUBLE; - if(dt instanceof FloatType) - return T_FLOAT; - if(dt instanceof LongType) - return T_LONG; - if(dt instanceof IntegerType) - return T_INT; - if(dt instanceof ShortType) - return T_SHORT; - if(dt instanceof ByteType) - return T_BYTE; - if(dt instanceof BooleanType) - return T_BOOLEAN; - if(dt instanceof StringType) - return T_STRING; + if( dt instanceof DoubleType ) return T_DOUBLE; + if( dt instanceof FloatType ) return T_FLOAT; + if( dt instanceof LongType ) return T_LONG; + if( dt instanceof IntegerType ) return T_INT; + if( dt instanceof ShortType ) return T_SHORT; + if( dt instanceof ByteType ) return T_BYTE; + if( dt instanceof BooleanType ) return T_BOOLEAN; + if( dt instanceof StringType ) return T_STRING; return -1; } @@ -441,10 +429,8 @@ private static void decodeColumnInto(ColumnReader creader, int maxDef, int nrow, } } - /** - * Floor on the adaptive writer target file size. Below this the per-file metadata/open overhead (and tiny-file - * proliferation) outweighs the extra read parallelism. - */ + /** Floor on the adaptive writer target file size. Below this the per-file metadata/open + * overhead (and tiny-file proliferation) outweighs the extra read parallelism. */ public static final long ADAPTIVE_WRITER_MIN_FILE_SIZE = 4L * 1024 * 1024; private static Configuration buildConf(Configuration base, int batchSize, long targetFileSize) { @@ -458,8 +444,9 @@ private static synchronized Configuration deltaConf() { Configuration base = ConfigurationManager.getCachedJobConf(); int batchSize = ConfigurationManager.getDeltaReaderBatchSize(); long targetFileSize = ConfigurationManager.getDeltaWriterTargetFileSize(); - if(cachedConf == null || cachedConfBase != base || cachedBatchSize != batchSize || - cachedTargetFileSize != targetFileSize) { + if(cachedConf == null || cachedConfBase != base + || cachedBatchSize != batchSize || cachedTargetFileSize != targetFileSize) + { cachedConf = buildConf(base, batchSize, targetFileSize); cachedConfBase = base; cachedBatchSize = batchSize; @@ -473,11 +460,12 @@ public static Engine createEngine() { } /** - * Compute the parquet target data-file size (bytes) for writing a table of the given estimated size. With adaptive - * sizing enabled the writer aims for roughly one data file per expected parallel reader (so the native per-file - * parallel read can use all threads): never above the configured target, and never below - * {@code ADAPTIVE_WRITER_MIN_FILE_SIZE} unless the configured target is itself smaller than that floor (in which - * case the configured target wins). + * Compute the parquet target data-file size (bytes) for writing a table of the given + * estimated size. With adaptive sizing enabled the writer aims for roughly one data + * file per expected parallel reader (so the native per-file parallel read can use all + * threads): never above the configured target, and never below + * {@code ADAPTIVE_WRITER_MIN_FILE_SIZE} unless the configured target is itself smaller + * than that floor (in which case the configured target wins). * * @param estimatedBytes estimate of the table's size (the block in-memory size is a fine proxy) * @return the target max parquet data-file size in bytes @@ -488,7 +476,7 @@ public static long adaptiveWriterTargetFileSize(long estimatedBytes) { return configured; int par = Math.max(1, OptimizerUtils.getParallelBinaryReadParallelism()); long perReader = Math.max(1, estimatedBytes / par); - // never above the configured cap, never below the floor (unless the cap itself is lower) + //never above the configured cap, never below the floor (unless the cap itself is lower) long target = Math.min(configured, Math.max(ADAPTIVE_WRITER_MIN_FILE_SIZE, perReader)); if(LOG.isDebugEnabled()) LOG.debug("Delta adaptive file size: est=" + estimatedBytes + "B par=" + par + " -> target=" + target @@ -497,24 +485,24 @@ public static long adaptiveWriterTargetFileSize(long estimatedBytes) { } /** - * Create an engine for writing a table of the given estimated size, configured with an adaptive target data-file - * size (see {@link #adaptiveWriterTargetFileSize(long)}). A fresh (uncached) configuration is built since writes - * happen once per table, not per data file. + * Create an engine for writing a table of the given estimated size, configured with an + * adaptive target data-file size (see {@link #adaptiveWriterTargetFileSize(long)}). A fresh + * (uncached) configuration is built since writes happen once per table, not per data file. * * @param estimatedBytes estimate of the table's size (the block in-memory size is a fine proxy) * @return a Delta Kernel engine for the write */ public static Engine createWriteEngine(long estimatedBytes) { - // the reader batch size is irrelevant on the write path but is set to keep the - // conf shape identical to deltaConf(); only the target file size matters here. + //the reader batch size is irrelevant on the write path but is set to keep the + //conf shape identical to deltaConf(); only the target file size matters here. Configuration c = buildConf(ConfigurationManager.getCachedJobConf(), ConfigurationManager.getDeltaReaderBatchSize(), adaptiveWriterTargetFileSize(estimatedBytes)); return DefaultEngine.create(c); } /** - * Resolve a (possibly relative) path to a fully-qualified URI so the kernel's default engine can locate the table - * on the right filesystem. + * Resolve a (possibly relative) path to a fully-qualified URI so the + * kernel's default engine can locate the table on the right filesystem. * * @param fname input path * @return fully-qualified table path @@ -531,9 +519,11 @@ public static String qualify(String fname) { } /** - * Opened latest snapshot of a Delta table: the logical schema plus everything needed to (re)read its data files, - * including the list of per-data-file scan rows. Delta Kernel scan-file rows are self-contained (the kernel's - * distributed design serializes them to workers), so they can be retained and read independently / in parallel. + * Opened latest snapshot of a Delta table: the logical schema plus everything + * needed to (re)read its data files, including the list of per-data-file scan + * rows. Delta Kernel scan-file rows are self-contained (the kernel's + * distributed design serializes them to workers), so they can be retained and + * read independently / in parallel. */ public static final class ScanHandle { public final StructType schema; @@ -541,18 +531,19 @@ public static final class ScanHandle { public final StructType physicalReadSchema; public final List scanFiles; /** - * Per-file record counts taken from the Delta {@code numRecords} statistic, aligned with {@link #scanFiles}; - * {@code -1} where the statistic is absent. + * Per-file record counts taken from the Delta {@code numRecords} statistic, + * aligned with {@link #scanFiles}; {@code -1} where the statistic is absent. */ public final long[] numRecords; /** - * Per-file flag indicating a deletion vector is present (so the live row count differs from - * {@link #numRecords}), aligned with {@link #scanFiles}. + * Per-file flag indicating a deletion vector is present (so the live row + * count differs from {@link #numRecords}), aligned with {@link #scanFiles}. */ public final boolean[] hasDeletionVector; - private ScanHandle(StructType schema, Row scanState, StructType physicalReadSchema, List scanFiles, - long[] numRecords, boolean[] hasDeletionVector) { + private ScanHandle(StructType schema, Row scanState, StructType physicalReadSchema, + List scanFiles, long[] numRecords, boolean[] hasDeletionVector) + { this.schema = schema; this.scanState = scanState; this.physicalReadSchema = physicalReadSchema; @@ -562,12 +553,13 @@ private ScanHandle(StructType schema, Row scanState, StructType physicalReadSche } /** - * @return true iff every data file carries a {@code numRecords} statistic and none has a deletion vector, i.e. - * exact per-file row offsets can be derived from metadata without reading the data. + * @return true iff every data file carries a {@code numRecords} statistic + * and none has a deletion vector, i.e. exact per-file row offsets + * can be derived from metadata without reading the data. */ public boolean hasExactRowCounts() { - for(int i = 0; i < numRecords.length; i++) - if(numRecords[i] < 0 || hasDeletionVector[i]) + for( int i=0; i scanFileIter = (scan instanceof ScanImpl) ? ((ScanImpl) scan) - .getScanFiles(engine, true) : scan.getScanFiles(engine); + //request the scan files WITH per-file statistics (numRecords) so callers can + //pre-size output and place rows without reading the data; harmless extra + //column for the data-read path. Fall back to the stats-less iterator if the + //concrete scan does not support it. + CloseableIterator scanFileIter = (scan instanceof ScanImpl) + ? ((ScanImpl) scan).getScanFiles(engine, true) + : scan.getScanFiles(engine); List files = new ArrayList<>(); List recs = new ArrayList<>(); List dvs = new ArrayList<>(); - try(CloseableIterator scanFiles = scanFileIter) { - while(scanFiles.hasNext()) { + try( CloseableIterator scanFiles = scanFileIter ) { + while( scanFiles.hasNext() ) { FilteredColumnarBatch scanFileBatch = scanFiles.next(); - try(CloseableIterator scanFileRows = scanFileBatch.getRows()) { - while(scanFileRows.hasNext()) { + try( CloseableIterator scanFileRows = scanFileBatch.getRows() ) { + while( scanFileRows.hasNext() ) { Row scanFileRow = scanFileRows.next(); files.add(scanFileRow); recs.add(numRecords(scanFileRow)); @@ -615,7 +609,7 @@ public static ScanHandle openScan(Engine engine, String tablePath) throws IOExce } long[] numRecords = new long[recs.size()]; boolean[] hasDv = new boolean[dvs.size()]; - for(int i = 0; i < numRecords.length; i++) { + for( int i=0; i physicalData = engine.getParquetHandler() .readParquetFiles(Utils.singletonCloseableIterator(dataFile), physicalReadSchema, Optional.empty()); - try(CloseableIterator logicalData = Scan.transformPhysicalData(engine, scanState, - scanFileRow, physicalData)) { - while(logicalData.hasNext()) + try( CloseableIterator logicalData = + Scan.transformPhysicalData(engine, scanState, scanFileRow, physicalData) ) + { + while( logicalData.hasNext() ) consumeBatch(logicalData.next(), consumer); } } /** - * Scan the latest snapshot of a Delta table sequentially, invoking the batch consumer for every data batch. The - * consumer is created lazily from the table schema (so callers can size buffers / derive per-column types up - * front). + * Scan the latest snapshot of a Delta table sequentially, invoking the batch + * consumer for every data batch. The consumer is created lazily from the table + * schema (so callers can size buffers / derive per-column types up front). * * @param engine delta kernel engine * @param tablePath fully-qualified table path @@ -679,10 +677,11 @@ public static void readScanFile(Engine engine, Row scanState, StructType physica * @throws IOException on read failure */ public static StructType scan(Engine engine, String tablePath, Function consumerFactory) - throws IOException { + throws IOException + { ScanHandle h = openScan(engine, tablePath); BatchConsumer consumer = consumerFactory.apply(h.schema); - for(Row scanFileRow : h.scanFiles) + for( Row scanFileRow : h.scanFiles ) readScanFile(engine, h.scanState, h.physicalReadSchema, scanFileRow, consumer); return h.schema; } @@ -691,27 +690,28 @@ private static void consumeBatch(FilteredColumnarBatch fcb, BatchConsumer consum ColumnarBatch batch = fcb.getData(); int ncol = batch.getSchema().length(); ColumnVector[] cols = new ColumnVector[ncol]; - for(int c = 0; c < ncol; c++) + for( int c=0; c all rows live) + //materialize the deletion/selection mask once (null => all rows live) Optional selVector = fcb.getSelectionVector(); boolean[] selected = null; - if(selVector.isPresent()) { + if( selVector.isPresent() ) { ColumnVector sv = selVector.get(); selected = new boolean[size]; - for(int r = 0; r < size; r++) + for( int r=0; r logicalData) throws IOException { - // replace any existing table at the path (the other SystemDS writers delete - // the output first; the caching layer does not do it on our behalf) + CloseableIterator logicalData) throws IOException + { + //replace any existing table at the path (the other SystemDS writers delete + //the output first; the caching layer does not do it on our behalf) HDFSTool.deleteFileIfExistOnHDFS(tablePath); Table table = Table.forPath(engine, tablePath); - TransactionBuilder txnBuilder = table.createTransactionBuilder(engine, ENGINE_INFO, Operation.CREATE_TABLE) + TransactionBuilder txnBuilder = table + .createTransactionBuilder(engine, ENGINE_INFO, Operation.CREATE_TABLE) .withSchema(engine, schema); Transaction txn = txnBuilder.build(engine); Row txnState = txn.getTransactionState(engine); - CloseableIterator physicalData = Transaction.transformLogicalData(engine, txnState, - logicalData, Collections.emptyMap()); - DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, Collections.emptyMap()); + CloseableIterator physicalData = + Transaction.transformLogicalData(engine, txnState, logicalData, Collections.emptyMap()); + DataWriteContext writeContext = + Transaction.getWriteContext(engine, txnState, Collections.emptyMap()); CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles(writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns()); - CloseableIterator appendActions = Transaction.generateAppendActions(engine, txnState, dataFiles, - writeContext); + CloseableIterator appendActions = + Transaction.generateAppendActions(engine, txnState, dataFiles, writeContext); txn.commit(engine, CloseableIterable.inMemoryIterable(appendActions)); } } diff --git a/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java b/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java index 55d8f8f7c2d..58a98741975 100644 --- a/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java +++ b/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java @@ -33,27 +33,29 @@ import io.delta.kernel.types.StructType; /** - * Single-threaded native Delta Lake reader for matrices, built on the Spark-free Delta Kernel library. It opens the - * latest snapshot of a Delta table directory, reads its parquet data files through the kernel's default engine - * (honoring deletion vectors), and materializes the numeric columns into a dense {@link MatrixBlock}. + * Single-threaded native Delta Lake reader for matrices, built on the + * Spark-free Delta Kernel library. It opens the latest snapshot of a Delta + * table directory, reads its parquet data files through the kernel's default + * engine (honoring deletion vectors), and materializes the numeric columns + * into a dense {@link MatrixBlock}. * - *

- * Only numeric columns (double/float/long/int/short/byte/boolean) are supported, matching the all-double nature of a - * SystemDS matrix. Dimensions do not need to be known up front: the row count is discovered while scanning and the - * column count is taken from the table schema. - *

+ *

Only numeric columns (double/float/long/int/short/byte/boolean) are + * supported, matching the all-double nature of a SystemDS matrix. Dimensions + * do not need to be known up front: the row count is discovered while scanning + * and the column count is taken from the table schema.

*/ public class ReaderDelta extends MatrixReader { @Override public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int blen, long estnnz) - throws IOException, DMLRuntimeException { + throws IOException, DMLRuntimeException + { Engine engine = DeltaKernelUtils.createEngine(); String tablePath = DeltaKernelUtils.qualify(fname); - // Scan column-at-a-time into one row-major buffer per batch (no per-row - // allocation, no boxing, no per-cell set()). Buffers are concatenated into - // the dense output via bulk array copies below. + //Scan column-at-a-time into one row-major buffer per batch (no per-row + //allocation, no boxing, no per-cell set()). Buffers are concatenated into + //the dense output via bulk array copies below. ArrayList batches = new ArrayList<>(); int[] nrowH = new int[1]; StructType schema = DeltaKernelUtils.scan(engine, tablePath, sch -> { @@ -70,7 +72,7 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl long lestnnz = (estnnz >= 0) ? estnnz : (long) nrow * ncol; MatrixBlock ret = createOutputMatrixBlock(nrow, ncol, Math.max(nrow, 1), lestnnz, true, false); - if(nrow > 0 && ncol > 0) + if( nrow > 0 && ncol > 0 ) fillDense(ret, batches); ret.recomputeNonZeros(); ret.examSparsity(); @@ -81,14 +83,14 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl static int[] columnTypes(StructType schema) { int ncol = schema.length(); int[] types = new int[ncol]; - for(int c = 0; c < ncol; c++) + for( int c=0; c batches) { DenseBlock db = ret.getDenseBlock(); - if(db.isContiguous()) { + if( db.isContiguous() ) { double[] dv = db.valuesAt(0); int off = 0; - for(double[] buf : batches) { + for( double[] buf : batches ) { System.arraycopy(buf, 0, dv, off, buf.length); off += buf.length; } } else { - // rare large multi-block fallback: route each row through the block API + //rare large multi-block fallback: route each row through the block API int ncol = ret.getNumColumns(); int r = 0; - for(double[] buf : batches) { + for( double[] buf : batches ) { int rowsInBuf = buf.length / ncol; - for(int i = 0; i < rowsInBuf; i++, r++) - for(int c = 0; c < ncol; c++) + for( int i=0; i - * The expensive part of a Delta read is the parquet decode, which the kernel performs per data file; parallelizing - * across files is therefore the natural way to bridge the gap to the (near-raw) binary reader. A table backed by a - * single data file (the default for tables <= the parquet target file size) cannot be split this way, so the reader - * transparently falls back to the sequential {@link ReaderDelta} path in that case. - *

+ *

The expensive part of a Delta read is the parquet decode, which the kernel + * performs per data file; parallelizing across files is therefore the natural + * way to bridge the gap to the (near-raw) binary reader. A table backed by a + * single data file (the default for tables <= the parquet target file size) + * cannot be split this way, so the reader transparently falls back to the + * sequential {@link ReaderDelta} path in that case.

*/ public class ReaderDeltaParallel extends ReaderDelta { @@ -56,27 +57,28 @@ public ReaderDeltaParallel() { @Override public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int blen, long estnnz) - throws IOException, DMLRuntimeException { + throws IOException, DMLRuntimeException + { Engine engine = DeltaKernelUtils.createEngine(); String tablePath = DeltaKernelUtils.qualify(fname); DeltaKernelUtils.ScanHandle handle = DeltaKernelUtils.openScan(engine, tablePath); final int nfiles = handle.scanFiles.size(); - // nothing to gain from parallelism for single-file (or empty) tables - if(_numThreads <= 1 || nfiles <= 1) + //nothing to gain from parallelism for single-file (or empty) tables + if( _numThreads <= 1 || nfiles <= 1 ) return super.readMatrixFromHDFS(fname, rlen, clen, blen, estnnz); final int ncol = handle.schema.length(); final int[] types = columnTypes(handle.schema); - // fast path: exact per-file row counts are known from metadata and the dense - // output fits a single contiguous array -> pre-size once and let each thread - // decode directly into its slice (no intermediate buffers, no serial copy). - if(useDirectPath(handle)) { + //fast path: exact per-file row counts are known from metadata and the dense + //output fits a single contiguous array -> pre-size once and let each thread + //decode directly into its slice (no intermediate buffers, no serial copy). + if( useDirectPath(handle) ) { long total = 0; - for(long r : handle.numRecords) + for( long r : handle.numRecords ) total += r; - if(total > 0 && (long) total * ncol <= Integer.MAX_VALUE) + if( total > 0 && (long) total * ncol <= Integer.MAX_VALUE ) return readDirect(fname, handle, ncol, types, (int) total, estnnz); } @@ -84,9 +86,11 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl } /** - * Whether the metadata-driven direct-write fast path can be used for this table (exact per-file row counts and no - * deletion vectors). Visible for testing: the buffered fallback is otherwise only reachable for tables lacking row - * statistics or carrying deletion vectors, which the SystemDS Delta writer never produces. + * Whether the metadata-driven direct-write fast path can be used for this + * table (exact per-file row counts and no deletion vectors). Visible for + * testing: the buffered fallback is otherwise only reachable for tables + * lacking row statistics or carrying deletion vectors, which the SystemDS + * Delta writer never produces. * * @param handle the opened scan handle * @return true if the direct path is applicable @@ -96,37 +100,38 @@ protected boolean useDirectPath(DeltaKernelUtils.ScanHandle handle) { } /** - * Fast path: each thread decodes one data file straight into the final dense array at a metadata-derived row - * offset. Single allocation, fully parallel. + * Fast path: each thread decodes one data file straight into the final dense + * array at a metadata-derived row offset. Single allocation, fully parallel. */ - private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, int ncol, int[] types, int nrow, - long estnnz) throws IOException { + private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, + int ncol, int[] types, int nrow, long estnnz) throws IOException + { final int nfiles = handle.scanFiles.size(); final int[] rowOffset = new int[nfiles]; int acc = 0; - for(int i = 0; i < nfiles; i++) { + for( int i=0; i> tasks = new ArrayList<>(nfiles); - for(int i = 0; i < nfiles; i++) { + for( int i=0; i { int[] cur = new int[] {base}; Engine eng = DeltaKernelUtils.createEngine(); DeltaKernelUtils.readScanFile(eng, handle.scanState, handle.physicalReadSchema, scanFileRow, (cols, size, selected) -> { - if(cur[0] + DeltaKernelUtils.countSelected(size, selected) > limit) + if( cur[0] + DeltaKernelUtils.countSelected(size, selected) > limit ) throw new DMLRuntimeException("Delta file produced more rows than its " + "numRecords statistic; refusing parallel direct read of " + fname); cur[0] += extractBatchInto(cols, size, selected, types, ncol, dv, cur[0]); @@ -142,17 +147,19 @@ private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, } /** - * Fallback path: decode each file in parallel into per-file buffers (used when row counts are unknown, deletion - * vectors are present, or the matrix exceeds a single contiguous array), then concatenate in file order. + * Fallback path: decode each file in parallel into per-file buffers (used when + * row counts are unknown, deletion vectors are present, or the matrix exceeds a + * single contiguous array), then concatenate in file order. */ - private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handle, int ncol, int[] types, - long estnnz) throws IOException { + private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handle, + int ncol, int[] types, long estnnz) throws IOException + { final int nfiles = handle.scanFiles.size(); @SuppressWarnings("unchecked") final ArrayList[] fileBufs = new ArrayList[nfiles]; final int[] fileRows = new int[nfiles]; ArrayList> tasks = new ArrayList<>(nfiles); - for(int i = 0; i < nfiles; i++) { + for( int i=0; i { @@ -172,15 +179,15 @@ private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handl awaitFileTasks(tasks, fname); int nrow = 0; - for(int i = 0; i < nfiles; i++) + for( int i=0; i ordered = new ArrayList<>(); - for(int i = 0; i < nfiles; i++) + for( int i=0; i= 0) ? estnnz : (long) nrow * ncol; MatrixBlock ret = createOutputMatrixBlock(nrow, ncol, Math.max(nrow, 1), lestnnz, true, false); - if(nrow > 0 && ncol > 0) + if( nrow > 0 && ncol > 0 ) fillDense(ret, ordered); ret.recomputeNonZeros(_numThreads); ret.examSparsity(); @@ -188,15 +195,16 @@ private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handl } /** - * Run one decode task per data file on the shared common thread pool and await completion. Full parallelism is - * requested (the task count, one per data file, naturally caps concurrency); this avoids the per-thread pool-size - * caching in {@code CommonThreadPool.get(k)} that could otherwise throttle this reader to a smaller pool created - * earlier on the same thread. + * Run one decode task per data file on the shared common thread pool and await + * completion. Full parallelism is requested (the task count, one per data file, + * naturally caps concurrency); this avoids the per-thread pool-size caching in + * {@code CommonThreadPool.get(k)} that could otherwise throttle this reader to a + * smaller pool created earlier on the same thread. */ private void awaitFileTasks(List> tasks, String fname) throws IOException { ExecutorService pool = CommonThreadPool.get(_numThreads); try { - for(Future f : pool.invokeAll(tasks)) + for( Future f : pool.invokeAll(tasks) ) f.get(); } catch(Exception ex) { diff --git a/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java b/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java index 602b540c407..55ea8a54297 100644 --- a/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java +++ b/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java @@ -39,54 +39,60 @@ import io.delta.kernel.utils.CloseableIterator; /** - * Single-threaded native Delta Lake writer for matrices, built on the Spark-free Delta Kernel library. It creates a - * Delta table at the target directory with an all-double schema {@code c0..c(n-1)} (replacing any existing table at - * that path), streams the {@link MatrixBlock} rows as columnar batches into parquet data files via the kernel's default - * engine, and commits the corresponding add-file actions to the transaction log. + * Single-threaded native Delta Lake writer for matrices, built on the + * Spark-free Delta Kernel library. It creates a Delta table at the target + * directory with an all-double schema {@code c0..c(n-1)} (replacing any existing + * table at that path), streams the {@link MatrixBlock} rows as columnar batches + * into parquet data files via the kernel's default engine, and commits the + * corresponding add-file actions to the transaction log. */ public class WriterDelta extends MatrixWriter { @Override public void writeMatrixToHDFS(MatrixBlock src, String fname, long rlen, long clen, int blen, long nnz, boolean diag) - throws IOException { - if(src.getNumRows() != rlen || src.getNumColumns() != clen) - throw new IOException("Matrix dimensions mismatch with metadata: (" + src.getNumRows() + "x" - + src.getNumColumns() + ") vs (" + rlen + "x" + clen + ")."); + throws IOException + { + if( src.getNumRows() != rlen || src.getNumColumns() != clen ) + throw new IOException("Matrix dimensions mismatch with metadata: (" + + src.getNumRows() + "x" + src.getNumColumns() + ") vs (" + rlen + "x" + clen + ")."); int ncol = (int) clen; int nrow = (int) rlen; int batchRows = ConfigurationManager.getDeltaWriterBatchSize(); - // fast path: a contiguous dense block lets the column views read straight - // from the backing double[] (avoids per-cell MatrixBlock.get dispatch). - double[] dense = (!src.isInSparseFormat() && src.getDenseBlock() != null && - src.getDenseBlock().isContiguous()) ? src.getDenseBlockValues() : null; - // size data files adaptively (toward one file per parallel reader) for faster parallel reads. - // Delta writes every cell as a double, so size by the dense footprint rather than the (possibly - // sparse) in-memory size, which would understate the on-disk table for sparse inputs. + //fast path: a contiguous dense block lets the column views read straight + //from the backing double[] (avoids per-cell MatrixBlock.get dispatch). + double[] dense = (!src.isInSparseFormat() && src.getDenseBlock() != null + && src.getDenseBlock().isContiguous()) ? src.getDenseBlockValues() : null; + //size data files adaptively (toward one file per parallel reader) for faster parallel reads. + //Delta writes every cell as a double, so size by the dense footprint rather than the (possibly + //sparse) in-memory size, which would understate the on-disk table for sparse inputs. long estimatedBytes = (long) nrow * ncol * 8L; Engine engine = DeltaKernelUtils.createWriteEngine(estimatedBytes); - DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), buildSchema(ncol), - new MatrixBatchIterator(src, dense, nrow, ncol, batchRows)); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), + buildSchema(ncol), new MatrixBatchIterator(src, dense, nrow, ncol, batchRows)); } @Override - public void writeEmptyMatrixToHDFS(String fname, long rlen, long clen, int blen) throws IOException { - // empty table: create with schema but no data files + public void writeEmptyMatrixToHDFS(String fname, long rlen, long clen, int blen) + throws IOException + { + //empty table: create with schema but no data files Engine engine = DeltaKernelUtils.createEngine(); - DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), buildSchema((int) clen), - CloseableIterable.emptyIterable().iterator()); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), + buildSchema((int) clen), CloseableIterable.emptyIterable().iterator()); } private static StructType buildSchema(int ncol) { StructType schema = new StructType(); - for(int c = 0; c < ncol; c++) + for( int c=0; c stream, long rlen, long clen, - int blen) throws IOException { + public long writeMatrixFromStream(String fname, OOCStream stream, long rlen, long clen, int blen) + throws IOException + { throw new UnsupportedOperationException("Out-of-core stream write is not supported for the Delta format."); } @@ -116,18 +122,18 @@ public boolean hasNext() { @Override public FilteredColumnarBatch next() { - if(!hasNext()) + if( !hasNext() ) throw new NoSuchElementException(); int size = Math.min(_batchRows, _nrow - _pos); ColumnarBatch batch = new MatrixColumnarBatch(_mb, _dense, _schema, _pos, size, _ncol); _pos += size; - // no selection vector: all rows in the batch are written + //no selection vector: all rows in the batch are written return new FilteredColumnarBatch(batch, Optional.empty()); } @Override public void close() { - // nothing to release + //nothing to release } } @@ -156,7 +162,7 @@ public StructType getSchema() { @Override public ColumnVector getColumnVector(int ordinal) { - if(ordinal < 0 || ordinal >= _ncol) + if( ordinal < 0 || ordinal >= _ncol ) throw new IndexOutOfBoundsException("column ordinal " + ordinal); return new MatrixColumnVector(_mb, _dense, _rowStart, _size, _ncol, ordinal); } @@ -202,14 +208,16 @@ public boolean isNullAt(int rowId) { @Override public double getDouble(int rowId) { - // dense contiguous single block => index fits in int (getDenseBlockValues - // is only handed over for single-block dense matrices) - return (_dense != null) ? _dense[(_rowStart + rowId) * _ncol + _col] : _mb.get(_rowStart + rowId, _col); + //dense contiguous single block => index fits in int (getDenseBlockValues + //is only handed over for single-block dense matrices) + return (_dense != null) + ? _dense[(_rowStart + rowId) * _ncol + _col] + : _mb.get(_rowStart + rowId, _col); } @Override public void close() { - // nothing to release + //nothing to release } } } diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java index 0bf9f7d87ba..5f478979104 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java @@ -977,7 +977,7 @@ public static MatrixBlock rmempty(MatrixBlock in, MatrixBlock ret, boolean rows, if(ret == null) ret = new MatrixBlock(); MatrixBlock ret2 = rmemptyEarlyAbort(in, ret, rows, emptyReturn, select); - if(ret2 != null) + if(ret2 != null ) return ret2; // core removeEmpty return rmemptyUnsafe(in, ret, rows, emptyReturn, select); @@ -985,7 +985,7 @@ public static MatrixBlock rmempty(MatrixBlock in, MatrixBlock ret, boolean rows, public static MatrixBlock rmemptyUnsafe(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, MatrixBlock select) { - if(rows) + if( rows ) return removeEmptyRows(in, ret, select, emptyReturn); else // cols return removeEmptyColumns(in, ret, select, emptyReturn); @@ -999,15 +999,14 @@ public static MatrixBlock rmemptyUnsafe(MatrixBlock in, MatrixBlock ret, boolean * @param rows If removing based on rows, or columns * @param emptyReturn Return a row/column of zeros for empty input * @param select An optional selection vector - * @return The early-abort result, or {@code null} if no early termination applies and the caller must continue. For - * the select-all case the returned block is the input {@code in} itself (a shallow alias, not a copy). + * @return The early-abort result, or {@code null} if no early termination applies and the caller must continue. + * For the select-all case the returned block is the input {@code in} itself (a shallow alias, not a copy). */ - public static MatrixBlock rmemptyEarlyAbort(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, - MatrixBlock select) { - // check for empty inputs - // (the semantics of removeEmpty are that for an empty m-by-n matrix, the output - // is an empty 1-by-n or m-by-1 matrix because we don't allow matrices with dims 0) - if(in.isEmptyBlock(false) && select == null) { + public static MatrixBlock rmemptyEarlyAbort(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, MatrixBlock select){ + //check for empty inputs + //(the semantics of removeEmpty are that for an empty m-by-n matrix, the output + //is an empty 1-by-n or m-by-1 matrix because we don't allow matrices with dims 0) + if( in.isEmptyBlock(false) && select == null ) { int n = emptyReturn ? 1 : 0; if( rows ) ret.reset(n, in.clen, in.sparse); @@ -3652,7 +3651,7 @@ private static MatrixBlock removeEmptyRows(MatrixBlock in, MatrixBlock ret, Matr /** * Remove selected rows, based on the boolean array given. Note this function is internal use only, and require a * boolean vector to be constructed first. - * + * * @param in Input to remove rows from * @param ret Output to assign the result into * @param emptyReturn If the output is allowed to be empty. @@ -3673,9 +3672,9 @@ public static MatrixBlock removeEmptyRows(MatrixBlock in, MatrixBlock ret, boole ret.reset(rlen2, n, sp); if( in.isEmptyBlock(false) ) return ret; - - if(SHALLOW_COPY_REORG && m == rlen2 && selectNull) { - // the condition m==rlen2 is not enough with non-empty 1-row input but empty + + if( SHALLOW_COPY_REORG && m == rlen2 && selectNull ) { + // the condition m==rlen2 is not enough with non-empty 1-row input but empty // 1-row select vector because if emptyReturn should output a single empty row ret.sparse = in.sparse; if( ret.sparse ) @@ -3715,9 +3714,10 @@ else if( !in.sparse && !ret.sparse ) //DENSE <- DENSE ci++; } } - - // check sparsity - ret.nonZeros = (selectNull) ? in.nonZeros : ret.recomputeNonZeros(); + + //check sparsity + ret.nonZeros = (selectNull) ? + in.nonZeros : ret.recomputeNonZeros(); ret.examSparsity(); return ret; diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java index c450ecbe1f5..7525dab2f7f 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java @@ -4756,13 +4756,13 @@ public static double computeIQMCorrection(double sum, double sum_wt, } /** - * Pick the quantiles out of this matrix. If this matrix contains two columns it is weighted quantile picking. If a - * single column it is unweighted. - * + * Pick the quantiles out of this matrix. If this matrix contains two columns it is weighted quantile picking. + * If a single column it is unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantiles The quantiles to pick - * @param ret The result matrix + * @param ret The result matrix * @return The result matrix */ public final MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret) { @@ -4792,11 +4792,11 @@ public MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret, boolean av } /** - * Pick the median value from this matrix. If this matrix has two columns it is weighted picking using the weight - * column, otherwise it is unweighted over the single column. - * + * Pick the median value from this matrix. If this matrix has two columns it is weighted picking using the + * weight column, otherwise it is unweighted over the single column. + * * Note the values are assumed to be sorted. - * + * * @return The median value */ public double median() { @@ -4807,11 +4807,10 @@ public double median() { } /** - * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is - * unweighted. - * + * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantile The quantile to pick * @return The quantile */ @@ -4820,13 +4819,12 @@ public final double pickValue(double quantile){ } /** - * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is - * unweighted. - * + * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantile The quantile to pick - * @param average If the quantile is averaged. + * @param average If the quantile is averaged. * @return The quantile */ public final double pickValue(double quantile, boolean average) { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java index 9ea39bd1e2f..9e3494a7d48 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java @@ -34,8 +34,8 @@ import java.util.function.Function; /** - * Small future implementation for OOC hot paths. It supports multiple synchronous subscribers without the - * completion-stage support of {@link java.util.concurrent.CompletableFuture}. + * Small future implementation for OOC hot paths. It supports multiple synchronous subscribers without + * the completion-stage support of {@link java.util.concurrent.CompletableFuture}. */ public class OOCFuture { private Subscriber _subscribers; @@ -240,7 +240,7 @@ private static void accept(Function mapper, Consu if(resultError == null) { try { @SuppressWarnings("unchecked") - R mapped = mapper == null ? (R) value : mapper.apply(value); + R mapped = mapper == null ? (R)value : mapper.apply(value); result = mapped; } catch(Throwable t) { @@ -345,7 +345,8 @@ public T get() throws InterruptedException, ExecutionException { } @Override - public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + public T get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { try { return mapper.apply(source.get(timeout, unit)); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java index df981f40223..94411242df1 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java @@ -25,22 +25,20 @@ public class CloseableQueue { private final BlockingQueue queue = new LinkedBlockingQueue<>(); - private final Object POISON = new Object(); // sentinel + private final Object POISON = new Object(); // sentinel private volatile boolean closed = false; - public CloseableQueue() { - } + public CloseableQueue() { } /** * Enqueue if the queue is not closed. - * * @return false if already closed */ public boolean enqueueIfOpen(T task) throws InterruptedException { - if(task == null) + if (task == null) throw new IllegalArgumentException("null tasks not allowed"); - synchronized(this) { - if(closed) + synchronized (this) { + if (closed) return false; queue.put(task); } @@ -49,12 +47,12 @@ public boolean enqueueIfOpen(T task) throws InterruptedException { @SuppressWarnings("unchecked") public T take() throws InterruptedException { - if(closed && queue.isEmpty()) + if (closed && queue.isEmpty()) return null; Object x = queue.take(); - if(x == POISON) + if (x == POISON) return null; return (T) x; @@ -62,31 +60,33 @@ public T take() throws InterruptedException { /** * Poll with max timeout. - * - * @return item, or null if: - timeout, or - queue has been closed and this consumer reached its poison pill + * @return item, or null if: + * - timeout, or + * - queue has been closed and this consumer reached its poison pill */ @SuppressWarnings("unchecked") public T poll(long timeout, TimeUnit unit) throws InterruptedException { - if(closed && queue.isEmpty()) + if (closed && queue.isEmpty()) return null; Object x = queue.poll(timeout, unit); - if(x == null) - return null; // timeout + if (x == null) + return null; // timeout - if(x == POISON) + if (x == POISON) return null; return (T) x; } /** - * Close queue for N consumers. Each consumer will receive exactly one poison pill and then should stop. + * Close queue for N consumers. + * Each consumer will receive exactly one poison pill and then should stop. */ public boolean close() throws InterruptedException { - synchronized(this) { - if(closed) - return false; // idempotent + synchronized (this) { + if (closed) + return false; // idempotent closed = true; } queue.put(POISON); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java index ccc73ff6160..ee02b28404c 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java @@ -78,7 +78,7 @@ public void readFully(byte[] b) throws IOException { } @Override - public void readFully(byte[] b, int off, int len) throws IOException { + public void readFully(byte [] b, int off, int len) throws IOException { if(len < 0) throw new IndexOutOfBoundsException(); @@ -127,12 +127,12 @@ public int readUnsignedByte() throws IOException { @Override public short readShort() throws IOException { if(_count - _pos >= 2) { - short ret = (short) baToShort(_buff, _pos); + short ret = (short)baToShort(_buff, _pos); _pos += 2; return ret; } readFully(_tmp, 0, 2); - return (short) baToShort(_tmp, 0); + return (short)baToShort(_tmp, 0); } @Override @@ -142,7 +142,7 @@ public int readUnsignedShort() throws IOException { @Override public char readChar() throws IOException { - return (char) readUnsignedShort(); + return (char)readUnsignedShort(); } @Override @@ -222,7 +222,7 @@ public long readDoubleArray(int len, double[] varr) throws IOException { @Override public long readSparseRows(int rlen, long nnz, SparseBlock rows) throws IOException { if(rows instanceof SparseBlockCSR) { - ((SparseBlockCSR) rows).initSparse(rlen, (int) nnz, this); + ((SparseBlockCSR)rows).initSparse(rlen, (int)nnz, this); return nnz; } @@ -256,7 +256,7 @@ private void refill() throws IOException { } private int getRefillLength() { - int pageOffset = (int) (_filePos & PAGE_MASK); + int pageOffset = (int)(_filePos & PAGE_MASK); if(pageOffset == 0) return _bufflen; return Math.min(_bufflen, PAGE_SIZE - pageOffset); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java index 22b7b23706c..9854a94586b 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java @@ -65,7 +65,7 @@ long getFlushedPosition() { public void write(int b) throws IOException { if(_count >= _bufflen) flushBuffer(); - _buff[_count++] = (byte) b; + _buff[_count++] = (byte)b; _position++; } @@ -109,7 +109,7 @@ public void close() throws IOException { public void writeBoolean(boolean v) throws IOException { if(_count >= _bufflen) flushBuffer(); - _buff[_count++] = (byte) (v ? 1 : 0); + _buff[_count++] = (byte)(v ? 1 : 0); _position++; } @@ -153,7 +153,7 @@ public void writeFloat(float v) throws IOException { public void writeByte(int v) throws IOException { if(_count + 1 > _bufflen) flushBuffer(); - _buff[_count++] = (byte) v; + _buff[_count++] = (byte)v; _position++; } @@ -194,18 +194,18 @@ public void writeUTF(String s) throws IOException { flushBuffer(); final char c = s.charAt(i); if(c >= 0x0001 && c <= 0x007F) { - _buff[_count++] = (byte) c; + _buff[_count++] = (byte)c; _position++; } else if(c >= 0x0800) { - _buff[_count++] = (byte) (0xE0 | ((c >> 12) & 0x0F)); - _buff[_count++] = (byte) (0x80 | ((c >> 6) & 0x3F)); - _buff[_count++] = (byte) (0x80 | (c & 0x3F)); + _buff[_count++] = (byte)(0xE0 | ((c >> 12) & 0x0F)); + _buff[_count++] = (byte)(0x80 | ((c >> 6) & 0x3F)); + _buff[_count++] = (byte)(0x80 | (c & 0x3F)); _position += 3; } else { - _buff[_count++] = (byte) (0xC0 | ((c >> 6) & 0x1F)); - _buff[_count++] = (byte) (0x80 | (c & 0x3F)); + _buff[_count++] = (byte)(0xC0 | ((c >> 6) & 0x1F)); + _buff[_count++] = (byte)(0x80 | (c & 0x3F)); _position += 2; } } @@ -213,7 +213,7 @@ else if(c >= 0x0800) { @Override public void writeDoubleArray(int len, double[] varr) throws IOException { - for(int i = 0; i < len;) { + for(int i = 0; i < len; ) { if(_count >= _bufflen) flushBuffer(); int lblen = Math.min(len - i, (_bufflen - _count) / 8); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java index 6c13a062561..ab28df0ef0f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java @@ -50,10 +50,10 @@ public interface OOCIOHandler { void registerSourceLocation(BlockKey key, SourceBlockDescriptor descriptor); /** - * Schedule an asynchronous read from an external source into the provided target stream. The returned future - * completes when either EOF is reached or the requested byte budget is exhausted. When the budget is reached and - * keepOpenOnLimit is true, the target stream is kept open and a continuation token is provided so the caller can - * resume. + * Schedule an asynchronous read from an external source into the provided target stream. + * The returned future completes when either EOF is reached or the requested byte budget + * is exhausted. When the budget is reached and keepOpenOnLimit is true, the target stream + * is kept open and a continuation token is provided so the caller can resume. */ CompletableFuture scheduleSourceRead(SourceReadRequest request); @@ -62,8 +62,7 @@ public interface OOCIOHandler { */ CompletableFuture continueSourceRead(SourceReadContinuation continuation, long maxBytesInFlight); - interface SourceReadContinuation { - } + interface SourceReadContinuation {} class SourceReadRequest { public final String path; @@ -76,8 +75,9 @@ class SourceReadRequest { public final boolean keepOpenOnLimit; public final OOCStream target; - public SourceReadRequest(String path, Types.FileFormat format, long rows, long cols, int blen, long estNnz, - long maxBytesInFlight, boolean keepOpenOnLimit, OOCStream target) { + public SourceReadRequest(String path, Types.FileFormat format, long rows, long cols, + int blen, long estNnz, long maxBytesInFlight, boolean keepOpenOnLimit, + OOCStream target) { this.path = path; this.format = format; this.rows = rows; @@ -113,8 +113,9 @@ class SourceBlockDescriptor { public final int recordLength; public final long serializedSize; - public SourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, - int recordLength, long serializedSize) { + public SourceBlockDescriptor(String path, Types.FileFormat format, + MatrixIndexes indexes, long offset, int recordLength, + long serializedSize) { this.path = path; this.format = format; this.indexes = indexes; @@ -128,8 +129,8 @@ class GroupSourceBlockDescriptor extends SourceBlockDescriptor { public final List blocks; public final int count; - public GroupSourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, - int recordLength, long serializedSize, List blocks) { + public GroupSourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, int recordLength, + long serializedSize, List blocks) { super(path, format, indexes, offset, recordLength, serializedSize); this.blocks = blocks; this.count = blocks.size(); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java index 55ac038205f..e9487f7bd45 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java @@ -81,7 +81,7 @@ public class OOCMatrixIOHandler implements OOCIOHandler { private final AtomicLong _readSeq = new AtomicLong(0); // Spill related structures - private final ConcurrentHashMap _spillLocations = new ConcurrentHashMap<>(); + private final ConcurrentHashMap _spillLocations = new ConcurrentHashMap<>(); private final ConcurrentHashMap _partitions = new ConcurrentHashMap<>(); private final ConcurrentHashMap _sourceLocations = new ConcurrentHashMap<>(); private final AtomicInteger _partitionCounter = new AtomicInteger(0); @@ -97,21 +97,38 @@ public class OOCMatrixIOHandler implements OOCIOHandler { @SuppressWarnings("unchecked") public OOCMatrixIOHandler() { this._spillDir = LocalFileUtils.getUniqueWorkingDir("ooc_stream"); - _writeExec = new ThreadPoolExecutor(WRITER_SIZE, WRITER_SIZE, 0L, TimeUnit.MILLISECONDS, + _writeExec = new ThreadPoolExecutor( + WRITER_SIZE, + WRITER_SIZE, + 0L, + TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); - _readExec = new ThreadPoolExecutor(READER_SIZE, READER_SIZE, 0L, TimeUnit.MILLISECONDS, + _readExec = new ThreadPoolExecutor( + READER_SIZE, + READER_SIZE, + 0L, + TimeUnit.MILLISECONDS, new PriorityBlockingQueue<>()); - _srcReadExec = new ThreadPoolExecutor(READER_SIZE, READER_SIZE, 0L, TimeUnit.MILLISECONDS, + _srcReadExec = new ThreadPoolExecutor( + READER_SIZE, + READER_SIZE, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(100000)); + _deleteExec = new ThreadPoolExecutor( + 1, + 1, + 0L, + TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); - _deleteExec = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); _q = new CloseableQueue[WRITER_SIZE]; _wCtr = new AtomicLong(0); - _started = new AtomicBoolean(false); + _started = new AtomicBoolean(false); } private synchronized void start() { - if(_started.compareAndSet(false, true)) { - for(int i = 0; i < WRITER_SIZE; i++) { + if (_started.compareAndSet(false, true)) { + for (int i = 0; i < WRITER_SIZE; i++) { final int finalIdx = i; _q[i] = new CloseableQueue<>(); _writeExec.submit(() -> evictTask(_q[finalIdx])); @@ -122,7 +139,7 @@ private synchronized void start() { @Override public void shutdown() { boolean started = _started.get(); - if(started) { + if (started) { try { for(int i = 0; i < WRITER_SIZE; i++) { if(_q[i] != null) @@ -142,7 +159,7 @@ public void shutdown() { _deleteExec.shutdownNow(); _spillLocations.clear(); _partitions.clear(); - if(started) + if (started) LocalFileUtils.deleteFileIfExists(_spillDir); } @@ -152,7 +169,7 @@ public CompletableFuture scheduleEviction(BlockEntry block) { CompletableFuture future = new CompletableFuture<>(); try { long q = _wCtr.getAndAdd(block.getSize()) / OVERFLOW; - int i = (int) (q % WRITER_SIZE); + int i = (int)(q % WRITER_SIZE); if(!_q[i].enqueueIfOpen(new Tuple2<>(block, future))) future.completeExceptionally(new DMLRuntimeException("OOC writer queue is closed")); } @@ -172,8 +189,7 @@ public OOCFuture scheduleRead(final BlockEntry block) { ReadTask task = new ReadTask(block, future, _readSeq.getAndIncrement(), pinnedPartitionId); _pendingReads.put(block.getKey(), task); _readExec.execute(task); - } - catch(RejectedExecutionException e) { + } catch (RejectedExecutionException e) { unpinPartitionForRead(pinnedPartitionId); _pendingReads.remove(block.getKey()); future.completeExceptionally(e); @@ -183,12 +199,12 @@ public OOCFuture scheduleRead(final BlockEntry block) { @Override public void prioritizeRead(BlockKey key, double priority) { - if(priority == 0) + if (priority == 0) return; ReadTask task = _pendingReads.get(key); - if(task == null) + if (task == null) return; - if(_readExec.getQueue().remove(task)) { + if (_readExec.getQueue().remove(task)) { task.addPriority(priority); _readExec.getQueue().offer(task); } @@ -212,9 +228,8 @@ public CompletableFuture scheduleSourceRead(SourceReadRequest } @Override - public CompletableFuture continueSourceRead(SourceReadContinuation continuation, - long maxBytesInFlight) { - if(!(continuation instanceof SourceReadState state)) { + public CompletableFuture continueSourceRead(SourceReadContinuation continuation, long maxBytesInFlight) { + if (!(continuation instanceof SourceReadState state)) { CompletableFuture failed = new CompletableFuture<>(); failed.completeExceptionally(new DMLRuntimeException("Unsupported continuation type: " + continuation)); return failed; @@ -225,8 +240,8 @@ public CompletableFuture continueSourceRead(SourceReadContinua private CompletableFuture submitSourceRead(SourceReadRequest request, SourceReadState state, long maxBytesInFlight) { if(request.format != Types.FileFormat.BINARY) - return CompletableFuture - .failedFuture(new DMLRuntimeException("Unsupported format for source read: " + request.format)); + return CompletableFuture.failedFuture( + new DMLRuntimeException("Unsupported format for source read: " + request.format)); return readBinarySourceParallel(request, state, maxBytesInFlight); } @@ -320,18 +335,17 @@ private CompletableFuture readBinarySourceParallel(SourceReadR return result; } - private void completeResult(CompletableFuture future, AtomicLong bytesRead, - AtomicBoolean budgetHit, AtomicReference error, SourceReadRequest request, Path[] files, - AtomicLongArray filePositions, AtomicIntegerArray completed, - ConcurrentLinkedDeque descriptors) { + private void completeResult(CompletableFuture future, AtomicLong bytesRead, AtomicBoolean budgetHit, + AtomicReference error, SourceReadRequest request, Path[] files, AtomicLongArray filePositions, + AtomicIntegerArray completed, ConcurrentLinkedDeque descriptors) { Throwable err = error.get(); - if(err != null) { + if (err != null) { future.completeExceptionally(err instanceof Exception ? err : new Exception(err)); return; } try { - if(budgetHit.get()) { + if (budgetHit.get()) { if(!request.keepOpenOnLimit) { closeTarget(request.target, false); } @@ -351,29 +365,29 @@ private void completeResult(CompletableFuture future, AtomicLo private void readSequenceFile(JobConf job, Path path, SourceReadRequest request, int fileIdx, AtomicLongArray filePositions, AtomicIntegerArray completed, AtomicBoolean stop, AtomicBoolean budgetHit, - AtomicLong bytesRead, long byteLimit, Object budgetLock, - ConcurrentLinkedDeque descriptors) throws IOException { + AtomicLong bytesRead, long byteLimit, Object budgetLock, ConcurrentLinkedDeque descriptors) + throws IOException { MatrixIndexes key = new MatrixIndexes(); try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { long pos = filePositions.get(fileIdx); - if(pos > 0) + if (pos > 0) reader.seek(pos); long ioStart = DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; while(!stop.get()) { long recordStart = reader.getPosition(); MatrixBlock value = new MatrixBlock(); - if(!reader.next(key, value)) + if (!reader.next(key, value)) break; long recordEnd = reader.getPosition(); long blockSize = value.getExactSerializedSize(); boolean shouldBreak = false; synchronized(budgetLock) { - if(stop.get()) + if (stop.get()) shouldBreak = true; - else if(bytesRead.get() + blockSize > byteLimit) { + else if (bytesRead.get() + blockSize > byteLimit) { stop.set(true); budgetHit.set(true); shouldBreak = true; @@ -384,7 +398,7 @@ else if(bytesRead.get() + blockSize > byteLimit) { MatrixIndexes outIdx = new MatrixIndexes(key); IndexedMatrixValue imv = new IndexedMatrixValue(outIdx, value); SourceBlockDescriptor descriptor = new SourceBlockDescriptor(path.toString(), request.format, outIdx, - recordStart, (int) (recordEnd - recordStart), blockSize); + recordStart, (int)(recordEnd - recordStart), blockSize); if(request.target instanceof SourceOOCStream src) src.enqueue(imv, descriptor); @@ -393,23 +407,22 @@ else if(bytesRead.get() + blockSize > byteLimit) { descriptors.add(descriptor); filePositions.set(fileIdx, reader.getPosition()); - if(DMLScript.OOC_LOG_EVENTS) { + if (DMLScript.OOC_LOG_EVENTS) { long currTime = System.nanoTime(); OOCEventLog.onDiskReadEvent(_srcReadCallerId, ioStart, currTime, blockSize); ioStart = currTime; } - if(shouldBreak) + if (shouldBreak) break; // Note that we knowingly go over limit, which could result in READER_SIZE*8MB overshoot } - if(!stop.get()) + if (!stop.get()) completed.set(fileIdx, 1); } } - private void closeTarget(org.apache.sysds.runtime.instructions.ooc.OOCStream target, - boolean close) { + private void closeTarget(org.apache.sysds.runtime.instructions.ooc.OOCStream target, boolean close) { if(close) { try { target.closeInput(); @@ -424,10 +437,10 @@ private void loadFromDisk(BlockEntry block) { String key = block.getKey().toFileKey(); SourceBlockDescriptor src = _sourceLocations.get(block.getKey()); - if(src != null) { + if (src != null) { long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; loadFromSource(block, src); - if(DMLScript.OOC_STATISTICS) { + if (DMLScript.OOC_STATISTICS) { Statistics.incrementOOCLoadFromDisk(); Statistics.accumulateOOCLoadFromDiskTime(System.nanoTime() - ioStart); Statistics.accumulateOOCLoadFromDiskBytes(block.getSize()); @@ -438,36 +451,34 @@ private void loadFromDisk(BlockEntry block) { long ioDuration = 0; // 1. find the blocks address (spill location) SpillLocation sloc = _spillLocations.get(key); - if(sloc == null) + if (sloc == null) throw new DMLRuntimeException("Failed to load spill location for: " + key); PartitionFile partFile = _partitions.get(sloc.partitionId); - if(partFile == null) + if (partFile == null) throw new DMLRuntimeException("Failed to load partition for: " + sloc.partitionId); String filename = partFile.filePath; SpillableObject obj; - try(RandomAccessFile raf = new RandomAccessFile(filename, "r")) { + try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { raf.seek(sloc.offset); DataInput dis = new OOCBufferedDataInputStream(raf); long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; obj = SpillableObjectRegistry.read(dis); - if(DMLScript.OOC_STATISTICS) + if (DMLScript.OOC_STATISTICS) ioDuration = System.nanoTime() - ioStart; - } - catch(ClosedByInterruptException ignored) { + } catch (ClosedByInterruptException ignored) { return; - } - catch(IOException e) { + } catch (IOException e) { throw new RuntimeException(e); } block.setDataUnsafe(obj); - if(DMLScript.OOC_STATISTICS) { + if (DMLScript.OOC_STATISTICS) { Statistics.incrementOOCLoadFromDisk(); Statistics.accumulateOOCLoadFromDiskTime(ioDuration); Statistics.accumulateOOCLoadFromDiskBytes(block.getSize()); @@ -475,23 +486,22 @@ private void loadFromDisk(BlockEntry block) { } private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { - if(src.format != Types.FileFormat.BINARY) + if (src.format != Types.FileFormat.BINARY) throw new DMLRuntimeException("Unsupported format for source read: " + src.format); JobConf job = new JobConf(ConfigurationManager.getCachedJobConf()); Path path = new Path(src.path); - if(src instanceof OOCIOHandler.GroupSourceBlockDescriptor gsrc) { + if (src instanceof OOCIOHandler.GroupSourceBlockDescriptor gsrc) { List values = new ArrayList<>(gsrc.count); try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { reader.seek(gsrc.offset); - for(int i = 0; i < gsrc.blocks.size(); i++) { + for (int i = 0; i < gsrc.blocks.size(); i++) { SourceBlockDescriptor d = gsrc.blocks.get(i); MatrixIndexes ix = new MatrixIndexes(); MatrixBlock mb = new MatrixBlock(); - if(!reader.next(ix, mb)) - throw new DMLRuntimeException( - "Failed to read source block at offset " + d.offset + " in " + d.path); + if (!reader.next(ix, mb)) + throw new DMLRuntimeException("Failed to read source block at offset " + d.offset + " in " + d.path); values.add(new IndexedMatrixValue(ix, mb)); } } @@ -506,9 +516,8 @@ private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { reader.seek(src.offset); - if(!reader.next(ix, mb)) - throw new DMLRuntimeException( - "Failed to read source block at offset " + src.offset + " in " + src.path); + if (!reader.next(ix, mb)) + throw new DMLRuntimeException("Failed to read source block at offset " + src.offset + " in " + src.path); } catch(IOException e) { throw new DMLRuntimeException(e); @@ -521,7 +530,7 @@ private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { private void evictTask(CloseableQueue>> q) { long byteCtr = 0; - while(!q.isFinished()) { + while (!q.isFinished()) { // --- 1. WRITE PHASE --- int partitionId = _partitionCounter.getAndIncrement(); @@ -563,17 +572,17 @@ private void evictTask(CloseableQueue } byteCtr += wrote; - if(byteCtr >= MAX_PARTITION_SIZE) { + if (byteCtr >= MAX_PARTITION_SIZE) { closePartition = true; byteCtr = 0; break; } - if(DMLScript.OOC_LOG_EVENTS) + if (DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskWriteEvent(_evictCallerId, ioStart, System.nanoTime(), wrote); } - if(!closePartition && q.close()) { + if (!closePartition && q.close()) { while((tpl = q.take()) != null) { long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; BlockEntry entry = tpl._1(); @@ -586,7 +595,7 @@ private void evictTask(CloseableQueue Statistics.accumulateOOCEvictionWriteTime(System.nanoTime() - ioStart); } - if(DMLScript.OOC_LOG_EVENTS) + if (DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskWriteEvent(_evictCallerId, ioStart, System.nanoTime(), wrote); } } @@ -608,13 +617,13 @@ private void evictTask(CloseableQueue } private long writeOut(int partitionId, BlockEntry entry, CompletableFuture future, - OOCBufferedDataOutputStream dos, ConcurrentLinkedDeque>> flushQueue) - throws IOException { + OOCBufferedDataOutputStream dos, + ConcurrentLinkedDeque>> flushQueue) throws IOException { String key = entry.getKey().toFileKey(); boolean alreadySpilled = _spillLocations.containsKey(key); - if(!alreadySpilled) { + if (!alreadySpilled) { long offsetBefore = dos.getPosition(); if(future.isCancelled()) @@ -647,10 +656,9 @@ private long writeOut(int partitionId, BlockEntry entry, CompletableFuture return 0; } - private void flushQueue(long offset, - ConcurrentLinkedDeque>> flushQueue) { + private void flushQueue(long offset, ConcurrentLinkedDeque>> flushQueue) { Tuple3> tmp; - while((tmp = flushQueue.peek()) != null && tmp._2() <= offset) { + while ((tmp = flushQueue.peek()) != null && tmp._2() <= offset) { flushQueue.poll(); tmp._3().complete(null); } @@ -769,14 +777,12 @@ public void run() { try { long ioStart = DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; loadFromDisk(_block); - if(DMLScript.OOC_LOG_EVENTS) + if (DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskReadEvent(_readCallerId, ioStart, System.nanoTime(), _block.getSize()); _future.complete(_block); - } - catch(Throwable e) { + } catch (Throwable e) { _future.completeExceptionally(e); - } - finally { + } finally { unpinPartitionForRead(_pinnedPartitionId); } } @@ -784,12 +790,15 @@ public void run() { @Override public int compareTo(ReadTask other) { int byPriority = Double.compare(other._priority, _priority); - if(byPriority != 0) + if (byPriority != 0) return byPriority; return Long.compare(_sequence, other._sequence); } } + + + private static class SpillLocation { // structure of spillLocation: file, offset final int partitionId; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java index 3458838c071..a93b1d18f2a 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java @@ -24,10 +24,8 @@ import java.io.IOException; public interface SpillableObject { - boolean tryWrite(DataOutput out) throws IOException; - - void read(DataInput in) throws IOException; - + boolean tryWrite(DataOutput out) throws IOException; + void read(DataInput in) throws IOException; long size(); default void discard() { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java index 0d318d83d94..d8ff79dd797 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java @@ -34,16 +34,14 @@ public interface OOCCacheScheduler { /** * Requests a single block from the cache. - * * @param key the requested key associated to the block * @return the available BlockEntry */ CompletableFuture request(BlockKey key); /** - * Tries to request a single block from the cache. Immediately returns the entry if present, otherwise null without - * scheduling reads. - * + * Tries to request a single block from the cache. + * Immediately returns the entry if present, otherwise null without scheduling reads. * @param key the requested key associated to the block * @return the available BlockEntry or null */ @@ -54,16 +52,14 @@ default BlockEntry tryRequest(BlockKey key) { /** * Requests a list of blocks from the cache that must be available at the same time. - * * @param keys the requested keys associated to the block * @return the list of available BlockEntries */ CompletableFuture> request(List keys); /** - * Tries to request a list of blocks from the cache that must be available at the same time. Immediately returns the - * list of entries if present, otherwise null without scheduling reads. - * + * Tries to request a list of blocks from the cache that must be available at the same time. + * Immediately returns the list of entries if present, otherwise null without scheduling reads. * @param keys the requested keys associated to the block * @return the list of available BlockEntries */ @@ -80,25 +76,24 @@ default BlockEntry tryRequest(BlockKey key) { List tryRequestAnyOf(List keys, int n, List selectionOut); /** - * Adds the given priority to any pending request accessing the key. Multi-requests are prioritized partially. + * Adds the given priority to any pending request accessing the key. + * Multi-requests are prioritized partially. */ void prioritize(BlockKey key, double priority); /** - * Places a new block in the cache. Note that objects are immutable and cannot be overwritten. The object data - * should now only be accessed via cache, as ownership has been transferred. - * - * @param key the associated key of the block + * Places a new block in the cache. Note that objects are immutable and cannot be overwritten. + * The object data should now only be accessed via cache, as ownership has been transferred. + * @param key the associated key of the block * @param data the block data * @param size the size of the data */ BlockKey put(BlockKey key, Object data, long size); /** - * Places a new block in the cache and returns a pinned handle. Note that objects are immutable and cannot be - * overwritten. - * - * @param key the associated key of the block + * Places a new block in the cache and returns a pinned handle. + * Note that objects are immutable and cannot be overwritten. + * @param key the associated key of the block * @param data the block data * @param size the size of the data */ @@ -106,11 +101,8 @@ default BlockEntry tryRequest(BlockKey key) { interface HandoverHandle { BlockKey getKey(); - boolean isCommitted(); - CompletableFuture getCompletionFuture(); - OOCStream.QueueCallback reclaim(); } @@ -139,30 +131,27 @@ BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.SourceBlockDescriptor descriptor); /** - * Notifies the cache that there is another reference to the same block key. This will prevent forget(key) from - * removing the block from cache. A block will only be forgotten after all referencing instances called forget(key). - * + * Notifies the cache that there is another reference to the same block key. + * This will prevent forget(key) from removing the block from cache. + * A block will only be forgotten after all referencing instances called forget(key). * @param key */ void addReference(BlockKey key); /** * Forgets a block from the cache. - * * @param key the associated key of the block */ void forget(BlockKey key); /** * Pins a BlockEntry in cache to prevent eviction. - * * @param entry the entry to be pinned */ void pin(BlockEntry entry); /** * Unpins a pinned block. - * * @param entry the entry to be unpinned */ void unpin(BlockEntry entry); @@ -203,7 +192,8 @@ BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, void updateLimits(long evictionLimit, long hardLimit); /** - * Creates a snapshot of the cache. Should only be used for debugging or diagnoses. + * Creates a snapshot of the cache. + * Should only be used for debugging or diagnoses. */ Collection snapshot(); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java index 9b0acc97b35..96de368ccc9 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java @@ -80,7 +80,7 @@ public class OOCLRUCacheScheduler implements OOCCacheScheduler { public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long hardLimit, long readBuffer) { this._ioHandler = ioHandler; this._cache = new LinkedHashMap<>(1024, 0.75f, true); - this._evictionCache = new HashMap<>(); + this._evictionCache = new HashMap<>(); this._deferredReadRequests = new DeferredReadQueue(); this._processingReadRequests = new ArrayDeque<>(); this._pendingHandovers = new ArrayDeque<>(); @@ -103,7 +103,7 @@ public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long har this._maintenanceNeedsIncr = new AtomicBoolean(false); this._callerId = DMLScript.OOC_LOG_EVENTS ? OOCEventLog.registerCaller("LRUCacheScheduler") : 0; - if(DMLScript.OOC_LOG_EVENTS) { + if (DMLScript.OOC_LOG_EVENTS) { OOCEventLog.putRunSetting("CacheEvictionLimit", _evictionLimit); OOCEventLog.putRunSetting("CacheHardLimit", _hardLimit); } @@ -111,7 +111,7 @@ public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long har @Override public CompletableFuture request(BlockKey key) { - if(!this._running) + if (!this._running) throw new IllegalStateException("Cache scheduler has been shut down."); Statistics.incrementOOCEvictionGet(); @@ -120,21 +120,21 @@ public CompletableFuture request(BlockKey key) { boolean couldPin = false; synchronized(this) { entry = _cache.get(key); - if(entry == null) + if (entry == null) entry = _evictionCache.get(key); - if(entry == null) + if (entry == null) throw new IllegalArgumentException("Could not find requested block with key " + key); synchronized(entry) { - if(entry.getState().isAvailable()) { - if(pinEntryWithAccounting(entry) == 0) + if (entry.getState().isAvailable()) { + if (pinEntryWithAccounting(entry) == 0) throw new IllegalStateException(); couldPin = true; } } } - if(couldPin) { + if (couldPin) { // Then we could pin the required entry and can terminate return CompletableFuture.completedFuture(entry); } @@ -184,7 +184,7 @@ public CompletableFuture> request(List keys) { } public CompletableFuture> request(List keys, boolean onlyIfAvailable) { - if(!this._running) + if (!this._running) throw new IllegalStateException("Cache scheduler has been shut down."); Statistics.incrementOOCEvictionGet(keys.size()); @@ -193,11 +193,11 @@ public CompletableFuture> request(List keys, boolean boolean allAvailable = true; synchronized(this) { - for(BlockKey key : keys) { + for (BlockKey key : keys) { BlockEntry entry = _cache.get(key); - if(entry == null) + if (entry == null) entry = _evictionCache.get(key); - if(entry == null) + if (entry == null) throw new IllegalArgumentException("Could not find requested block with key " + key); synchronized(entry) { @@ -217,7 +217,7 @@ public CompletableFuture> request(List keys, boolean } } - if(allAvailable) { + if (allAvailable) { // Then we could pin all entries return CompletableFuture.completedFuture(entries); } @@ -226,12 +226,12 @@ public CompletableFuture> request(List keys, boolean return null; // Schedule deferred read otherwise - final CompletableFuture> future = new CompletableFuture<>(); + final CompletableFuture> future = new CompletableFuture<>(); DeferredReadRequest request = new DeferredReadRequest(future, entries); - for(int i = 0; i < entries.size(); i++) { + for (int i = 0; i < entries.size(); i++) { BlockEntry entry = entries.get(i); synchronized(entry) { - if(entry.getState().isAvailable()) { + if (entry.getState().isAvailable()) { entry.addRetainHint(); request.markRetainHinted(i); } @@ -243,9 +243,9 @@ public CompletableFuture> request(List keys, boolean @Override public void prioritize(BlockKey key, double priority) { - if(!this._running) + if (!this._running) return; - if(priority == 0) + if (priority == 0) return; synchronized(this) { @@ -267,18 +267,18 @@ private void scheduleDeferredRead(DeferredReadRequest deferredReadRequest) { if(entry.getState().isAvailable()) readyCount++; BlockReadState state = _blockReads.get(entry.getKey()); - if(state != null) + if (state != null) score += state.priority; } - if(!deferredReadRequest.getEntries().isEmpty()) + if (!deferredReadRequest.getEntries().isEmpty()) score /= deferredReadRequest.getEntries().size(); - if(!deferredReadRequest.getEntries().isEmpty()) + if (!deferredReadRequest.getEntries().isEmpty()) score += ((double) readyCount) / deferredReadRequest.getEntries().size(); deferredReadRequest.setPriorityScore(score); _deferredReadRequests.add(deferredReadRequest); _deferredReadCountHint = _deferredReadRequests.size(); } - onCacheSizeChanged(true); // Apply pressure from deferred read demand. + onCacheSizeChanged(true); // Apply pressure from deferred read demand. onCacheSizeChanged(false); // Attempt to schedule deferred reads. } @@ -317,8 +317,7 @@ public void putSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.S } @Override - public BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, - OOCIOHandler.SourceBlockDescriptor descriptor) { + public BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.SourceBlockDescriptor descriptor) { return put(key, data, size, true, descriptor); } @@ -334,24 +333,23 @@ public void addReference(BlockKey key) { } } - private BlockEntry put(BlockKey key, Object data, long size, boolean pin, - OOCIOHandler.SourceBlockDescriptor descriptor) { - if(!this._running) + private BlockEntry put(BlockKey key, Object data, long size, boolean pin, OOCIOHandler.SourceBlockDescriptor descriptor) { + if (!this._running) throw new IllegalStateException(); - if(data == null) + if (data == null) throw new IllegalArgumentException(); - if(descriptor != null) + if (descriptor != null) _ioHandler.registerSourceLocation(key, descriptor); Statistics.incrementOOCEvictionPut(); BlockEntry entry = new BlockEntry(key, size, data); - if(descriptor != null) + if (descriptor != null) entry.setState(BlockState.WARM); - if(pin) + if (pin) entry.pin(); synchronized(this) { BlockEntry avail = _cache.putIfAbsent(key, entry); - if(avail != null || _evictionCache.containsKey(key)) + if (avail != null || _evictionCache.containsKey(key)) throw new IllegalStateException("Cannot overwrite existing entries: " + key); _cacheSize += size; if(pin) { @@ -366,7 +364,7 @@ private BlockEntry put(BlockKey key, Object data, long size, boolean pin, @Override public void forget(BlockKey key) { - if(!this._running) + if (!this._running) return; final MutableObject mEntry = new MutableObject<>(); BlockEntry entry; @@ -383,7 +381,7 @@ public void forget(BlockKey key) { return e; }); - if(mEntry.getValue() == null) { + if (mEntry.getValue() == null) { _evictionCache.compute(key, (k, e) -> { if(e == null) return null; @@ -397,10 +395,10 @@ public void forget(BlockKey key) { entry = mEntry.getValue(); - if(entry != null) { + if (entry != null) { synchronized(entry) { - shouldScheduleDeletion = entry.getState().isBackedByDisk() || - entry.getState() == BlockState.EVICTING; + shouldScheduleDeletion = entry.getState().isBackedByDisk() + || entry.getState() == BlockState.EVICTING; cacheSizeDelta = transitionMemState(entry, BlockState.REMOVED); if(entry.isPinned() && entry.getDataUnsafe() != null) _pinnedBytes -= entry.getSize(); @@ -410,9 +408,9 @@ public void forget(BlockKey key) { } } } - if(cacheSizeDelta != 0) + if (cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); - if(shouldScheduleDeletion) + if (shouldScheduleDeletion) _ioHandler.scheduleDeletion(entry); } @@ -426,11 +424,11 @@ public void pin(BlockEntry entry) { synchronized(this) { synchronized(entry) { int pinCount = pinEntryWithAccounting(entry); - if(pinCount == 0) + if (pinCount == 0) throw new IllegalStateException("Could not pin the requested entry: " + entry.getKey()); } // Access element in cache for Lru - // _cache.get(entry.getKey()); + //_cache.get(entry.getKey()); } } @@ -444,26 +442,26 @@ public void unpin(BlockEntry entry) { synchronized(entry) { if(!unpinEntryWithAccounting(entry)) return; - if(_cacheSize <= _evictionLimit) + if (_cacheSize <= _evictionLimit) return; // Nothing to do - if(entry.isPinned()) + if (entry.isPinned()) return; // Pin state changed so we cannot evict - if(entry.getState().isAvailable() && entry.getState().isBackedByDisk()) { - if(entry.getRetainHintCount() > 0) { + if (entry.getState().isAvailable() && entry.getState().isBackedByDisk()) { + if (entry.getRetainHintCount() > 0) { shouldCheckEviction = true; } else { - cacheSizeDelta = transitionMemState(entry, BlockState.COLD); + cacheSizeDelta = transitionMemState(entry, BlockState.COLD); long cleared = entry.clear(); - if(cleared != entry.getSize()) + if (cleared != entry.getSize()) throw new IllegalStateException(); _cache.remove(entry.getKey()); _evictionCache.put(entry.getKey(), entry); } } - else if(entry.getState() == BlockState.HOT) { - if(entry.getRetainHintCount() > 0) { + else if (entry.getState() == BlockState.HOT) { + if (entry.getRetainHintCount() > 0) { shouldCheckEviction = true; } else { @@ -472,9 +470,9 @@ else if(entry.getState() == BlockState.HOT) { } } } - if(cacheSizeDelta != 0) + if (cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); - else if(shouldCheckEviction) + else if (shouldCheckEviction) onCacheSizeChanged(true); } @@ -510,11 +508,10 @@ public synchronized void shutdown() { System.out.println("[WARN] Cache still holds " + _cache.size() + " / " + _evictionCache.size() + " blocks"); Set cachedStreams = _cache.keySet().stream().map(BlockKey::getStreamId).collect(Collectors.toSet()); - Set evictedStreams = _evictionCache.keySet().stream().map(BlockKey::getStreamId) - .collect(Collectors.toSet()); + Set evictedStreams = _evictionCache.keySet().stream().map(BlockKey::getStreamId).collect(Collectors.toSet()); cachedStreams.addAll(evictedStreams); - System.out.println("[WARN] Affected stream IDs: " + cachedStreams + ", Pinned: " - + _cache.values().stream().mapToInt(e -> e.isPinned() ? 1 : 0).sum()); + System.out.println("[WARN] Affected stream IDs: " + cachedStreams + ", Pinned: " + _cache.values().stream().mapToInt( + e -> e.isPinned() ? 1 : 0).sum()); } _cache.clear(); _evictionCache.clear(); @@ -578,8 +575,7 @@ private void runMaintenanceLoop() { do { _maintenanceRequested.set(false); onCacheSizeChangedInternal(_maintenanceNeedsIncr.getAndSet(false)); - } - while(_maintenanceRequested.get()); + } while(_maintenanceRequested.get()); } finally { _maintenanceRunning.set(false); @@ -595,8 +591,7 @@ private void onCacheSizeChangedInternal(boolean incr) { if(incr) onCacheSizeIncremented(); else - while(onCacheSizeDecremented()) { - } + while(onCacheSizeDecremented()) {} while(processPendingHandovers()) { onCacheSizeIncremented(); } @@ -606,23 +601,18 @@ private void onCacheSizeChangedInternal(boolean incr) { } private synchronized void sanityCheck() { - if(_cacheSize > _hardLimit * 1.1) { - if(!_warnThrottling) { + if (_cacheSize > _hardLimit * 1.1) { + if (!_warnThrottling) { _warnThrottling = true; - System.out.println( - "[WARN] Cache hard limit exceeded by over 10%: " + String.format("%.2f", _cacheSize / 1000000.0) - + "MB (-" + String.format("%.2f", _bytesUpForEviction / 1000000.0) + "MB) > " - + String.format("%.2f", _hardLimit / 1000000.0) + "MB"); + System.out.println("[WARN] Cache hard limit exceeded by over 10%: " + String.format("%.2f", _cacheSize/1000000.0) + "MB (-" + String.format("%.2f", _bytesUpForEviction/1000000.0) + "MB) > " + String.format("%.2f", _hardLimit/1000000.0) + "MB"); } } - else if(_warnThrottling && _cacheSize < _hardLimit) { + else if (_warnThrottling && _cacheSize < _hardLimit) { _warnThrottling = false; - System.out.println("[INFO] Cache within limit: " + String.format("%.2f", _cacheSize / 1000000.0) + "MB (-" - + String.format("%.2f", _bytesUpForEviction / 1000000.0) + "MB) <= " - + String.format("%.2f", _hardLimit / 1000000.0) + "MB"); + System.out.println("[INFO] Cache within limit: " + String.format("%.2f", _cacheSize/1000000.0) + "MB (-" + String.format("%.2f", _bytesUpForEviction/1000000.0) + "MB) <= " + String.format("%.2f", _hardLimit/1000000.0) + "MB"); } - if(!SANITY_CHECKS) + if (!SANITY_CHECKS) return; int pinned = 0; @@ -635,16 +625,16 @@ else if(_warnThrottling && _cacheSize < _hardLimit) { long actualPinnedEvictingBytes = 0; long actualWarmPinnedBytes = 0; long actualReadingReservedBytes = 0; - for(BlockEntry entry : _cache.values()) { - if(entry.isPinned()) { + for (BlockEntry entry : _cache.values()) { + if (entry.isPinned()) { pinned++; actualPinnedBytes += entry.getSize(); if(entry.getState() == BlockState.WARM) actualWarmPinnedBytes += entry.getSize(); } - if(entry.getState().isBackedByDisk()) + if (entry.getState().isBackedByDisk()) backedByDisk++; - if(entry.getState() == BlockState.EVICTING) { + if (entry.getState() == BlockState.EVICTING) { evicting++; upForEviction += entry.getSize(); if(entry.isPinned()) @@ -652,44 +642,43 @@ else if(_warnThrottling && _cacheSize < _hardLimit) { } if(entry.getState() == BlockState.READING) actualReadingReservedBytes += entry.getSize(); - if(!entry.getState().isAvailable()) + if (!entry.getState().isAvailable()) throw new IllegalStateException(); total++; actualCacheSize += entry.getSize(); } - for(BlockEntry entry : _evictionCache.values()) { - if(entry.getState().isAvailable()) + for (BlockEntry entry : _evictionCache.values()) { + if (entry.getState().isAvailable()) throw new IllegalStateException("Invalid eviction state: " + entry.getState()); - if(entry.getState() == BlockState.EVICTING && entry.isPinned()) + if (entry.getState() == BlockState.EVICTING && entry.isPinned()) actualPinnedEvictingBytes += entry.getSize(); - if(entry.getState() == BlockState.READING) + if (entry.getState() == BlockState.READING) actualCacheSize += entry.getSize(); - if(entry.getState() == BlockState.READING) + if (entry.getState() == BlockState.READING) actualReadingReservedBytes += entry.getSize(); - if(entry.isPinned()) { + if (entry.isPinned()) { actualPinnedBytes += entry.getSize(); if(entry.getState() == BlockState.WARM) actualWarmPinnedBytes += entry.getSize(); } } - if(actualCacheSize != _cacheSize) + if (actualCacheSize != _cacheSize) throw new IllegalStateException(actualCacheSize + " != " + _cacheSize); - if(upForEviction != _bytesUpForEviction) + if (upForEviction != _bytesUpForEviction) throw new IllegalStateException(upForEviction + " != " + _bytesUpForEviction); - if(actualPinnedBytes != _pinnedBytes) + if (actualPinnedBytes != _pinnedBytes) throw new IllegalStateException(actualPinnedBytes + " != " + _pinnedBytes); - if(actualPinnedEvictingBytes != _pinnedEvictingBytes) + if (actualPinnedEvictingBytes != _pinnedEvictingBytes) throw new IllegalStateException(actualPinnedEvictingBytes + " != " + _pinnedEvictingBytes); - if(_pinnedEvictingBytes > _bytesUpForEviction) + if (_pinnedEvictingBytes > _bytesUpForEviction) throw new IllegalStateException(_pinnedEvictingBytes + " > " + _bytesUpForEviction); if(actualWarmPinnedBytes != _warmPinnedBytes) throw new IllegalStateException(actualWarmPinnedBytes + " != " + _warmPinnedBytes); - if(actualReadingReservedBytes != _readingReservedBytes) + if (actualReadingReservedBytes != _readingReservedBytes) throw new IllegalStateException(actualReadingReservedBytes + " != " + _readingReservedBytes); System.out.println("=========="); - System.out.println("Limit: " + _evictionLimit / 1000 + "KB"); - System.out.println("Memory: (" + _cacheSize / 1000 + "KB - " + _bytesUpForEviction / 1000 + "KB) / " - + _hardLimit / 1000 + "KB"); + System.out.println("Limit: " + _evictionLimit/1000 + "KB"); + System.out.println("Memory: (" + _cacheSize/1000 + "KB - " + _bytesUpForEviction/1000 + "KB) / " + _hardLimit/1000 + "KB"); System.out.println("Pinned: " + pinned + " / " + total); System.out.println("Disk backed: " + backedByDisk + " / " + total); System.out.println("Evicting: " + evicting + " / " + total); @@ -706,11 +695,10 @@ private void onCacheSizeIncremented() { if(pressure <= _evictionLimit) return; // Nothing to do - long overshoot = Math.max((long) (0.1 * _evictionLimit), 10000000); + long overshoot = Math.max((long)(0.1 * _evictionLimit), 10000000); long lowLimit = _evictionLimit - _readBuffer - overshoot; - // System.out.println("[CACHE] Claiming " + (pressure + overshoot - _evictionLimit)/1000 + "kB (last claim - // was " + (System.currentTimeMillis() - _lastEvictRun) + "ms ago)"); + //System.out.println("[CACHE] Claiming " + (pressure + overshoot - _evictionLimit)/1000 + "kB (last claim was " + (System.currentTimeMillis() - _lastEvictRun) + "ms ago)"); // Scan for values that can be evicted Collection entries = _cache.values(); @@ -725,8 +713,8 @@ private void onCacheSizeIncremented() { break; synchronized(entry) { - // if(entry.isPinned()) - // continue; + //if(entry.isPinned()) + // continue; if(!allowRetainHint && entry.getRetainHintCount() > 0) continue; if(entry.getState() == BlockState.COLD || entry.getState() == BlockState.EVICTING) @@ -760,12 +748,12 @@ private void onCacheSizeIncremented() { _lastEvictRun = System.currentTimeMillis(); } - for(BlockEntry entry : upForEvictionNeedsWrite) + for (BlockEntry entry : upForEvictionNeedsWrite) evict(entry, true); - for(BlockEntry entry : upForEvictionNoWrite) + for (BlockEntry entry : upForEvictionNoWrite) evict(entry, false); - if(cacheSizeDelta != 0) + if (cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); } @@ -825,7 +813,7 @@ private boolean onCacheSizeDecremented() { throw new IllegalStateException(); req.setPinned(idx); } - else if(entry.getState() == BlockState.READING) { + else if (entry.getState() == BlockState.READING) { req.schedule(idx); registerWaiter(entry.getKey(), req, idx); reading = true; @@ -848,7 +836,7 @@ else if(entry.getState() == BlockState.READING) { if(allReserved) { _deferredReadRequests.poll(); _deferredReadCountHint = _deferredReadRequests.size(); - if(!toRead.isEmpty()) + if (!toRead.isEmpty()) _processingReadRequests.add(req); } @@ -955,17 +943,17 @@ private void onEvicted(final BlockEntry entry) { if(tmp != null && tmp != entry) throw new IllegalStateException(); tmp = _evictionCache.put(entry.getKey(), entry); - if(tmp != null) + if (tmp != null) throw new IllegalStateException(); sanityCheck(); } - if(cacheSizeDelta != 0) + if (cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); } private void clearRetainHints(DeferredReadRequest request) { - for(int i = 0; i < request.getEntries().size(); i++) { - if(!request.isRetainHinted(i)) + for (int i = 0; i < request.getEntries().size(); i++) { + if (!request.isRetainHinted(i)) continue; BlockEntry entry = request.getEntries().get(i); synchronized(entry) { @@ -975,12 +963,12 @@ private void clearRetainHints(DeferredReadRequest request) { } /** - * Cleanly transitions state of a BlockEntry and handles accounting. Requires both the scheduler object and the - * entry to be locked: + * Cleanly transitions state of a BlockEntry and handles accounting. + * Requires both the scheduler object and the entry to be locked: */ private long transitionMemState(BlockEntry entry, BlockState newState) { BlockState oldState = entry.getState(); - if(oldState == newState) + if (oldState == newState) return 0; long sz = entry.getSize(); @@ -988,7 +976,7 @@ private long transitionMemState(BlockEntry entry, BlockState newState) { boolean pinned = entry.isPinned(); // Remove old contribution - switch(oldState) { + switch (oldState) { case REMOVED: throw new IllegalStateException(); case HOT: @@ -1012,7 +1000,7 @@ private long transitionMemState(BlockEntry entry, BlockState newState) { } // Add new contribution - switch(newState) { + switch (newState) { case REMOVED: case COLD: break; @@ -1068,7 +1056,6 @@ private int pinEntryWithAccounting(BlockEntry entry) { /** * Requires scheduler lock and entry lock. - * * @return true if this call transitioned pin count to zero. */ private boolean unpinEntryWithAccounting(BlockEntry entry) { diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java b/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java index f04534481c9..1f731fc3aa5 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java @@ -66,8 +66,8 @@ protected boolean isHashCol(int colID) { } /** - * Domain size of a dummycoded source column: the hash domain K from the meta cell for feature-hashed columns, - * otherwise the column's {@code numDistinct} (0 when unset). + * Domain size of a dummycoded source column: the hash domain K from the meta cell for + * feature-hashed columns, otherwise the column's {@code numDistinct} (0 when unset). * * @param meta transform meta frame * @param colID 1-based column id of the dummycoded source column @@ -144,13 +144,13 @@ public FrameBlock decode(final MatrixBlock in, final FrameBlock out, final int k try { final List> tasks = new ArrayList<>(); int blz = Math.max((in.getNumRows() + k) / k, 1000); - - for(int i = 0; i < in.getNumRows(); i += blz) { + + for(int i = 0; i < in.getNumRows(); i += blz){ final int start = i; final int end = Math.min(in.getNumRows(), i + blz); tasks.add(pool.submit(() -> decode(in, out, start, end))); } - + for(Future f : tasks) f.get(); return out; diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java index 01a502154cd..a286c03dce8 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java @@ -72,10 +72,10 @@ public void decode(MatrixBlock in, FrameBlock out, int rl, int ru) { final double val = in.get(i, _srcCols[j] - 1); if(!Double.isNaN(val)){ final int key = (int) Math.round(val); - if(key == 0) { + if(key == 0){ a.set(i, _binMins[j][key]); } - else { + else{ double bmin = _binMins[j][key - 1]; double bmax = _binMaxs[j][key - 1]; double oval = bmin + (bmax - bmin) / 2 // bin center diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java index 8aa0b60d990..ee1a33c49fd 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java @@ -34,7 +34,7 @@ /** * Simple atomic decoder for dummycoded columns. This decoder builds internally inverted column mappings from the given * frame meta data. - * + * */ public class DecoderDummycode extends Decoder { private static final long serialVersionUID = 4758831042891032129L; diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java index fc123657032..8f6c45d63e8 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java @@ -64,15 +64,15 @@ public static Decoder createDecoder(String spec, String[] colnames, ValueType[] try { //parse transform specification JSONObject jSpec = new JSONObject(spec); - - // create decoders 'bin', 'recode', 'hash', 'dummy', and 'pass-through' + + //create decoders 'bin', 'recode', 'hash', 'dummy', and 'pass-through' List binIDs = TfMetaUtils.parseBinningColIDs(jSpec, colnames, minCol, maxCol); - List rcIDs = Arrays.asList(ArrayUtils - .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.RECODE.toString(), minCol, maxCol))); - List hcIDs = Arrays.asList(ArrayUtils - .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.HASH.toString(), minCol, maxCol))); - List dcIDs = Arrays.asList(ArrayUtils - .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.DUMMYCODE.toString(), minCol, maxCol))); + List rcIDs = Arrays.asList(ArrayUtils.toObject( + TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.RECODE.toString(), minCol, maxCol))); + List hcIDs = Arrays.asList(ArrayUtils.toObject( + TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.HASH.toString(), minCol, maxCol))); + List dcIDs = Arrays.asList(ArrayUtils.toObject( + TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.DUMMYCODE.toString(), minCol, maxCol))); // only specially treat the columns with both recode and dictionary rcIDs = unionDistinct(rcIDs, dcIDs); // hashing is a lossy, one-way transform with no inverse recode map, so hash columns @@ -106,18 +106,29 @@ public static Decoder createDecoder(String spec, String[] colnames, ValueType[] // collect all the decoders in one list. List ldecoders = new ArrayList<>(); - - if(!binIDs.isEmpty()) { - ldecoders.add(new DecoderBin(schema, ArrayUtils.toPrimitive(binIDs.toArray(new Integer[0])), + + if( !binIDs.isEmpty() ) { + ldecoders.add(new DecoderBin(schema, + ArrayUtils.toPrimitive(binIDs.toArray(new Integer[0])), ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); } if( !dcIDs.isEmpty() ) { ldecoders.add(new DecoderDummycode(schema, ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); } - - // create composite decoder of all created decoders - // and initialize with given meta data (recode, dummy, bin) + if( !rcIDs.isEmpty() ) { + // recode on output (after dummycode rebuilds the categorical columns) when dummycoding is present + ldecoders.add(new DecoderRecode(schema, !dcIDs.isEmpty(), + ArrayUtils.toPrimitive(rcIDs.toArray(new Integer[0])))); + } + if( !ptIDs.isEmpty() ) { + ldecoders.add(new DecoderPassThrough(schema, + ArrayUtils.toPrimitive(ptIDs.toArray(new Integer[0])), + ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); + } + + //create composite decoder of all created decoders + //and initialize with given meta data (recode, dummy, bin) decoder = new DecoderComposite(schema, ldecoders); decoder.setColnames(colnames); decoder.initMetaData(meta); @@ -136,7 +147,7 @@ else if( decoder instanceof DecoderRecode ) return DecoderType.Recode.ordinal(); else if( decoder instanceof DecoderPassThrough ) return DecoderType.PassThrough.ordinal(); - else if(decoder instanceof DecoderBin) + else if( decoder instanceof DecoderBin ) return DecoderType.Bin.ordinal(); throw new DMLRuntimeException("Unsupported decoder type: " + decoder.getClass().getCanonicalName()); @@ -147,14 +158,10 @@ public static Decoder createInstance(int type) { // create instance switch(dtype) { - case Bin: - return new DecoderBin(); - case Dummycode: - return new DecoderDummycode(null, null); - case PassThrough: - return new DecoderPassThrough(null, null, null); - case Recode: - return new DecoderRecode(null, false, null); + case Bin: return new DecoderBin(); + case Dummycode: return new DecoderDummycode(null, null); + case PassThrough: return new DecoderPassThrough(null, null, null); + case Recode: return new DecoderRecode(null, false, null); default: throw new DMLRuntimeException("Unsupported Encoder Type used: " + dtype); } diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java index d8d624122a0..11dd2c7faa5 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java @@ -126,20 +126,20 @@ public void initMetaData(FrameBlock meta) { _rcMaps = new HashMap[_colList.length]; for( int j=0; j<_colList.length; j++ ) { HashMap map = new HashMap<>(); - for(int i = 0; i < meta.getNumRows(); i++) { - try { + for( int i=0; i= 0} both the minimum and - * maximum fraction digits are pinned to {@code decimal}, so values are printed with exactly that many decimals; - * otherwise the {@link DecimalFormat} defaults apply. - * + * Creates a non-grouping {@link DecimalFormat} for printing values. When {@code decimal >= 0} + * both the minimum and maximum fraction digits are pinned to {@code decimal}, so values are + * printed with exactly that many decimals; otherwise the {@link DecimalFormat} defaults apply. * @param decimal number of decimal places to print, -1 for default * @return a configured {@link DecimalFormat} */ private static DecimalFormat createDecimalFormat(int decimal) { DecimalFormat df = new DecimalFormat(); df.setGroupingUsed(false); - if(decimal >= 0) { + if (decimal >= 0) { df.setMinimumFractionDigits(decimal); df.setMaximumFractionDigits(decimal); } diff --git a/src/main/java/org/apache/sysds/utils/DoubleParser.java b/src/main/java/org/apache/sysds/utils/DoubleParser.java index 2252b7270c8..c0122f8061f 100644 --- a/src/main/java/org/apache/sysds/utils/DoubleParser.java +++ b/src/main/java/org/apache/sysds/utils/DoubleParser.java @@ -200,7 +200,7 @@ public static double parseFloatingPointLiteral(String str, int offset, int endIn // : is the first character after numbers. // 0 is the first number. // we use the last position, since this is not allowed to be other values than a number. - if(str.charAt(endIndex - 1) > '9' || str.charAt(endIndex - 1) < '0') + if(str.charAt(endIndex - 1) > '9' || str.charAt(endIndex - 1) < '0') return Double.parseDouble(str); final double val = parseDecFloatLiteral(str, index, offset, endIndex); diff --git a/src/main/java/org/apache/sysds/utils/SettingsChecker.java b/src/main/java/org/apache/sysds/utils/SettingsChecker.java index 10d1a42b246..c5f14a7043b 100644 --- a/src/main/java/org/apache/sysds/utils/SettingsChecker.java +++ b/src/main/java/org/apache/sysds/utils/SettingsChecker.java @@ -106,24 +106,25 @@ private static long maxMemMachineOSX() { } private static long maxMemMachineWin() { - // try modern powershell, otherwise wmic, log errors as warning but avoid crashes + //try modern powershell, otherwise wmic, log errors as warning but avoid crashes long tmp = maxMemMachineWin(true); - if(tmp < 0) + if( tmp < 0 ) tmp = maxMemMachineWin(false); return tmp; } - + private static long maxMemMachineWin(boolean modern) { int startIx = modern ? 3 : 1; - String command = modern ? "powershell Get-CimInstance -ClassName Win32_PhysicalMemory | Select-Object Capacity" : "wmic memorychip get capacity"; // in - // bytes + String command = modern ? + "powershell Get-CimInstance -ClassName Win32_PhysicalMemory | Select-Object Capacity" : + "wmic memorychip get capacity"; //in bytes try { Runtime rt = Runtime.getRuntime(); Process pr = rt.exec(command); String[] memStr = new String(pr.getInputStream().readAllBytes(), StandardCharsets.UTF_8).split("\n"); //skip header, and aggregate DIMM capacities long capacity = 0; - for(int i = startIx; i < memStr.length; i++) { + for( int i=startIx; i 0 ) capacity += Long.parseLong(tmp); diff --git a/src/test/java/org/apache/sysds/performance/Main.java b/src/test/java/org/apache/sysds/performance/Main.java index a941e90f724..0622e789baa 100644 --- a/src/test/java/org/apache/sysds/performance/Main.java +++ b/src/test/java/org/apache/sysds/performance/Main.java @@ -243,9 +243,10 @@ private static void run17(String[] args) throws Exception { } /** - * Repeatedly read the same on-disk Delta frame table (written once as setup). Args: - * {@code 18 [mode] [targetFileSizeMB]} where mode is one of serial|parallel|both (default parallel) - * and an omitted target file size uses the adaptive default sizing. + * Repeatedly read the same on-disk Delta frame table (written once as setup). + * Args: {@code 18 [mode] [targetFileSizeMB]} + * where mode is one of serial|parallel|both (default parallel) and an omitted + * target file size uses the adaptive default sizing. */ private static void run18(String[] args) throws Exception { int rows = Integer.parseInt(args[1]); diff --git a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java index 4aac0685698..36ea11b3e2f 100644 --- a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java @@ -117,8 +117,8 @@ public abstract class AutomatedTestBase { public static final double GPU_TOLERANCE = 1e-9; /** - * Default deadline (ms) for federated worker/monitoring readiness waits and a few legacy {@code sleep()} calls. - * {@link FederatedWorkerUtils} enforces its own minimum floor. + * Default deadline (ms) for federated worker/monitoring readiness waits and a few legacy + * {@code sleep()} calls. {@link FederatedWorkerUtils} enforces its own minimum floor. */ public static final int FED_WORKER_WAIT = 3000; @@ -1761,9 +1761,9 @@ private static Process spawnLocalFedWorker(int port, String[] addArgs) { /** * Start a new JVM for a federated monitoring backend at the port. * - *

- * Returns once the backend's TCP port accepts connections (Netty's bind has completed), or throws a - * {@link RuntimeException} once the {@link FederatedWorkerUtils} readiness floor elapses. + *

Returns once the backend's TCP port accepts connections (Netty's bind has completed), or + * throws a {@link RuntimeException} once the {@link FederatedWorkerUtils} readiness floor + * elapses. * * @param port Port to use for the JVM * @param addArgs Extra CLI args to append, or null @@ -1776,10 +1776,10 @@ protected Process startLocalFedMonitoring(int port, String[] addArgs) { /** * Start a new JVM for a federated monitoring backend at the port. * - *

- * Returns once the backend's TCP port accepts connections, or throws a {@link RuntimeException} after - * {@code timeoutMs} elapses. The monitoring server opens the port after Netty's {@code bind().sync()} returns; a - * successful TCP connect therefore signals that the HTTP listener is ready to accept requests. + *

Returns once the backend's TCP port accepts connections, or throws a + * {@link RuntimeException} after {@code timeoutMs} elapses. The monitoring server opens the + * port after Netty's {@code bind().sync()} returns; a successful TCP connect therefore signals + * that the HTTP listener is ready to accept requests. * * @param port Port to use for the JVM * @param addArgs Extra CLI args to append, or null @@ -1798,9 +1798,8 @@ private static Process spawnLocalFedMonitoring(int port, String[] addArgs) { String separator = System.getProperty("file.separator"); String classpath = System.getProperty("java.class.path"); String path = System.getProperty("java.home") + separator + "bin" + separator + "java"; - String[] args = ArrayUtils.addAll( - new String[] {path, "-cp", classpath, DMLScript.class.getName(), "-fedMonitoring", Integer.toString(port)}, - addArgs); + String[] args = ArrayUtils.addAll(new String[] {path, "-cp", classpath, DMLScript.class.getName(), + "-fedMonitoring", Integer.toString(port)}, addArgs); try { return new ProcessBuilder(args).start(); } diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java index 0c7cd2046e5..f14a614b583 100644 --- a/src/test/java/org/apache/sysds/test/TestUtils.java +++ b/src/test/java/org/apache/sysds/test/TestUtils.java @@ -2941,9 +2941,10 @@ public static void writeTestScalar(String file, double value) { } } + /** * Write scalar to file - * + * * @param file File to write to * @param value Value to write */ @@ -3518,9 +3519,9 @@ public static void shutdownThread(Thread t) { // Bounded join: workers are daemon threads, so even if one ignores the interrupt // we must not block cleanup (and the JVM) indefinitely waiting for it. t.join(THREAD_SHUTDOWN_JOIN_MS); - if(t.isAlive()) - LOG.warn("Federated worker thread " + t.getName() + " did not stop within " - + THREAD_SHUTDOWN_JOIN_MS + "ms; leaving it as a daemon."); + if( t.isAlive() ) + LOG.warn("Federated worker thread " + t.getName() + + " did not stop within " + THREAD_SHUTDOWN_JOIN_MS + "ms; leaving it as a daemon."); } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java b/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java index d76c741d870..07ec9752928 100644 --- a/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java +++ b/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java @@ -67,10 +67,10 @@ public void setUp() { /** * Compile a DML script string into a runtime {@link Program} without executing it. * - * @param dmlScript the DML source - * @param args named command-line arguments ($name -> value), may be null - * @param mode the global execution mode (e.g. {@link ExecMode#HYBRID}) - * @param localMaxMem the local memory budget in bytes used for memory-based exec-type decisions + * @param dmlScript the DML source + * @param args named command-line arguments ($name -> value), may be null + * @param mode the global execution mode (e.g. {@link ExecMode#HYBRID}) + * @param localMaxMem the local memory budget in bytes used for memory-based exec-type decisions * @return the compiled runtime program */ protected Program compile(String dmlScript, Map args, ExecMode mode, long localMaxMem) { @@ -145,7 +145,8 @@ else if(pb instanceof FunctionProgramBlock) { /** All instructions whose opcode equals {@code opcode} (exact match). */ protected List getByOpcode(Program prog, String opcode) { - return getInstructions(prog).stream().filter(i -> opcode.equals(i.getOpcode())).collect(Collectors.toList()); + return getInstructions(prog).stream().filter(i -> opcode.equals(i.getOpcode())) + .collect(Collectors.toList()); } protected static boolean isSpark(Instruction inst) { @@ -168,13 +169,13 @@ protected void assertCP(Program prog, String opcode) { private void assertExecType(Program prog, String opcode, boolean expectSpark) { List matches = getByOpcode(prog, opcode); - Assert.assertFalse( - "Expected at least one '" + opcode + "' instruction but found none.\n" + Explain.explain(prog), - matches.isEmpty()); + Assert.assertFalse("Expected at least one '" + opcode + "' instruction but found none.\n" + + Explain.explain(prog), matches.isEmpty()); for(Instruction inst : matches) { boolean spark = isSpark(inst); - Assert.assertEquals("Instruction '" + opcode + "' expected exec type " + (expectSpark ? "SPARK" : "CP") - + " but was " + (spark ? "SPARK" : "CP") + ".\n" + Explain.explain(prog), expectSpark, spark); + Assert.assertEquals("Instruction '" + opcode + "' expected exec type " + + (expectSpark ? "SPARK" : "CP") + " but was " + (spark ? "SPARK" : "CP") + ".\n" + + Explain.explain(prog), expectSpark, spark); } } diff --git a/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java b/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java index 7b45120d8f1..0b3889db908 100644 --- a/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java +++ b/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java @@ -33,13 +33,14 @@ */ public class SparkTransitiveExecTypeCompileTest extends CompilerTestBase { - private static final String DML_HEADER = "X = rand(rows=20000000, cols=8, seed=1);\n" + // ~1.2GB -> rand and - // colSums run on Spark - "v = colSums(X);\n"; // 1x8 Spark-resident vector (opcode uack+) + private static final String DML_HEADER = + "X = rand(rows=20000000, cols=8, seed=1);\n" + // ~1.2GB -> rand and colSums run on Spark + "v = colSums(X);\n"; // 1x8 Spark-resident vector (opcode uack+) @Test public void singleConsumerUnaryPulledIntoSpark() { - String dml = DML_HEADER + "r = round(v);\n" + // sole consumer of the Spark-resident vector -> pulled into Spark + String dml = DML_HEADER + + "r = round(v);\n" + // sole consumer of the Spark-resident vector -> pulled into Spark "print(sum(r));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); @@ -49,58 +50,62 @@ public void singleConsumerUnaryPulledIntoSpark() { @Test public void multiConsumerUnaryStaysCP() { - String dml = DML_HEADER + "a = round(v);\n" + // v now has two consumers (round + abs) ... - "b = abs(v);\n" + "print(sum(a) + sum(b));\n"; + String dml = DML_HEADER + + "a = round(v);\n" + // v now has two consumers (round + abs) ... + "b = abs(v);\n" + + "print(sum(a) + sum(b));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uack+"); // input still has a Spark output ... - assertCP(prog, "round"); // ... but the multi-parent guard keeps both unaries in CP + assertCP(prog, "round"); // ... but the multi-parent guard keeps both unaries in CP assertCP(prog, "abs"); } // A tall, Spark-resident column vector that is still small enough (40 KB) to be CP by memory // estimate: rowSums over a very wide matrix runs on Spark, but its 1-column result fits in CP. - private static final String TALL_VECTOR_HEADER = "X = rand(rows=5000, cols=200000, seed=1);\n" + // ~8GB -> rand and - // rowSums run - // on Spark - "c = rowSums(X);\n"; // 5000x1 Spark-resident vector (opcode uark+) + private static final String TALL_VECTOR_HEADER = + "X = rand(rows=5000, cols=200000, seed=1);\n" + // ~8GB -> rand and rowSums run on Spark + "c = rowSums(X);\n"; // 5000x1 Spark-resident vector (opcode uark+) @Test public void cumulativeUnaryStaysCP() { - String dml = TALL_VECTOR_HEADER + "r = cumsum(c);\n" + // sole consumer of the Spark-resident vector, CP by - // estimate ... + String dml = TALL_VECTOR_HEADER + + "r = cumsum(c);\n" + // sole consumer of the Spark-resident vector, CP by estimate ... "print(as.scalar(r[2500,1]));\n"; // ... consume via indexing (avoids the sum(cumsum) rewrite) Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); - assertSpark(prog, "uark+"); // input genuinely has a Spark output + assertSpark(prog, "uark+"); // input genuinely has a Spark output assertCP(prog, "ucumk+"); // ... but cumulative ops are excluded from the transitive pull } @Test public void singleConsumerBinaryPulledIntoSpark() { - String dml = TALL_VECTOR_HEADER + "r = c + 2.0;\n" + // matrix-scalar on the Spark-resident vector, sole - // consumer -> pulled into Spark + String dml = TALL_VECTOR_HEADER + + "r = c + 2.0;\n" + // matrix-scalar on the Spark-resident vector, sole consumer -> pulled into Spark "print(as.scalar(r[2500,1]));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uark+"); // input genuinely has a Spark output (multi-block column vector) - assertSpark(prog, "+"); // matrix-scalar binary pulled into Spark (CP by estimate, single consumer) + assertSpark(prog, "+"); // matrix-scalar binary pulled into Spark (CP by estimate, single consumer) } @Test public void multiConsumerBinaryStaysCP() { - String dml = TALL_VECTOR_HEADER + "a = c + 2.0;\n" + // c now has two consumers (+ and *) ... - "b = c * 3.0;\n" + "print(as.scalar(a[2500,1]) + as.scalar(b[2500,1]));\n"; + String dml = TALL_VECTOR_HEADER + + "a = c + 2.0;\n" + // c now has two consumers (+ and *) ... + "b = c * 3.0;\n" + + "print(as.scalar(a[2500,1]) + as.scalar(b[2500,1]));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uark+"); // input still has a Spark output ... - assertCP(prog, "+"); // ... but the multi-parent guard keeps both binaries in CP + assertCP(prog, "+"); // ... but the multi-parent guard keeps both binaries in CP assertCP(prog, "*"); } @Test public void transitiveDisabledUnaryStaysCP() { - String dml = DML_HEADER + "r = round(v);\n" + // pullable unary, but flag is off + String dml = DML_HEADER + + "r = round(v);\n" + // pullable unary, but flag is off "print(sum(r));\n"; Program prog = compileWithTransitive(dml, false); @@ -110,7 +115,8 @@ public void transitiveDisabledUnaryStaysCP() { @Test public void transitiveDisabledBinaryStaysCP() { - String dml = TALL_VECTOR_HEADER + "r = c + 2.0;\n" + // pullable matrix-scalar, but flag is off + String dml = TALL_VECTOR_HEADER + + "r = c + 2.0;\n" + // pullable matrix-scalar, but flag is off "print(as.scalar(r[2500,1]));\n"; Program prog = compileWithTransitive(dml, false); diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java index 22f867819c3..083a29f965b 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java @@ -43,8 +43,7 @@ /** * Tests the {@code order} (sort) reorg operation on compressed matrices. A single column held in a single column group * is sorted ascending while staying compressed (via {@link org.apache.sysds.runtime.compress.lib.CLALibSort}); every - * other configuration falls back to a decompressed reorg. In all cases the result must match the uncompressed - * reference. + * other configuration falls back to a decompressed reorg. In all cases the result must match the uncompressed reference. */ public class CompressedSortTest { @@ -264,7 +263,8 @@ private void runCompressed(MatrixBlock mb, CompressionType ct) { assertEquals("Expected a single column group", 1, cmb.getColGroups().size()); MatrixBlock actual = cmb.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); - assertTrue("Expected the sorted result to stay compressed for " + ct, actual instanceof CompressedMatrixBlock); + assertTrue("Expected the sorted result to stay compressed for " + ct, + actual instanceof CompressedMatrixBlock); MatrixBlock expected = mb.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); TestUtils.compareMatrices(expected, CompressedMatrixBlock.getUncompressed(actual, "sort"), 0.0, "sort " + ct); diff --git a/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java index 49cbbdd248d..833128ad9f0 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java @@ -62,8 +62,8 @@ public static void setup() { } /** - * Build a compressed matrix backed by a single DDC column group spanning all {@code nCol} columns. This guarantees - * a single (non-uncompressed) column group, which is what triggers the mm-chain fast path for wide enough matrices. + * Build a compressed matrix backed by a single DDC column group spanning all {@code nCol} columns. This guarantees a + * single (non-uncompressed) column group, which is what triggers the mm-chain fast path for wide enough matrices. */ private static CompressedMatrixBlock singleDDC(int nRow, int nCol, int nVal, int seed) { Random r = new Random(seed); diff --git a/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java b/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java index c6afd5a3543..549010a78cb 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java @@ -43,8 +43,8 @@ /** * Drive the solve opcode through {@link BinaryMatrixMatrixCPInstruction} with compressed inputs to cover the - * commons-math matrix-matrix branch that decompresses compressed left/right operands before solving. The script-level - * solve tests only ever see uncompressed inputs, so this branch is otherwise unreached. + * commons-math matrix-matrix branch that decompresses compressed left/right operands before solving. The + * script-level solve tests only ever see uncompressed inputs, so this branch is otherwise unreached. */ public class CompressedBinaryMatrixMatrixSolveTest { @@ -72,8 +72,8 @@ public void solveCompressedLeftCompressedRight() { assertTrue("A must compress to exercise the compressed-left path", aC instanceof CompressedMatrixBlock); CompressedMatrixBlock bC = CompressedMatrixBlockFactory.createConstant(n, 2, 1.0); - MatrixBlock expected = LibCommonsMath.matrixMatrixOperations(CompressedMatrixBlock.getUncompressed(aC), - CompressedMatrixBlock.getUncompressed(bC), SOLVE); + MatrixBlock expected = LibCommonsMath.matrixMatrixOperations( + CompressedMatrixBlock.getUncompressed(aC), CompressedMatrixBlock.getUncompressed(bC), SOLVE); MatrixBlock actual = runSolve(aC, bC); TestUtils.compareMatricesBitAvgDistance(expected, actual, 0, 0, SOLVE); @@ -94,8 +94,8 @@ public void solveCompressedLeftDenseRight() { assertTrue("A must compress to exercise the compressed-left path", aC instanceof CompressedMatrixBlock); MatrixBlock b = TestUtils.round(TestUtils.generateTestMatrixBlock(n, 1, -5, 5, 1.0, 7)); - MatrixBlock expected = LibCommonsMath.matrixMatrixOperations(CompressedMatrixBlock.getUncompressed(aC), b, - SOLVE); + MatrixBlock expected = LibCommonsMath.matrixMatrixOperations( + CompressedMatrixBlock.getUncompressed(aC), b, SOLVE); MatrixBlock actual = runSolve(aC, b); TestUtils.compareMatricesBitAvgDistance(expected, actual, 0, 0, SOLVE); @@ -119,8 +119,7 @@ private static BinaryMatrixMatrixCPInstruction solveInstruction() { } private static MatrixObject matrixObject(String name, MatrixBlock mb) { - MatrixCharacteristics mc = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), 1000, - mb.getNonZeros()); + MatrixCharacteristics mc = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), 1000, mb.getNonZeros()); MatrixObject mo = new MatrixObject(ValueType.FP64, "/dev/null/" + name, new MetaDataFormat(mc, FileFormat.BINARY), mb); return mo; diff --git a/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java b/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java index 0d2f37e319c..8907c82f1d1 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java @@ -50,9 +50,7 @@ public class OffsetClassInitConcurrencyTest { private static final String[] INIT_TARGETS = {PKG + "AOffset", PKG + "OffsetEmpty", PKG + "OffsetChar", PKG + "OffsetByte", PKG + "OffsetSingle", PKG + "OffsetTwo"}; - /** - * Whether a class-init cycle deadlocks depends on thread timing, so repeat to make a regression reliable to catch. - */ + /** Whether a class-init cycle deadlocks depends on thread timing, so repeat to make a regression reliable to catch. */ private static final int ROUNDS = 20; /** A real init deadlock never resolves; a healthy round finishes in milliseconds, so this bound is generous. */ diff --git a/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java b/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java index f7dffa9021c..3493da7d2b1 100644 --- a/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java +++ b/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java @@ -35,10 +35,12 @@ public class SparkContextReferenceCountTest { /** - * Two DML executions sharing the JVM-wide singleton spark context (as happens with surefire parallel tests, - * threadCount>1). When the first execution finishes and calls close(), the shared context must stay alive - * because the second execution still has in-flight work. Before reference counting, close() stopped the context - * unconditionally, which cancelled the second execution's spark job and wedged it until the test watchdog. + * Two DML executions sharing the JVM-wide singleton spark context (as happens + * with surefire parallel tests, threadCount>1). When the first execution + * finishes and calls close(), the shared context must stay alive because the + * second execution still has in-flight work. Before reference counting, + * close() stopped the context unconditionally, which cancelled the second + * execution's spark job and wedged it until the test watchdog. */ @Test public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { @@ -62,13 +64,16 @@ public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { // context that B still uses SparkExecutionContext.exitSparkExecution(); ecA.close(); - assertFalse("shared context must stay alive while another execution is active", sc.sc().isStopped()); - assertEquals("B's job must still run on the live context", 10L, rdd.reduce(Integer::sum).longValue()); + assertFalse("shared context must stay alive while another execution is active", + sc.sc().isStopped()); + assertEquals("B's job must still run on the live context", + 10L, rdd.reduce(Integer::sum).longValue()); // B finishes last: releasing the final registration lets close() stop it SparkExecutionContext.exitSparkExecution(); ecB.close(); - assertTrue("shared context must be stopped once the last execution closes", sc.sc().isStopped()); + assertTrue("shared context must be stopped once the last execution closes", + sc.sc().isStopped()); } finally { // drain any remaining registrations and stop the context so a failed @@ -82,9 +87,10 @@ public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { } /** - * An unpaired close() (a caller that borrows the shared context but never registered via enterSparkExecution()) - * must not stop a context another execution still uses. This fails on the old unconditional-stop code, which tore - * the context down out from under the active execution. + * An unpaired close() (a caller that borrows the shared context but never + * registered via enterSparkExecution()) must not stop a context another + * execution still uses. This fails on the old unconditional-stop code, which + * tore the context down out from under the active execution. */ @Test public void unpairedCloseDoesNotStopAContextStillInUse() { @@ -100,12 +106,14 @@ public void unpairedCloseDoesNotStopAContextStillInUse() { // borrows the shared context): close() must not stop a context in use unregistered = ExecutionContextFactory.createSparkExecutionContext(); unregistered.close(); - assertFalse("unpaired close() must not stop a context still in use", sc.sc().isStopped()); + assertFalse("unpaired close() must not stop a context still in use", + sc.sc().isStopped()); // the registered execution finishing stops the context as the last user SparkExecutionContext.exitSparkExecution(); active.close(); - assertTrue("context must stop once the last registered execution closes", sc.sc().isStopped()); + assertTrue("context must stop once the last registered execution closes", + sc.sc().isStopped()); } finally { SparkExecutionContext.exitSparkExecution(); diff --git a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java index 459936fa24c..2c854b4a81b 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java @@ -78,18 +78,17 @@ public MatrixBlock getMatrixBlock(long id) { } /** - * Poll the federated worker until the matrix at {@code id} is observed as a {@link CompressedMatrixBlock}, or - * {@link #COMPRESS_TIMEOUT_MS} elapses. + * Poll the federated worker until the matrix at {@code id} is observed as a + * {@link CompressedMatrixBlock}, or {@link #COMPRESS_TIMEOUT_MS} elapses. * - *

- * Federated workers compress asynchronously after a PUT/READ_VAR (see - * {@code CompressedMatrixBlockFactory.compressAsync}), so a {@code getMatrixBlock} fired right after the operation - * can race against the in-flight compression and return the uncompressed block. Tests that need to observe the - * compressed form should poll instead of sleeping a fixed amount. + *

Federated workers compress asynchronously after a PUT/READ_VAR (see + * {@code CompressedMatrixBlockFactory.compressAsync}), so a {@code getMatrixBlock} fired right + * after the operation can race against the in-flight compression and return the uncompressed + * block. Tests that need to observe the compressed form should poll instead of sleeping a fixed + * amount. * - *

- * On timeout this returns the most recent (uncompressed) read so the caller can produce a meaningful assertion - * failure naming the variable. + *

On timeout this returns the most recent (uncompressed) read so the caller can produce a + * meaningful assertion failure naming the variable. * * @param id federated variable id * @return the matrix block, compressed if compression finished in time, otherwise the latest read diff --git a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java index d3e4485bdf4..2b5ff327ef3 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java @@ -68,11 +68,13 @@ public void verifySameOrAlsoCompressedAsLocalCompress() { // federated. Compression on the worker is async; poll only when we expect compression to // match the local result, otherwise a single read is enough. final long id = putMatrixBlock(mb); - final MatrixBlock mbr = (mbcLocal instanceof CompressedMatrixBlock) ? awaitCompressed(id) : getMatrixBlock(id); + final MatrixBlock mbr = (mbcLocal instanceof CompressedMatrixBlock) + ? awaitCompressed(id) + : getMatrixBlock(id); if(mbcLocal instanceof CompressedMatrixBlock && !(mbr instanceof CompressedMatrixBlock)) - fail("Invalid result, the federated site did not compress the matrix block within " + COMPRESS_TIMEOUT_MS - + "ms"); + fail("Invalid result, the federated site did not compress the matrix block within " + + COMPRESS_TIMEOUT_MS + "ms"); TestUtils.compareMatricesBitAvgDistance(mbcLocal, mbr, 0, 0, "Not equivalent matrix block returned from federated site"); diff --git a/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java b/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java index 0de3b6db640..60587bf51a2 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java @@ -42,7 +42,7 @@ public void test100x100() { @Test public void testDecimalClampsFractionDigits() { - FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); + FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); f.ensureAllocatedColumns(1); f.set(0, 0, 5.244058388023880); // decimal=2 must print exactly two fraction digits, not DecimalFormat's default max of 3 @@ -53,10 +53,10 @@ public void testDecimalClampsFractionDigits() { @Test public void testDecimalPadsAndRounds() { - FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); + FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); f.ensureAllocatedColumns(2); - f.set(0, 0, 22.0); // integer-valued: padded up to the requested digits - f.set(1, 0, 5.244058388023880); // rounded at the last requested digit + f.set(0, 0, 22.0); // integer-valued: padded up to the requested digits + f.set(1, 0, 5.244058388023880); // rounded at the last requested digit String out = DataConverter.toString(f, false, " ", "\n", 2, 1, 4); assertTrue("expected 22.0000 padded: " + out, out.contains("22.0000\n")); assertTrue("expected 5.2441 rounded: " + out, out.contains("5.2441\n")); @@ -64,10 +64,10 @@ public void testDecimalPadsAndRounds() { @Test public void testNegativeDecimalUsesDefaultFormatting() { - FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); + FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); f.ensureAllocatedColumns(2); - f.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained - f.set(1, 0, 5.244058388023880); // default cap of three fraction digits + f.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained + f.set(1, 0, 5.244058388023880); // default cap of three fraction digits // decimal < 0 leaves DecimalFormat unconstrained (no min/max fraction digits set) String out = DataConverter.toString(f, false, " ", "\n", 2, 1, -1); assertTrue("expected unpadded 22: " + out, out.contains("22\n")); diff --git a/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java b/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java index 7d67c04983b..43a53879f17 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java @@ -142,7 +142,8 @@ public void safeCastWarnsOnlyOnce() { // the fallback warning must be logged exactly once across both conversions final List log = LoggingUtils.reinsert(appender); long warnings = log.stream() - .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")).count(); + .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")) + .count(); assertEquals(1, warnings); } @@ -152,7 +153,8 @@ public void strictThrowsWhenWarnCastDisabled() { setWarnCast(false); FrameBlock fb = mixedFrame(); - Exception e = assertThrows(DMLRuntimeException.class, () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 1)); + Exception e = assertThrows(DMLRuntimeException.class, + () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 1)); assertTrue(e.getMessage().contains("Failed to convert FrameBlock to MatrixBlock")); } @@ -162,7 +164,8 @@ public void strictThrowsParallelWhenWarnCastDisabled() { setWarnCast(false); FrameBlock fb = mixedFrame(); - Exception e = assertThrows(DMLRuntimeException.class, () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 4)); + Exception e = assertThrows(DMLRuntimeException.class, + () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 4)); assertTrue(e.getMessage().contains("Failed to convert FrameBlock to MatrixBlock")); } @@ -182,7 +185,8 @@ public void warnCastValidFrameConvertsWithoutFallback() { final List log = LoggingUtils.reinsert(appender); long warnings = log.stream() - .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")).count(); + .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")) + .count(); assertEquals(0, warnings); } diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java index 982941bb190..ccba674707b 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java @@ -74,8 +74,8 @@ private void runDecode(String spec, int nCol, int nCat) { try { FrameBlock data = categoricalFrame(ROWS, nCol, nCat, 17); - MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), data.getNumColumns(), - null); + MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), + data.getNumColumns(), null); MatrixBlock encoded = encoder.encode(data, 1); Decoder decoder = buildDecoder(data, spec, encoder); @@ -122,8 +122,8 @@ public void singleThreadEqualsParallelManyCategories() { public void decoderIsComposite() { FrameBlock data = categoricalFrame(100, 2, 3, 1); String spec = "{recode:[C1], dummycode:[C2]}"; - MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), data.getNumColumns(), - null); + MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), + data.getNumColumns(), null); encoder.encode(data, 1); Decoder decoder = buildDecoder(data, spec, encoder); if(!(decoder instanceof DecoderComposite)) diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java index 412686c1e83..d9c540f54c5 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java @@ -47,9 +47,9 @@ import org.junit.Test; /** - * Unit tests that drive the get_categorical_mask instruction directly to exercise the defensive code paths - * (distinct-count prefix in the metadata frame, default column metadata, non id-based specs and the unsupported opcode - * guard) that the script-level transform tests cannot reach. + * Unit tests that drive the get_categorical_mask instruction directly to exercise the defensive code + * paths (distinct-count prefix in the metadata frame, default column metadata, non id-based specs and + * the unsupported opcode guard) that the script-level transform tests cannot reach. */ public class GetCategoricalMaskInstructionTest { protected static final Log LOG = LogFactory.getLog(GetCategoricalMaskInstructionTest.class.getName()); @@ -210,8 +210,7 @@ public void imputeAndOmitAreAccepted() { // impute and omit do not change the output column count or categorical flag, so a spec that // only adds them on top of a recoded column must still succeed and mark that column categorical FrameBlock meta = new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}); - MatrixBlock res = run(meta, - "{\"ids\": true, \"recode\": [1], \"impute\": [{\"id\": 1, \"method\": \"global_mode\"}], \"omit\": [1]}"); + MatrixBlock res = run(meta, "{\"ids\": true, \"recode\": [1], \"impute\": [{\"id\": 1, \"method\": \"global_mode\"}], \"omit\": [1]}"); assertEquals(1, res.getNumRows()); assertEquals(1, res.getNumColumns()); @@ -231,7 +230,8 @@ public void unsupportedOpcodeThrows() { // any frame-scalar binary opcode other than get_categorical_mask must be rejected ExecutionContext ec = ExecutionContextFactory.createContext(); ec.setAutoCreateVars(true); - ec.setVariable("F", frameObject(new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}))); + ec.setVariable("F", frameObject(new FrameBlock(new ValueType[] {ValueType.STRING}, + new String[][] {{"a"}}))); assertThrowsMessage("Unsupported operation", () -> maskInstruction("+").processInstruction(ec)); } @@ -263,9 +263,9 @@ private static void assertMask(MatrixBlock res, double[] expected) { } /** - * Build a single-row metadata frame of nCol string columns. A positive distinct[i] is written to that column's - * metadata as the recode distinct count (the path real transformencode uses), while a zero leaves the column with - * default metadata (a continuous / non-dummycoded column). + * Build a single-row metadata frame of nCol string columns. A positive distinct[i] is written to + * that column's metadata as the recode distinct count (the path real transformencode uses), while + * a zero leaves the column with default metadata (a continuous / non-dummycoded column). */ private static FrameBlock metaWithDistinct(int nCol, int[] distinct) { ValueType[] schema = new ValueType[nCol]; @@ -290,8 +290,7 @@ private static MatrixBlock run(FrameBlock meta, String spec) { private static BinaryFrameScalarCPInstruction maskInstruction(String opcode) { String in1 = InstructionUtils.concatOperandParts("F", DataType.FRAME.name(), ValueType.STRING.name(), "false"); - String in2 = InstructionUtils.concatOperandParts("spec", DataType.SCALAR.name(), ValueType.STRING.name(), - "true"); + String in2 = InstructionUtils.concatOperandParts("spec", DataType.SCALAR.name(), ValueType.STRING.name(), "true"); String out = InstructionUtils.concatOperandParts("out", DataType.MATRIX.name(), ValueType.FP64.name(), "false"); String str = InstructionUtils.concatOperands("CP", opcode, in1, in2, out); return (BinaryFrameScalarCPInstruction) BinaryCPInstruction.parseInstruction(str); diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java index 645c4dd09bd..b2d31f43b83 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -44,10 +44,10 @@ import org.junit.Test; /** - * Exact inverse correctness tests for the transform decoders. Recode and dummycode are lossless category encodings, so - * a decode of the encoded matrix must reconstruct the original categorical frame. These tests assert exact - * reconstruction for the dense path, the sparse path, and the parallel path so that the dummycode sparse binary search - * and the parallel block split are validated against ground truth rather than only against each other. + * Exact inverse correctness tests for the transform decoders. Recode and dummycode are lossless category encodings, so a + * decode of the encoded matrix must reconstruct the original categorical frame. These tests assert exact reconstruction + * for the dense path, the sparse path, and the parallel path so that the dummycode sparse binary search and the parallel + * block split are validated against ground truth rather than only against each other. */ public class TransformDecodeRoundTripTest { protected static final Log LOG = LogFactory.getLog(TransformDecodeRoundTripTest.class.getName()); @@ -59,9 +59,9 @@ public void setUp() { } private static FrameBlock categoricalFrame() { - final String[] values = new String[] {"apple", "banana", "apple", "cherry", "banana", "date", "apple", "cherry", - "date", "banana", "elderberry", "apple", "fig", "banana", "cherry", "apple", "date", "fig", "elderberry", - "banana"}; + final String[] values = new String[] { + "apple", "banana", "apple", "cherry", "banana", "date", "apple", "cherry", "date", "banana", "elderberry", + "apple", "fig", "banana", "cherry", "apple", "date", "fig", "elderberry", "banana"}; final FrameBlock f = new FrameBlock(new ValueType[] {ValueType.STRING}); f.ensureAllocatedColumns(values.length); for(int i = 0; i < values.length; i++) @@ -117,8 +117,8 @@ public void binWithDummycodeOnOtherColumnConsistency() { /** * Dummycode on an earlier column (1) shifts the bin column (2) to the right in the encoded matrix. The bin decoder - * must walk the dummycode domain sizes to recover the bin column's true source position. This drives the non-magic - * offset branch of the bin source-column mapping. + * must walk the dummycode domain sizes to recover the bin column's true source position. This drives the + * non-magic offset branch of the bin source-column mapping. */ @Test public void binAfterDummycodeOnEarlierColumnConsistency() { @@ -128,9 +128,9 @@ public void binAfterDummycodeOnEarlierColumnConsistency() { } /** - * Same right-shift as above, but the earlier column is feature-hashed before being dummycoded. The hash domain size - * K is stored as a plain integer in the single meta cell, so the bin source-column mapping reads it (instead of - * numDistinct) to compute the offset. + * Same right-shift as above, but the earlier column is feature-hashed before being dummycoded. The hash domain + * size K is stored as a plain integer in the single meta cell, so the bin source-column mapping reads it (instead + * of numDistinct) to compute the offset. */ @Test public void binAfterHashDummycodeOnEarlierColumnConsistency() { @@ -211,10 +211,10 @@ public void binDecodeZeroCodeUsesFirstBinBoundary() { } /** - * Spark broadcasts the decoder to executors via Java serialization without re-running initMetaData, so the decoder - * must round-trip all of its decode state through writeExternal/readExternal. Decode with a freshly deserialized - * decoder and assert it matches the in-memory decode. Covers plain bin and bin-with-dummycode (the latter exercises - * the serialized _srcCols/_dcCols source-column mapping). + * Spark broadcasts the decoder to executors via Java serialization without re-running initMetaData, so the + * decoder must round-trip all of its decode state through writeExternal/readExternal. Decode with a freshly + * deserialized decoder and assert it matches the in-memory decode. Covers plain bin and bin-with-dummycode + * (the latter exercises the serialized _srcCols/_dcCols source-column mapping). */ @Test public void binDecoderSurvivesSerialization() { @@ -481,7 +481,8 @@ public void parallelDecodeWrapsWorkerException() { fail("expected the parallel decode wrapper to propagate the worker failure"); } catch(DMLRuntimeException expected) { - assertNotNull("parallel decode wrapper must retain the worker exception as cause", expected.getCause()); + assertNotNull("parallel decode wrapper must retain the worker exception as cause", + expected.getCause()); } } catch(Exception e) { diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java index 5745a35d506..54bd1679716 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java index 64012c06bbf..8d7ad14539a 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java @@ -57,15 +57,17 @@ import io.delta.kernel.types.TimestampType; /** - * Targeted tests for the error/defensive branches of the native Delta matrix read/write code that the round-trip and - * interop tests do not reach: malformed per-file statistics, unsupported column types, unsupported stream operations, + * Targeted tests for the error/defensive branches of the native Delta matrix + * read/write code that the round-trip and interop tests do not reach: malformed + * per-file statistics, unsupported column types, unsupported stream operations, * bad table paths, and the non-dense writer input path. * - *

- * A few of these branches guard against inputs that the SystemDS writer and the Delta Kernel scan API never produce in - * a normal round trip (e.g. a statistics JSON without {@code numRecords}, or a column type code outside the supported - * set). They are exercised here by mocking the Delta Kernel data objects and invoking the (package-private) helpers - * reflectively, rather than widening their production visibility purely for testing. + *

A few of these branches guard against inputs that the SystemDS writer and + * the Delta Kernel scan API never produce in a normal round trip (e.g. a + * statistics JSON without {@code numRecords}, or a column type code outside the + * supported set). They are exercised here by mocking the Delta Kernel data + * objects and invoking the (package-private) helpers reflectively, rather than + * widening their production visibility purely for testing. */ public class DeltaMatrixCoverageTest { @@ -87,8 +89,8 @@ public void qualifyRejectsUnknownFilesystemScheme() { @Test public void typeCodeReturnsNegativeForUnsupportedTypes() { - // non-numeric / unsupported Delta types must map to the sentinel -1 so the - // reader can reject them with a clear message rather than mis-decoding. + //non-numeric / unsupported Delta types must map to the sentinel -1 so the + //reader can reject them with a clear message rather than mis-decoding. assertEquals(-1, DeltaKernelUtils.typeCode(DateType.DATE)); assertEquals(-1, DeltaKernelUtils.typeCode(TimestampType.TIMESTAMP)); assertEquals(-1, DeltaKernelUtils.typeCode(BinaryType.BINARY)); @@ -110,17 +112,17 @@ public void writerRejectsStreamWrite() throws Exception { @Test public void numRecordsHandlesAbsentNullAndMalformedStats() throws Exception { - // no "stats" field at all -> -1 + //no "stats" field at all -> -1 assertEquals(-1, numRecords(addFileRow(new StructType().add("path", StringType.STRING), false, null))); - // stats column present but null-at -> -1 + //stats column present but null-at -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), true, null))); - // stats string explicitly null -> -1 + //stats string explicitly null -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, null))); - // malformed JSON -> JsonProcessingException -> -1 + //malformed JSON -> JsonProcessingException -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, "{not valid json"))); - // valid JSON but no numRecords field -> -1 + //valid JSON but no numRecords field -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, "{\"minValues\":{}}"))); - // well-formed stats -> the parsed count + //well-formed stats -> the parsed count assertEquals(1234L, numRecords(addFileRow(statsSchema(), false, "{\"numRecords\":1234}"))); } @@ -129,8 +131,8 @@ public void getDoubleValueRejectsUnknownTypeCode() throws Exception { Method m = ReaderDelta.class.getDeclaredMethod("getDoubleValue", ColumnVector.class, int.class, int.class); m.setAccessible(true); try { - // type code outside the supported T_* set; the switch default must throw - // before touching the (null) vector. + //type code outside the supported T_* set; the switch default must throw + //before touching the (null) vector. m.invoke(null, (ColumnVector) null, 0, 999); fail("expected a DMLRuntimeException for an unsupported type code"); } @@ -154,9 +156,9 @@ public void numericTypeCodeRejectsNonNumericType() throws Exception { @Test public void parallelReadWrapsFileFailure() throws Exception { - // a per-file decode failure in the parallel reader must surface as a single - // clear IOException (the awaitFileTasks catch), not a raw executor error. - // Provoke it by deleting one data file after the table (and its log) exist. + //a per-file decode failure in the parallel reader must surface as a single + //clear IOException (the awaitFileTasks catch), not a raw executor error. + //Provoke it by deleting one data file after the table (and its log) exist. MatrixBlock in = TestUtils.generateTestMatrixBlock(100_000, 8, -10, 10, 1.0, 13); in.recomputeNonZeros(); DMLConfig conf = new DMLConfig(); @@ -165,14 +167,15 @@ public void parallelReadWrapsFileFailure() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_fail_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, - in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); - // delete one parquet data file; the transaction log still references it, - // so the scan enumerates it but the decode task fails. + //delete one parquet data file; the transaction log still references it, + //so the scan enumerates it but the decode task fails. File victim; - try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { - victim = s.filter(p -> p.toString().endsWith(".parquet")).findFirst().map(Path::toFile).orElse(null); + try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + victim = s.filter(p -> p.toString().endsWith(".parquet")) + .findFirst().map(Path::toFile).orElse(null); } assertTrue("expected at least one data file to delete", victim != null && victim.delete()); @@ -197,8 +200,8 @@ public void parallelReadWrapsFileFailure() throws Exception { @Test public void sparseFormatMatrixRoundTrips() throws Exception { - // a sparse-backed MatrixBlock takes the writer's non-contiguous path (no - // direct double[] view), exercising MatrixColumnVector.get via MatrixBlock. + //a sparse-backed MatrixBlock takes the writer's non-contiguous path (no + //direct double[] view), exercising MatrixColumnVector.get via MatrixBlock. MatrixBlock in = TestUtils.generateTestMatrixBlock(2000, 7, -5, 5, 0.05, 13); in.recomputeNonZeros(); in.examSparsity(); @@ -207,8 +210,8 @@ public void sparseFormatMatrixRoundTrips() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_sparse_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, - in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", in.getNumRows(), out.getNumRows()); assertEquals("cols", in.getNumColumns(), out.getNumColumns()); @@ -221,15 +224,15 @@ public void sparseFormatMatrixRoundTrips() throws Exception { @Test public void fillDenseHandlesNonContiguousBlock() throws Exception { - // the dense fill normally hits the contiguous fast path; force a multi-block - // (non-contiguous) dense block so the row-by-row fallback is exercised. Such - // blocks only arise for matrices beyond a single contiguous array, so we - // shrink the per-block allocation cap to provoke it on a tiny matrix. + //the dense fill normally hits the contiguous fast path; force a multi-block + //(non-contiguous) dense block so the row-by-row fallback is exercised. Such + //blocks only arise for matrices beyond a single contiguous array, so we + //shrink the per-block allocation cap to provoke it on a tiny matrix. int rows = 5, cols = 4; int savedMaxAlloc = DenseBlockLDRB.MAX_ALLOC; DenseBlock db; try { - DenseBlockLDRB.MAX_ALLOC = 2 * cols; // ~2 rows per block -> multiple blocks + DenseBlockLDRB.MAX_ALLOC = 2 * cols; //~2 rows per block -> multiple blocks db = new DenseBlockLFP64(new int[] {rows, cols}); } finally { @@ -237,14 +240,14 @@ public void fillDenseHandlesNonContiguousBlock() throws Exception { } assertTrue("expected a non-contiguous (multi-block) dense block", !db.isContiguous()); - // two row-major batches (3 rows + 2 rows) covering all 5 rows + //two row-major batches (3 rows + 2 rows) covering all 5 rows double[] b0 = new double[3 * cols]; double[] b1 = new double[2 * cols]; - for(int r = 0; r < 3; r++) - for(int c = 0; c < cols; c++) + for( int r = 0; r < 3; r++ ) + for( int c = 0; c < cols; c++ ) b0[r * cols + c] = cell(r, c); - for(int r = 0; r < 2; r++) - for(int c = 0; c < cols; c++) + for( int r = 0; r < 2; r++ ) + for( int c = 0; c < cols; c++ ) b1[r * cols + c] = cell(3 + r, c); java.util.ArrayList batches = new java.util.ArrayList<>(); batches.add(b0); @@ -255,8 +258,8 @@ public void fillDenseHandlesNonContiguousBlock() throws Exception { m.setAccessible(true); m.invoke(null, ret, batches); - for(int r = 0; r < rows; r++) - for(int c = 0; c < cols; c++) + for( int r = 0; r < rows; r++ ) + for( int c = 0; c < cols; c++ ) assertEquals("r" + r + " c" + c, cell(r, c), ret.getDenseBlock().get(r, c), 0.0); } @@ -273,8 +276,8 @@ private static StructType statsSchema() { } /** - * Build a mocked scan-file row whose AddFile child has the given schema, null flag and (when not null) stats - * string, matching what {@code numRecords} reads. + * Build a mocked scan-file row whose AddFile child has the given schema, null + * flag and (when not null) stats string, matching what {@code numRecords} reads. */ private static Row addFileRow(StructType addSchema, boolean statsNull, String statsValue) { Row outer = mock(Row.class); @@ -282,12 +285,12 @@ private static Row addFileRow(StructType addSchema, boolean statsNull, String st when(outer.getStruct(InternalScanFileUtils.ADD_FILE_ORDINAL)).thenReturn(add); when(add.getSchema()).thenReturn(addSchema); int statsOrd = addSchema.fieldNames().indexOf("stats"); - if(statsOrd >= 0) { + if( statsOrd >= 0 ) { when(add.isNullAt(statsOrd)).thenReturn(statsNull); - if(!statsNull) + if( !statsNull ) when(add.getString(statsOrd)).thenReturn(statsValue); } - return outer; // the scan-file row numRecords consumes (its AddFile child is 'add') + return outer; //the scan-file row numRecords consumes (its AddFile child is 'add') } private static long numRecords(Row scanFileRow) throws Exception { diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java index 65c4ec21c0f..54a3bcf6334 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java @@ -63,19 +63,20 @@ import io.delta.kernel.utils.CloseableIterator; /** - * Direct (no DML) round-trip tests for the native Delta Kernel based matrix reader/writer. Each test writes a - * MatrixBlock to a fresh local Delta table directory and reads it back, asserting dimensions and values match. + * Direct (no DML) round-trip tests for the native Delta Kernel based matrix + * reader/writer. Each test writes a MatrixBlock to a fresh local Delta table + * directory and reads it back, asserting dimensions and values match. */ public class DeltaMatrixReadWriteTest { - // small writer target file size (bytes) used to force a multi-file table - // layout cheaply, instead of brute-forcing huge row counts. + //small writer target file size (bytes) used to force a multi-file table + //layout cheaply, instead of brute-forcing huge row counts. private static final long SMALL_TARGET_FILE_SIZE = 256L * 1024; private static final int ROWS_MULTI_FILE = 100_000; private static MatrixBlock writeThenRead(MatrixBlock in) throws Exception { Path dir = Files.createTempDirectory("sysds_delta_"); - // WriterDelta creates the table at the given (empty) directory + //WriterDelta creates the table at the given (empty) directory String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { WriterDelta writer = new WriterDelta(); @@ -91,9 +92,9 @@ private static MatrixBlock writeThenRead(MatrixBlock in) throws Exception { @Test public void parallelReadMatchesSerialMultiFile() throws Exception { - // force a multi-file table cheaply via a small writer target file size - // (rather than a huge row count), so the parallel per-file path is - // actually exercised rather than falling back to serial. + //force a multi-file table cheaply via a small writer target file size + //(rather than a huge row count), so the parallel per-file path is + //actually exercised rather than falling back to serial. MatrixBlock in = TestUtils.generateTestMatrixBlock(ROWS_MULTI_FILE, 8, -10, 10, 1.0, 13); in.recomputeNonZeros(); @@ -103,18 +104,21 @@ public void parallelReadMatchesSerialMultiFile() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_par_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, - in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); - // sanity: confirm the table really is split across multiple files + //sanity: confirm the table really is split across multiple files long files; - try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { + try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } - assertTrue("expected a multi-file Delta table to exercise the parallel path, got " + files, files > 1); + assertTrue("expected a multi-file Delta table to exercise the parallel path, got " + files, + files > 1); - MatrixBlock serial = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); - MatrixBlock parallel = new ReaderDeltaParallel().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + MatrixBlock serial = new ReaderDelta() + .readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + MatrixBlock parallel = new ReaderDeltaParallel() + .readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", serial.getNumRows(), parallel.getNumRows()); assertEquals("cols", serial.getNumColumns(), parallel.getNumColumns()); @@ -131,8 +135,8 @@ public void parallelReadMatchesSerialMultiFile() throws Exception { public void roundTripDenseSmall() throws Exception { MatrixBlock in = new MatrixBlock(3, 4, false); double v = 1.0; - for(int i = 0; i < 3; i++) - for(int j = 0; j < 4; j++) + for( int i=0; i<3; i++ ) + for( int j=0; j<4; j++ ) in.set(i, j, v++); in.recomputeNonZeros(); @@ -153,7 +157,7 @@ public void roundTripDenseRandom() throws Exception { @Test public void roundTripSparseRandom() throws Exception { - // values written are dense parquet, but exercise a sparse-ish input + //values written are dense parquet, but exercise a sparse-ish input MatrixBlock in = TestUtils.generateTestMatrixBlock(1200, 9, -5, 5, 0.1, 13); in.recomputeNonZeros(); MatrixBlock out = writeThenRead(in); @@ -164,7 +168,7 @@ public void roundTripSparseRandom() throws Exception { @Test public void roundTripMultiBatch() throws Exception { - // more rows than the writer batch size (4096) to exercise chunking + //more rows than the writer batch size (4096) to exercise chunking MatrixBlock in = TestUtils.generateTestMatrixBlock(10000, 5, 0, 100, 1.0, 1); MatrixBlock out = writeThenRead(in); assertEquals("rows", 10000, out.getNumRows()); @@ -178,9 +182,8 @@ public void readDiscoversUnknownDimensions() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, - in.getNonZeros()); - // pass -1 dimensions: the reader must discover them from the table + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + //pass -1 dimensions: the reader must discover them from the table MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 123, out.getNumRows()); assertEquals("cols", 6, out.getNumColumns()); @@ -209,21 +212,22 @@ public void emptyMatrixRoundTrip() throws Exception { @Test public void readNonDoubleNumericColumns() throws Exception { - // tables produced by external tools (or the frame writer) can carry - // long/int/boolean columns; the matrix reader must coerce them to double + //tables produced by external tools (or the frame writer) can carry + //long/int/boolean columns; the matrix reader must coerce them to double Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] longVals = {1, -2, 1_000_000_000L, 0}; - double[] intVals = {7, -8, 123456, 0}; + double[] intVals = {7, -8, 123456, 0}; double[] boolVals = {1, 0, 1, 0}; - writeTypedColumns(tablePath, new DataType[] {LongType.LONG, IntegerType.INTEGER, BooleanType.BOOLEAN}, + writeTypedColumns(tablePath, + new DataType[] {LongType.LONG, IntegerType.INTEGER, BooleanType.BOOLEAN}, new double[][] {longVals, intVals, boolVals}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 3, out.getNumColumns()); - for(int r = 0; r < 4; r++) { + for( int r=0; r<4; r++ ) { assertEquals("long col r" + r, longVals[r], out.get(r, 0), 0.0); assertEquals("int col r" + r, intVals[r], out.get(r, 1), 0.0); assertEquals("bool col r" + r, boolVals[r], out.get(r, 2), 0.0); @@ -236,14 +240,14 @@ public void readNonDoubleNumericColumns() throws Exception { @Test public void rewriteSamePathReplacesData() throws Exception { - // writing to a path that already holds a Delta table must fully replace it + //writing to a path that already holds a Delta table must fully replace it Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { MatrixBlock first = TestUtils.generateTestMatrixBlock(50, 8, 0, 100, 1.0, 1); new WriterDelta().writeMatrixToHDFS(first, tablePath, 50, 8, -1, first.getNonZeros()); - // second write has different dimensions and values + //second write has different dimensions and values MatrixBlock second = TestUtils.generateTestMatrixBlock(20, 3, -5, 5, 1.0, 2); new WriterDelta().writeMatrixToHDFS(second, tablePath, 20, 3, -1, second.getNonZeros()); @@ -259,10 +263,10 @@ public void rewriteSamePathReplacesData() throws Exception { @Test public void parallelBufferedPathMatchesSerial() throws Exception { - // the direct fast path is always taken for SystemDS-written tables (exact - // row stats, no deletion vectors); force the buffered fallback to exercise - // its per-file decode + serial concatenation and assert it matches serial. - // force a multi-file table cheaply via a small writer target file size. + //the direct fast path is always taken for SystemDS-written tables (exact + //row stats, no deletion vectors); force the buffered fallback to exercise + //its per-file decode + serial concatenation and assert it matches serial. + //force a multi-file table cheaply via a small writer target file size. MatrixBlock in = TestUtils.generateTestMatrixBlock(ROWS_MULTI_FILE, 8, -10, 10, 1.0, 23); in.recomputeNonZeros(); @@ -272,22 +276,19 @@ public void parallelBufferedPathMatchesSerial() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_buf_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, - in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); long files; - try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { + try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } assertTrue("expected a multi-file Delta table, got " + files, files > 1); MatrixBlock serial = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); - // subclass that always declines the direct path -> readBuffered() + //subclass that always declines the direct path -> readBuffered() MatrixBlock buffered = new ReaderDeltaParallel() { - @Override - protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { - return false; - } + @Override protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { return false; } }.readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", serial.getNumRows(), buffered.getNumRows()); @@ -303,30 +304,30 @@ protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { @Test public void writerTargetFileSizeConfigProducesMoreFiles() throws Exception { - // a smaller configured target file size must make the writer roll more - // data files for the same matrix (the lever the parallel reader relies on). + //a smaller configured target file size must make the writer roll more + //data files for the same matrix (the lever the parallel reader relies on). MatrixBlock in = TestUtils.generateTestMatrixBlock(400_000, 16, -10, 10, 1.0, 7); in.recomputeNonZeros(); - // isolate the override in a fresh thread-local config (restored in finally) + //isolate the override in a fresh thread-local config (restored in finally) DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(1L * 1024 * 1024)); ConfigurationManager.setLocalConfig(conf); Path dir = Files.createTempDirectory("sysds_delta_cfg_"); try { - assertEquals("config getter reflects the override", 1L * 1024 * 1024, - ConfigurationManager.getDeltaWriterTargetFileSize()); + assertEquals("config getter reflects the override", + 1L * 1024 * 1024, ConfigurationManager.getDeltaWriterTargetFileSize()); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, - in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); long files; - try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { + try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } assertTrue("expected >1 data file with a 1MB target, got " + files, files > 1); - // data still round-trips correctly with the custom layout + //data still round-trips correctly with the custom layout MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); TestUtils.compareMatrices(in, out, 1e-12, "small-target-roundtrip"); } @@ -338,20 +339,21 @@ public void writerTargetFileSizeConfigProducesMoreFiles() throws Exception { @Test public void readerBatchSizeConfigRoundTrips() throws Exception { - // a non-default reader batch size must not change the result (more, smaller - // batches exercise the per-batch extract/concatenate loop more often). + //a non-default reader batch size must not change the result (more, smaller + //batches exercise the per-batch extract/concatenate loop more often). MatrixBlock in = TestUtils.generateTestMatrixBlock(5000, 7, -10, 10, 1.0, 11); - // isolate the override in a fresh thread-local config (restored in finally) + //isolate the override in a fresh thread-local config (restored in finally) DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_READER_BATCH_SIZE, "128"); ConfigurationManager.setLocalConfig(conf); Path dir = Files.createTempDirectory("sysds_delta_bs_"); try { - assertEquals("config getter reflects the override", 128, ConfigurationManager.getDeltaReaderBatchSize()); + assertEquals("config getter reflects the override", + 128, ConfigurationManager.getDeltaReaderBatchSize()); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, - in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, + in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); TestUtils.compareMatrices(in, out, 1e-12, "small-batch-roundtrip"); } @@ -363,13 +365,14 @@ public void readerBatchSizeConfigRoundTrips() throws Exception { @Test public void factoryRoutesDeltaToParallelWhenEnabled() { - // the factory must pick the parallel reader iff parallel CP read is enabled + //the factory must pick the parallel reader iff parallel CP read is enabled CompilerConfig cc = ConfigurationManager.getCompilerConfig(); try { cc.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, true); ConfigurationManager.setLocalConfig(cc); MatrixReader par = MatrixReaderFactory.createMatrixReader(FileFormat.DELTA); - assertTrue("expected ReaderDeltaParallel when parallel read enabled", par instanceof ReaderDeltaParallel); + assertTrue("expected ReaderDeltaParallel when parallel read enabled", + par instanceof ReaderDeltaParallel); cc.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, false); ConfigurationManager.setLocalConfig(cc); @@ -384,18 +387,20 @@ public void factoryRoutesDeltaToParallelWhenEnabled() { @Test public void readFloatColumnsCoercedToDouble() throws Exception { - // float columns must be widened to double on read (exact-representable values) + //float columns must be widened to double on read (exact-representable values) Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] f0 = {1.5, -2.25, 0.0, 1024.5}; double[] f1 = {-0.5, 3.75, 100.125, -7.0}; - writeTypedColumns(tablePath, new DataType[] {FloatType.FLOAT, FloatType.FLOAT}, new double[][] {f0, f1}); + writeTypedColumns(tablePath, + new DataType[] {FloatType.FLOAT, FloatType.FLOAT}, + new double[][] {f0, f1}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 2, out.getNumColumns()); - for(int r = 0; r < 4; r++) { + for( int r=0; r<4; r++ ) { assertEquals("f0 r" + r, f0[r], out.get(r, 0), 0.0); assertEquals("f1 r" + r, f1[r], out.get(r, 1), 0.0); } @@ -407,20 +412,21 @@ public void readFloatColumnsCoercedToDouble() throws Exception { @Test public void readShortByteColumnsCoercedToDouble() throws Exception { - // short/byte columns must be coerced to double on read, exercising the - // T_SHORT / T_BYTE branches of ReaderDelta.getDoubleValue. + //short/byte columns must be coerced to double on read, exercising the + //T_SHORT / T_BYTE branches of ReaderDelta.getDoubleValue. Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] shortVals = {1, -2, 30000, 0}; - double[] byteVals = {7, -8, 120, 0}; - writeTypedColumns(tablePath, new DataType[] {ShortType.SHORT, ByteType.BYTE}, + double[] byteVals = {7, -8, 120, 0}; + writeTypedColumns(tablePath, + new DataType[] {ShortType.SHORT, ByteType.BYTE}, new double[][] {shortVals, byteVals}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 2, out.getNumColumns()); - for(int r = 0; r < 4; r++) { + for( int r=0; r<4; r++ ) { assertEquals("short col r" + r, shortVals[r], out.get(r, 0), 0.0); assertEquals("byte col r" + r, byteVals[r], out.get(r, 1), 0.0); } @@ -432,8 +438,8 @@ public void readShortByteColumnsCoercedToDouble() throws Exception { @Test public void writerRejectsDimensionMismatch() throws Exception { - // WriterDelta validates that the passed rlen/clen match the MatrixBlock - // and rejects a mismatch with an IOException. + //WriterDelta validates that the passed rlen/clen match the MatrixBlock + //and rejects a mismatch with an IOException. MatrixBlock in = TestUtils.generateTestMatrixBlock(10, 4, -1, 1, 1.0, 5); in.recomputeNonZeros(); Path dir = Files.createTempDirectory("sysds_delta_"); @@ -453,18 +459,18 @@ public void writerRejectsDimensionMismatch() throws Exception { @Test public void readNullCellsBecomeZero() throws Exception { - // nullable numeric columns with null cells must read back as 0.0 + //nullable numeric columns with null cells must read back as 0.0 Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - double[] vals = {3.0, 7.0, 9.0, 11.0}; - boolean[] nulls = {false, true, false, true}; + double[] vals = {3.0, 7.0, 9.0, 11.0}; + boolean[] nulls = {false, true, false, true}; writeNullableDoubleColumn(tablePath, vals, nulls); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 1, out.getNumColumns()); - for(int r = 0; r < 4; r++) + for( int r=0; r<4; r++ ) assertEquals("r" + r, nulls[r] ? 0.0 : vals[r], out.get(r, 0), 0.0); } finally { @@ -474,7 +480,7 @@ public void readNullCellsBecomeZero() throws Exception { @Test public void readStringColumnRejected() throws Exception { - // string columns cannot back an all-double matrix -> reader must reject them + //string columns cannot back an all-double matrix -> reader must reject them Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { @@ -497,10 +503,9 @@ public void readStringColumnRejected() throws Exception { private static void writeTypedColumns(String tablePath, DataType[] types, double[][] vals) throws Exception { Engine engine = DeltaKernelUtils.createEngine(); StructType schema = new StructType(); - for(int c = 0; c < types.length; c++) + for( int c=0; c singleton(FilteredColumnarBatch fcb) { return new CloseableIterator() { private boolean _done = false; - - @Override - public boolean hasNext() { - return !_done; - } - - @Override - public FilteredColumnarBatch next() { - if(_done) - throw new NoSuchElementException(); + @Override public boolean hasNext() { return !_done; } + @Override public FilteredColumnarBatch next() { + if( _done ) throw new NoSuchElementException(); _done = true; return fcb; } - - @Override - public void close() { - } + @Override public void close() {} }; } - /** - * Minimal in-memory columnar batch backed by per-column double[] values, with an optional per-column null mask - * ({@code nulls==null} => no nulls). - */ + /** Minimal in-memory columnar batch backed by per-column double[] values, with + * an optional per-column null mask ({@code nulls==null} => no nulls). */ private static class TypedBatch implements ColumnarBatch { private final StructType _schema; private final DataType[] _types; private final double[][] _vals; private final boolean[][] _nulls; - TypedBatch(StructType schema, DataType[] types, double[][] vals, boolean[][] nulls) { - _schema = schema; - _types = types; - _vals = vals; - _nulls = nulls; + _schema = schema; _types = types; _vals = vals; _nulls = nulls; } - - @Override - public StructType getSchema() { - return _schema; - } - - @Override - public int getSize() { - return _vals[0].length; - } - - @Override - public ColumnVector getColumnVector(int ordinal) { - return new TypedVector(_types[ordinal], _vals[ordinal], _nulls == null ? null : _nulls[ordinal]); + @Override public StructType getSchema() { return _schema; } + @Override public int getSize() { return _vals[0].length; } + @Override public ColumnVector getColumnVector(int ordinal) { + return new TypedVector(_types[ordinal], _vals[ordinal], + _nulls == null ? null : _nulls[ordinal]); } } @@ -599,98 +568,28 @@ private static class TypedVector implements ColumnVector { private final DataType _type; private final double[] _vals; private final boolean[] _nulls; - - TypedVector(DataType type, double[] vals, boolean[] nulls) { - _type = type; - _vals = vals; - _nulls = nulls; - } - - @Override - public DataType getDataType() { - return _type; - } - - @Override - public int getSize() { - return _vals.length; - } - - @Override - public boolean isNullAt(int rowId) { - return _nulls != null && _nulls[rowId]; - } - - @Override - public double getDouble(int rowId) { - return _vals[rowId]; - } - - @Override - public float getFloat(int rowId) { - return (float) _vals[rowId]; - } - - @Override - public long getLong(int rowId) { - return (long) _vals[rowId]; - } - - @Override - public int getInt(int rowId) { - return (int) _vals[rowId]; - } - - @Override - public short getShort(int rowId) { - return (short) _vals[rowId]; - } - - @Override - public byte getByte(int rowId) { - return (byte) _vals[rowId]; - } - - @Override - public boolean getBoolean(int rowId) { - return _vals[rowId] != 0; - } - - @Override - public void close() { - } + TypedVector(DataType type, double[] vals, boolean[] nulls) { _type = type; _vals = vals; _nulls = nulls; } + @Override public DataType getDataType() { return _type; } + @Override public int getSize() { return _vals.length; } + @Override public boolean isNullAt(int rowId) { return _nulls != null && _nulls[rowId]; } + @Override public double getDouble(int rowId) { return _vals[rowId]; } + @Override public float getFloat(int rowId) { return (float) _vals[rowId]; } + @Override public long getLong(int rowId) { return (long) _vals[rowId]; } + @Override public int getInt(int rowId) { return (int) _vals[rowId]; } + @Override public short getShort(int rowId) { return (short) _vals[rowId]; } + @Override public byte getByte(int rowId) { return (byte) _vals[rowId]; } + @Override public boolean getBoolean(int rowId) { return _vals[rowId] != 0; } + @Override public void close() {} } /** Column view exposing a String[] as a Delta string column. */ private static class StringVector implements ColumnVector { private final String[] _vals; - - StringVector(String[] vals) { - _vals = vals; - } - - @Override - public DataType getDataType() { - return StringType.STRING; - } - - @Override - public int getSize() { - return _vals.length; - } - - @Override - public boolean isNullAt(int rowId) { - return _vals[rowId] == null; - } - - @Override - public String getString(int rowId) { - return _vals[rowId]; - } - - @Override - public void close() { - } + StringVector(String[] vals) { _vals = vals; } + @Override public DataType getDataType() { return StringType.STRING; } + @Override public int getSize() { return _vals.length; } + @Override public boolean isNullAt(int rowId) { return _vals[rowId] == null; } + @Override public String getString(int rowId) { return _vals[rowId]; } + @Override public void close() {} } } diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java index 45194d12b5a..2d79b79f2dd 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java @@ -49,22 +49,24 @@ import org.junit.Test; /** - * Cross-engine interoperability tests for the native (Delta Kernel based) matrix reader/writer against the reference - * Delta implementation (Delta's Spark connector, {@code delta-spark}, pulled in test-only). + * Cross-engine interoperability tests for the native (Delta Kernel based) matrix + * reader/writer against the reference Delta implementation (Delta's Spark + * connector, {@code delta-spark}, pulled in test-only). * - *

- * The other Delta matrix tests round-trip exclusively through SystemDS' own Kernel-based read/write paths, so they - * cannot catch a table that SystemDS writes in a way other Delta engines reject (or vice versa). These tests close that - * gap by routing data through two independent engines: + *

The other Delta matrix tests round-trip exclusively through SystemDS' own + * Kernel-based read/write paths, so they cannot catch a table that SystemDS + * writes in a way other Delta engines reject (or vice versa). These tests close + * that gap by routing data through two independent engines: *

    - *
  • SystemDS writes -> Spark/Delta reads (our output is spec-compliant), and
    • - *
  • Spark/Delta writes -> SystemDS reads, including a multi-file layout and a table with deletion vectors / a - * second commit that the SystemDS writer never produces itself.
    • + *
  • SystemDS writes -> Spark/Delta reads (our output is spec-compliant), and
    • + *
  • Spark/Delta writes -> SystemDS reads, including a multi-file layout and a + * table with deletion vectors / a second commit that the SystemDS writer + * never produces itself.
    • *
* - *

- * Row order is never assumed: every table carries a unique id in column 0 and comparisons are keyed by that id, since - * neither engine guarantees row order across files. + *

Row order is never assumed: every table carries a unique id in column 0 and + * comparisons are keyed by that id, since neither engine guarantees row order + * across files. */ @net.jcip.annotations.NotThreadSafe public class DeltaMatrixSparkInteropTest { @@ -73,19 +75,23 @@ public class DeltaMatrixSparkInteropTest { @BeforeClass public static void startSpark() { - // each test class runs in its own fork (surefire reuseForks=false), so this - // is the only SparkSession in the JVM and gets the Delta extensions injected. + //each test class runs in its own fork (surefire reuseForks=false), so this + //is the only SparkSession in the JVM and gets the Delta extensions injected. SparkSession.clearActiveSession(); SparkSession.clearDefaultSession(); - spark = SparkSession.builder().appName("sysds-delta-interop").master("local[2]") - .config("spark.ui.enabled", "false").config("spark.sql.shuffle.partitions", "2") + spark = SparkSession.builder() + .appName("sysds-delta-interop") + .master("local[2]") + .config("spark.ui.enabled", "false") + .config("spark.sql.shuffle.partitions", "2") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") - .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog").getOrCreate(); + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") + .getOrCreate(); } @AfterClass public static void stopSpark() { - if(spark != null) + if( spark != null ) spark.stop(); SparkSession.clearActiveSession(); SparkSession.clearDefaultSession(); @@ -94,13 +100,13 @@ public static void stopSpark() { @Test public void systemdsWriteSparkReadMultiFile() throws Exception { - // SystemDS writes a (forced) multi-file Delta table; the reference Delta - // engine (Spark) must read every data file back with matching values. + //SystemDS writes a (forced) multi-file Delta table; the reference Delta + //engine (Spark) must read every data file back with matching values. int rows = 500, cols = 5; MatrixBlock in = indexedMatrix(rows, cols); - // small target file size -> multiple parquet data files (exercise that an - // external reader stitches all of our data files, not just the first). + //small target file size -> multiple parquet data files (exercise that an + //external reader stitches all of our data files, not just the first). DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(16L * 1024)); ConfigurationManager.setLocalConfig(conf); @@ -116,10 +122,10 @@ public void systemdsWriteSparkReadMultiFile() throws Exception { List read = df.collectAsList(); assertEquals(rows, read.size()); - for(Row r : read) { + for( Row r : read ) { int id = (int) Math.round(r.getDouble(0)); assertTrue("id in range: " + id, id >= 0 && id < rows); - for(int c = 0; c < cols; c++) + for( int c = 0; c < cols; c++ ) assertEquals("r" + id + " c" + c, in.get(id, c), r.getDouble(c), 1e-9); } } @@ -131,10 +137,10 @@ public void systemdsWriteSparkReadMultiFile() throws Exception { @Test public void sparkWriteSystemdsReadMultiFile() throws Exception { - // the reference Delta engine writes a multi-file table; both the serial and - // parallel SystemDS readers must reconstruct it (coercing long ids to double). + //the reference Delta engine writes a multi-file table; both the serial and + //parallel SystemDS readers must reconstruct it (coercing long ids to double). int rows = 600, cols = 4; - Dataset df = indexedDataFrame(rows, cols).repartition(3); // -> multiple data files + Dataset df = indexedDataFrame(rows, cols).repartition(3); //-> multiple data files Path dir = Files.createTempDirectory("sysds_delta_p2s_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { @@ -142,10 +148,10 @@ public void sparkWriteSystemdsReadMultiFile() throws Exception { assertTrue("spark should have written a multi-file table", countParquet(tablePath) > 1); Map expected = expectedById(rows, cols); - assertMatchesById(new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, - "serial"); - assertMatchesById(new ReaderDeltaParallel().readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, - "parallel"); + assertMatchesById(new ReaderDelta() + .readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, "serial"); + assertMatchesById(new ReaderDeltaParallel() + .readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, "parallel"); } finally { FileUtils.deleteQuietly(dir.toFile()); @@ -154,15 +160,15 @@ public void sparkWriteSystemdsReadMultiFile() throws Exception { @Test public void sparkDeletionVectorsSystemdsRead() throws Exception { - // a Delta table with deletion vectors + a second commit (the DELETE) is a - // layout the SystemDS writer never emits; the readers must honor the DV and - // return only the surviving rows. This exercises the hasDeletionVector path. + //a Delta table with deletion vectors + a second commit (the DELETE) is a + //layout the SystemDS writer never emits; the readers must honor the DV and + //return only the surviving rows. This exercises the hasDeletionVector path. int rows = 400, cols = 3, deleteBelow = 50; Path dir = Files.createTempDirectory("sysds_delta_dv_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - // enable deletion vectors for tables created in this block, then delete a - // row range so Delta records a DV rather than rewriting the data files. + //enable deletion vectors for tables created in this block, then delete a + //row range so Delta records a DV rather than rewriting the data files. spark.conf().set(DV_DEFAULT, "true"); indexedDataFrame(rows, cols).write().format("delta").save(tablePath); spark.sql("DELETE FROM delta.`" + tablePath + "` WHERE c0 < " + deleteBelow); @@ -179,20 +185,21 @@ public void sparkDeletionVectorsSystemdsRead() throws Exception { assertMatchesById(parallel, expected, cols, "parallel-dv"); } finally { - // fresh fork per test class, so simply clearing the override is enough + //fresh fork per test class, so simply clearing the override is enough spark.conf().unset(DV_DEFAULT); FileUtils.deleteQuietly(dir.toFile()); } } - private static final String DV_DEFAULT = "spark.databricks.delta.properties.defaults.enableDeletionVectors"; + private static final String DV_DEFAULT = + "spark.databricks.delta.properties.defaults.enableDeletionVectors"; /** Matrix whose column 0 is the row index and remaining columns are exact doubles. */ private static MatrixBlock indexedMatrix(int rows, int cols) { MatrixBlock mb = new MatrixBlock(rows, cols, false); - for(int r = 0; r < rows; r++) { + for( int r = 0; r < rows; r++ ) { mb.set(r, 0, r); - for(int c = 1; c < cols; c++) + for( int c = 1; c < cols; c++ ) mb.set(r, c, value(r, c)); } mb.recomputeNonZeros(); @@ -202,15 +209,15 @@ private static MatrixBlock indexedMatrix(int rows, int cols) { /** Spark DataFrame mirroring {@link #indexedMatrix} with columns c0..c(cols-1) as doubles. */ private static Dataset indexedDataFrame(int rows, int cols) { StructField[] fields = new StructField[cols]; - for(int c = 0; c < cols; c++) + for( int c = 0; c < cols; c++ ) fields[c] = DataTypes.createStructField("c" + c, DataTypes.DoubleType, false); StructType schema = DataTypes.createStructType(fields); List data = new ArrayList<>(rows); - for(int r = 0; r < rows; r++) { + for( int r = 0; r < rows; r++ ) { Object[] vals = new Object[cols]; vals[0] = (double) r; - for(int c = 1; c < cols; c++) + for( int c = 1; c < cols; c++ ) vals[c] = value(r, c); data.add(RowFactory.create(vals)); } @@ -224,10 +231,10 @@ private static double value(int row, int col) { private static Map expectedById(int rows, int cols) { Map exp = new HashMap<>(rows); - for(int r = 0; r < rows; r++) { + for( int r = 0; r < rows; r++ ) { double[] row = new double[cols]; row[0] = r; - for(int c = 1; c < cols; c++) + for( int c = 1; c < cols; c++ ) row[c] = value(r, c); exp.put(r, row); } @@ -239,25 +246,25 @@ private static void assertMatchesById(MatrixBlock out, Map ex assertEquals(tag + " rows", expected.size(), out.getNumRows()); assertEquals(tag + " cols", cols, out.getNumColumns()); boolean[] seen = new boolean[expected.size() == 0 ? 0 : maxId(expected) + 1]; - for(int r = 0; r < out.getNumRows(); r++) { + for( int r = 0; r < out.getNumRows(); r++ ) { int id = (int) Math.round(out.get(r, 0)); double[] exp = expected.get(id); assertTrue(tag + ": unexpected/duplicate id " + id, exp != null && id < seen.length && !seen[id]); seen[id] = true; - for(int c = 0; c < cols; c++) + for( int c = 0; c < cols; c++ ) assertEquals(tag + " id" + id + " c" + c, exp[c], out.get(r, c), 1e-9); } } private static int maxId(Map expected) { int m = 0; - for(int id : expected.keySet()) + for( int id : expected.keySet() ) m = Math.max(m, id); return m; } private static long countParquet(String tablePath) throws Exception { - try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { + try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { return s.filter(p -> p.toString().endsWith(".parquet")).count(); } } diff --git a/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java b/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java index ae19b291882..472b61d8cd6 100644 --- a/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java +++ b/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -26,10 +26,11 @@ /** * Tests the single-column (unweighted) branch of {@link MatrixBlock#pickValue(double, boolean)} and - * {@link MatrixBlock#median()}. The values are assumed to be sorted in ascending order, mirroring the contract used by - * the quantile pick instructions. The unweighted branch uses the same ceil-based rank as the two-column weighted branch - * (with an implicit weight of 1 per value), so a single column yields the same quantile as the equivalent (value, - * weight) representation. The two-column (weighted) branch is exercised separately through the compressed sort tests. + * {@link MatrixBlock#median()}. The values are assumed to be sorted in ascending order, mirroring the contract used + * by the quantile pick instructions. The unweighted branch uses the same ceil-based rank as the two-column weighted + * branch (with an implicit weight of 1 per value), so a single column yields the same quantile as the equivalent + * (value, weight) representation. The two-column (weighted) branch is exercised separately through the compressed + * sort tests. */ public class QuantilePickTest { diff --git a/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java b/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java index 05aca41ebc7..5c9ed821e78 100644 --- a/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java +++ b/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java @@ -30,7 +30,7 @@ public class TensorToStringTest { @Test public void testDecimalClampsFractionDigits() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 1}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 1}); tb.allocateBlock(); tb.set(0, 0, 5.244058388023880); // decimal=2 must print exactly two fraction digits, not DecimalFormat's default max of 3 @@ -41,10 +41,10 @@ public void testDecimalClampsFractionDigits() { @Test public void testDecimalPadsAndRounds() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 2}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 2}); tb.allocateBlock(); - tb.set(0, 0, 22.0); // integer-valued: padded up to the requested digits - tb.set(0, 1, 5.244058388023880); // rounded at the last requested digit + tb.set(0, 0, 22.0); // integer-valued: padded up to the requested digits + tb.set(0, 1, 5.244058388023880); // rounded at the last requested digit String out = DataConverter.toString(tb, false, " ", "\n", "[", "]", 1, 2, 4); assertTrue("expected 22.0000 padded: " + out, out.contains("22.0000")); assertTrue("expected 5.2441 rounded: " + out, out.contains("5.2441")); @@ -52,10 +52,10 @@ public void testDecimalPadsAndRounds() { @Test public void testNegativeDecimalUsesDefaultFormatting() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 2}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 2}); tb.allocateBlock(); - tb.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained - tb.set(0, 1, 5.244058388023880); // default cap of three fraction digits + tb.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained + tb.set(0, 1, 5.244058388023880); // default cap of three fraction digits // decimal < 0 leaves DecimalFormat unconstrained (no min/max fraction digits set) String out = DataConverter.toString(tb, false, " ", "\n", "[", "]", 1, 2, -1); assertTrue("expected unpadded 22: " + out, out.contains("22")); diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java index 426de81263f..810e2614e7c 100644 --- a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java +++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java @@ -52,12 +52,18 @@ public class QuantileTest extends AutomatedTestBase public void setUp() { TestUtils.clearAssertionInformation(); - addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"})); - addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"})); - addTestConfiguration(TEST_NAME3, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] {"R"})); - addTestConfiguration(TEST_NAME4, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] {"R"})); - addTestConfiguration(TEST_NAME5, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] {"R"})); - addTestConfiguration(TEST_NAME6, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] {"R"})); + addTestConfiguration(TEST_NAME1, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"})); + addTestConfiguration(TEST_NAME2, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"})); + addTestConfiguration(TEST_NAME3, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] {"R"})); + addTestConfiguration(TEST_NAME4, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] {"R"})); + addTestConfiguration(TEST_NAME5, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] {"R"})); + addTestConfiguration(TEST_NAME6, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] {"R"})); } @Test diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java index 35cfb2982fc..e70aedcabe4 100644 --- a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java +++ b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java @@ -59,8 +59,7 @@ private void runSTEPGlmTest(ExecType instType) { // runTest executes the script; fails if the DML script invokes stop() runTest(true, false, null, -1); - } - finally { + } finally { rtplatform = platformOld; } } diff --git a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java index d8cfc4d9fd7..5de429a3c53 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java @@ -91,12 +91,10 @@ public void testBackendPerformance() throws InterruptedException { taskFutures.forEach(res -> { try { Assert.assertEquals("Stats parsed correctly", res.get().statusCode(), 200); - } - catch(InterruptedException e) { + } catch (InterruptedException e) { Thread.currentThread().interrupt(); Assert.fail("Interrupted while fetching statistics: " + e.getMessage()); - } - catch(ExecutionException e) { + } catch (ExecutionException e) { Assert.fail("Failed to fetch statistics: " + e.getMessage()); } }); diff --git a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java index 1cab99ee1ab..f8acdd07930 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java @@ -375,8 +375,9 @@ public void federatedLogicalTest(String testname, Type op_type, ExecMode execMod int port2 = single_fed_worker ? 0 : getRandomAvailablePort(); int port3 = single_fed_worker ? 0 : getRandomAvailablePort(); int port4 = single_fed_worker ? 0 : getRandomAvailablePort(); - Process[] workers = startLocalFedWorkers( - single_fed_worker ? new int[] {port1} : new int[] {port1, port2, port3, port4}); + Process[] workers = startLocalFedWorkers(single_fed_worker + ? new int[] {port1} + : new int[] {port1, port2, port3, port4}); try { if(!isAlive(workers)) diff --git a/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java b/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java index 9d2bb583a4f..dbbf199f8fa 100644 --- a/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java +++ b/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java @@ -72,26 +72,27 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod } if(et == ExecType.SPARK) { - rtplatform = ExecMode.SPARK; + rtplatform = ExecMode.SPARK; } else { // rtplatform = (et==ExecType.MR)? ExecMode.HADOOP : ExecMode.SINGLE_NODE; - rtplatform = ExecMode.HYBRID; + rtplatform = ExecMode.HYBRID; } if( rtplatform == ExecMode.SPARK ) DMLScript.USE_LOCAL_SPARK_CONFIG = true; config.addVariable("rows", rows); - config.addVariable("cols", cols); + config.addVariable("cols", cols); - long rowstart = 816, rowend = 1229, colstart = 967, colend = 1009; - // long rowstart=2, rowend=4, colstart=9, colend=10; + long rowstart=816, rowend=1229, colstart=967, colend=1009; + // long rowstart=2, rowend=4, colstart=9, colend=10; /* - * Random rand=new Random(System.currentTimeMillis()); rowstart=(long)(rand.nextDouble()*((double)rows))+1; - * rowend=(long)(rand.nextDouble()*((double)(rows-rowstart+1)))+rowstart; - * colstart=(long)(rand.nextDouble()*((double)cols))+1; - * colend=(long)(rand.nextDouble()*((double)(cols-colstart+1)))+colstart; - */ + Random rand=new Random(System.currentTimeMillis()); + rowstart=(long)(rand.nextDouble()*((double)rows))+1; + rowend=(long)(rand.nextDouble()*((double)(rows-rowstart+1)))+rowstart; + colstart=(long)(rand.nextDouble()*((double)cols))+1; + colend=(long)(rand.nextDouble()*((double)(cols-colstart+1)))+colstart; + */ config.addVariable("rowstart", rowstart); config.addVariable("rowend", rowend); config.addVariable("colstart", colstart); @@ -118,20 +119,17 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod double sparsity=1.0;//rand.nextDouble(); double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, System.currentTimeMillis()); writeInputMatrix("A", A, true); - - sparsity = 0.1;// rand.nextDouble(); - double[][] B = getRandomMatrix((int) (rowend - rowstart + 1), (int) (colend - colstart + 1), min, max, - sparsity, System.currentTimeMillis()); + + sparsity=0.1;//rand.nextDouble(); + double[][] B = getRandomMatrix((int)(rowend-rowstart+1), (int)(colend-colstart+1), min, max, sparsity, System.currentTimeMillis()); writeInputMatrix("B", B, true); - - sparsity = 0.5;// rand.nextDouble(); - double[][] C = getRandomMatrix((int) (rowend), (int) (cols - colstart + 1), min, max, sparsity, - System.currentTimeMillis()); + + sparsity=0.5;//rand.nextDouble(); + double[][] C = getRandomMatrix((int)(rowend), (int)(cols-colstart+1), min, max, sparsity, System.currentTimeMillis()); writeInputMatrix("C", C, true); - - sparsity = 0.01;// rand.nextDouble(); - double[][] D = getRandomMatrix(rows, (int) (colend - colstart + 1), min, max, sparsity, - System.currentTimeMillis()); + + sparsity=0.01;//rand.nextDouble(); + double[][] D = getRandomMatrix(rows, (int)(colend-colstart+1), min, max, sparsity, System.currentTimeMillis()); writeInputMatrix("D", D, true); /* @@ -140,9 +138,9 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod * While loop iteration - 10 jobs * Final output write - 1 job */ - // boolean exceptionExpected = false; - // int expectedNumberOfJobs = 12; - // runTest(exceptionExpected, null, expectedNumberOfJobs); + //boolean exceptionExpected = false; + //int expectedNumberOfJobs = 12; + //runTest(exceptionExpected, null, expectedNumberOfJobs); boolean exceptionExpected = false; int expectedNumberOfJobs = -1; runTest(true, exceptionExpected, null, expectedNumberOfJobs); diff --git a/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java b/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java index fad1cc123c8..a1b51ee9d81 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java @@ -81,9 +81,8 @@ public void testDoubleScalarWrite() { fullDMLScriptName = HOME + "ScalarComputeWrite.dml"; runTest(true, false, null, -1); - double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)).doubleValue(); - Assert.assertEquals("Computation test for Doubles failed: Values not equal: " + double_scalar - + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar); + double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1,1)).doubleValue(); + Assert.assertEquals("Computation test for Doubles failed: Values not equal: " + double_scalar + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar); } @Test @@ -123,7 +122,8 @@ public void testIntScalarRead() { programArgs = new String[]{"-args", String.valueOf(int_scalar), output("a.scalar")}; runTest(true, false, null, -1); - int int_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)).intValue(); + int int_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)) + .get(new CellIndex(1,1)).intValue(); Assert.assertEquals("Values not equal: " + int_scalar + Opcodes.NOTEQUAL.toString() + int_out_scalar, int_scalar, int_out_scalar); @@ -143,8 +143,8 @@ public void testDoubleScalarRead() { programArgs = new String[]{ "-args", String.valueOf(double_scalar), output("a.scalar") }; runTest(true, false, null, -1); - double double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)) - .doubleValue(); + double double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)) + .get(new CellIndex(1,1)).doubleValue(); Assert.assertEquals("Values not equal: " + double_scalar + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar, 0); diff --git a/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java b/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java index 20ae7aa63ea..a4013c3672d 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java @@ -35,14 +35,14 @@ /** * End-to-end DML test of the native Delta read/write path. * - *

- * The write and the read are run as two separate SystemDS executions on purpose. If they shared a single - * script/process, SystemDS would reuse the still-materialized in-memory matrix for the subsequent read and never invoke - * {@link org.apache.sysds.runtime.io.ReaderDelta} at all (verified: the cache reports 0 HDFS hits in that case). - * Splitting the executions forces a genuine read from disk, and we additionally assert via {@link CacheStatistics} that - * the read run actually performed HDFS reads (the Delta table + the text reference) rather than serving the matrix from - * cache. - *

+ *

The write and the read are run as two separate SystemDS executions + * on purpose. If they shared a single script/process, SystemDS would reuse the + * still-materialized in-memory matrix for the subsequent read and never invoke + * {@link org.apache.sysds.runtime.io.ReaderDelta} at all (verified: the cache + * reports 0 HDFS hits in that case). Splitting the executions forces a genuine + * read from disk, and we additionally assert via {@link CacheStatistics} that + * the read run actually performed HDFS reads (the Delta table + the text + * reference) rather than serving the matrix from cache.

*/ public class DeltaReadWriteTest extends AutomatedTestBase { @@ -54,8 +54,10 @@ public class DeltaReadWriteTest extends AutomatedTestBase { @Override public void setUp() { TestUtils.clearAssertionInformation(); - addTestConfiguration(WRITE_NAME, new TestConfiguration(TEST_CLASS_DIR, WRITE_NAME, new String[] {"ref"})); - addTestConfiguration(READ_NAME, new TestConfiguration(TEST_CLASS_DIR, READ_NAME, new String[] {"R"})); + addTestConfiguration(WRITE_NAME, + new TestConfiguration(TEST_CLASS_DIR, WRITE_NAME, new String[] { "ref" })); + addTestConfiguration(READ_NAME, + new TestConfiguration(TEST_CLASS_DIR, READ_NAME, new String[] { "R" })); } @Test @@ -82,16 +84,17 @@ private void runDeltaRoundTrip(int rows, int cols, double sparsity) { String deltaPath = output("deltaTable"); String refPath = output("ref"); fullDMLScriptName = HOME + WRITE_NAME + ".dml"; - programArgs = new String[] {"-stats", "-args", String.valueOf(rows), String.valueOf(cols), - String.valueOf(sparsity), deltaPath, refPath}; + programArgs = new String[] { "-stats", "-args", + String.valueOf(rows), String.valueOf(cols), String.valueOf(sparsity), + deltaPath, refPath }; runTest(true, false, null, -1); // the write run must have materialized two matrices to disk (the Delta // table under test + the text reference); WriterDelta genuinely hitting // HDFS is what produces these write-side cache statistics. long hdfsWrites = CacheStatistics.getHDFSWrites(); - assertTrue("expected >= 2 HDFS writes in the write run (delta + reference), got " + hdfsWrites, - hdfsWrites >= 2); + assertTrue("expected >= 2 HDFS writes in the write run (delta + reference), got " + + hdfsWrites, hdfsWrites >= 2); // and a real Delta table (transaction log) must have been created assertTrue("missing Delta transaction log under " + deltaPath, new File(deltaPath, "_delta_log").isDirectory()); @@ -99,18 +102,19 @@ private void runDeltaRoundTrip(int rows, int cols, double sparsity) { // ---- phase 2: fresh execution reads the Delta table and compares ---- getAndLoadTestConfiguration(READ_NAME); fullDMLScriptName = HOME + READ_NAME + ".dml"; - programArgs = new String[] {"-stats", "-args", deltaPath, refPath, output("R")}; + programArgs = new String[] { "-stats", "-args", + deltaPath, refPath, output("R") }; runTest(true, false, null, -1); // the read run must have materialized two matrices from disk (the Delta // table under test + the text reference); a cached/short-circuited read // would report fewer HDFS hits and fail here. long hdfsReads = CacheStatistics.getHDFSHits(); - assertTrue("expected >= 2 HDFS reads in the read run (delta + reference), got " + hdfsReads, - hdfsReads >= 2); + assertTrue("expected >= 2 HDFS reads in the read run (delta + reference), got " + + hdfsReads, hdfsReads >= 2); HashMap R = readDMLMatrixFromOutputDir("R"); - // text-cell output omits exact zeros, so a missing cell means 0.0 + //text-cell output omits exact zeros, so a missing cell means 0.0 double diff = R.getOrDefault(new CellIndex(1, 1), 0.0); double nrow = R.getOrDefault(new CellIndex(1, 2), 0.0); double ncol = R.getOrDefault(new CellIndex(1, 3), 0.0); diff --git a/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java index a844321c249..cc1412b1606 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java @@ -49,9 +49,10 @@ public class FrameParquetSchemaTest extends AutomatedTestBase { @Override public void setUp() { - addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"Rout"})); + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[]{"Rout"})); } + /** * Test for sequential writer and reader * diff --git a/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java b/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java index 6e6e4665f5e..dfb3d8a19de 100644 --- a/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java @@ -42,8 +42,12 @@ */ @net.jcip.annotations.NotThreadSafe public class JMLConnectionTest extends AutomatedTestBase { - public static final String META = "{\"data_type\": \"matrix\",\n" + " \"value_type\": \"double\", \n" - + " \"rows\": 1,\n" + " \"cols\": 1,\n" + " \"nnz\": 1,\n" + " \"format\": \"csv\"}"; + public static final String META = "{\"data_type\": \"matrix\",\n" + + " \"value_type\": \"double\", \n" + + " \"rows\": 1,\n" + + " \"cols\": 1,\n" + + " \"nnz\": 1,\n" + + " \"format\": \"csv\"}"; private final static String TEST_NAME = "JMLConnectionTest"; private final static String TEST_DIR = "functions/jmlc/"; @@ -95,14 +99,12 @@ public void testConnectionInvalidInName() throws DMLException { conn.gatherMemStats(false); Assert.assertFalse(DMLScript.STATISTICS); - try(conn) { - conn.prepareScript("printx('hello')", new String[] {"$inScalar1", null}, new String[] {null}); + try (conn) { + conn.prepareScript("printx('hello')", new String[]{"$inScalar1", null}, new String[]{null}); throw new AssertionError("Test should have thrown a LanguageException"); - } - catch(LanguageException e) { + } catch (LanguageException e) { Assert.assertTrue(e.getMessage().startsWith("Invalid variable names")); - } - finally { + } finally { DMLScript.STATISTICS = oldStat; DMLScript.JMLC_MEM_STATISTICS = oldJMLCStat; } @@ -110,24 +112,21 @@ public void testConnectionInvalidInName() throws DMLException { @Test public void testConnectionParseLanguageException() { - try(Connection conn = new Connection()) { - conn.prepareScript("printx('hello')", new String[] {}, new String[] {}); + try (Connection conn = new Connection()) { + conn.prepareScript("printx('hello')", new String[]{}, new String[]{}); throw new AssertionError("Test should have thrown a DMLException"); - } - catch(DMLException e) { + } catch (DMLException e) { Throwable cause = e.getCause(); - Assert.assertTrue(cause.getMessage().startsWith( - "ERROR: [line 1:0] -> printx('hello') -- function printx is undefined in namespace .builtinNS")); + Assert.assertTrue(cause.getMessage().startsWith("ERROR: [line 1:0] -> printx('hello') -- function printx is undefined in namespace .builtinNS")); } } @Test public void testConnectionParseException() { - try(Connection conn = new Connection()) { - conn.prepareScript("print('hello'", new String[] {}, new String[] {}); + try (Connection conn = new Connection()) { + conn.prepareScript("print('hello'", new String[]{}, new String[]{}); throw new AssertionError("Test should have thrown a ParseException"); - } - catch(Exception e) { + } catch (Exception e) { Assert.assertEquals("ParseException", e.getClass().getSimpleName()); } } @@ -145,11 +144,10 @@ public void testConnectionClose() { @Test public void testReadScriptHDFS() { - try(Connection conn = new Connection()) { + try (Connection conn = new Connection()) { conn.readScript("hdfs://localhost:9000/Test"); - } - catch(IOException e) { - Assert.assertEquals("ConnectException", e.getClass().getSimpleName()); + } catch (IOException e) { + Assert.assertEquals("ConnectException",e.getClass().getSimpleName()); } } diff --git a/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java b/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java index 2178884ef5b..4852220861e 100644 --- a/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java @@ -111,16 +111,17 @@ public void federatedReuse(String test) { // Run reference dml script with normal matrix. Reuse of ba+*. fullDMLScriptName = HOME + test + "Reference.dml"; - programArgs = new String[] {"-stats", "-lineage", "reuse_full", "-nvargs", "X1=" + input("X1"), - "X2=" + input("X2"), "Y1=" + input("Y1"), "Y2=" + input("Y2"), "Z=" + expected("Z")}; + programArgs = new String[] {"-stats", "-lineage", "reuse_full", + "-nvargs", "X1=" + input("X1"), "X2=" + input("X2"), "Y1=" + input("Y1"), + "Y2=" + input("Y2"), "Z=" + expected("Z")}; runTest(true, false, null, -1); long mmCount = Statistics.getCPHeavyHitterCount(Opcodes.MMULT.toString()); // Run actual dml script with federated matrix // The fed workers reuse ba+* fullDMLScriptName = HOME + test + ".dml"; - programArgs = new String[] {"-stats", "-lineage", "reuse_full", "-nvargs", - "X1=" + TestUtils.federatedAddress(port1, input("X1")), + programArgs = new String[] {"-stats","-lineage", "reuse_full", + "-nvargs", "X1=" + TestUtils.federatedAddress(port1, input("X1")), "X2=" + TestUtils.federatedAddress(port2, input("X2")), "Y1=" + TestUtils.federatedAddress(port1, input("Y1")), "Y2=" + TestUtils.federatedAddress(port2, input("Y2")), "r=" + rows, "c=" + cols, "Z=" + output("Z")}; @@ -128,12 +129,12 @@ public void federatedReuse(String test) { long mmCount_fed = Statistics.getCPHeavyHitterCount(Opcodes.MMULT.toString()); long fedMMCount = Statistics.getCPHeavyHitterCount("fed_ba+*"); - // compare results + // compare results compareResults(1e-9); // compare matrix multiplication count - // #federated execution of ba+* = #threads times #non-federated execution of ba+* (after reuse) - Assert.assertTrue("Violated reuse count: " + mmCount_fed + " == " + mmCount * 2, - mmCount_fed == mmCount * 2); // #threads = 2 + // #federated execution of ba+* = #threads times #non-federated execution of ba+* (after reuse) + Assert.assertTrue("Violated reuse count: "+mmCount_fed+" == "+mmCount*2, + mmCount_fed == mmCount * 2); // #threads = 2 switch(test) { case TEST_NAME1: // If the o/p is federated, fed_ba+* will be called everytime diff --git a/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java b/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java index c86eb0f4941..eca3628a89b 100644 --- a/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java @@ -121,8 +121,9 @@ private void runTriUDFReuse(ExecMode execMode) { // Run reference dml script with normal matrix fullDMLScriptName = HOME + TEST_NAME + "Reference.dml"; - programArgs = new String[] {"-lineage", "reuse_full", "-stats", "100", "-args", input("X1"), input("X2"), - input("X3"), input("X4"), Boolean.toString(rowPartitioned).toUpperCase(), expected("S")}; + programArgs = new String[] {"-lineage", "reuse_full", "-stats", "100", "-args", + input("X1"), input("X2"), input("X3"), input("X4"), + Boolean.toString(rowPartitioned).toUpperCase(), expected("S")}; runTest(null); // Run actual dml script with federated matrix diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java index 3ffdfe1d30b..18ca2fbc454 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java @@ -272,79 +272,82 @@ protected void toStringTestHelper(ExecMode platform, String testName, String exp } @Test - public void testPrintWithDecimal() { + public void testPrintWithDecimal(){ String testName = "ToString12"; String decimalPoints = "2"; String value = "22"; String expectedOutput = "22.00\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } + @Test - public void testPrintWithDecimal2() { + public void testPrintWithDecimal2(){ String testName = "ToString12"; String decimalPoints = "2"; String value = "5.244058388023880"; String expectedOutput = "5.24\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } + @Test - public void testPrintWithDecimal3() { + public void testPrintWithDecimal3(){ String testName = "ToString12"; String decimalPoints = "10"; String value = "5.244058388023880"; String expectedOutput = "5.2440583880\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } + @Test - public void testPrintWithDecimal4() { + public void testPrintWithDecimal4(){ String testName = "ToString12"; String decimalPoints = "4"; String value = "5.244058388023880"; String expectedOutput = "5.2441\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } + @Test - public void testPrintWithDecimal5() { + public void testPrintWithDecimal5(){ String testName = "ToString12"; String decimalPoints = "10"; String value = "0.000000008023880"; String expectedOutput = "0.0000000080\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - protected void toStringTestHelper2(ExecMode platform, String testName, String expectedOutput, String decimalPoints, - String value) { + protected void toStringTestHelper2(ExecMode platform, String testName, String expectedOutput, String decimalPoints, String value) { ExecMode platformOld = rtplatform; - + rtplatform = platform; boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - if(rtplatform == ExecMode.SPARK) + if (rtplatform == ExecMode.SPARK) DMLScript.USE_LOCAL_SPARK_CONFIG = true; try { // Create and load test configuration getAndLoadTestConfiguration(testName); String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testName + ".dml"; - programArgs = new String[] {"-args", output(OUTPUT_NAME), value, decimalPoints}; + programArgs = new String[]{"-args", output(OUTPUT_NAME), value, decimalPoints}; // Run DML and R scripts runTest(true, false, null, -1); diff --git a/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java index 26143dc16ee..770c5b7c5bf 100644 --- a/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java @@ -76,9 +76,10 @@ public ReshapeTest(int rlen, int clen, int rows, int cols, boolean rowWise) { @Parameterized.Parameters(name = "{0}x{1} {2}x{3} rowWise {4}") public static Iterable getParams() { - int[][][] dims = {{{1000, 1000}, {1, 1000000}}, // single row/col - {{3000, 4000}, {1500, 8000}}, // partialBlocks - {{2400, 1400}, {800, 4200}} // fullBlocks + int[][][] dims = { + {{1000, 1000}, {1, 1000000}}, // single row/col + {{3000, 4000}, {1500, 8000}}, // partialBlocks + {{2400, 1400}, {800, 4200}} // fullBlocks }; ArrayList params = new ArrayList<>(); @@ -116,8 +117,7 @@ public void runTestMatrixReshapeOOC() { double[][] X = getRandomMatrix(rlen, clen, 0, 1, 1, 7); MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(Types.FileFormat.BINARY); - writer.writeMatrixToHDFS(DataConverter.convertToMatrixBlock(X), input(INPUT_NAME), rlen, clen, 1000, - rlen * clen); + writer.writeMatrixToHDFS(DataConverter.convertToMatrixBlock(X), input(INPUT_NAME), rlen, clen, 1000, rlen * clen); HDFSTool.writeMetaDataFile(input(INPUT_NAME + ".mtd"), Types.ValueType.FP64, new MatrixCharacteristics(rlen, clen, blen, rlen * clen), Types.FileFormat.BINARY); @@ -143,8 +143,8 @@ public void runTestMatrixReshapeOOC() { runTest(true, false, null, -1); // compare results - MatrixBlock actual = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME), Types.FileFormat.BINARY, rows, - cols, blen); + MatrixBlock actual = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME), + Types.FileFormat.BINARY, rows, cols, blen); MatrixBlock expected = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME + "_target"), Types.FileFormat.BINARY, rows, cols, blen); diff --git a/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java index 49a52587cde..5f42db7d733 100644 --- a/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java @@ -335,9 +335,9 @@ private void runTestMatrixReshape( ReshapeType type, boolean rowwise, boolean sp String.valueOf(trows), String.valueOf(tcols), output("Y") }; fullRScriptName = HOME + TEST_NAME + ".R"; - rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + trows + " " + tcols + " " - + expectedDir(); - + rCmd = "Rscript" + " " + fullRScriptName + " " + + inputDir() + " " + trows + " " + tcols + " " + expectedDir(); + double[][] X = getRandomMatrix(rows, cols, 0, 1, sparsity, 7); writeInputMatrix("X", X, true); diff --git a/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java index 69d6958f8a6..dcdafddcd47 100644 --- a/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java @@ -94,9 +94,9 @@ private void runVectorReshape(boolean sparse, ExecType et) String.valueOf(rows2), String.valueOf(cols2), output("R") }; fullRScriptName = HOME + TEST_NAME + ".R"; - rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + rows2 + " " + cols2 + " " - + expectedDir(); - + rCmd = "Rscript" + " " + fullRScriptName + " " + + inputDir() + " " + rows2 + " " + cols2 + " " + expectedDir(); + double sparsity = sparse ? sparsitySparse : sparsityDense; double[][] X = getRandomMatrix(rows1, cols1, 0, 1, sparsity, 7); writeInputMatrixWithMTD("X", X, true); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java index b16554045e4..60b491b8141 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java @@ -151,8 +151,8 @@ private void runTestMatrixChainDP(String testName) { overwriteCurrentConfig(dmlConfig); } catch(FileNotFoundException fnfe) { - Assert.fail( - "Could not find DML config file: " + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); + Assert.fail("Could not find DML config file: " + + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); } catch(IOException ioe) { Assert.fail("Could not overwrite the DML configuration file. " + ioe.getMessage()); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java index 96c479e206d..bf9acd9e52a 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java @@ -123,8 +123,8 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { overwriteCurrentConfig(dmlConfig); } catch(FileNotFoundException fnfe) { - Assert.fail( - "Could not find DML config file: " + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); + Assert.fail("Could not find DML config file: " + + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); } catch(IOException ioe) { Assert.fail("Could not overwrite the DML configuration file. " + ioe.getMessage()); @@ -132,7 +132,8 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + TEST_NAME + ".dml"; - programArgs = new String[] {"-explain", "hops", "-stats", "-args", input("X"), input("Y"), output("R")}; + programArgs = new String[] {"-explain", "hops", "-stats", + "-args", input("X"), input("Y"), output("R")}; fullRScriptName = HOME + TEST_NAME + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java index 15b80e49618..e8e885f905f 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java @@ -74,7 +74,7 @@ public void testRewriteQuantizationFusedCompressionNoRewrite() { /** * Unified method to test both scalar and matrix scale factors. - * + * * @param testname Test name * @param rewrites Whether to enable fusion rewrites * @param isScalar Whether the scale factor is a scalar or a matrix diff --git a/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java b/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java index 39266f5f3d3..30681f373e4 100644 --- a/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java +++ b/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -106,8 +106,7 @@ public void testHash2() throws Exception { @Test public void testHash3() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, - new ValueType[] {ValueType.UINT8, ValueType.INT64, ValueType.UINT8}, 32); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.INT64,ValueType.UINT8}, 32); MatrixBlock expected = new MatrixBlock(1, 7, new double[] {1, 1, 1, 0, 1, 1, 1}); String spec = "{\"ids\": true, \"dummycode\": [1,3], \"hash\": [1,3], \"K\": 3}"; @@ -115,11 +114,11 @@ public void testHash3() throws Exception { } + @Test public void testHybrid1() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, - new ValueType[] {ValueType.UINT8, ValueType.INT64, ValueType.UINT8, ValueType.BOOLEAN}, 32); - MatrixBlock expected = new MatrixBlock(1, 9, new double[] {1, 1, 1, 0, 1, 1, 1, 1, 1}); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.INT64,ValueType.UINT8, ValueType.BOOLEAN}, 32); + MatrixBlock expected = new MatrixBlock(1, 9, new double[] {1, 1, 1, 0, 1, 1, 1,1,1}); String spec = "{\"ids\": true, \"dummycode\": [1,3,4], \"hash\": [1,3], \"K\": 3}"; runTransformTest(fb, spec, expected); @@ -128,9 +127,8 @@ public void testHybrid1() throws Exception { @Test public void testHybrid2() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, - new ValueType[] {ValueType.UINT8, ValueType.BOOLEAN, ValueType.UINT8, ValueType.BOOLEAN}, 32); - MatrixBlock expected = new MatrixBlock(1, 10, new double[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.BOOLEAN,ValueType.UINT8, ValueType.BOOLEAN}, 32); + MatrixBlock expected = new MatrixBlock(1, 10, new double[] {1, 1, 1, 1,1, 1, 1, 1,1,1}); String spec = "{\"ids\": true, \"dummycode\": [1,2,3,4], \"hash\": [1,3], \"K\": 3}"; runTransformTest(fb, spec, expected); @@ -141,7 +139,7 @@ private void runTransformTest(FrameBlock fb, String spec, MatrixBlock expected) try { getAndLoadTestConfiguration(TEST_NAME1); - + String inF = input("F-In"); String inS = input("spec"); diff --git a/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java b/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java index cd28649dc42..8c4ba6ae8ad 100644 --- a/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java +++ b/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java @@ -283,8 +283,7 @@ private String[][] readTwoColumnStringCSV(String s) { out[1][i] = in.getString(i, 1); } return out; - } - catch(IOException e) { + } catch (IOException e) { throw new RuntimeException(e); } } diff --git a/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java b/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java index ae13cbd510f..d3d71d820d6 100644 --- a/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java +++ b/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java @@ -92,7 +92,7 @@ private void runVectorizationTest( String testName, boolean rewrites ) runTest(true, false, null, -1); runRScript(true); - // compare results + //compare results HashMap dmlfile = readDMLMatrixFromOutputDir("R"); HashMap rfile = readRMatrixFromExpectedDir("R"); TestUtils.compareMatrices(dmlfile, rfile, 1e-14, "DML", "R"); From b0124fa60f5edaabff1bcea2cb1683a6ef8bf0ce Mon Sep 17 00:00:00 2001 From: bruno Date: Wed, 2 Sep 2026 10:46:44 +0200 Subject: [PATCH 132/132] dev/format-changed.sh --- .../java/org/apache/sysds/api/DMLScript.java | 10 +- .../org/apache/sysds/common/Builtins.java | 376 +++++------------- .../java/org/apache/sysds/hops/BinaryOp.java | 4 +- src/main/java/org/apache/sysds/hops/Hop.java | 14 +- .../java/org/apache/sysds/hops/UnaryOp.java | 9 +- .../sysds/hops/estim/EstimationUtils.java | 12 +- .../sysds/hops/rewrite/ProgramRewriter.java | 6 +- ...riteMatrixMultChainOptimizationSparse.java | 19 +- .../parser/BuiltinFunctionExpression.java | 24 -- .../apache/sysds/parser/DMLTranslator.java | 87 +--- .../apache/sysds/parser/DataExpression.java | 206 +++------- .../compress/CompressedMatrixBlock.java | 2 +- .../runtime/compress/colgroup/AColGroup.java | 17 +- .../compress/colgroup/AColGroupValue.java | 1 - .../runtime/compress/colgroup/ASDCZero.java | 6 +- .../compress/colgroup/ColGroupDDC.java | 4 +- .../compress/colgroup/ColGroupEmpty.java | 7 +- .../colgroup/ColGroupLinearFunctional.java | 2 +- .../compress/colgroup/ColGroupOLE.java | 2 +- .../compress/colgroup/ColGroupRLE.java | 4 +- .../compress/colgroup/ColGroupSDCFOR.java | 4 +- .../colgroup/ColGroupUncompressed.java | 13 +- .../colgroup/ColGroupUncompressedArray.java | 2 +- .../dictionary/AIdentityDictionary.java | 4 +- .../colgroup/dictionary/DeltaDictionary.java | 4 +- .../colgroup/dictionary/IDictionary.java | 6 +- .../dictionary/IdentityDictionary.java | 4 +- .../dictionary/IdentityDictionarySlice.java | 4 +- .../compress/colgroup/mapping/AMapToData.java | 2 +- .../compress/colgroup/offset/AOffset.java | 10 +- .../compress/colgroup/offset/OffsetEmpty.java | 1 + .../compress/lib/CLALibBinaryCellOp.java | 6 +- .../runtime/compress/lib/CLALibMMChain.java | 2 +- .../compress/lib/CLALibRemoveEmpty.java | 15 +- .../runtime/compress/lib/CLALibSort.java | 8 +- .../controlprogram/caching/FrameObject.java | 4 +- .../controlprogram/caching/MatrixObject.java | 2 +- .../context/SparkExecutionContext.java | 34 +- .../federated/FederatedWorker.java | 3 +- .../frame/data/columns/ArrayFactory.java | 22 +- .../frame/data/lib/MatrixBlockFromFrame.java | 4 +- .../runtime/functionobjects/Builtin.java | 139 +++---- .../instructions/cp/BinaryCPInstruction.java | 2 +- .../cp/BinaryFrameScalarCPInstruction.java | 10 +- .../cp/BinaryMatrixMatrixCPInstruction.java | 4 +- .../cp/ParameterizedBuiltinCPInstruction.java | 7 +- .../instructions/ooc/ReorgOOCInstruction.java | 8 +- .../ooc/ReshapeOOCInstruction.java | 69 ++-- .../spark/QuantilePickSPInstruction.java | 3 +- .../spark/data/IndexedMatrixValue.java | 7 +- .../sysds/runtime/io/DeltaKernelUtils.java | 257 ++++++------ .../apache/sysds/runtime/io/ReaderDelta.java | 107 ++--- .../sysds/runtime/io/ReaderDeltaParallel.java | 96 ++--- .../apache/sysds/runtime/io/WriterDelta.java | 74 ++-- .../runtime/matrix/data/LibMatrixReorg.java | 34 +- .../runtime/matrix/data/MatrixBlock.java | 34 +- .../sysds/runtime/ooc/cache/OOCFuture.java | 9 +- .../runtime/ooc/cache/io/CloseableQueue.java | 38 +- .../cache/io/OOCBufferedDataInputStream.java | 12 +- .../cache/io/OOCBufferedDataOutputStream.java | 20 +- .../runtime/ooc/cache/io/OOCIOHandler.java | 25 +- .../ooc/cache/io/OOCMatrixIOHandler.java | 161 ++++---- .../runtime/ooc/cache/io/SpillableObject.java | 6 +- .../ooc/cache/legacy/OOCCacheScheduler.java | 44 +- .../cache/legacy/OOCLRUCacheScheduler.java | 207 +++++----- .../runtime/transform/decode/Decoder.java | 10 +- .../runtime/transform/decode/DecoderBin.java | 4 +- .../transform/decode/DecoderDummycode.java | 2 +- .../transform/decode/DecoderFactory.java | 53 ++- .../transform/decode/DecoderRecode.java | 18 +- .../sysds/runtime/util/CommonThreadPool.java | 8 +- .../sysds/runtime/util/DataConverter.java | 9 +- .../org/apache/sysds/utils/DoubleParser.java | 2 +- .../apache/sysds/utils/SettingsChecker.java | 13 +- .../org/apache/sysds/performance/Main.java | 7 +- .../apache/sysds/test/AutomatedTestBase.java | 23 +- .../java/org/apache/sysds/test/TestUtils.java | 9 +- .../component/compile/CompilerTestBase.java | 21 +- .../SparkTransitiveExecTypeCompileTest.java | 50 +-- .../compress/CompressedSortTest.java | 6 +- .../compress/lib/CLALibMMChainTest.java | 4 +- ...CompressedBinaryMatrixMatrixSolveTest.java | 15 +- .../OffsetClassInitConcurrencyTest.java | 4 +- .../SparkContextReferenceCountTest.java | 32 +- .../component/federated/FedWorkerBase.java | 19 +- .../federated/FedWorkerMatrixCompress.java | 8 +- .../component/frame/FrameToStringTest.java | 14 +- .../frame/MatrixFromFrameSafeCastTest.java | 12 +- .../frame/transform/DecoderCompositeTest.java | 8 +- .../GetCategoricalMaskInstructionTest.java | 21 +- .../TransformDecodeRoundTripTest.java | 39 +- .../frame/transform/TransformDecodeTest.java | 4 +- .../component/io/DeltaMatrixCoverageTest.java | 97 +++-- .../io/DeltaMatrixReadWriteTest.java | 333 ++++++++++------ .../io/DeltaMatrixSparkInteropTest.java | 105 +++-- .../component/matrix/QuantilePickTest.java | 13 +- .../component/tensor/TensorToStringTest.java | 14 +- .../functions/binary/matrix/QuantileTest.java | 18 +- .../builtin/part2/BuiltinSTEPGlmTest.java | 3 +- .../FederatedBackendPerformanceTest.java | 6 +- .../part4/FederatedLogicalTest.java | 5 +- .../functions/indexing/LeftIndexingTest.java | 48 +-- .../sysds/test/functions/io/ScalarIOTest.java | 12 +- .../io/delta/DeltaReadWriteTest.java | 40 +- .../io/parquet/FrameParquetSchemaTest.java | 3 +- .../functions/jmlc/JMLConnectionTest.java | 42 +- .../functions/lineage/FedFullReuseTest.java | 17 +- .../functions/lineage/FedUDFReuseTest.java | 5 +- .../test/functions/misc/ToStringTest.java | 33 +- .../sysds/test/functions/ooc/ReshapeTest.java | 14 +- .../functions/reorg/MatrixReshapeTest.java | 6 +- .../functions/reorg/VectorReshapeTest.java | 6 +- .../rewrite/RewriteMatrixChainDPTest.java | 4 +- .../RewriteMatrixMultChainOptSparseTest.java | 7 +- ...writeQuantizationFusedCompressionTest.java | 2 +- .../transform/GetCategoricalMaskTest.java | 20 +- .../TransformFrameEncodeBagOfWords.java | 3 +- .../vect/LeftIndexingChainUpdateTest.java | 2 +- 118 files changed, 1654 insertions(+), 1958 deletions(-) diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java b/src/main/java/org/apache/sysds/api/DMLScript.java index a7a175bb7b6..0bb1e9b462d 100644 --- a/src/main/java/org/apache/sysds/api/DMLScript.java +++ b/src/main/java/org/apache/sysds/api/DMLScript.java @@ -508,9 +508,9 @@ private static void execute(String dmlScriptStr, String fnameOptConfig, Map inHops1 = new ArrayList<>(); - inHops1.add(expr); - inHops1.add(expr2); - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), inHops1); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - case AVG_POOL: - case MAX_POOL: { - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForPoolingForwardIM2COL(expr, source, 1, hops)); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - case AVG_POOL_BACKWARD: - case MAX_POOL_BACKWARD: { - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForConvOpPoolingCOL2IM(expr, source, 1, hops)); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - case CONV2D: - case CONV2D_BACKWARD_FILTER: - case CONV2D_BACKWARD_DATA: { - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForConvOp(expr, source, 1, hops)); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - - case ROW_COUNT_DISTINCT: - currBuiltinOp = new AggUnaryOp(target.getName(), - DataType.MATRIX, target.getValueType(), AggOp.COUNT_DISTINCT, Direction.Row, expr); - break; - - case COL_COUNT_DISTINCT: - currBuiltinOp = new AggUnaryOp(target.getName(), - DataType.MATRIX, target.getValueType(), AggOp.COUNT_DISTINCT, Direction.Col, expr); - break; - - case GET_CATEGORICAL_MASK: - currBuiltinOp = new BinaryOp(target.getName(), DataType.MATRIX, ValueType.FP64, OpOp2.GET_CATEGORICAL_MASK, expr, expr2); - break; - default: - throw new ParseException("Unsupported builtin function type: "+source.getOpCode()); - } - - boolean isConvolution = source.getOpCode() == Builtins.CONV2D || source.getOpCode() == Builtins.CONV2D_BACKWARD_DATA || - source.getOpCode() == Builtins.CONV2D_BACKWARD_FILTER || - source.getOpCode() == Builtins.MAX_POOL || source.getOpCode() == Builtins.MAX_POOL_BACKWARD || - source.getOpCode() == Builtins.AVG_POOL || source.getOpCode() == Builtins.AVG_POOL_BACKWARD; - if( !isConvolution) { + boolean isConvolution = source.getOpCode() == Builtins.CONV2D || + source.getOpCode() == Builtins.CONV2D_BACKWARD_DATA || + source.getOpCode() == Builtins.CONV2D_BACKWARD_FILTER || source.getOpCode() == Builtins.MAX_POOL || + source.getOpCode() == Builtins.MAX_POOL_BACKWARD || source.getOpCode() == Builtins.AVG_POOL || + source.getOpCode() == Builtins.AVG_POOL_BACKWARD; + if(!isConvolution) { // Since the dimension of output doesnot match that of input variable for these operations setIdentifierParams(currBuiltinOp, source.getOutput()); } diff --git a/src/main/java/org/apache/sysds/parser/DataExpression.java b/src/main/java/org/apache/sysds/parser/DataExpression.java index 68a3d1b7ffe..3d3a90b4f6f 100644 --- a/src/main/java/org/apache/sysds/parser/DataExpression.java +++ b/src/main/java/org/apache/sysds/parser/DataExpression.java @@ -1176,52 +1176,72 @@ else if( getVarParam(READNNZPARAM) != null ) { boolean isHDF5 = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.HDF5.toString())); - boolean isCOG = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.COG.toString())); + // handle all csv default parameters + handleCSVDefaultParam(DELIM_DELIMITER, ValueType.STRING, conditional); + handleCSVDefaultParam(DELIM_FILL_VALUE, ValueType.FP64, conditional); + handleCSVDefaultParam(DELIM_HAS_HEADER_ROW, ValueType.BOOLEAN, conditional); + handleCSVDefaultParam(DELIM_FILL, ValueType.BOOLEAN, conditional); + handleCSVDefaultParam(DELIM_NA_STRINGS, ValueType.STRING, conditional); + } - // Delta tables are self-describing (schema + dimensions discovered from the - // transaction log at read time), so dimensions are optional like CSV. - boolean isDelta = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.DELTA.toString())); + boolean isLIBSVM = false; + isLIBSVM = (formatTypeString != null && + formatTypeString.equalsIgnoreCase(FileFormat.LIBSVM.toString())); + if(isLIBSVM) { + // Handle libsvm file format + shouldReadMTD = true; + + // only allow IO_FILENAME, READROWPARAM, READCOLPARAM + // as valid parameters + if(!inferredFormatType) { + for(String key : _varParams.keySet()) { + if(!(key.equals(IO_FILENAME) || key.equals(FORMAT_TYPE) || key.equals(READROWPARAM) || + key.equals(READCOLPARAM) || key.equals(READNNZPARAM) || key.equals(DATATYPEPARAM) || + key.equals(VALUETYPEPARAM) || key.equals(DELIM_DELIMITER) || + key.equals(LIBSVM_INDEX_DELIM))) { + String msg = "Only parameters allowed are: " + IO_FILENAME + "," + READROWPARAM + "," + + READCOLPARAM + DELIM_DELIMITER + "," + LIBSVM_INDEX_DELIM; + + raiseValidateError( + "Invalid parameter " + key + " in read statement: " + toString() + ". " + msg, + conditional, LanguageErrorCodes.INVALID_PARAMETERS); + } + } + } + // handle all default parameters + handleCSVDefaultParam(DELIM_DELIMITER, ValueType.STRING, conditional); + handleCSVDefaultParam(LIBSVM_INDEX_DELIM, ValueType.STRING, conditional); + } - dataTypeString = (getVarParam(DATATYPEPARAM) == null) ? null : getVarParam(DATATYPEPARAM).toString(); - - if ( dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE) - || dataTypeString.equalsIgnoreCase(Statement.FRAME_DATA_TYPE)) { - - boolean isMatrix = false; - if ( dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE)) + boolean isHDF5 = (formatTypeString != null && + formatTypeString.equalsIgnoreCase(FileFormat.HDF5.toString())); + + boolean isCOG = (formatTypeString != null && + formatTypeString.equalsIgnoreCase(FileFormat.COG.toString())); + + // Delta tables are self-describing (schema + dimensions discovered from the + // transaction log at read time), so dimensions are optional like CSV. + boolean isDelta = (formatTypeString != null && + formatTypeString.equalsIgnoreCase(FileFormat.DELTA.toString())); + + dataTypeString = (getVarParam(DATATYPEPARAM) == null) ? null : getVarParam(DATATYPEPARAM).toString(); + + if(dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE) || + dataTypeString.equalsIgnoreCase(Statement.FRAME_DATA_TYPE)) { + + boolean isMatrix = false; + if(dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE)) isMatrix = true; - - // set data type - getOutput().setDataType(isMatrix ? DataType.MATRIX : DataType.FRAME); - - // set number non-zeros - Expression ennz = getVarParam("nnz"); - long nnz = -1; - if( ennz != null ) { - nnz = Long.valueOf(ennz.toString()); - getOutput().setNnz(nnz); - } - // Following dimension checks must be done when data type = MATRIX_DATA_TYPE - // initialize size of target data identifier to UNKNOWN - getOutput().setDimensions(-1, -1); - - if (!isCSV && !isLIBSVM && !isHDF5 && !isCOG && !isDelta && ConfigurationManager.getCompilerConfig() - .getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) //skip check for csv/libsvm/delta format / jmlc api - && (getVarParam(READROWPARAM) == null || getVarParam(READCOLPARAM) == null) ) { - raiseValidateError("Missing or incomplete dimension information in read statement: " - + mtdFileName, conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - - if (getVarParam(READROWPARAM) instanceof ConstIdentifier - && getVarParam(READCOLPARAM) instanceof ConstIdentifier) - { - // these are strings that are long values - Long dim1 = (getVarParam(READROWPARAM) == null) ? null : Long.valueOf( getVarParam(READROWPARAM).toString()); - Long dim2 = (getVarParam(READCOLPARAM) == null) ? null : Long.valueOf( getVarParam(READCOLPARAM).toString()); - if ( !isCSV && !isDelta && (dim1 < 0 || dim2 < 0) && ConfigurationManager - .getCompilerConfig().getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) ) { - raiseValidateError("Invalid dimension information in read statement", conditional, LanguageErrorCodes.INVALID_PARAMETERS); + // set data type + getOutput().setDataType(isMatrix ? DataType.MATRIX : DataType.FRAME); + + // set number non-zeros + Expression ennz = getVarParam("nnz"); + long nnz = -1; + if(ennz != null) { + nnz = Long.valueOf(ennz.toString()); + getOutput().setNnz(nnz); } // set dim1 and dim2 values @@ -1252,104 +1272,10 @@ && getVarParam(READCOLPARAM) instanceof ConstIdentifier) catch(Exception ex) { raiseValidateError("Invalid format '" + fmt+ "' in statement: " + toString(), conditional); } - - if (getVarParam(ROWBLOCKCOUNTPARAM) instanceof ConstIdentifier && getVarParam(COLUMNBLOCKCOUNTPARAM) instanceof ConstIdentifier) { - Integer rowBlockCount = (getVarParam(ROWBLOCKCOUNTPARAM) == null) ? - null : Integer.valueOf(getVarParam(ROWBLOCKCOUNTPARAM).toString()); - getOutput().setBlocksize(rowBlockCount != null ? rowBlockCount : -1); - } - - // block dimensions must be -1x-1 when format="text" - // NOTE MB: disabled validate of default blocksize for inputs w/ format="binary" - // because we automatically introduce reblocks if blocksizes don't match - if ( (getOutput().getFileFormat().isTextFormat() || !isMatrix) && getOutput().getBlocksize() != -1 ){ - raiseValidateError("Invalid block dimensions (" + getOutput().getBlocksize() + ") when format=" + getVarParam(FORMAT_TYPE) + " in \"" + this.toString() + "\".", conditional); - } - - } - else if ( dataTypeString.equalsIgnoreCase(Statement.SCALAR_DATA_TYPE)) { - getOutput().setDataType(DataType.SCALAR); - getOutput().setNnz(-1L); - } - else if ( dataTypeString.equalsIgnoreCase(DataType.LIST.name())) { - getOutput().setDataType(DataType.LIST); - } - else{ - raiseValidateError("Unknown Data Type " + dataTypeString + ". Valid values: " - + Statement.SCALAR_DATA_TYPE +", " + Statement.MATRIX_DATA_TYPE+", " + Statement.FRAME_DATA_TYPE - +", " + DataType.LIST.name().toLowerCase(), conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - - // handle value type parameter - if (getVarParam(VALUETYPEPARAM) != null && !(getVarParam(VALUETYPEPARAM) instanceof StringIdentifier)){ - raiseValidateError("for read method, parameter " + VALUETYPEPARAM + " can only be a string. " + - "Valid values are: " + Statement.DOUBLE_VALUE_TYPE +", " + Statement.INT_VALUE_TYPE + ", " + Statement.BOOLEAN_VALUE_TYPE + ", " + Statement.STRING_VALUE_TYPE, conditional); - } - // Identify the value type (used only for read method) - String valueTypeString = getVarParam(VALUETYPEPARAM) == null ? null : getVarParam(VALUETYPEPARAM).toString(); - if (valueTypeString != null) { - if (valueTypeString.equalsIgnoreCase(Statement.DOUBLE_VALUE_TYPE)) - getOutput().setValueType(ValueType.FP64); - else if (valueTypeString.equalsIgnoreCase(Statement.STRING_VALUE_TYPE)) - getOutput().setValueType(ValueType.STRING); - else if (valueTypeString.equalsIgnoreCase(Statement.INT_VALUE_TYPE)) - getOutput().setValueType(ValueType.INT64); - else if (valueTypeString.equalsIgnoreCase(Statement.BOOLEAN_VALUE_TYPE)) - getOutput().setValueType(ValueType.BOOLEAN); - else if (valueTypeString.equalsIgnoreCase(ValueType.UNKNOWN.name())) - getOutput().setValueType(ValueType.UNKNOWN); - else { - raiseValidateError("Unknown Value Type " + valueTypeString - + ". Valid values are: " + Statement.DOUBLE_VALUE_TYPE +", " + Statement.INT_VALUE_TYPE + ", " + Statement.BOOLEAN_VALUE_TYPE + ", " + Statement.STRING_VALUE_TYPE, conditional); - } - } else { - getOutput().setValueType(ValueType.FP64); - } - - break; - - case WRITE: - - // for CSV format, if no delimiter specified THEN set default "," - if (getVarParam(FORMAT_TYPE) == null || checkFormatType(FileFormat.CSV) ){ - if (getVarParam(DELIM_DELIMITER) == null) { - addVarParam(DELIM_DELIMITER, new StringIdentifier(DEFAULT_DELIM_DELIMITER, this)); - } - if (getVarParam(DELIM_HAS_HEADER_ROW) == null) { - addVarParam(DELIM_HAS_HEADER_ROW, new BooleanIdentifier(DEFAULT_DELIM_HAS_HEADER_ROW, this)); - } - if (getVarParam(DELIM_SPARSE) == null) { - addVarParam(DELIM_SPARSE, new BooleanIdentifier(DEFAULT_DELIM_SPARSE, this)); - } - } - - // for LIBSVM format, add the default separators if not specified - if (getVarParam(FORMAT_TYPE) == null || checkFormatType(FileFormat.LIBSVM)) { - if(getVarParam(DELIM_DELIMITER) == null) { - addVarParam(DELIM_DELIMITER, new StringIdentifier(DEFAULT_DELIM_DELIMITER, this)); - } - if(getVarParam(LIBSVM_INDEX_DELIM) == null) { - addVarParam(LIBSVM_INDEX_DELIM, new StringIdentifier(DEFAULT_LIBSVM_INDEX_DELIM, this)); - } - if(getVarParam(DELIM_SPARSE) == null) { - addVarParam(DELIM_SPARSE, new BooleanIdentifier(DEFAULT_DELIM_SPARSE, this)); - } - } - - //validate read filename - if (getVarParam(FORMAT_TYPE) == null || FileFormat.isTextFormat(getVarParam(FORMAT_TYPE).toString()) - || checkFormatType(FileFormat.DELTA)) //delta: columnar, no block layout - getOutput().setBlocksize(-1); - else if (checkFormatType(FileFormat.BINARY, FileFormat.COMPRESSED, FileFormat.UNKNOWN)) { - if( getVarParam(ROWBLOCKCOUNTPARAM)!=null ) - getOutput().setBlocksize(Integer.parseInt(getVarParam(ROWBLOCKCOUNTPARAM).toString())); - else - getOutput().setBlocksize(ConfigurationManager.getBlocksize()); - } - else if( getVarParam(FORMAT_TYPE) instanceof StringIdentifier ) //literal format - raiseValidateError("Invalid format " + getVarParam(FORMAT_TYPE) - + " in statement: " + toString(), conditional); - break; + else if(getVarParam(FORMAT_TYPE) instanceof StringIdentifier) // literal format + raiseValidateError("Invalid format " + getVarParam(FORMAT_TYPE) + " in statement: " + toString(), + conditional); + break; case RAND: diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java index d0ba5363939..042e0dc0328 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java @@ -484,7 +484,7 @@ public static CompressedMatrixBlock read(DataInput in) throws IOException { long nonZeros = in.readLong(); boolean overlappingColGroups = in.readBoolean(); List groups = ColGroupIO.readGroups(in, rlen); - CompressedMatrixBlock ret = new CompressedMatrixBlock(rlen, clen, nonZeros, overlappingColGroups, groups); + CompressedMatrixBlock ret = new CompressedMatrixBlock(rlen, clen, nonZeros, overlappingColGroups, groups); LOG.debug("Compressed read serialization time: " + t.stop()); return ret; } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java index 354325e293b..66d4e78cb0f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java @@ -402,7 +402,8 @@ public final AColGroup rightMultByMatrix(MatrixBlock right) { * @param cru The right hand side column upper * @param nRows The number of rows in this column group */ - public void rightDecompressingMult(MatrixBlock right, MatrixBlock ret, int rl, int ru, int nRows, int crl, int cru) { + public void rightDecompressingMult(MatrixBlock right, MatrixBlock ret, int rl, int ru, int nRows, int crl, + int cru) { throw new NotImplementedException( "not supporting right Decompressing Multiply on class: " + this.getClass().getSimpleName()); } @@ -977,9 +978,9 @@ public AColGroup[] splitReshapePushDown(final int multiplier, final int nRow, fi /** * Sort the values of the column group according to double comparison operations and return as another compressed * group. - * + * * This sorting assumes that the column group is sorted independently of everything else. - * + * * @return The sorted group */ public abstract AColGroup sort(); @@ -996,9 +997,9 @@ public String toString() { /** * Return a new column group containing only the selected rows in the given boolean vector. - * + * * Whenever possible only modify the index structure, not the dictionary of the column groups. - * + * * @param selectV The selection vector * @param rOut The number of rows in the output * @return The new column group @@ -1007,9 +1008,9 @@ public String toString() { /** * Return a new column group containing only the selected columns in the given boolean vector. - * + * * Whenever possible only modify the column index, and reduce the dictionaries of the column groups. - * + * * @param selectV The selection vector * @return The new column group, or {@code null} if no column of this group is selected */ @@ -1045,7 +1046,7 @@ public AColGroup removeEmptyCols(boolean[] selectV) { /** * Using the selection of columns, slice out those and return in a new column group with the given column indexes. * Ideally this method should only modify the dictionaries. - * + * * @param newColumnIDs the new column indexes * @param selectedColumns The selected columns of this column group (guaranteed < current number of columns) * @return A new Column group diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java index d825b91f089..d610c1b586c 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java @@ -210,7 +210,6 @@ public void clear() { counts = null; } - @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java index 30de5e120c5..794d90c0d11 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java @@ -212,8 +212,8 @@ public void decompressToSparseBlock(SparseBlock sb, int rl, int ru, int offR, in // TODO make sparse decompression where the iterator is known in argument decompressToSparseBlockSparseDictionary(sb, rl, ru, offR, offC, mb.getSparseBlock()); else - decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, mb.getDenseBlockValues(), - it); + decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, + mb.getDenseBlockValues(), it); } else decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, _dict.getValues(), it); @@ -240,7 +240,7 @@ public void decompressToDenseBlockDenseDictionary(DenseBlock db, int rl, int ru, } public abstract void decompressToSparseBlockDenseDictionaryWithProvidedIterator(SparseBlock db, int rl, int ru, - int offR, int offC, double[] values, AIterator it); + int offR, int offC, double[] values, AIterator it); public abstract void decompressToDenseBlockDenseDictionaryWithProvidedIterator(DenseBlock db, int rl, int ru, int offR, int offC, double[] values, AIterator it); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java index b316e48474a..d643cae440c 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java @@ -674,8 +674,8 @@ private void defaultRightDecompressingMult(MatrixBlock right, MatrixBlock ret, i } } - final void vectMM(double aa, double[] b, double[] c, int endT, int jd, int crl, int cru, int offOut, int k, int vLen, - DoubleVector vVec) { + final void vectMM(double aa, double[] b, double[] c, int endT, int jd, int crl, int cru, int offOut, int k, + int vLen, DoubleVector vVec) { vVec = vVec.broadcast(aa); final int offj = k * jd; final int end = endT + offj; diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java index 64114a054ab..d5ad55772c7 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java @@ -478,14 +478,13 @@ public AColGroup combineWithSameIndex(int nRow, int nCol, List right) return new ColGroupEmpty(combinedIndex); } - @Override - public AColGroup removeEmptyRows(boolean[] selectV, int rOut){ + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { return this; } - @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { return new ColGroupEmpty(newColumnIDs); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java index fa8aa104ffb..e0bea3c3696 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java @@ -747,7 +747,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java index a251d828b5f..b4f0c144a73 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java @@ -738,7 +738,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java index 347cea9c0da..43df7fa3b94 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java @@ -1195,9 +1195,9 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { throw new NotImplementedException("Unimplemented method 'removeEmptyRows'"); } - + @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java index 815ecacf378..4566106a3e2 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java @@ -634,8 +634,8 @@ protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList s for(int i = 0; i < selectedColumns.size(); i++) { ref[i] = _reference[selectedColumns.get(i)]; } - return ColGroupSDCFOR.create(newColumnIDs, _numRows, _dict.sliceColumns(selectedColumns, getNumCols()), _indexes, _data, null, - ref); + return ColGroupSDCFOR.create(newColumnIDs, _numRows, _dict.sliceColumns(selectedColumns, getNumCols()), + _indexes, _data, null, ref); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java index 611add6480f..9797087f8c3 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java @@ -85,7 +85,7 @@ public class ColGroupUncompressed extends AColGroup { /** * Do not use this constructor of column group uncompressed, instead use the create constructor. - * + * * @param mb The contained data. * @param colIndexes Column indexes for this Columngroup */ @@ -96,9 +96,10 @@ protected ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes) { /** * Do not use this constructor of column group quantization-fused uncompressed, instead use the create constructor. - * + * * @param mb The contained data. - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire + * matrix * @param colIndexes Column indexes for this Columngroup */ protected ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes, double[] scaleFactors) { @@ -138,7 +139,8 @@ public static AColGroup create(MatrixBlock mb, IColIndex colIndexes) { * * @param mb The MB / data to contain in the uncompressed column * @param colIndexes The column indexes for the group - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire + * matrix * @return An Uncompressed Column group */ public static AColGroup createQuantized(MatrixBlock mb, IColIndex colIndexes, double[] scaleFactors) { @@ -157,7 +159,8 @@ public static AColGroup createQuantized(MatrixBlock mb, IColIndex colIndexes, do * @param rawBlock The uncompressed block; uncompressed data must be present at the time that the constructor is * called * @param transposed Says if the input matrix raw block have been transposed. - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire + * matrix * @return AColGroup. */ public static AColGroup createQuantized(IColIndex colIndexes, MatrixBlock rawBlock, boolean transposed, diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java index 51e26a3f9d2..de8a740ceb2 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java @@ -290,7 +290,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java index a7e715b59b8..6e66ef6ef9b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java @@ -76,8 +76,8 @@ public double[] productAllRowsToDoubleWithDefault(double[] defaultTuple) { return ret; } - @Override - public int[] sort(){ + @Override + public int[] sort() { throw new NotImplementedException(); } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java index 9a0412145f0..7ebba2f1a76 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java @@ -138,8 +138,8 @@ public IDictionary clone() { throw new NotImplementedException(); } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { throw new NotImplementedException(); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java index c8ddfc4883a..b5e1a99355b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java @@ -1055,7 +1055,7 @@ public IDictionary rightMMPreAggSparse(int numVals, SparseBlock b, IColIndex thi /** * Slice out the selected columns given of this encoded group. - * + * * @param selectedColumns The columns to slice out and return as a new matrix. * @param nCol The number of columns in this dictionary. * @return The returned matrix @@ -1064,9 +1064,9 @@ public IDictionary rightMMPreAggSparse(int numVals, SparseBlock b, IColIndex thi /** * Sort the values of this dictionary via an index of how the values mapped previously. - * + * * In practice this design means we can reuse the previous dictionary for the resulting column group - * + * * @return The sorted index. */ public int[] sort(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java index c2540de959a..4337da7307f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java @@ -541,8 +541,8 @@ public String getString(int colIndexes) { return "IdentityMatrix of size: " + nRowCol + " with empty: " + withEmpty; } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { return getMBDict().sliceColumns(selectedColumns, nCol); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java index c7f642edfd0..47628b43d2a 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java @@ -311,8 +311,8 @@ public String getString(int colIndexes) { return toString(); } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { return getMBDict().sliceColumns(selectedColumns, nCol); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java index 83a74972db7..6d516713689 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java @@ -1064,7 +1064,7 @@ public AMapToData removeEmpty(final boolean[] selectV, final int rOut) { /** * Use the offsets of the select vector to choose which values to keep. - * + * * @param select The row indexes to keep * @return A New MapToData */ diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java index f65876b7f37..bf8ee7f9ee1 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java @@ -56,11 +56,11 @@ public abstract class AOffset implements Serializable { protected static final Log LOG = LogFactory.getLog(AOffset.class.getName()); /** - * Lazy holder for the cached empty slice. The empty slice is built on first use rather than in AOffset's - * static initializer: instantiating the OffsetEmpty subclass from AOffset's {@code } forms a - * superclass/subclass class-initialization cycle that deadlocks when several threads first touch the offset - * classes concurrently (e.g. parallel tests). Deferring it to first use guarantees AOffset is already - * initialized by the time OffsetEmpty is loaded, so no cycle exists. + * Lazy holder for the cached empty slice. The empty slice is built on first use rather than in AOffset's static + * initializer: instantiating the OffsetEmpty subclass from AOffset's {@code } forms a superclass/subclass + * class-initialization cycle that deadlocks when several threads first touch the offset classes concurrently (e.g. + * parallel tests). Deferring it to first use guarantees AOffset is already initialized by the time OffsetEmpty is + * loaded, so no cycle exists. */ private static final class EmptySliceHolder { static final OffsetSliceInfo EMPTY_SLICE = new OffsetSliceInfo(-1, -1, new OffsetEmpty()); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java index 866168ded2f..37ff41cf817 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java @@ -76,6 +76,7 @@ public int getOffsetToLast() { public long getInMemorySize() { return estimateInMemorySize(); } + @Override public boolean equals(AOffset b) { return b instanceof OffsetEmpty; diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java index d981ab87838..7953322350e 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java @@ -139,7 +139,8 @@ private static boolean isDoubleCompressedOpApplicable(CompressedMatrixBlock m1, m1.getColGroups().get(0) instanceof ColGroupDDC && !((CompressedMatrixBlock) that).isOverlapping() && ((CompressedMatrixBlock) that).getColGroups().get(0) instanceof ColGroupDDC && ((IMapToDataGroup) m1.getColGroups().get(0)) - .getMapToData() == ((IMapToDataGroup) ((CompressedMatrixBlock) that).getColGroups().get(0)).getMapToData(); + .getMapToData() == ((IMapToDataGroup) ((CompressedMatrixBlock) that).getColGroups().get(0)) + .getMapToData(); } private static CompressedMatrixBlock doubleCompressedBinaryOp(BinaryOperator op, CompressedMatrixBlock m1, @@ -1062,7 +1063,8 @@ public Long call() { return _ret.recomputeNonZeros(_rl, _ru - 1); } - private final void processBlock(final int rl, final int ru, final List groups, final AIterator[] its) { + private final void processBlock(final int rl, final int ru, final List groups, + final AIterator[] its) { decompressToTmpBlock(rl, ru, tmp.getSparseBlock(), groups, its); // decompressing multiple column groups can leave the temp rows with unsorted column indices, so sort // before reading them in stored order into the (column-sorted) output sparse block. diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java index cc7953f8c5d..a91b75ae73c 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java @@ -96,7 +96,7 @@ public static MatrixBlock mmChain(CompressedMatrixBlock x, MatrixBlock v, Matrix if(x.isEmpty()) return returnEmpty(x, out); - if(ctype == ChainType.XtXv && x.getColGroups().size() < 5 && x.getNumColumns()> 30){ + if(ctype == ChainType.XtXv && x.getColGroups().size() < 5 && x.getNumColumns() > 30) { MatrixBlock tmp = CLALibTSMM.leftMultByTransposeSelf(x, k); return tmp.aggregateBinaryOperations(tmp, v, out, InstructionUtils.getMatMultOperator(k)); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java index 3755e4040e7..802eddffcb8 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java @@ -36,7 +36,7 @@ public class CLALibRemoveEmpty { /** * CP rmempty operation (single input, single output matrix) - * + * * @param in The input matrix * @param ret The output matrix * @param rows If we are removing based on rows, or columns. @@ -66,13 +66,13 @@ private static MatrixBlock rmEmptyCols(CompressedMatrixBlock in, MatrixBlock ret int cOut = (int) select.getNonZeros(); if(cOut == -1) cOut = (int) select.recomputeNonZeros(); - if(cOut == 0){ + if(cOut == 0) { ret.reset(in.getNumRows(), !emptyReturn ? 0 : 1); return ret; } - final boolean[] selectV = DataConverter - .convertToBooleanVector(CompressedMatrixBlock.getUncompressed(select, "decompressing selection in rmempty")); + final boolean[] selectV = DataConverter.convertToBooleanVector( + CompressedMatrixBlock.getUncompressed(select, "decompressing selection in rmempty")); final List inG = in.getColGroups(); final List retG = new ArrayList<>(inG.size()); @@ -102,18 +102,17 @@ private static MatrixBlock rmEmptyRows(CompressedMatrixBlock in, MatrixBlock ret int rOut = (int) select.getNonZeros(); if(rOut == -1) rOut = (int) select.recomputeNonZeros(); - if(rOut == 0){ + if(rOut == 0) { ret.reset(!emptyReturn ? 0 : 1, in.getNumColumns()); return ret; } - // TODO: add optimization to avoid linear scan and make selectV indexes, if selection is small relative to number + // TODO: add optimization to avoid linear scan and make selectV indexes, if selection is small relative to + // number // of rows // TODO: add decompress to boolean vector. final boolean[] selectV = DataConverter.convertToBooleanVector(select); - - final List inG = in.getColGroups(); final List retG = new ArrayList<>(inG.size()); try { diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java index b94f11ae723..5ae7bd5103b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java @@ -40,10 +40,10 @@ private CLALibSort() { /** * Sort (order) a compressed matrix in place of the {@code order} built-in, while keeping the result compressed. * - * The compressed fast-path only supports the case the user can benefit from: a single column held in a single column - * group, sorted ascending and returning the sorted values (not the index permutation). For everything else (multiple - * columns, multiple column groups, descending order, index return, or a column-group encoding without a sort - * implementation) this returns {@code null} so the caller can fall back to a decompressed reorg. + * The compressed fast-path only supports the case the user can benefit from: a single column held in a single + * column group, sorted ascending and returning the sorted values (not the index permutation). For everything else + * (multiple columns, multiple column groups, descending order, index return, or a column-group encoding without a + * sort implementation) this returns {@code null} so the caller can fall back to a decompressed reorg. * * @param mb the compressed matrix to sort * @param fn the sort specification carried by the reorg operator diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java index 87d14dbf87e..9ccaa474f39 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java @@ -208,8 +208,8 @@ protected FrameBlock readBlobFromHDFS(String fname, long[] dims) throws IOExcept if(data == null) throw new IOException("Unable to load frame from file: " + fname); - //Delta and CSV discover dimensions (and Delta also schema) at read time, so - //refresh the cached metadata to reflect the materialized frame block. + // Delta and CSV discover dimensions (and Delta also schema) at read time, so + // refresh the cached metadata to reflect the materialized frame block. if(iimd.getFileFormat() == FileFormat.CSV || iimd.getFileFormat() == FileFormat.DELTA) { _metaData = _metaData instanceof MetaDataFormat ? new MetaDataFormat(data.getDataCharacteristics(), iimd.getFileFormat()) : new MetaData(data.getDataCharacteristics()); diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java index 28fa70f7741..4331da2b426 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java @@ -454,7 +454,7 @@ protected MatrixBlock readBlobFromHDFS(String fname, long[] dims) throws IOExcep rlen, clen, blen, mc.getNonZeros(), getFileFormatProperties()); if(iimd.getFileFormat() == FileFormat.CSV || iimd.getFileFormat() == FileFormat.DELTA) { - //dimensions/nnz are discovered at read time for these self-describing formats + // dimensions/nnz are discovered at read time for these self-describing formats _metaData = _metaData instanceof MetaDataFormat ? new MetaDataFormat(newData.getDataCharacteristics(), iimd.getFileFormat()) : new MetaData(newData.getDataCharacteristics()); } diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java index b52f3777e1f..fbae4925c66 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java @@ -122,9 +122,9 @@ public class SparkExecutionContext extends ExecutionContext //singleton spark context (as there can be only one spark context per JVM) private static JavaSparkContext _spctx = null; - //registered users of the singleton context (guarded by the - //SparkExecutionContext.class monitor); maintained by enterSparkExecution()/ - //exitSparkExecution(), and close() only stops the context once it hits zero + // registered users of the singleton context (guarded by the + // SparkExecutionContext.class monitor); maintained by enterSparkExecution()/ + // exitSparkExecution(), and close() only stops the context once it hits zero private static int _activeExecutions = 0; //registry of parallelized RDDs to enforce that at any time, we spent at most @@ -175,8 +175,8 @@ public synchronized static JavaSparkContext getSparkContextStatic() { initSparkContext(); if(_spctx.sc().isStopped()){ _spctx = null; - //the previous context was stopped; reset the active-execution count so a - //stale registration cannot skip a future legitimate stop of the new one + // the previous context was stopped; reset the active-execution count so a + // stale registration cannot skip a future legitimate stop of the new one _activeExecutions = 0; initSparkContext(); } @@ -196,16 +196,15 @@ public synchronized static boolean isSparkContextCreated() { public static void resetSparkContextStatic() { synchronized(SparkExecutionContext.class) { _spctx = null; - //force-discarding the shared context: drop the active-execution count so - //a stale registration cannot skip a future legitimate stop + // force-discarding the shared context: drop the active-execution count so + // a stale registration cannot skip a future legitimate stop _activeExecutions = 0; } } /** - * Registers an active user of the shared spark context. Must be balanced by a - * later {@link #exitSparkExecution()} so a concurrent execution cannot stop the - * context while this one still has in-flight jobs. + * Registers an active user of the shared spark context. Must be balanced by a later {@link #exitSparkExecution()} + * so a concurrent execution cannot stop the context while this one still has in-flight jobs. */ public static void enterSparkExecution() { synchronized(SparkExecutionContext.class) { @@ -214,9 +213,8 @@ public static void enterSparkExecution() { } /** - * Releases an active user previously registered via {@link #enterSparkExecution()}. - * Only adjusts the count; the actual teardown is left to {@link #close()}, which - * stops the context once no registered execution remains. + * Releases an active user previously registered via {@link #enterSparkExecution()}. Only adjusts the count; the + * actual teardown is left to {@link #close()}, which stops the context once no registered execution remains. */ public static void exitSparkExecution() { synchronized(SparkExecutionContext.class) { @@ -227,13 +225,13 @@ public static void exitSparkExecution() { public void close() { synchronized(SparkExecutionContext.class) { - //keep the shared context alive while a registered execution still uses - //it; close() never changes the count, so an unpaired close() (a caller - //that never entered) cannot stop a context another execution is using + // keep the shared context alive while a registered execution still uses + // it; close() never changes the count, so an unpaired close() (a caller + // that never entered) cannot stop a context another execution is using if(_activeExecutions > 0) { if(LOG.isDebugEnabled()) - LOG.debug("Keeping shared spark context alive; " + _activeExecutions - + " execution(s) still active"); + LOG.debug( + "Keeping shared spark context alive; " + _activeExecutions + " execution(s) still active"); return; } if(_spctx != null) { diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java index 682cc8e3fff..c502817e026 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java @@ -95,8 +95,7 @@ private void run() { int par_conn = ConfigurationManager.getDMLConfig().getIntValue(DMLConfig.FEDERATED_PAR_CONN); final int EVENT_LOOP_THREADS = (par_conn > 0) ? par_conn : InfrastructureAnalyzer.getLocalParallelism(); // Daemon event loops so a leaked in-JVM (test) worker cannot block JVM exit. - NioEventLoopGroup bossGroup = new NioEventLoopGroup(1, - new DefaultThreadFactory("fed-worker-boss", true)); + NioEventLoopGroup bossGroup = new NioEventLoopGroup(1, new DefaultThreadFactory("fed-worker-boss", true)); ThreadPoolExecutor workerTPE = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 10, TimeUnit.SECONDS, new SynchronousQueue(true), new DefaultThreadFactory("fed-worker-pool", true)); NioEventLoopGroup workerGroup = new NioEventLoopGroup(EVENT_LOOP_THREADS, workerTPE); diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java index 80a5d699dfa..ebf05972b87 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java @@ -125,13 +125,15 @@ public static RaggedArray create(T[] col, int m) { /** * Wrap a fully populated raw typed column array into an {@link Array} of the given value type. The runtime type of - * {@code col} must match the primitive backing type of {@code vt} (e.g. {@code double[]} for {@link ValueType#FP64}, - * {@code String[]} for {@link ValueType#STRING}). + * {@code col} must match the primitive backing type of {@code vt} (e.g. {@code double[]} for + * {@link ValueType#FP64}, {@code String[]} for {@link ValueType#STRING}). * - *

For {@link ValueType#BOOLEAN} this mirrors {@link #allocateBoolean(int)}: a {@code boolean[]} longer than - * {@link #bitSetSwitchPoint} is bit-packed into a compact {@link BitSetArray} (so a bulk decoder that fills a - * plain {@code boolean[]} still ends up with the same representation as every other frame allocation path), - * while shorter columns stay a plain {@link BooleanArray}.

+ *

+ * For {@link ValueType#BOOLEAN} this mirrors {@link #allocateBoolean(int)}: a {@code boolean[]} longer than + * {@link #bitSetSwitchPoint} is bit-packed into a compact {@link BitSetArray} (so a bulk decoder that fills a plain + * {@code boolean[]} still ends up with the same representation as every other frame allocation path), while shorter + * columns stay a plain {@link BooleanArray}. + *

* * @param vt the value type of the column * @param col the backing array to wrap @@ -168,10 +170,10 @@ public static Array create(ValueType vt, Object col) { /** * Allocate the raw backing array for a column of the given value type: the inverse of - * {@link #create(ValueType, Object)}. Returns {@code double[]} for {@link ValueType#FP64}, - * {@code int[]} for INT32/UINT/HASH32, {@code long[]} for INT64/HASH64, {@code String[]} for STRING, etc. The - * runtime array type matches what {@link #create(ValueType, Object)} expects, so a bulk decoder can fill this - * primitive array directly and then wrap it via {@code create(vt, backing)}. + * {@link #create(ValueType, Object)}. Returns {@code double[]} for {@link ValueType#FP64}, {@code int[]} for + * INT32/UINT/HASH32, {@code long[]} for INT64/HASH64, {@code String[]} for STRING, etc. The runtime array type + * matches what {@link #create(ValueType, Object)} expects, so a bulk decoder can fill this primitive array directly + * and then wrap it via {@code create(vt, backing)}. * * @param vt the value type of the column * @param nRow the number of rows to allocate diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java b/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java index 9ff58065d97..95be95117e2 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java @@ -41,7 +41,7 @@ public class MatrixBlockFromFrame { public static Boolean WARNED_FOR_FAILED_CAST = false; - private MatrixBlockFromFrame(){ + private MatrixBlockFromFrame() { // private constructor for code coverage. } @@ -115,7 +115,7 @@ private static long convert(FrameBlock frame, MatrixBlock mb, int n, int rl, int return convertStrict(frame, mb, n, rl, ru); } catch(NumberFormatException | DMLRuntimeException e) { - synchronized(WARNED_FOR_FAILED_CAST){ + synchronized(WARNED_FOR_FAILED_CAST) { if(!WARNED_FOR_FAILED_CAST) { LOG.error( "Failed to convert to Matrix because of number format errors, falling back to NaN on incompatible cells", diff --git a/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java b/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java index eed2c58f78c..c12a187cf17 100644 --- a/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java +++ b/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java @@ -30,32 +30,13 @@ import jdk.incubator.vector.VectorSpecies; -/** - * Class with pre-defined set of objects. This class can not be instantiated elsewhere. - * - * Notes on commons.math FastMath: - * * FastMath uses lookup tables and interpolation instead of native calls. - * * The memory overhead for those tables is roughly 48KB in total (acceptable) - * * Micro and application benchmarks showed significantly (30%-3x) performance improvements - * for most operations; without loss of accuracy. - * * atan / sqrt were 20% slower in FastMath and hence, we use Math there - * * round / abs were equivalent in FastMath and hence, we use Math there - * * Finally, there is just one argument against FastMath - The comparison heavily depends - * on the JVM. For example, currently the IBM JDK JIT compiles to HW instructions for sqrt - * which makes this operation very efficient; as soon as other operations like log/exp are - * similarly compiled, we should rerun the micro benchmarks, and switch back if necessary. - * - */ -public class Builtin extends ValueFunction -{ - private static final long serialVersionUID = 3836744687789840574L; - - public enum BuiltinCode { AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, ATAN, LOG, LOG_NZ, MIN, - MAX, ABS, SIGN, SQRT, EXP, PLOGP, PRINT, PRINTF, NROW, NCOL, LENGTH, LINEAGE, ROUND, MAXINDEX, MININDEX, - STOP, CEIL, FLOOR, CUMSUM, ROWCUMSUM, CUMPROD, CUMMIN, CUMMAX, CUMSUMPROD, INVERSE, SPROP, SIGMOID, EVAL, LIST, - TYPEOF, APPLY_SCHEMA, DETECTSCHEMA, ISNA, ISNAN, ISINF, DROP_INVALID_TYPE, - DROP_INVALID_LENGTH, VALUE_SWAP, FRAME_ROW_REPLICATE, GET_CATEGORICAL_MASK, - MAP, COUNT_DISTINCT, COUNT_DISTINCT_APPROX, UNIQUE} + public enum BuiltinCode { + AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, ATAN, LOG, LOG_NZ, MIN, MAX, ABS, SIGN, SQRT, EXP, PLOGP, + PRINT, PRINTF, NROW, NCOL, LENGTH, LINEAGE, ROUND, MAXINDEX, MININDEX, STOP, CEIL, FLOOR, CUMSUM, ROWCUMSUM, + CUMPROD, CUMMIN, CUMMAX, CUMSUMPROD, INVERSE, SPROP, SIGMOID, EVAL, LIST, TYPEOF, APPLY_SCHEMA, DETECTSCHEMA, + ISNA, ISNAN, ISINF, DROP_INVALID_TYPE, DROP_INVALID_LENGTH, VALUE_SWAP, FRAME_ROW_REPLICATE, + GET_CATEGORICAL_MASK, MAP, COUNT_DISTINCT, COUNT_DISTINCT_APPROX, UNIQUE + } private static final VectorSpecies SPECIES = DoubleVector.SPECIES_PREFERRED; private static final int vLen = SPECIES.length(); @@ -68,59 +49,59 @@ public enum BuiltinCode { AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, static public HashMap String2BuiltinCode; static { String2BuiltinCode = new HashMap<>(); - String2BuiltinCode.put( "autoDiff" , BuiltinCode.AUTODIFF); - String2BuiltinCode.put( "sin" , BuiltinCode.SIN); - String2BuiltinCode.put( "cos" , BuiltinCode.COS); - String2BuiltinCode.put( "tan" , BuiltinCode.TAN); - String2BuiltinCode.put( "sinh" , BuiltinCode.SINH); - String2BuiltinCode.put( "cosh" , BuiltinCode.COSH); - String2BuiltinCode.put( "tanh" , BuiltinCode.TANH); - String2BuiltinCode.put( "asin" , BuiltinCode.ASIN); - String2BuiltinCode.put( "acos" , BuiltinCode.ACOS); - String2BuiltinCode.put( "atan" , BuiltinCode.ATAN); - String2BuiltinCode.put( "log" , BuiltinCode.LOG); - String2BuiltinCode.put( "log_nz" , BuiltinCode.LOG_NZ); - String2BuiltinCode.put( "min" , BuiltinCode.MIN); - String2BuiltinCode.put( "max" , BuiltinCode.MAX); - String2BuiltinCode.put( "maxindex", BuiltinCode.MAXINDEX); - String2BuiltinCode.put( "minindex", BuiltinCode.MININDEX); - String2BuiltinCode.put( "abs" , BuiltinCode.ABS); - String2BuiltinCode.put( "sign" , BuiltinCode.SIGN); - String2BuiltinCode.put( "sqrt" , BuiltinCode.SQRT); - String2BuiltinCode.put( "exp" , BuiltinCode.EXP); - String2BuiltinCode.put( "plogp" , BuiltinCode.PLOGP); - String2BuiltinCode.put( "print" , BuiltinCode.PRINT); - String2BuiltinCode.put( "printf" , BuiltinCode.PRINTF); - String2BuiltinCode.put( "eval" , BuiltinCode.EVAL); - String2BuiltinCode.put( "list" , BuiltinCode.LIST); - String2BuiltinCode.put( "nrow" , BuiltinCode.NROW); - String2BuiltinCode.put( "ncol" , BuiltinCode.NCOL); - String2BuiltinCode.put( "length" , BuiltinCode.LENGTH); - String2BuiltinCode.put( "round" , BuiltinCode.ROUND); - String2BuiltinCode.put( "stop" , BuiltinCode.STOP); - String2BuiltinCode.put( "ceil" , BuiltinCode.CEIL); - String2BuiltinCode.put( "floor" , BuiltinCode.FLOOR); - String2BuiltinCode.put( "ucumk+" , BuiltinCode.CUMSUM); - String2BuiltinCode.put( "urowcumk+" , BuiltinCode.ROWCUMSUM); - String2BuiltinCode.put( "ucum*" , BuiltinCode.CUMPROD); - String2BuiltinCode.put( "ucumk+*", BuiltinCode.CUMSUMPROD); - String2BuiltinCode.put( "ucummin", BuiltinCode.CUMMIN); - String2BuiltinCode.put( "ucummax", BuiltinCode.CUMMAX); - String2BuiltinCode.put( "inverse", BuiltinCode.INVERSE); - String2BuiltinCode.put( "sprop", BuiltinCode.SPROP); - String2BuiltinCode.put( "sigmoid", BuiltinCode.SIGMOID); - String2BuiltinCode.put( "typeOf", BuiltinCode.TYPEOF); - String2BuiltinCode.put( "detectSchema", BuiltinCode.DETECTSCHEMA); - String2BuiltinCode.put( "isna", BuiltinCode.ISNA); - String2BuiltinCode.put( "isnan", BuiltinCode.ISNAN); - String2BuiltinCode.put( "isinf", BuiltinCode.ISINF); - String2BuiltinCode.put( "dropInvalidType", BuiltinCode.DROP_INVALID_TYPE); - String2BuiltinCode.put( "freplicate", BuiltinCode.FRAME_ROW_REPLICATE); - String2BuiltinCode.put( "dropInvalidLength", BuiltinCode.DROP_INVALID_LENGTH); - String2BuiltinCode.put( "_map", BuiltinCode.MAP); - String2BuiltinCode.put( "valueSwap", BuiltinCode.VALUE_SWAP); - String2BuiltinCode.put( "applySchema", BuiltinCode.APPLY_SCHEMA); - String2BuiltinCode.put( "get_categorical_mask", BuiltinCode.GET_CATEGORICAL_MASK); + String2BuiltinCode.put("autoDiff", BuiltinCode.AUTODIFF); + String2BuiltinCode.put("sin", BuiltinCode.SIN); + String2BuiltinCode.put("cos", BuiltinCode.COS); + String2BuiltinCode.put("tan", BuiltinCode.TAN); + String2BuiltinCode.put("sinh", BuiltinCode.SINH); + String2BuiltinCode.put("cosh", BuiltinCode.COSH); + String2BuiltinCode.put("tanh", BuiltinCode.TANH); + String2BuiltinCode.put("asin", BuiltinCode.ASIN); + String2BuiltinCode.put("acos", BuiltinCode.ACOS); + String2BuiltinCode.put("atan", BuiltinCode.ATAN); + String2BuiltinCode.put("log", BuiltinCode.LOG); + String2BuiltinCode.put("log_nz", BuiltinCode.LOG_NZ); + String2BuiltinCode.put("min", BuiltinCode.MIN); + String2BuiltinCode.put("max", BuiltinCode.MAX); + String2BuiltinCode.put("maxindex", BuiltinCode.MAXINDEX); + String2BuiltinCode.put("minindex", BuiltinCode.MININDEX); + String2BuiltinCode.put("abs", BuiltinCode.ABS); + String2BuiltinCode.put("sign", BuiltinCode.SIGN); + String2BuiltinCode.put("sqrt", BuiltinCode.SQRT); + String2BuiltinCode.put("exp", BuiltinCode.EXP); + String2BuiltinCode.put("plogp", BuiltinCode.PLOGP); + String2BuiltinCode.put("print", BuiltinCode.PRINT); + String2BuiltinCode.put("printf", BuiltinCode.PRINTF); + String2BuiltinCode.put("eval", BuiltinCode.EVAL); + String2BuiltinCode.put("list", BuiltinCode.LIST); + String2BuiltinCode.put("nrow", BuiltinCode.NROW); + String2BuiltinCode.put("ncol", BuiltinCode.NCOL); + String2BuiltinCode.put("length", BuiltinCode.LENGTH); + String2BuiltinCode.put("round", BuiltinCode.ROUND); + String2BuiltinCode.put("stop", BuiltinCode.STOP); + String2BuiltinCode.put("ceil", BuiltinCode.CEIL); + String2BuiltinCode.put("floor", BuiltinCode.FLOOR); + String2BuiltinCode.put("ucumk+", BuiltinCode.CUMSUM); + String2BuiltinCode.put("urowcumk+", BuiltinCode.ROWCUMSUM); + String2BuiltinCode.put("ucum*", BuiltinCode.CUMPROD); + String2BuiltinCode.put("ucumk+*", BuiltinCode.CUMSUMPROD); + String2BuiltinCode.put("ucummin", BuiltinCode.CUMMIN); + String2BuiltinCode.put("ucummax", BuiltinCode.CUMMAX); + String2BuiltinCode.put("inverse", BuiltinCode.INVERSE); + String2BuiltinCode.put("sprop", BuiltinCode.SPROP); + String2BuiltinCode.put("sigmoid", BuiltinCode.SIGMOID); + String2BuiltinCode.put("typeOf", BuiltinCode.TYPEOF); + String2BuiltinCode.put("detectSchema", BuiltinCode.DETECTSCHEMA); + String2BuiltinCode.put("isna", BuiltinCode.ISNA); + String2BuiltinCode.put("isnan", BuiltinCode.ISNAN); + String2BuiltinCode.put("isinf", BuiltinCode.ISINF); + String2BuiltinCode.put("dropInvalidType", BuiltinCode.DROP_INVALID_TYPE); + String2BuiltinCode.put("freplicate", BuiltinCode.FRAME_ROW_REPLICATE); + String2BuiltinCode.put("dropInvalidLength", BuiltinCode.DROP_INVALID_LENGTH); + String2BuiltinCode.put("_map", BuiltinCode.MAP); + String2BuiltinCode.put("valueSwap", BuiltinCode.VALUE_SWAP); + String2BuiltinCode.put("applySchema", BuiltinCode.APPLY_SCHEMA); + String2BuiltinCode.put("get_categorical_mask", BuiltinCode.GET_CATEGORICAL_MASK); } protected Builtin(BuiltinCode bf) { diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java index 86184f47be6..08d28512d5c 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java @@ -59,7 +59,7 @@ else if (in1.getDataType() == DataType.TENSOR && in2.getDataType() == DataType.T return new BinaryTensorTensorCPInstruction(operator, in1, in2, out, opcode, str); else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.FRAME) return new BinaryFrameFrameCPInstruction(operator, in1, in2, out, opcode, str); - else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.SCALAR) + else if(in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.SCALAR) return new BinaryFrameScalarCPInstruction(operator, in1, in2, out, opcode, str); else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.MATRIX) return new BinaryFrameMatrixCPInstruction(operator, in1, in2, out, opcode, str); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java index 193894fd9bc..de76fca18b8 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java @@ -37,8 +37,8 @@ public class BinaryFrameScalarCPInstruction extends BinaryCPInstruction { // private static final Log LOG = LogFactory.getLog(BinaryFrameFrameCPInstruction.class.getName()); - private static final TfMethod[] UNSUPPORTED_MASK_METHODS = new TfMethod[] {TfMethod.BIN, - TfMethod.WORD_EMBEDDING, TfMethod.BAG_OF_WORDS, TfMethod.UDF}; + private static final TfMethod[] UNSUPPORTED_MASK_METHODS = new TfMethod[] {TfMethod.BIN, TfMethod.WORD_EMBEDDING, + TfMethod.BAG_OF_WORDS, TfMethod.UDF}; protected BinaryFrameScalarCPInstruction(MultiThreadedOperator op, CPOperand in1, CPOperand in2, CPOperand out, String opcode, String istr) { @@ -96,9 +96,9 @@ public void processGetCategorical(ExecutionContext ec, FrameBlock f, ScalarObjec } /** - * Accumulates, per input column, how many output columns it expands to (lengths) and whether those - * output columns are categorical (categorical). The arrays are allocated lazily: a column that no - * method touches keeps the implicit default of a single, non-categorical output column. + * Accumulates, per input column, how many output columns it expands to (lengths) and whether those output columns + * are categorical (categorical). The arrays are allocated lazily: a column that no method touches keeps the + * implicit default of a single, non-categorical output column. */ private static final class CategoricalMask { private final FrameBlock f; diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java index d76dbe0d45e..2c8093c3717 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java @@ -80,10 +80,10 @@ public void processInstruction(ExecutionContext ec) { retBlock = inBlock1; } else { - if(LibCommonsMath.isSupportedMatrixMatrixOperation(getOpcode()) ){ + if(LibCommonsMath.isSupportedMatrixMatrixOperation(getOpcode())) { if(compressedLeft) inBlock1 = CompressedMatrixBlock.getUncompressed(inBlock1, getOpcode()); - + if(compressedRight) inBlock2 = CompressedMatrixBlock.getUncompressed(inBlock2, getOpcode()); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java index e53958ac4b8..97ae151ecc0 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java @@ -350,9 +350,10 @@ else if(opcode.equalsIgnoreCase(Opcodes.TRANSFORMDECODE.toString())) { String[] colnames = meta.getColumnNames(); // compute transformdecode - Decoder decoder = DecoderFactory - .createDecoder(getParameterMap().get("spec"), colnames, null, meta, data.getNumColumns()); - FrameBlock fbout = decoder.decode(data, new FrameBlock(decoder.getSchema()), InfrastructureAnalyzer.getLocalParallelism()); + Decoder decoder = DecoderFactory.createDecoder(getParameterMap().get("spec"), colnames, null, meta, + data.getNumColumns()); + FrameBlock fbout = decoder.decode(data, new FrameBlock(decoder.getSchema()), + InfrastructureAnalyzer.getLocalParallelism()); fbout.setColumnNames(Arrays.copyOfRange(colnames, 0, fbout.getNumColumns())); // release locks diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java index 40a677e5d71..04353806ca3 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java @@ -45,8 +45,8 @@ protected ReorgOOCInstruction(ReorgOperator op, CPOperand in1, CPOperand out, St this(op, in1, out, null, null, null, opcode, istr); } - private ReorgOOCInstruction(Operator op, CPOperand in, CPOperand out, CPOperand col, CPOperand desc, CPOperand ixret, - String opcode, String istr) { + private ReorgOOCInstruction(Operator op, CPOperand in, CPOperand out, CPOperand col, CPOperand desc, + CPOperand ixret, String opcode, String istr) { super(OOCType.Reorg, op, in, out, opcode, istr); _col = col; _desc = desc; @@ -76,8 +76,8 @@ else if(opcode.equalsIgnoreCase(Opcodes.SORT.toString())) { CPOperand desc = new CPOperand(parts[3]); CPOperand ixret = new CPOperand(parts[4]); int k = Integer.parseInt(parts[6]); - return new ReorgOOCInstruction(new ReorgOperator(new SortIndex(1, false, false), k), - in, out, col, desc, ixret, opcode, str); + return new ReorgOOCInstruction(new ReorgOperator(new SortIndex(1, false, false), k), in, out, col, desc, + ixret, opcode, str); } else throw new NotImplementedException(); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java index 7590438b949..091c6785b3c 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java @@ -128,17 +128,20 @@ public void processInstruction(ExecutionContext ec) { long r = tmp.getIndexes().getRowIndex(); long c = tmp.getIndexes().getColumnIndex(); // adapt index to new position in row - return new IndexedMatrixValue(new MatrixIndexes(1, (r - 1) * numBlocksPerRowIn + c), tmp.getValue()); + return new IndexedMatrixValue(new MatrixIndexes(1, (r - 1) * numBlocksPerRowIn + c), + tmp.getValue()); }); } else { f.join(); - reshapeFullColBlocks(rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, singleRowBlocks, qOut); + reshapeFullColBlocks(rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, + singleRowBlocks, qOut); } } else { f.join(); - reshapePartialColBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, singleRowBlocks, qOut); + reshapePartialColBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, + numBlocksPerColOut, singleRowBlocks, qOut); } } else { @@ -166,17 +169,20 @@ public void processInstruction(ExecutionContext ec) { long r = tmp.getIndexes().getRowIndex(); long c = tmp.getIndexes().getColumnIndex(); // adapt index to new position in col - return new IndexedMatrixValue(new MatrixIndexes((c - 1) * numBlocksPerColIn + r, 1), tmp.getValue()); + return new IndexedMatrixValue(new MatrixIndexes((c - 1) * numBlocksPerColIn + r, 1), + tmp.getValue()); }); } else { f.join(); - reshapeFullRowBlocks(rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, singleColBlocks, qOut); + reshapeFullRowBlocks(rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, + singleColBlocks, qOut); } } else { f.join(); - reshapePartialRowBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, singleColBlocks, qOut); + reshapePartialRowBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, + numBlocksPerColOut, singleColBlocks, qOut); } } } @@ -226,7 +232,8 @@ private void reshapeFullColBlocks(long rows, long cols, int blen, int numBlocksP } private void reshapePartialColBlocks(long rlen, long clen, long rows, long cols, int blen, int numBlocksPerRowIn, - int numBlocksPerRowOut, int numBlocksPerColOut, OOCStream singleRowBlocks, OOCStream qOut) { + int numBlocksPerRowOut, int numBlocksPerColOut, OOCStream singleRowBlocks, + OOCStream qOut) { // use cache for accessing input rows by index CachingStream singleRowBlockCache = new CachingStream(singleRowBlocks); singleRowBlockCache.incrSubscriberCount(1); @@ -237,14 +244,16 @@ private void reshapePartialColBlocks(long rlen, long clen, long rows, long cols, int r = 0; // allocate row of output blocks - MatrixBlock[] outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut,true); + MatrixBlock[] outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, + true); int offsetOut = 0; int localColsOut = (cols > blen) ? blen : (int) cols; // iterate through input rows and add to row of output blocks for(int i = 1; i <= rlen; i++) { for(int j = 1; j <= numBlocksPerRowIn; j++) { - try(OOCStream.QueueCallback qcb = singleRowBlockCache.findCached(new MatrixIndexes(i, j))) { + try(OOCStream.QueueCallback qcb = singleRowBlockCache + .findCached(new MatrixIndexes(i, j))) { MatrixBlock blk = (MatrixBlock) qcb.get().getValue(); int offsetIn = 0; @@ -278,10 +287,12 @@ else if(remIn == remOut) { if(r == outputBlockRow[0].getNumRows()) { // enqueue filled output blocks and allocate new ones for(int b = 0; b < outputBlockRow.length; b++) - qOut.enqueue(new IndexedMatrixValue(new MatrixIndexes(br + 1, b + 1), outputBlockRow[b])); + qOut.enqueue( + new IndexedMatrixValue(new MatrixIndexes(br + 1, b + 1), outputBlockRow[b])); br++; // allocate new block row - outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, true); + outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, + numBlocksPerColOut, true); r = 0; } bc = 0; @@ -341,7 +352,8 @@ private void reshapeFullRowBlocks(long rows, long cols, int blen, int numBlocksP } private void reshapePartialRowBlocks(long rlen, long clen, long rows, long cols, int blen, int numBlocksPerRowOut, - int numBlocksPerColIn, int numBlocksPerColOut, OOCStream singleColBlocks, OOCStream qOut) { + int numBlocksPerColIn, int numBlocksPerColOut, OOCStream singleColBlocks, + OOCStream qOut) { // use cache for accessing input cols by index CachingStream singleRowBlockCache = new CachingStream(singleColBlocks); singleRowBlockCache.incrSubscriberCount(1); @@ -352,14 +364,16 @@ private void reshapePartialRowBlocks(long rlen, long clen, long rows, long cols, int c = 0; // allocate col of output blocks - MatrixBlock[] outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, false); + MatrixBlock[] outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, + false); int offsetOut = 0; int localRowsOut = (rows > blen) ? blen : (int) rows; // iterate through input cols and add to col of output blocks for(int j = 1; j <= clen; j++) { for(int i = 1; i <= numBlocksPerColIn; i++) { - try(OOCStream.QueueCallback qcb = singleRowBlockCache.findCached(new MatrixIndexes(i, j))) { + try(OOCStream.QueueCallback qcb = singleRowBlockCache + .findCached(new MatrixIndexes(i, j))) { MatrixBlock blk = (MatrixBlock) qcb.get().getValue(); int offsetIn = 0; @@ -394,11 +408,13 @@ else if(remIn == remOut) { // enqueue filled output blocks and allocate new ones for(int b = 0; b < outputBlockCol.length; b++) { outputBlockCol[b].recomputeNonZeros(); - qOut.enqueue(new IndexedMatrixValue(new MatrixIndexes(b + 1, bc + 1), outputBlockCol[b])); + qOut.enqueue( + new IndexedMatrixValue(new MatrixIndexes(b + 1, bc + 1), outputBlockCol[b])); } bc++; // allocate new block col - outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, false); + outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, + numBlocksPerColOut, false); c = 0; } br = 0; @@ -411,16 +427,20 @@ else if(remIn == remOut) { qOut.closeInput(); } - private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int blen, int numBlocksPerRow, int numBlocksPerCol, boolean isBlockRowSlice) { + private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int blen, int numBlocksPerRow, + int numBlocksPerCol, boolean isBlockRowSlice) { int num = isBlockRowSlice ? numBlocksPerRow : numBlocksPerCol; MatrixBlock[] res = new MatrixBlock[num]; // full inner blocks, adjust for outer blocks - int localRows = ((!isBlockRowSlice || idx == numBlocksPerCol - 1) && rows % blen != 0) ? (int) rows % blen : blen; - int localCols = ((isBlockRowSlice || idx == numBlocksPerRow - 1) && cols % blen != 0) ? (int) cols % blen : blen; + int localRows = ((!isBlockRowSlice || idx == numBlocksPerCol - 1) && rows % blen != 0) ? (int) rows % + blen : blen; + int localCols = ((isBlockRowSlice || idx == numBlocksPerRow - 1) && cols % blen != 0) ? (int) cols % + blen : blen; for(int k = 0; k < num - 1; k++) { - res[k] = isBlockRowSlice ? new MatrixBlock(localRows, blen, false) : new MatrixBlock(blen, localCols, false); + res[k] = isBlockRowSlice ? new MatrixBlock(localRows, blen, false) : new MatrixBlock(blen, localCols, + false); res[k].allocateDenseBlock(); } res[num - 1] = new MatrixBlock(localRows, localCols, false); @@ -428,10 +448,13 @@ private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int ble return res; } - private void setOutputEntries(MatrixBlock src, MatrixBlock dest, int idx, int srcOffset, int destOffset, int length, boolean rowWise) { + private void setOutputEntries(MatrixBlock src, MatrixBlock dest, int idx, int srcOffset, int destOffset, int length, + boolean rowWise) { if(rowWise) - ((DenseBlockFP64) dest.getDenseBlock()).setPartialRow(src.getDenseBlock(), idx, srcOffset, destOffset, length); + ((DenseBlockFP64) dest.getDenseBlock()).setPartialRow(src.getDenseBlock(), idx, srcOffset, destOffset, + length); else - ((DenseBlockFP64) dest.getDenseBlock()).setPartialCol(src.getDenseBlock(), idx, srcOffset, destOffset, length); + ((DenseBlockFP64) dest.getDenseBlock()).setPartialCol(src.getDenseBlock(), idx, srcOffset, destOffset, + length); } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java index 75f84882478..e25219b80ba 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java @@ -114,8 +114,7 @@ public void processInstruction(ExecutionContext ec) { case VALUEPICK: { if( input2.isScalar() ) { ScalarObject quantile = ec.getScalarInput(input2); - double[] wt = getWeightedQuantileSummary(in, mc, - new double[] {quantile.getDoubleValue()}, true); + double[] wt = getWeightedQuantileSummary(in, mc, new double[] {quantile.getDoubleValue()}, true); ec.setScalarOutput(output.getName(), new DoubleObject(wt[3])); } else { diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java index 2f83caa5526..f007558ebdc 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java @@ -30,8 +30,7 @@ import org.apache.sysds.runtime.matrix.data.MatrixValue; import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; -public class IndexedMatrixValue implements SpillableObject, Serializable -{ +public class IndexedMatrixValue implements SpillableObject, Serializable { private static final long serialVersionUID = 6723389820806752110L; private MatrixIndexes _indexes = null; @@ -110,8 +109,8 @@ public void discard() { @Override public void read(DataInput dataInput) throws IOException { - _indexes = new MatrixIndexes(); - _value = new MatrixBlock(); + _indexes = new MatrixIndexes(); + _value = new MatrixBlock(); _indexes.readFields(dataInput); _value.readFields(dataInput); } diff --git a/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java index c3b9351d3d3..cc8491d4515 100644 --- a/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java +++ b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java @@ -87,11 +87,10 @@ import io.delta.kernel.utils.FileStatus; /** - * Shared helpers for the native (Spark-free) Delta Lake read/write paths used - * by both the matrix and frame readers/writers. Centralizes engine creation, - * path qualification, the scan loop (snapshot -> data files -> logical - * columnar batches, honoring deletion vectors), and the write transaction - * (logical data -> parquet -> commit). + * Shared helpers for the native (Spark-free) Delta Lake read/write paths used by both the matrix and frame + * readers/writers. Centralizes engine creation, path qualification, the scan loop (snapshot -> data files -> + * logical columnar batches, honoring deletion vectors), and the write transaction (logical data -> parquet -> + * commit). */ public class DeltaKernelUtils { @@ -102,23 +101,29 @@ public class DeltaKernelUtils { /** Reused thread-safe JSON reader for the per-file Delta stats (numRecords). */ private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); - /** Delta Kernel config key: number of rows per parquet read batch, overridable via - * {@link org.apache.sysds.conf.DMLConfig#DELTA_READER_BATCH_SIZE}. */ + /** + * Delta Kernel config key: number of rows per parquet read batch, overridable via + * {@link org.apache.sysds.conf.DMLConfig#DELTA_READER_BATCH_SIZE}. + */ private static final String CONF_READER_BATCH_SIZE = "delta.kernel.default.parquet.reader.batch-size"; - /** Delta Kernel config key: target size (bytes) at which the writer rolls a new data file, overridable via - * {@link org.apache.sysds.conf.DMLConfig#DELTA_WRITER_TARGET_FILE_SIZE}. */ + /** + * Delta Kernel config key: target size (bytes) at which the writer rolls a new data file, overridable via + * {@link org.apache.sysds.conf.DMLConfig#DELTA_WRITER_TARGET_FILE_SIZE}. + */ private static final String CONF_WRITER_TARGET_FILE_SIZE = "delta.kernel.default.parquet.writer.targetMaxFileSize"; - /** Internal Delta column type codes shared by the matrix and frame readers to - * dispatch boxing-free primitive column access. */ - public static final int T_DOUBLE = 0; - public static final int T_FLOAT = 1; - public static final int T_LONG = 2; - public static final int T_INT = 3; - public static final int T_SHORT = 4; - public static final int T_BYTE = 5; + /** + * Internal Delta column type codes shared by the matrix and frame readers to dispatch boxing-free primitive column + * access. + */ + public static final int T_DOUBLE = 0; + public static final int T_FLOAT = 1; + public static final int T_LONG = 2; + public static final int T_INT = 3; + public static final int T_SHORT = 4; + public static final int T_BYTE = 5; public static final int T_BOOLEAN = 6; - public static final int T_STRING = 7; + public static final int T_STRING = 7; /** * Parquet physical type each {@code T_*} column is stored as, indexed by type code (delta int/short/byte columns @@ -128,22 +133,22 @@ public class DeltaKernelUtils { PrimitiveTypeName.INT64, PrimitiveTypeName.INT32, PrimitiveTypeName.INT32, PrimitiveTypeName.INT32, PrimitiveTypeName.BOOLEAN, PrimitiveTypeName.BINARY}; - //derived configuration cached to avoid copying the (large) base conf on every - //engine creation (createEngine is called once per data file in parallel reads); - //rebuilt whenever the base conf or the relevant SystemDS settings change. + // derived configuration cached to avoid copying the (large) base conf on every + // engine creation (createEngine is called once per data file in parallel reads); + // rebuilt whenever the base conf or the relevant SystemDS settings change. private static Configuration cachedConf; private static Configuration cachedConfBase; private static int cachedBatchSize; private static long cachedTargetFileSize; - private DeltaKernelUtils() {} + private DeltaKernelUtils() { + } /** - * Consumes a whole columnar batch. {@code selected} is {@code null} when all - * {@code size} rows are live; otherwise {@code selected[r]} indicates whether - * row {@code r} survived the deletion/selection vector. Batch-level consumption - * lets callers extract data column-at-a-time (cache friendly, boxing free) - * instead of paying a per-row callback. + * Consumes a whole columnar batch. {@code selected} is {@code null} when all {@code size} rows are live; otherwise + * {@code selected[r]} indicates whether row {@code r} survived the deletion/selection vector. Batch-level + * consumption lets callers extract data column-at-a-time (cache friendly, boxing free) instead of paying a per-row + * callback. */ @FunctionalInterface public interface BatchConsumer { @@ -151,22 +156,29 @@ public interface BatchConsumer { } /** - * Map a Delta Kernel {@link DataType} to an internal type code (see the - * {@code T_*} constants). Returned once per column so the per-cell read loop - * can switch on a primitive int instead of repeating {@code instanceof} checks. + * Map a Delta Kernel {@link DataType} to an internal type code (see the {@code T_*} constants). Returned once per + * column so the per-cell read loop can switch on a primitive int instead of repeating {@code instanceof} checks. * * @param dt the Delta column data type * @return the matching {@code T_*} code, or {@code -1} if the type is not supported */ public static int typeCode(DataType dt) { - if( dt instanceof DoubleType ) return T_DOUBLE; - if( dt instanceof FloatType ) return T_FLOAT; - if( dt instanceof LongType ) return T_LONG; - if( dt instanceof IntegerType ) return T_INT; - if( dt instanceof ShortType ) return T_SHORT; - if( dt instanceof ByteType ) return T_BYTE; - if( dt instanceof BooleanType ) return T_BOOLEAN; - if( dt instanceof StringType ) return T_STRING; + if(dt instanceof DoubleType) + return T_DOUBLE; + if(dt instanceof FloatType) + return T_FLOAT; + if(dt instanceof LongType) + return T_LONG; + if(dt instanceof IntegerType) + return T_INT; + if(dt instanceof ShortType) + return T_SHORT; + if(dt instanceof ByteType) + return T_BYTE; + if(dt instanceof BooleanType) + return T_BOOLEAN; + if(dt instanceof StringType) + return T_STRING; return -1; } @@ -429,8 +441,10 @@ private static void decodeColumnInto(ColumnReader creader, int maxDef, int nrow, } } - /** Floor on the adaptive writer target file size. Below this the per-file metadata/open - * overhead (and tiny-file proliferation) outweighs the extra read parallelism. */ + /** + * Floor on the adaptive writer target file size. Below this the per-file metadata/open overhead (and tiny-file + * proliferation) outweighs the extra read parallelism. + */ public static final long ADAPTIVE_WRITER_MIN_FILE_SIZE = 4L * 1024 * 1024; private static Configuration buildConf(Configuration base, int batchSize, long targetFileSize) { @@ -444,9 +458,8 @@ private static synchronized Configuration deltaConf() { Configuration base = ConfigurationManager.getCachedJobConf(); int batchSize = ConfigurationManager.getDeltaReaderBatchSize(); long targetFileSize = ConfigurationManager.getDeltaWriterTargetFileSize(); - if(cachedConf == null || cachedConfBase != base - || cachedBatchSize != batchSize || cachedTargetFileSize != targetFileSize) - { + if(cachedConf == null || cachedConfBase != base || cachedBatchSize != batchSize || + cachedTargetFileSize != targetFileSize) { cachedConf = buildConf(base, batchSize, targetFileSize); cachedConfBase = base; cachedBatchSize = batchSize; @@ -460,12 +473,11 @@ public static Engine createEngine() { } /** - * Compute the parquet target data-file size (bytes) for writing a table of the given - * estimated size. With adaptive sizing enabled the writer aims for roughly one data - * file per expected parallel reader (so the native per-file parallel read can use all - * threads): never above the configured target, and never below - * {@code ADAPTIVE_WRITER_MIN_FILE_SIZE} unless the configured target is itself smaller - * than that floor (in which case the configured target wins). + * Compute the parquet target data-file size (bytes) for writing a table of the given estimated size. With adaptive + * sizing enabled the writer aims for roughly one data file per expected parallel reader (so the native per-file + * parallel read can use all threads): never above the configured target, and never below + * {@code ADAPTIVE_WRITER_MIN_FILE_SIZE} unless the configured target is itself smaller than that floor (in which + * case the configured target wins). * * @param estimatedBytes estimate of the table's size (the block in-memory size is a fine proxy) * @return the target max parquet data-file size in bytes @@ -476,7 +488,7 @@ public static long adaptiveWriterTargetFileSize(long estimatedBytes) { return configured; int par = Math.max(1, OptimizerUtils.getParallelBinaryReadParallelism()); long perReader = Math.max(1, estimatedBytes / par); - //never above the configured cap, never below the floor (unless the cap itself is lower) + // never above the configured cap, never below the floor (unless the cap itself is lower) long target = Math.min(configured, Math.max(ADAPTIVE_WRITER_MIN_FILE_SIZE, perReader)); if(LOG.isDebugEnabled()) LOG.debug("Delta adaptive file size: est=" + estimatedBytes + "B par=" + par + " -> target=" + target @@ -485,24 +497,24 @@ public static long adaptiveWriterTargetFileSize(long estimatedBytes) { } /** - * Create an engine for writing a table of the given estimated size, configured with an - * adaptive target data-file size (see {@link #adaptiveWriterTargetFileSize(long)}). A fresh - * (uncached) configuration is built since writes happen once per table, not per data file. + * Create an engine for writing a table of the given estimated size, configured with an adaptive target data-file + * size (see {@link #adaptiveWriterTargetFileSize(long)}). A fresh (uncached) configuration is built since writes + * happen once per table, not per data file. * * @param estimatedBytes estimate of the table's size (the block in-memory size is a fine proxy) * @return a Delta Kernel engine for the write */ public static Engine createWriteEngine(long estimatedBytes) { - //the reader batch size is irrelevant on the write path but is set to keep the - //conf shape identical to deltaConf(); only the target file size matters here. + // the reader batch size is irrelevant on the write path but is set to keep the + // conf shape identical to deltaConf(); only the target file size matters here. Configuration c = buildConf(ConfigurationManager.getCachedJobConf(), ConfigurationManager.getDeltaReaderBatchSize(), adaptiveWriterTargetFileSize(estimatedBytes)); return DefaultEngine.create(c); } /** - * Resolve a (possibly relative) path to a fully-qualified URI so the - * kernel's default engine can locate the table on the right filesystem. + * Resolve a (possibly relative) path to a fully-qualified URI so the kernel's default engine can locate the table + * on the right filesystem. * * @param fname input path * @return fully-qualified table path @@ -519,11 +531,9 @@ public static String qualify(String fname) { } /** - * Opened latest snapshot of a Delta table: the logical schema plus everything - * needed to (re)read its data files, including the list of per-data-file scan - * rows. Delta Kernel scan-file rows are self-contained (the kernel's - * distributed design serializes them to workers), so they can be retained and - * read independently / in parallel. + * Opened latest snapshot of a Delta table: the logical schema plus everything needed to (re)read its data files, + * including the list of per-data-file scan rows. Delta Kernel scan-file rows are self-contained (the kernel's + * distributed design serializes them to workers), so they can be retained and read independently / in parallel. */ public static final class ScanHandle { public final StructType schema; @@ -531,19 +541,18 @@ public static final class ScanHandle { public final StructType physicalReadSchema; public final List scanFiles; /** - * Per-file record counts taken from the Delta {@code numRecords} statistic, - * aligned with {@link #scanFiles}; {@code -1} where the statistic is absent. + * Per-file record counts taken from the Delta {@code numRecords} statistic, aligned with {@link #scanFiles}; + * {@code -1} where the statistic is absent. */ public final long[] numRecords; /** - * Per-file flag indicating a deletion vector is present (so the live row - * count differs from {@link #numRecords}), aligned with {@link #scanFiles}. + * Per-file flag indicating a deletion vector is present (so the live row count differs from + * {@link #numRecords}), aligned with {@link #scanFiles}. */ public final boolean[] hasDeletionVector; - private ScanHandle(StructType schema, Row scanState, StructType physicalReadSchema, - List scanFiles, long[] numRecords, boolean[] hasDeletionVector) - { + private ScanHandle(StructType schema, Row scanState, StructType physicalReadSchema, List scanFiles, + long[] numRecords, boolean[] hasDeletionVector) { this.schema = schema; this.scanState = scanState; this.physicalReadSchema = physicalReadSchema; @@ -553,13 +562,12 @@ private ScanHandle(StructType schema, Row scanState, StructType physicalReadSche } /** - * @return true iff every data file carries a {@code numRecords} statistic - * and none has a deletion vector, i.e. exact per-file row offsets - * can be derived from metadata without reading the data. + * @return true iff every data file carries a {@code numRecords} statistic and none has a deletion vector, i.e. + * exact per-file row offsets can be derived from metadata without reading the data. */ public boolean hasExactRowCounts() { - for( int i=0; i scanFileIter = (scan instanceof ScanImpl) - ? ((ScanImpl) scan).getScanFiles(engine, true) - : scan.getScanFiles(engine); + // request the scan files WITH per-file statistics (numRecords) so callers can + // pre-size output and place rows without reading the data; harmless extra + // column for the data-read path. Fall back to the stats-less iterator if the + // concrete scan does not support it. + CloseableIterator scanFileIter = (scan instanceof ScanImpl) ? ((ScanImpl) scan) + .getScanFiles(engine, true) : scan.getScanFiles(engine); List files = new ArrayList<>(); List recs = new ArrayList<>(); List dvs = new ArrayList<>(); - try( CloseableIterator scanFiles = scanFileIter ) { - while( scanFiles.hasNext() ) { + try(CloseableIterator scanFiles = scanFileIter) { + while(scanFiles.hasNext()) { FilteredColumnarBatch scanFileBatch = scanFiles.next(); - try( CloseableIterator scanFileRows = scanFileBatch.getRows() ) { - while( scanFileRows.hasNext() ) { + try(CloseableIterator scanFileRows = scanFileBatch.getRows()) { + while(scanFileRows.hasNext()) { Row scanFileRow = scanFileRows.next(); files.add(scanFileRow); recs.add(numRecords(scanFileRow)); @@ -609,7 +615,7 @@ public static ScanHandle openScan(Engine engine, String tablePath) throws IOExce } long[] numRecords = new long[recs.size()]; boolean[] hasDv = new boolean[dvs.size()]; - for( int i=0; i physicalData = engine.getParquetHandler() .readParquetFiles(Utils.singletonCloseableIterator(dataFile), physicalReadSchema, Optional.empty()); - try( CloseableIterator logicalData = - Scan.transformPhysicalData(engine, scanState, scanFileRow, physicalData) ) - { - while( logicalData.hasNext() ) + try(CloseableIterator logicalData = Scan.transformPhysicalData(engine, scanState, + scanFileRow, physicalData)) { + while(logicalData.hasNext()) consumeBatch(logicalData.next(), consumer); } } /** - * Scan the latest snapshot of a Delta table sequentially, invoking the batch - * consumer for every data batch. The consumer is created lazily from the table - * schema (so callers can size buffers / derive per-column types up front). + * Scan the latest snapshot of a Delta table sequentially, invoking the batch consumer for every data batch. The + * consumer is created lazily from the table schema (so callers can size buffers / derive per-column types up + * front). * * @param engine delta kernel engine * @param tablePath fully-qualified table path @@ -677,11 +679,10 @@ public static void readScanFile(Engine engine, Row scanState, StructType physica * @throws IOException on read failure */ public static StructType scan(Engine engine, String tablePath, Function consumerFactory) - throws IOException - { + throws IOException { ScanHandle h = openScan(engine, tablePath); BatchConsumer consumer = consumerFactory.apply(h.schema); - for( Row scanFileRow : h.scanFiles ) + for(Row scanFileRow : h.scanFiles) readScanFile(engine, h.scanState, h.physicalReadSchema, scanFileRow, consumer); return h.schema; } @@ -690,28 +691,27 @@ private static void consumeBatch(FilteredColumnarBatch fcb, BatchConsumer consum ColumnarBatch batch = fcb.getData(); int ncol = batch.getSchema().length(); ColumnVector[] cols = new ColumnVector[ncol]; - for( int c=0; c all rows live) + // materialize the deletion/selection mask once (null => all rows live) Optional selVector = fcb.getSelectionVector(); boolean[] selected = null; - if( selVector.isPresent() ) { + if(selVector.isPresent()) { ColumnVector sv = selVector.get(); selected = new boolean[size]; - for( int r=0; r logicalData) throws IOException - { - //replace any existing table at the path (the other SystemDS writers delete - //the output first; the caching layer does not do it on our behalf) + CloseableIterator logicalData) throws IOException { + // replace any existing table at the path (the other SystemDS writers delete + // the output first; the caching layer does not do it on our behalf) HDFSTool.deleteFileIfExistOnHDFS(tablePath); Table table = Table.forPath(engine, tablePath); - TransactionBuilder txnBuilder = table - .createTransactionBuilder(engine, ENGINE_INFO, Operation.CREATE_TABLE) + TransactionBuilder txnBuilder = table.createTransactionBuilder(engine, ENGINE_INFO, Operation.CREATE_TABLE) .withSchema(engine, schema); Transaction txn = txnBuilder.build(engine); Row txnState = txn.getTransactionState(engine); - CloseableIterator physicalData = - Transaction.transformLogicalData(engine, txnState, logicalData, Collections.emptyMap()); - DataWriteContext writeContext = - Transaction.getWriteContext(engine, txnState, Collections.emptyMap()); + CloseableIterator physicalData = Transaction.transformLogicalData(engine, txnState, + logicalData, Collections.emptyMap()); + DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, Collections.emptyMap()); CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles(writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns()); - CloseableIterator appendActions = - Transaction.generateAppendActions(engine, txnState, dataFiles, writeContext); + CloseableIterator appendActions = Transaction.generateAppendActions(engine, txnState, dataFiles, + writeContext); txn.commit(engine, CloseableIterable.inMemoryIterable(appendActions)); } } diff --git a/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java b/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java index 58a98741975..55d8f8f7c2d 100644 --- a/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java +++ b/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java @@ -33,29 +33,27 @@ import io.delta.kernel.types.StructType; /** - * Single-threaded native Delta Lake reader for matrices, built on the - * Spark-free Delta Kernel library. It opens the latest snapshot of a Delta - * table directory, reads its parquet data files through the kernel's default - * engine (honoring deletion vectors), and materializes the numeric columns - * into a dense {@link MatrixBlock}. + * Single-threaded native Delta Lake reader for matrices, built on the Spark-free Delta Kernel library. It opens the + * latest snapshot of a Delta table directory, reads its parquet data files through the kernel's default engine + * (honoring deletion vectors), and materializes the numeric columns into a dense {@link MatrixBlock}. * - *

Only numeric columns (double/float/long/int/short/byte/boolean) are - * supported, matching the all-double nature of a SystemDS matrix. Dimensions - * do not need to be known up front: the row count is discovered while scanning - * and the column count is taken from the table schema.

+ *

+ * Only numeric columns (double/float/long/int/short/byte/boolean) are supported, matching the all-double nature of a + * SystemDS matrix. Dimensions do not need to be known up front: the row count is discovered while scanning and the + * column count is taken from the table schema. + *

*/ public class ReaderDelta extends MatrixReader { @Override public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int blen, long estnnz) - throws IOException, DMLRuntimeException - { + throws IOException, DMLRuntimeException { Engine engine = DeltaKernelUtils.createEngine(); String tablePath = DeltaKernelUtils.qualify(fname); - //Scan column-at-a-time into one row-major buffer per batch (no per-row - //allocation, no boxing, no per-cell set()). Buffers are concatenated into - //the dense output via bulk array copies below. + // Scan column-at-a-time into one row-major buffer per batch (no per-row + // allocation, no boxing, no per-cell set()). Buffers are concatenated into + // the dense output via bulk array copies below. ArrayList batches = new ArrayList<>(); int[] nrowH = new int[1]; StructType schema = DeltaKernelUtils.scan(engine, tablePath, sch -> { @@ -72,7 +70,7 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl long lestnnz = (estnnz >= 0) ? estnnz : (long) nrow * ncol; MatrixBlock ret = createOutputMatrixBlock(nrow, ncol, Math.max(nrow, 1), lestnnz, true, false); - if( nrow > 0 && ncol > 0 ) + if(nrow > 0 && ncol > 0) fillDense(ret, batches); ret.recomputeNonZeros(); ret.examSparsity(); @@ -83,14 +81,14 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl static int[] columnTypes(StructType schema) { int ncol = schema.length(); int[] types = new int[ncol]; - for( int c=0; c batches) { DenseBlock db = ret.getDenseBlock(); - if( db.isContiguous() ) { + if(db.isContiguous()) { double[] dv = db.valuesAt(0); int off = 0; - for( double[] buf : batches ) { + for(double[] buf : batches) { System.arraycopy(buf, 0, dv, off, buf.length); off += buf.length; } } else { - //rare large multi-block fallback: route each row through the block API + // rare large multi-block fallback: route each row through the block API int ncol = ret.getNumColumns(); int r = 0; - for( double[] buf : batches ) { + for(double[] buf : batches) { int rowsInBuf = buf.length / ncol; - for( int i=0; iThe expensive part of a Delta read is the parquet decode, which the kernel - * performs per data file; parallelizing across files is therefore the natural - * way to bridge the gap to the (near-raw) binary reader. A table backed by a - * single data file (the default for tables <= the parquet target file size) - * cannot be split this way, so the reader transparently falls back to the - * sequential {@link ReaderDelta} path in that case.

+ *

+ * The expensive part of a Delta read is the parquet decode, which the kernel performs per data file; parallelizing + * across files is therefore the natural way to bridge the gap to the (near-raw) binary reader. A table backed by a + * single data file (the default for tables <= the parquet target file size) cannot be split this way, so the reader + * transparently falls back to the sequential {@link ReaderDelta} path in that case. + *

*/ public class ReaderDeltaParallel extends ReaderDelta { @@ -57,28 +56,27 @@ public ReaderDeltaParallel() { @Override public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int blen, long estnnz) - throws IOException, DMLRuntimeException - { + throws IOException, DMLRuntimeException { Engine engine = DeltaKernelUtils.createEngine(); String tablePath = DeltaKernelUtils.qualify(fname); DeltaKernelUtils.ScanHandle handle = DeltaKernelUtils.openScan(engine, tablePath); final int nfiles = handle.scanFiles.size(); - //nothing to gain from parallelism for single-file (or empty) tables - if( _numThreads <= 1 || nfiles <= 1 ) + // nothing to gain from parallelism for single-file (or empty) tables + if(_numThreads <= 1 || nfiles <= 1) return super.readMatrixFromHDFS(fname, rlen, clen, blen, estnnz); final int ncol = handle.schema.length(); final int[] types = columnTypes(handle.schema); - //fast path: exact per-file row counts are known from metadata and the dense - //output fits a single contiguous array -> pre-size once and let each thread - //decode directly into its slice (no intermediate buffers, no serial copy). - if( useDirectPath(handle) ) { + // fast path: exact per-file row counts are known from metadata and the dense + // output fits a single contiguous array -> pre-size once and let each thread + // decode directly into its slice (no intermediate buffers, no serial copy). + if(useDirectPath(handle)) { long total = 0; - for( long r : handle.numRecords ) + for(long r : handle.numRecords) total += r; - if( total > 0 && (long) total * ncol <= Integer.MAX_VALUE ) + if(total > 0 && (long) total * ncol <= Integer.MAX_VALUE) return readDirect(fname, handle, ncol, types, (int) total, estnnz); } @@ -86,11 +84,9 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl } /** - * Whether the metadata-driven direct-write fast path can be used for this - * table (exact per-file row counts and no deletion vectors). Visible for - * testing: the buffered fallback is otherwise only reachable for tables - * lacking row statistics or carrying deletion vectors, which the SystemDS - * Delta writer never produces. + * Whether the metadata-driven direct-write fast path can be used for this table (exact per-file row counts and no + * deletion vectors). Visible for testing: the buffered fallback is otherwise only reachable for tables lacking row + * statistics or carrying deletion vectors, which the SystemDS Delta writer never produces. * * @param handle the opened scan handle * @return true if the direct path is applicable @@ -100,38 +96,37 @@ protected boolean useDirectPath(DeltaKernelUtils.ScanHandle handle) { } /** - * Fast path: each thread decodes one data file straight into the final dense - * array at a metadata-derived row offset. Single allocation, fully parallel. + * Fast path: each thread decodes one data file straight into the final dense array at a metadata-derived row + * offset. Single allocation, fully parallel. */ - private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, - int ncol, int[] types, int nrow, long estnnz) throws IOException - { + private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, int ncol, int[] types, int nrow, + long estnnz) throws IOException { final int nfiles = handle.scanFiles.size(); final int[] rowOffset = new int[nfiles]; int acc = 0; - for( int i=0; i> tasks = new ArrayList<>(nfiles); - for( int i=0; i { int[] cur = new int[] {base}; Engine eng = DeltaKernelUtils.createEngine(); DeltaKernelUtils.readScanFile(eng, handle.scanState, handle.physicalReadSchema, scanFileRow, (cols, size, selected) -> { - if( cur[0] + DeltaKernelUtils.countSelected(size, selected) > limit ) + if(cur[0] + DeltaKernelUtils.countSelected(size, selected) > limit) throw new DMLRuntimeException("Delta file produced more rows than its " + "numRecords statistic; refusing parallel direct read of " + fname); cur[0] += extractBatchInto(cols, size, selected, types, ncol, dv, cur[0]); @@ -147,19 +142,17 @@ private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, } /** - * Fallback path: decode each file in parallel into per-file buffers (used when - * row counts are unknown, deletion vectors are present, or the matrix exceeds a - * single contiguous array), then concatenate in file order. + * Fallback path: decode each file in parallel into per-file buffers (used when row counts are unknown, deletion + * vectors are present, or the matrix exceeds a single contiguous array), then concatenate in file order. */ - private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handle, - int ncol, int[] types, long estnnz) throws IOException - { + private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handle, int ncol, int[] types, + long estnnz) throws IOException { final int nfiles = handle.scanFiles.size(); @SuppressWarnings("unchecked") final ArrayList[] fileBufs = new ArrayList[nfiles]; final int[] fileRows = new int[nfiles]; ArrayList> tasks = new ArrayList<>(nfiles); - for( int i=0; i { @@ -179,15 +172,15 @@ private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handl awaitFileTasks(tasks, fname); int nrow = 0; - for( int i=0; i ordered = new ArrayList<>(); - for( int i=0; i= 0) ? estnnz : (long) nrow * ncol; MatrixBlock ret = createOutputMatrixBlock(nrow, ncol, Math.max(nrow, 1), lestnnz, true, false); - if( nrow > 0 && ncol > 0 ) + if(nrow > 0 && ncol > 0) fillDense(ret, ordered); ret.recomputeNonZeros(_numThreads); ret.examSparsity(); @@ -195,16 +188,15 @@ private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handl } /** - * Run one decode task per data file on the shared common thread pool and await - * completion. Full parallelism is requested (the task count, one per data file, - * naturally caps concurrency); this avoids the per-thread pool-size caching in - * {@code CommonThreadPool.get(k)} that could otherwise throttle this reader to a - * smaller pool created earlier on the same thread. + * Run one decode task per data file on the shared common thread pool and await completion. Full parallelism is + * requested (the task count, one per data file, naturally caps concurrency); this avoids the per-thread pool-size + * caching in {@code CommonThreadPool.get(k)} that could otherwise throttle this reader to a smaller pool created + * earlier on the same thread. */ private void awaitFileTasks(List> tasks, String fname) throws IOException { ExecutorService pool = CommonThreadPool.get(_numThreads); try { - for( Future f : pool.invokeAll(tasks) ) + for(Future f : pool.invokeAll(tasks)) f.get(); } catch(Exception ex) { diff --git a/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java b/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java index 55ea8a54297..602b540c407 100644 --- a/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java +++ b/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java @@ -39,60 +39,54 @@ import io.delta.kernel.utils.CloseableIterator; /** - * Single-threaded native Delta Lake writer for matrices, built on the - * Spark-free Delta Kernel library. It creates a Delta table at the target - * directory with an all-double schema {@code c0..c(n-1)} (replacing any existing - * table at that path), streams the {@link MatrixBlock} rows as columnar batches - * into parquet data files via the kernel's default engine, and commits the - * corresponding add-file actions to the transaction log. + * Single-threaded native Delta Lake writer for matrices, built on the Spark-free Delta Kernel library. It creates a + * Delta table at the target directory with an all-double schema {@code c0..c(n-1)} (replacing any existing table at + * that path), streams the {@link MatrixBlock} rows as columnar batches into parquet data files via the kernel's default + * engine, and commits the corresponding add-file actions to the transaction log. */ public class WriterDelta extends MatrixWriter { @Override public void writeMatrixToHDFS(MatrixBlock src, String fname, long rlen, long clen, int blen, long nnz, boolean diag) - throws IOException - { - if( src.getNumRows() != rlen || src.getNumColumns() != clen ) - throw new IOException("Matrix dimensions mismatch with metadata: (" - + src.getNumRows() + "x" + src.getNumColumns() + ") vs (" + rlen + "x" + clen + ")."); + throws IOException { + if(src.getNumRows() != rlen || src.getNumColumns() != clen) + throw new IOException("Matrix dimensions mismatch with metadata: (" + src.getNumRows() + "x" + + src.getNumColumns() + ") vs (" + rlen + "x" + clen + ")."); int ncol = (int) clen; int nrow = (int) rlen; int batchRows = ConfigurationManager.getDeltaWriterBatchSize(); - //fast path: a contiguous dense block lets the column views read straight - //from the backing double[] (avoids per-cell MatrixBlock.get dispatch). - double[] dense = (!src.isInSparseFormat() && src.getDenseBlock() != null - && src.getDenseBlock().isContiguous()) ? src.getDenseBlockValues() : null; - //size data files adaptively (toward one file per parallel reader) for faster parallel reads. - //Delta writes every cell as a double, so size by the dense footprint rather than the (possibly - //sparse) in-memory size, which would understate the on-disk table for sparse inputs. + // fast path: a contiguous dense block lets the column views read straight + // from the backing double[] (avoids per-cell MatrixBlock.get dispatch). + double[] dense = (!src.isInSparseFormat() && src.getDenseBlock() != null && + src.getDenseBlock().isContiguous()) ? src.getDenseBlockValues() : null; + // size data files adaptively (toward one file per parallel reader) for faster parallel reads. + // Delta writes every cell as a double, so size by the dense footprint rather than the (possibly + // sparse) in-memory size, which would understate the on-disk table for sparse inputs. long estimatedBytes = (long) nrow * ncol * 8L; Engine engine = DeltaKernelUtils.createWriteEngine(estimatedBytes); - DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), - buildSchema(ncol), new MatrixBatchIterator(src, dense, nrow, ncol, batchRows)); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), buildSchema(ncol), + new MatrixBatchIterator(src, dense, nrow, ncol, batchRows)); } @Override - public void writeEmptyMatrixToHDFS(String fname, long rlen, long clen, int blen) - throws IOException - { - //empty table: create with schema but no data files + public void writeEmptyMatrixToHDFS(String fname, long rlen, long clen, int blen) throws IOException { + // empty table: create with schema but no data files Engine engine = DeltaKernelUtils.createEngine(); - DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), - buildSchema((int) clen), CloseableIterable.emptyIterable().iterator()); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), buildSchema((int) clen), + CloseableIterable.emptyIterable().iterator()); } private static StructType buildSchema(int ncol) { StructType schema = new StructType(); - for( int c=0; c stream, long rlen, long clen, int blen) - throws IOException - { + public long writeMatrixFromStream(String fname, OOCStream stream, long rlen, long clen, + int blen) throws IOException { throw new UnsupportedOperationException("Out-of-core stream write is not supported for the Delta format."); } @@ -122,18 +116,18 @@ public boolean hasNext() { @Override public FilteredColumnarBatch next() { - if( !hasNext() ) + if(!hasNext()) throw new NoSuchElementException(); int size = Math.min(_batchRows, _nrow - _pos); ColumnarBatch batch = new MatrixColumnarBatch(_mb, _dense, _schema, _pos, size, _ncol); _pos += size; - //no selection vector: all rows in the batch are written + // no selection vector: all rows in the batch are written return new FilteredColumnarBatch(batch, Optional.empty()); } @Override public void close() { - //nothing to release + // nothing to release } } @@ -162,7 +156,7 @@ public StructType getSchema() { @Override public ColumnVector getColumnVector(int ordinal) { - if( ordinal < 0 || ordinal >= _ncol ) + if(ordinal < 0 || ordinal >= _ncol) throw new IndexOutOfBoundsException("column ordinal " + ordinal); return new MatrixColumnVector(_mb, _dense, _rowStart, _size, _ncol, ordinal); } @@ -208,16 +202,14 @@ public boolean isNullAt(int rowId) { @Override public double getDouble(int rowId) { - //dense contiguous single block => index fits in int (getDenseBlockValues - //is only handed over for single-block dense matrices) - return (_dense != null) - ? _dense[(_rowStart + rowId) * _ncol + _col] - : _mb.get(_rowStart + rowId, _col); + // dense contiguous single block => index fits in int (getDenseBlockValues + // is only handed over for single-block dense matrices) + return (_dense != null) ? _dense[(_rowStart + rowId) * _ncol + _col] : _mb.get(_rowStart + rowId, _col); } @Override public void close() { - //nothing to release + // nothing to release } } } diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java index 5f478979104..0bf9f7d87ba 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java @@ -977,7 +977,7 @@ public static MatrixBlock rmempty(MatrixBlock in, MatrixBlock ret, boolean rows, if(ret == null) ret = new MatrixBlock(); MatrixBlock ret2 = rmemptyEarlyAbort(in, ret, rows, emptyReturn, select); - if(ret2 != null ) + if(ret2 != null) return ret2; // core removeEmpty return rmemptyUnsafe(in, ret, rows, emptyReturn, select); @@ -985,7 +985,7 @@ public static MatrixBlock rmempty(MatrixBlock in, MatrixBlock ret, boolean rows, public static MatrixBlock rmemptyUnsafe(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, MatrixBlock select) { - if( rows ) + if(rows) return removeEmptyRows(in, ret, select, emptyReturn); else // cols return removeEmptyColumns(in, ret, select, emptyReturn); @@ -999,14 +999,15 @@ public static MatrixBlock rmemptyUnsafe(MatrixBlock in, MatrixBlock ret, boolean * @param rows If removing based on rows, or columns * @param emptyReturn Return a row/column of zeros for empty input * @param select An optional selection vector - * @return The early-abort result, or {@code null} if no early termination applies and the caller must continue. - * For the select-all case the returned block is the input {@code in} itself (a shallow alias, not a copy). + * @return The early-abort result, or {@code null} if no early termination applies and the caller must continue. For + * the select-all case the returned block is the input {@code in} itself (a shallow alias, not a copy). */ - public static MatrixBlock rmemptyEarlyAbort(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, MatrixBlock select){ - //check for empty inputs - //(the semantics of removeEmpty are that for an empty m-by-n matrix, the output - //is an empty 1-by-n or m-by-1 matrix because we don't allow matrices with dims 0) - if( in.isEmptyBlock(false) && select == null ) { + public static MatrixBlock rmemptyEarlyAbort(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, + MatrixBlock select) { + // check for empty inputs + // (the semantics of removeEmpty are that for an empty m-by-n matrix, the output + // is an empty 1-by-n or m-by-1 matrix because we don't allow matrices with dims 0) + if(in.isEmptyBlock(false) && select == null) { int n = emptyReturn ? 1 : 0; if( rows ) ret.reset(n, in.clen, in.sparse); @@ -3651,7 +3652,7 @@ private static MatrixBlock removeEmptyRows(MatrixBlock in, MatrixBlock ret, Matr /** * Remove selected rows, based on the boolean array given. Note this function is internal use only, and require a * boolean vector to be constructed first. - * + * * @param in Input to remove rows from * @param ret Output to assign the result into * @param emptyReturn If the output is allowed to be empty. @@ -3672,9 +3673,9 @@ public static MatrixBlock removeEmptyRows(MatrixBlock in, MatrixBlock ret, boole ret.reset(rlen2, n, sp); if( in.isEmptyBlock(false) ) return ret; - - if( SHALLOW_COPY_REORG && m == rlen2 && selectNull ) { - // the condition m==rlen2 is not enough with non-empty 1-row input but empty + + if(SHALLOW_COPY_REORG && m == rlen2 && selectNull) { + // the condition m==rlen2 is not enough with non-empty 1-row input but empty // 1-row select vector because if emptyReturn should output a single empty row ret.sparse = in.sparse; if( ret.sparse ) @@ -3714,10 +3715,9 @@ else if( !in.sparse && !ret.sparse ) //DENSE <- DENSE ci++; } } - - //check sparsity - ret.nonZeros = (selectNull) ? - in.nonZeros : ret.recomputeNonZeros(); + + // check sparsity + ret.nonZeros = (selectNull) ? in.nonZeros : ret.recomputeNonZeros(); ret.examSparsity(); return ret; diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java index 7525dab2f7f..c450ecbe1f5 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java @@ -4756,13 +4756,13 @@ public static double computeIQMCorrection(double sum, double sum_wt, } /** - * Pick the quantiles out of this matrix. If this matrix contains two columns it is weighted quantile picking. - * If a single column it is unweighted. - * + * Pick the quantiles out of this matrix. If this matrix contains two columns it is weighted quantile picking. If a + * single column it is unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantiles The quantiles to pick - * @param ret The result matrix + * @param ret The result matrix * @return The result matrix */ public final MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret) { @@ -4792,11 +4792,11 @@ public MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret, boolean av } /** - * Pick the median value from this matrix. If this matrix has two columns it is weighted picking using the - * weight column, otherwise it is unweighted over the single column. - * + * Pick the median value from this matrix. If this matrix has two columns it is weighted picking using the weight + * column, otherwise it is unweighted over the single column. + * * Note the values are assumed to be sorted. - * + * * @return The median value */ public double median() { @@ -4807,10 +4807,11 @@ public double median() { } /** - * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is unweighted. - * + * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is + * unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantile The quantile to pick * @return The quantile */ @@ -4819,12 +4820,13 @@ public final double pickValue(double quantile){ } /** - * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is unweighted. - * + * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is + * unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantile The quantile to pick - * @param average If the quantile is averaged. + * @param average If the quantile is averaged. * @return The quantile */ public final double pickValue(double quantile, boolean average) { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java index 9e3494a7d48..9ea39bd1e2f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java @@ -34,8 +34,8 @@ import java.util.function.Function; /** - * Small future implementation for OOC hot paths. It supports multiple synchronous subscribers without - * the completion-stage support of {@link java.util.concurrent.CompletableFuture}. + * Small future implementation for OOC hot paths. It supports multiple synchronous subscribers without the + * completion-stage support of {@link java.util.concurrent.CompletableFuture}. */ public class OOCFuture { private Subscriber _subscribers; @@ -240,7 +240,7 @@ private static void accept(Function mapper, Consu if(resultError == null) { try { @SuppressWarnings("unchecked") - R mapped = mapper == null ? (R)value : mapper.apply(value); + R mapped = mapper == null ? (R) value : mapper.apply(value); result = mapped; } catch(Throwable t) { @@ -345,8 +345,7 @@ public T get() throws InterruptedException, ExecutionException { } @Override - public T get(long timeout, TimeUnit unit) - throws InterruptedException, ExecutionException, TimeoutException { + public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { try { return mapper.apply(source.get(timeout, unit)); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java index 94411242df1..df981f40223 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java @@ -25,20 +25,22 @@ public class CloseableQueue { private final BlockingQueue queue = new LinkedBlockingQueue<>(); - private final Object POISON = new Object(); // sentinel + private final Object POISON = new Object(); // sentinel private volatile boolean closed = false; - public CloseableQueue() { } + public CloseableQueue() { + } /** * Enqueue if the queue is not closed. + * * @return false if already closed */ public boolean enqueueIfOpen(T task) throws InterruptedException { - if (task == null) + if(task == null) throw new IllegalArgumentException("null tasks not allowed"); - synchronized (this) { - if (closed) + synchronized(this) { + if(closed) return false; queue.put(task); } @@ -47,12 +49,12 @@ public boolean enqueueIfOpen(T task) throws InterruptedException { @SuppressWarnings("unchecked") public T take() throws InterruptedException { - if (closed && queue.isEmpty()) + if(closed && queue.isEmpty()) return null; Object x = queue.take(); - if (x == POISON) + if(x == POISON) return null; return (T) x; @@ -60,33 +62,31 @@ public T take() throws InterruptedException { /** * Poll with max timeout. - * @return item, or null if: - * - timeout, or - * - queue has been closed and this consumer reached its poison pill + * + * @return item, or null if: - timeout, or - queue has been closed and this consumer reached its poison pill */ @SuppressWarnings("unchecked") public T poll(long timeout, TimeUnit unit) throws InterruptedException { - if (closed && queue.isEmpty()) + if(closed && queue.isEmpty()) return null; Object x = queue.poll(timeout, unit); - if (x == null) - return null; // timeout + if(x == null) + return null; // timeout - if (x == POISON) + if(x == POISON) return null; return (T) x; } /** - * Close queue for N consumers. - * Each consumer will receive exactly one poison pill and then should stop. + * Close queue for N consumers. Each consumer will receive exactly one poison pill and then should stop. */ public boolean close() throws InterruptedException { - synchronized (this) { - if (closed) - return false; // idempotent + synchronized(this) { + if(closed) + return false; // idempotent closed = true; } queue.put(POISON); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java index ee02b28404c..ccc73ff6160 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java @@ -78,7 +78,7 @@ public void readFully(byte[] b) throws IOException { } @Override - public void readFully(byte [] b, int off, int len) throws IOException { + public void readFully(byte[] b, int off, int len) throws IOException { if(len < 0) throw new IndexOutOfBoundsException(); @@ -127,12 +127,12 @@ public int readUnsignedByte() throws IOException { @Override public short readShort() throws IOException { if(_count - _pos >= 2) { - short ret = (short)baToShort(_buff, _pos); + short ret = (short) baToShort(_buff, _pos); _pos += 2; return ret; } readFully(_tmp, 0, 2); - return (short)baToShort(_tmp, 0); + return (short) baToShort(_tmp, 0); } @Override @@ -142,7 +142,7 @@ public int readUnsignedShort() throws IOException { @Override public char readChar() throws IOException { - return (char)readUnsignedShort(); + return (char) readUnsignedShort(); } @Override @@ -222,7 +222,7 @@ public long readDoubleArray(int len, double[] varr) throws IOException { @Override public long readSparseRows(int rlen, long nnz, SparseBlock rows) throws IOException { if(rows instanceof SparseBlockCSR) { - ((SparseBlockCSR)rows).initSparse(rlen, (int)nnz, this); + ((SparseBlockCSR) rows).initSparse(rlen, (int) nnz, this); return nnz; } @@ -256,7 +256,7 @@ private void refill() throws IOException { } private int getRefillLength() { - int pageOffset = (int)(_filePos & PAGE_MASK); + int pageOffset = (int) (_filePos & PAGE_MASK); if(pageOffset == 0) return _bufflen; return Math.min(_bufflen, PAGE_SIZE - pageOffset); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java index 9854a94586b..22b7b23706c 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java @@ -65,7 +65,7 @@ long getFlushedPosition() { public void write(int b) throws IOException { if(_count >= _bufflen) flushBuffer(); - _buff[_count++] = (byte)b; + _buff[_count++] = (byte) b; _position++; } @@ -109,7 +109,7 @@ public void close() throws IOException { public void writeBoolean(boolean v) throws IOException { if(_count >= _bufflen) flushBuffer(); - _buff[_count++] = (byte)(v ? 1 : 0); + _buff[_count++] = (byte) (v ? 1 : 0); _position++; } @@ -153,7 +153,7 @@ public void writeFloat(float v) throws IOException { public void writeByte(int v) throws IOException { if(_count + 1 > _bufflen) flushBuffer(); - _buff[_count++] = (byte)v; + _buff[_count++] = (byte) v; _position++; } @@ -194,18 +194,18 @@ public void writeUTF(String s) throws IOException { flushBuffer(); final char c = s.charAt(i); if(c >= 0x0001 && c <= 0x007F) { - _buff[_count++] = (byte)c; + _buff[_count++] = (byte) c; _position++; } else if(c >= 0x0800) { - _buff[_count++] = (byte)(0xE0 | ((c >> 12) & 0x0F)); - _buff[_count++] = (byte)(0x80 | ((c >> 6) & 0x3F)); - _buff[_count++] = (byte)(0x80 | (c & 0x3F)); + _buff[_count++] = (byte) (0xE0 | ((c >> 12) & 0x0F)); + _buff[_count++] = (byte) (0x80 | ((c >> 6) & 0x3F)); + _buff[_count++] = (byte) (0x80 | (c & 0x3F)); _position += 3; } else { - _buff[_count++] = (byte)(0xC0 | ((c >> 6) & 0x1F)); - _buff[_count++] = (byte)(0x80 | (c & 0x3F)); + _buff[_count++] = (byte) (0xC0 | ((c >> 6) & 0x1F)); + _buff[_count++] = (byte) (0x80 | (c & 0x3F)); _position += 2; } } @@ -213,7 +213,7 @@ else if(c >= 0x0800) { @Override public void writeDoubleArray(int len, double[] varr) throws IOException { - for(int i = 0; i < len; ) { + for(int i = 0; i < len;) { if(_count >= _bufflen) flushBuffer(); int lblen = Math.min(len - i, (_bufflen - _count) / 8); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java index ab28df0ef0f..6c13a062561 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java @@ -50,10 +50,10 @@ public interface OOCIOHandler { void registerSourceLocation(BlockKey key, SourceBlockDescriptor descriptor); /** - * Schedule an asynchronous read from an external source into the provided target stream. - * The returned future completes when either EOF is reached or the requested byte budget - * is exhausted. When the budget is reached and keepOpenOnLimit is true, the target stream - * is kept open and a continuation token is provided so the caller can resume. + * Schedule an asynchronous read from an external source into the provided target stream. The returned future + * completes when either EOF is reached or the requested byte budget is exhausted. When the budget is reached and + * keepOpenOnLimit is true, the target stream is kept open and a continuation token is provided so the caller can + * resume. */ CompletableFuture scheduleSourceRead(SourceReadRequest request); @@ -62,7 +62,8 @@ public interface OOCIOHandler { */ CompletableFuture continueSourceRead(SourceReadContinuation continuation, long maxBytesInFlight); - interface SourceReadContinuation {} + interface SourceReadContinuation { + } class SourceReadRequest { public final String path; @@ -75,9 +76,8 @@ class SourceReadRequest { public final boolean keepOpenOnLimit; public final OOCStream target; - public SourceReadRequest(String path, Types.FileFormat format, long rows, long cols, - int blen, long estNnz, long maxBytesInFlight, boolean keepOpenOnLimit, - OOCStream target) { + public SourceReadRequest(String path, Types.FileFormat format, long rows, long cols, int blen, long estNnz, + long maxBytesInFlight, boolean keepOpenOnLimit, OOCStream target) { this.path = path; this.format = format; this.rows = rows; @@ -113,9 +113,8 @@ class SourceBlockDescriptor { public final int recordLength; public final long serializedSize; - public SourceBlockDescriptor(String path, Types.FileFormat format, - MatrixIndexes indexes, long offset, int recordLength, - long serializedSize) { + public SourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, + int recordLength, long serializedSize) { this.path = path; this.format = format; this.indexes = indexes; @@ -129,8 +128,8 @@ class GroupSourceBlockDescriptor extends SourceBlockDescriptor { public final List blocks; public final int count; - public GroupSourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, int recordLength, - long serializedSize, List blocks) { + public GroupSourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, + int recordLength, long serializedSize, List blocks) { super(path, format, indexes, offset, recordLength, serializedSize); this.blocks = blocks; this.count = blocks.size(); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java index e9487f7bd45..55ac038205f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java @@ -81,7 +81,7 @@ public class OOCMatrixIOHandler implements OOCIOHandler { private final AtomicLong _readSeq = new AtomicLong(0); // Spill related structures - private final ConcurrentHashMap _spillLocations = new ConcurrentHashMap<>(); + private final ConcurrentHashMap _spillLocations = new ConcurrentHashMap<>(); private final ConcurrentHashMap _partitions = new ConcurrentHashMap<>(); private final ConcurrentHashMap _sourceLocations = new ConcurrentHashMap<>(); private final AtomicInteger _partitionCounter = new AtomicInteger(0); @@ -97,38 +97,21 @@ public class OOCMatrixIOHandler implements OOCIOHandler { @SuppressWarnings("unchecked") public OOCMatrixIOHandler() { this._spillDir = LocalFileUtils.getUniqueWorkingDir("ooc_stream"); - _writeExec = new ThreadPoolExecutor( - WRITER_SIZE, - WRITER_SIZE, - 0L, - TimeUnit.MILLISECONDS, + _writeExec = new ThreadPoolExecutor(WRITER_SIZE, WRITER_SIZE, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); - _readExec = new ThreadPoolExecutor( - READER_SIZE, - READER_SIZE, - 0L, - TimeUnit.MILLISECONDS, + _readExec = new ThreadPoolExecutor(READER_SIZE, READER_SIZE, 0L, TimeUnit.MILLISECONDS, new PriorityBlockingQueue<>()); - _srcReadExec = new ThreadPoolExecutor( - READER_SIZE, - READER_SIZE, - 0L, - TimeUnit.MILLISECONDS, - new ArrayBlockingQueue<>(100000)); - _deleteExec = new ThreadPoolExecutor( - 1, - 1, - 0L, - TimeUnit.MILLISECONDS, + _srcReadExec = new ThreadPoolExecutor(READER_SIZE, READER_SIZE, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); + _deleteExec = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); _q = new CloseableQueue[WRITER_SIZE]; _wCtr = new AtomicLong(0); - _started = new AtomicBoolean(false); + _started = new AtomicBoolean(false); } private synchronized void start() { - if (_started.compareAndSet(false, true)) { - for (int i = 0; i < WRITER_SIZE; i++) { + if(_started.compareAndSet(false, true)) { + for(int i = 0; i < WRITER_SIZE; i++) { final int finalIdx = i; _q[i] = new CloseableQueue<>(); _writeExec.submit(() -> evictTask(_q[finalIdx])); @@ -139,7 +122,7 @@ private synchronized void start() { @Override public void shutdown() { boolean started = _started.get(); - if (started) { + if(started) { try { for(int i = 0; i < WRITER_SIZE; i++) { if(_q[i] != null) @@ -159,7 +142,7 @@ public void shutdown() { _deleteExec.shutdownNow(); _spillLocations.clear(); _partitions.clear(); - if (started) + if(started) LocalFileUtils.deleteFileIfExists(_spillDir); } @@ -169,7 +152,7 @@ public CompletableFuture scheduleEviction(BlockEntry block) { CompletableFuture future = new CompletableFuture<>(); try { long q = _wCtr.getAndAdd(block.getSize()) / OVERFLOW; - int i = (int)(q % WRITER_SIZE); + int i = (int) (q % WRITER_SIZE); if(!_q[i].enqueueIfOpen(new Tuple2<>(block, future))) future.completeExceptionally(new DMLRuntimeException("OOC writer queue is closed")); } @@ -189,7 +172,8 @@ public OOCFuture scheduleRead(final BlockEntry block) { ReadTask task = new ReadTask(block, future, _readSeq.getAndIncrement(), pinnedPartitionId); _pendingReads.put(block.getKey(), task); _readExec.execute(task); - } catch (RejectedExecutionException e) { + } + catch(RejectedExecutionException e) { unpinPartitionForRead(pinnedPartitionId); _pendingReads.remove(block.getKey()); future.completeExceptionally(e); @@ -199,12 +183,12 @@ public OOCFuture scheduleRead(final BlockEntry block) { @Override public void prioritizeRead(BlockKey key, double priority) { - if (priority == 0) + if(priority == 0) return; ReadTask task = _pendingReads.get(key); - if (task == null) + if(task == null) return; - if (_readExec.getQueue().remove(task)) { + if(_readExec.getQueue().remove(task)) { task.addPriority(priority); _readExec.getQueue().offer(task); } @@ -228,8 +212,9 @@ public CompletableFuture scheduleSourceRead(SourceReadRequest } @Override - public CompletableFuture continueSourceRead(SourceReadContinuation continuation, long maxBytesInFlight) { - if (!(continuation instanceof SourceReadState state)) { + public CompletableFuture continueSourceRead(SourceReadContinuation continuation, + long maxBytesInFlight) { + if(!(continuation instanceof SourceReadState state)) { CompletableFuture failed = new CompletableFuture<>(); failed.completeExceptionally(new DMLRuntimeException("Unsupported continuation type: " + continuation)); return failed; @@ -240,8 +225,8 @@ public CompletableFuture continueSourceRead(SourceReadContinua private CompletableFuture submitSourceRead(SourceReadRequest request, SourceReadState state, long maxBytesInFlight) { if(request.format != Types.FileFormat.BINARY) - return CompletableFuture.failedFuture( - new DMLRuntimeException("Unsupported format for source read: " + request.format)); + return CompletableFuture + .failedFuture(new DMLRuntimeException("Unsupported format for source read: " + request.format)); return readBinarySourceParallel(request, state, maxBytesInFlight); } @@ -335,17 +320,18 @@ private CompletableFuture readBinarySourceParallel(SourceReadR return result; } - private void completeResult(CompletableFuture future, AtomicLong bytesRead, AtomicBoolean budgetHit, - AtomicReference error, SourceReadRequest request, Path[] files, AtomicLongArray filePositions, - AtomicIntegerArray completed, ConcurrentLinkedDeque descriptors) { + private void completeResult(CompletableFuture future, AtomicLong bytesRead, + AtomicBoolean budgetHit, AtomicReference error, SourceReadRequest request, Path[] files, + AtomicLongArray filePositions, AtomicIntegerArray completed, + ConcurrentLinkedDeque descriptors) { Throwable err = error.get(); - if (err != null) { + if(err != null) { future.completeExceptionally(err instanceof Exception ? err : new Exception(err)); return; } try { - if (budgetHit.get()) { + if(budgetHit.get()) { if(!request.keepOpenOnLimit) { closeTarget(request.target, false); } @@ -365,29 +351,29 @@ private void completeResult(CompletableFuture future, AtomicLo private void readSequenceFile(JobConf job, Path path, SourceReadRequest request, int fileIdx, AtomicLongArray filePositions, AtomicIntegerArray completed, AtomicBoolean stop, AtomicBoolean budgetHit, - AtomicLong bytesRead, long byteLimit, Object budgetLock, ConcurrentLinkedDeque descriptors) - throws IOException { + AtomicLong bytesRead, long byteLimit, Object budgetLock, + ConcurrentLinkedDeque descriptors) throws IOException { MatrixIndexes key = new MatrixIndexes(); try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { long pos = filePositions.get(fileIdx); - if (pos > 0) + if(pos > 0) reader.seek(pos); long ioStart = DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; while(!stop.get()) { long recordStart = reader.getPosition(); MatrixBlock value = new MatrixBlock(); - if (!reader.next(key, value)) + if(!reader.next(key, value)) break; long recordEnd = reader.getPosition(); long blockSize = value.getExactSerializedSize(); boolean shouldBreak = false; synchronized(budgetLock) { - if (stop.get()) + if(stop.get()) shouldBreak = true; - else if (bytesRead.get() + blockSize > byteLimit) { + else if(bytesRead.get() + blockSize > byteLimit) { stop.set(true); budgetHit.set(true); shouldBreak = true; @@ -398,7 +384,7 @@ else if (bytesRead.get() + blockSize > byteLimit) { MatrixIndexes outIdx = new MatrixIndexes(key); IndexedMatrixValue imv = new IndexedMatrixValue(outIdx, value); SourceBlockDescriptor descriptor = new SourceBlockDescriptor(path.toString(), request.format, outIdx, - recordStart, (int)(recordEnd - recordStart), blockSize); + recordStart, (int) (recordEnd - recordStart), blockSize); if(request.target instanceof SourceOOCStream src) src.enqueue(imv, descriptor); @@ -407,22 +393,23 @@ else if (bytesRead.get() + blockSize > byteLimit) { descriptors.add(descriptor); filePositions.set(fileIdx, reader.getPosition()); - if (DMLScript.OOC_LOG_EVENTS) { + if(DMLScript.OOC_LOG_EVENTS) { long currTime = System.nanoTime(); OOCEventLog.onDiskReadEvent(_srcReadCallerId, ioStart, currTime, blockSize); ioStart = currTime; } - if (shouldBreak) + if(shouldBreak) break; // Note that we knowingly go over limit, which could result in READER_SIZE*8MB overshoot } - if (!stop.get()) + if(!stop.get()) completed.set(fileIdx, 1); } } - private void closeTarget(org.apache.sysds.runtime.instructions.ooc.OOCStream target, boolean close) { + private void closeTarget(org.apache.sysds.runtime.instructions.ooc.OOCStream target, + boolean close) { if(close) { try { target.closeInput(); @@ -437,10 +424,10 @@ private void loadFromDisk(BlockEntry block) { String key = block.getKey().toFileKey(); SourceBlockDescriptor src = _sourceLocations.get(block.getKey()); - if (src != null) { + if(src != null) { long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; loadFromSource(block, src); - if (DMLScript.OOC_STATISTICS) { + if(DMLScript.OOC_STATISTICS) { Statistics.incrementOOCLoadFromDisk(); Statistics.accumulateOOCLoadFromDiskTime(System.nanoTime() - ioStart); Statistics.accumulateOOCLoadFromDiskBytes(block.getSize()); @@ -451,34 +438,36 @@ private void loadFromDisk(BlockEntry block) { long ioDuration = 0; // 1. find the blocks address (spill location) SpillLocation sloc = _spillLocations.get(key); - if (sloc == null) + if(sloc == null) throw new DMLRuntimeException("Failed to load spill location for: " + key); PartitionFile partFile = _partitions.get(sloc.partitionId); - if (partFile == null) + if(partFile == null) throw new DMLRuntimeException("Failed to load partition for: " + sloc.partitionId); String filename = partFile.filePath; SpillableObject obj; - try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { + try(RandomAccessFile raf = new RandomAccessFile(filename, "r")) { raf.seek(sloc.offset); DataInput dis = new OOCBufferedDataInputStream(raf); long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; obj = SpillableObjectRegistry.read(dis); - if (DMLScript.OOC_STATISTICS) + if(DMLScript.OOC_STATISTICS) ioDuration = System.nanoTime() - ioStart; - } catch (ClosedByInterruptException ignored) { + } + catch(ClosedByInterruptException ignored) { return; - } catch (IOException e) { + } + catch(IOException e) { throw new RuntimeException(e); } block.setDataUnsafe(obj); - if (DMLScript.OOC_STATISTICS) { + if(DMLScript.OOC_STATISTICS) { Statistics.incrementOOCLoadFromDisk(); Statistics.accumulateOOCLoadFromDiskTime(ioDuration); Statistics.accumulateOOCLoadFromDiskBytes(block.getSize()); @@ -486,22 +475,23 @@ private void loadFromDisk(BlockEntry block) { } private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { - if (src.format != Types.FileFormat.BINARY) + if(src.format != Types.FileFormat.BINARY) throw new DMLRuntimeException("Unsupported format for source read: " + src.format); JobConf job = new JobConf(ConfigurationManager.getCachedJobConf()); Path path = new Path(src.path); - if (src instanceof OOCIOHandler.GroupSourceBlockDescriptor gsrc) { + if(src instanceof OOCIOHandler.GroupSourceBlockDescriptor gsrc) { List values = new ArrayList<>(gsrc.count); try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { reader.seek(gsrc.offset); - for (int i = 0; i < gsrc.blocks.size(); i++) { + for(int i = 0; i < gsrc.blocks.size(); i++) { SourceBlockDescriptor d = gsrc.blocks.get(i); MatrixIndexes ix = new MatrixIndexes(); MatrixBlock mb = new MatrixBlock(); - if (!reader.next(ix, mb)) - throw new DMLRuntimeException("Failed to read source block at offset " + d.offset + " in " + d.path); + if(!reader.next(ix, mb)) + throw new DMLRuntimeException( + "Failed to read source block at offset " + d.offset + " in " + d.path); values.add(new IndexedMatrixValue(ix, mb)); } } @@ -516,8 +506,9 @@ private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { reader.seek(src.offset); - if (!reader.next(ix, mb)) - throw new DMLRuntimeException("Failed to read source block at offset " + src.offset + " in " + src.path); + if(!reader.next(ix, mb)) + throw new DMLRuntimeException( + "Failed to read source block at offset " + src.offset + " in " + src.path); } catch(IOException e) { throw new DMLRuntimeException(e); @@ -530,7 +521,7 @@ private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { private void evictTask(CloseableQueue>> q) { long byteCtr = 0; - while (!q.isFinished()) { + while(!q.isFinished()) { // --- 1. WRITE PHASE --- int partitionId = _partitionCounter.getAndIncrement(); @@ -572,17 +563,17 @@ private void evictTask(CloseableQueue } byteCtr += wrote; - if (byteCtr >= MAX_PARTITION_SIZE) { + if(byteCtr >= MAX_PARTITION_SIZE) { closePartition = true; byteCtr = 0; break; } - if (DMLScript.OOC_LOG_EVENTS) + if(DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskWriteEvent(_evictCallerId, ioStart, System.nanoTime(), wrote); } - if (!closePartition && q.close()) { + if(!closePartition && q.close()) { while((tpl = q.take()) != null) { long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; BlockEntry entry = tpl._1(); @@ -595,7 +586,7 @@ private void evictTask(CloseableQueue Statistics.accumulateOOCEvictionWriteTime(System.nanoTime() - ioStart); } - if (DMLScript.OOC_LOG_EVENTS) + if(DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskWriteEvent(_evictCallerId, ioStart, System.nanoTime(), wrote); } } @@ -617,13 +608,13 @@ private void evictTask(CloseableQueue } private long writeOut(int partitionId, BlockEntry entry, CompletableFuture future, - OOCBufferedDataOutputStream dos, - ConcurrentLinkedDeque>> flushQueue) throws IOException { + OOCBufferedDataOutputStream dos, ConcurrentLinkedDeque>> flushQueue) + throws IOException { String key = entry.getKey().toFileKey(); boolean alreadySpilled = _spillLocations.containsKey(key); - if (!alreadySpilled) { + if(!alreadySpilled) { long offsetBefore = dos.getPosition(); if(future.isCancelled()) @@ -656,9 +647,10 @@ private long writeOut(int partitionId, BlockEntry entry, CompletableFuture return 0; } - private void flushQueue(long offset, ConcurrentLinkedDeque>> flushQueue) { + private void flushQueue(long offset, + ConcurrentLinkedDeque>> flushQueue) { Tuple3> tmp; - while ((tmp = flushQueue.peek()) != null && tmp._2() <= offset) { + while((tmp = flushQueue.peek()) != null && tmp._2() <= offset) { flushQueue.poll(); tmp._3().complete(null); } @@ -777,12 +769,14 @@ public void run() { try { long ioStart = DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; loadFromDisk(_block); - if (DMLScript.OOC_LOG_EVENTS) + if(DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskReadEvent(_readCallerId, ioStart, System.nanoTime(), _block.getSize()); _future.complete(_block); - } catch (Throwable e) { + } + catch(Throwable e) { _future.completeExceptionally(e); - } finally { + } + finally { unpinPartitionForRead(_pinnedPartitionId); } } @@ -790,15 +784,12 @@ public void run() { @Override public int compareTo(ReadTask other) { int byPriority = Double.compare(other._priority, _priority); - if (byPriority != 0) + if(byPriority != 0) return byPriority; return Long.compare(_sequence, other._sequence); } } - - - private static class SpillLocation { // structure of spillLocation: file, offset final int partitionId; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java index a93b1d18f2a..3458838c071 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java @@ -24,8 +24,10 @@ import java.io.IOException; public interface SpillableObject { - boolean tryWrite(DataOutput out) throws IOException; - void read(DataInput in) throws IOException; + boolean tryWrite(DataOutput out) throws IOException; + + void read(DataInput in) throws IOException; + long size(); default void discard() { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java index d8ff79dd797..0d318d83d94 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java @@ -34,14 +34,16 @@ public interface OOCCacheScheduler { /** * Requests a single block from the cache. + * * @param key the requested key associated to the block * @return the available BlockEntry */ CompletableFuture request(BlockKey key); /** - * Tries to request a single block from the cache. - * Immediately returns the entry if present, otherwise null without scheduling reads. + * Tries to request a single block from the cache. Immediately returns the entry if present, otherwise null without + * scheduling reads. + * * @param key the requested key associated to the block * @return the available BlockEntry or null */ @@ -52,14 +54,16 @@ default BlockEntry tryRequest(BlockKey key) { /** * Requests a list of blocks from the cache that must be available at the same time. + * * @param keys the requested keys associated to the block * @return the list of available BlockEntries */ CompletableFuture> request(List keys); /** - * Tries to request a list of blocks from the cache that must be available at the same time. - * Immediately returns the list of entries if present, otherwise null without scheduling reads. + * Tries to request a list of blocks from the cache that must be available at the same time. Immediately returns the + * list of entries if present, otherwise null without scheduling reads. + * * @param keys the requested keys associated to the block * @return the list of available BlockEntries */ @@ -76,24 +80,25 @@ default BlockEntry tryRequest(BlockKey key) { List tryRequestAnyOf(List keys, int n, List selectionOut); /** - * Adds the given priority to any pending request accessing the key. - * Multi-requests are prioritized partially. + * Adds the given priority to any pending request accessing the key. Multi-requests are prioritized partially. */ void prioritize(BlockKey key, double priority); /** - * Places a new block in the cache. Note that objects are immutable and cannot be overwritten. - * The object data should now only be accessed via cache, as ownership has been transferred. - * @param key the associated key of the block + * Places a new block in the cache. Note that objects are immutable and cannot be overwritten. The object data + * should now only be accessed via cache, as ownership has been transferred. + * + * @param key the associated key of the block * @param data the block data * @param size the size of the data */ BlockKey put(BlockKey key, Object data, long size); /** - * Places a new block in the cache and returns a pinned handle. - * Note that objects are immutable and cannot be overwritten. - * @param key the associated key of the block + * Places a new block in the cache and returns a pinned handle. Note that objects are immutable and cannot be + * overwritten. + * + * @param key the associated key of the block * @param data the block data * @param size the size of the data */ @@ -101,8 +106,11 @@ default BlockEntry tryRequest(BlockKey key) { interface HandoverHandle { BlockKey getKey(); + boolean isCommitted(); + CompletableFuture getCompletionFuture(); + OOCStream.QueueCallback reclaim(); } @@ -131,27 +139,30 @@ BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.SourceBlockDescriptor descriptor); /** - * Notifies the cache that there is another reference to the same block key. - * This will prevent forget(key) from removing the block from cache. - * A block will only be forgotten after all referencing instances called forget(key). + * Notifies the cache that there is another reference to the same block key. This will prevent forget(key) from + * removing the block from cache. A block will only be forgotten after all referencing instances called forget(key). + * * @param key */ void addReference(BlockKey key); /** * Forgets a block from the cache. + * * @param key the associated key of the block */ void forget(BlockKey key); /** * Pins a BlockEntry in cache to prevent eviction. + * * @param entry the entry to be pinned */ void pin(BlockEntry entry); /** * Unpins a pinned block. + * * @param entry the entry to be unpinned */ void unpin(BlockEntry entry); @@ -192,8 +203,7 @@ BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, void updateLimits(long evictionLimit, long hardLimit); /** - * Creates a snapshot of the cache. - * Should only be used for debugging or diagnoses. + * Creates a snapshot of the cache. Should only be used for debugging or diagnoses. */ Collection snapshot(); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java index 96de368ccc9..9b0acc97b35 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java @@ -80,7 +80,7 @@ public class OOCLRUCacheScheduler implements OOCCacheScheduler { public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long hardLimit, long readBuffer) { this._ioHandler = ioHandler; this._cache = new LinkedHashMap<>(1024, 0.75f, true); - this._evictionCache = new HashMap<>(); + this._evictionCache = new HashMap<>(); this._deferredReadRequests = new DeferredReadQueue(); this._processingReadRequests = new ArrayDeque<>(); this._pendingHandovers = new ArrayDeque<>(); @@ -103,7 +103,7 @@ public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long har this._maintenanceNeedsIncr = new AtomicBoolean(false); this._callerId = DMLScript.OOC_LOG_EVENTS ? OOCEventLog.registerCaller("LRUCacheScheduler") : 0; - if (DMLScript.OOC_LOG_EVENTS) { + if(DMLScript.OOC_LOG_EVENTS) { OOCEventLog.putRunSetting("CacheEvictionLimit", _evictionLimit); OOCEventLog.putRunSetting("CacheHardLimit", _hardLimit); } @@ -111,7 +111,7 @@ public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long har @Override public CompletableFuture request(BlockKey key) { - if (!this._running) + if(!this._running) throw new IllegalStateException("Cache scheduler has been shut down."); Statistics.incrementOOCEvictionGet(); @@ -120,21 +120,21 @@ public CompletableFuture request(BlockKey key) { boolean couldPin = false; synchronized(this) { entry = _cache.get(key); - if (entry == null) + if(entry == null) entry = _evictionCache.get(key); - if (entry == null) + if(entry == null) throw new IllegalArgumentException("Could not find requested block with key " + key); synchronized(entry) { - if (entry.getState().isAvailable()) { - if (pinEntryWithAccounting(entry) == 0) + if(entry.getState().isAvailable()) { + if(pinEntryWithAccounting(entry) == 0) throw new IllegalStateException(); couldPin = true; } } } - if (couldPin) { + if(couldPin) { // Then we could pin the required entry and can terminate return CompletableFuture.completedFuture(entry); } @@ -184,7 +184,7 @@ public CompletableFuture> request(List keys) { } public CompletableFuture> request(List keys, boolean onlyIfAvailable) { - if (!this._running) + if(!this._running) throw new IllegalStateException("Cache scheduler has been shut down."); Statistics.incrementOOCEvictionGet(keys.size()); @@ -193,11 +193,11 @@ public CompletableFuture> request(List keys, boolean boolean allAvailable = true; synchronized(this) { - for (BlockKey key : keys) { + for(BlockKey key : keys) { BlockEntry entry = _cache.get(key); - if (entry == null) + if(entry == null) entry = _evictionCache.get(key); - if (entry == null) + if(entry == null) throw new IllegalArgumentException("Could not find requested block with key " + key); synchronized(entry) { @@ -217,7 +217,7 @@ public CompletableFuture> request(List keys, boolean } } - if (allAvailable) { + if(allAvailable) { // Then we could pin all entries return CompletableFuture.completedFuture(entries); } @@ -226,12 +226,12 @@ public CompletableFuture> request(List keys, boolean return null; // Schedule deferred read otherwise - final CompletableFuture> future = new CompletableFuture<>(); + final CompletableFuture> future = new CompletableFuture<>(); DeferredReadRequest request = new DeferredReadRequest(future, entries); - for (int i = 0; i < entries.size(); i++) { + for(int i = 0; i < entries.size(); i++) { BlockEntry entry = entries.get(i); synchronized(entry) { - if (entry.getState().isAvailable()) { + if(entry.getState().isAvailable()) { entry.addRetainHint(); request.markRetainHinted(i); } @@ -243,9 +243,9 @@ public CompletableFuture> request(List keys, boolean @Override public void prioritize(BlockKey key, double priority) { - if (!this._running) + if(!this._running) return; - if (priority == 0) + if(priority == 0) return; synchronized(this) { @@ -267,18 +267,18 @@ private void scheduleDeferredRead(DeferredReadRequest deferredReadRequest) { if(entry.getState().isAvailable()) readyCount++; BlockReadState state = _blockReads.get(entry.getKey()); - if (state != null) + if(state != null) score += state.priority; } - if (!deferredReadRequest.getEntries().isEmpty()) + if(!deferredReadRequest.getEntries().isEmpty()) score /= deferredReadRequest.getEntries().size(); - if (!deferredReadRequest.getEntries().isEmpty()) + if(!deferredReadRequest.getEntries().isEmpty()) score += ((double) readyCount) / deferredReadRequest.getEntries().size(); deferredReadRequest.setPriorityScore(score); _deferredReadRequests.add(deferredReadRequest); _deferredReadCountHint = _deferredReadRequests.size(); } - onCacheSizeChanged(true); // Apply pressure from deferred read demand. + onCacheSizeChanged(true); // Apply pressure from deferred read demand. onCacheSizeChanged(false); // Attempt to schedule deferred reads. } @@ -317,7 +317,8 @@ public void putSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.S } @Override - public BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.SourceBlockDescriptor descriptor) { + public BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, + OOCIOHandler.SourceBlockDescriptor descriptor) { return put(key, data, size, true, descriptor); } @@ -333,23 +334,24 @@ public void addReference(BlockKey key) { } } - private BlockEntry put(BlockKey key, Object data, long size, boolean pin, OOCIOHandler.SourceBlockDescriptor descriptor) { - if (!this._running) + private BlockEntry put(BlockKey key, Object data, long size, boolean pin, + OOCIOHandler.SourceBlockDescriptor descriptor) { + if(!this._running) throw new IllegalStateException(); - if (data == null) + if(data == null) throw new IllegalArgumentException(); - if (descriptor != null) + if(descriptor != null) _ioHandler.registerSourceLocation(key, descriptor); Statistics.incrementOOCEvictionPut(); BlockEntry entry = new BlockEntry(key, size, data); - if (descriptor != null) + if(descriptor != null) entry.setState(BlockState.WARM); - if (pin) + if(pin) entry.pin(); synchronized(this) { BlockEntry avail = _cache.putIfAbsent(key, entry); - if (avail != null || _evictionCache.containsKey(key)) + if(avail != null || _evictionCache.containsKey(key)) throw new IllegalStateException("Cannot overwrite existing entries: " + key); _cacheSize += size; if(pin) { @@ -364,7 +366,7 @@ private BlockEntry put(BlockKey key, Object data, long size, boolean pin, OOCIOH @Override public void forget(BlockKey key) { - if (!this._running) + if(!this._running) return; final MutableObject mEntry = new MutableObject<>(); BlockEntry entry; @@ -381,7 +383,7 @@ public void forget(BlockKey key) { return e; }); - if (mEntry.getValue() == null) { + if(mEntry.getValue() == null) { _evictionCache.compute(key, (k, e) -> { if(e == null) return null; @@ -395,10 +397,10 @@ public void forget(BlockKey key) { entry = mEntry.getValue(); - if (entry != null) { + if(entry != null) { synchronized(entry) { - shouldScheduleDeletion = entry.getState().isBackedByDisk() - || entry.getState() == BlockState.EVICTING; + shouldScheduleDeletion = entry.getState().isBackedByDisk() || + entry.getState() == BlockState.EVICTING; cacheSizeDelta = transitionMemState(entry, BlockState.REMOVED); if(entry.isPinned() && entry.getDataUnsafe() != null) _pinnedBytes -= entry.getSize(); @@ -408,9 +410,9 @@ public void forget(BlockKey key) { } } } - if (cacheSizeDelta != 0) + if(cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); - if (shouldScheduleDeletion) + if(shouldScheduleDeletion) _ioHandler.scheduleDeletion(entry); } @@ -424,11 +426,11 @@ public void pin(BlockEntry entry) { synchronized(this) { synchronized(entry) { int pinCount = pinEntryWithAccounting(entry); - if (pinCount == 0) + if(pinCount == 0) throw new IllegalStateException("Could not pin the requested entry: " + entry.getKey()); } // Access element in cache for Lru - //_cache.get(entry.getKey()); + // _cache.get(entry.getKey()); } } @@ -442,26 +444,26 @@ public void unpin(BlockEntry entry) { synchronized(entry) { if(!unpinEntryWithAccounting(entry)) return; - if (_cacheSize <= _evictionLimit) + if(_cacheSize <= _evictionLimit) return; // Nothing to do - if (entry.isPinned()) + if(entry.isPinned()) return; // Pin state changed so we cannot evict - if (entry.getState().isAvailable() && entry.getState().isBackedByDisk()) { - if (entry.getRetainHintCount() > 0) { + if(entry.getState().isAvailable() && entry.getState().isBackedByDisk()) { + if(entry.getRetainHintCount() > 0) { shouldCheckEviction = true; } else { - cacheSizeDelta = transitionMemState(entry, BlockState.COLD); + cacheSizeDelta = transitionMemState(entry, BlockState.COLD); long cleared = entry.clear(); - if (cleared != entry.getSize()) + if(cleared != entry.getSize()) throw new IllegalStateException(); _cache.remove(entry.getKey()); _evictionCache.put(entry.getKey(), entry); } } - else if (entry.getState() == BlockState.HOT) { - if (entry.getRetainHintCount() > 0) { + else if(entry.getState() == BlockState.HOT) { + if(entry.getRetainHintCount() > 0) { shouldCheckEviction = true; } else { @@ -470,9 +472,9 @@ else if (entry.getState() == BlockState.HOT) { } } } - if (cacheSizeDelta != 0) + if(cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); - else if (shouldCheckEviction) + else if(shouldCheckEviction) onCacheSizeChanged(true); } @@ -508,10 +510,11 @@ public synchronized void shutdown() { System.out.println("[WARN] Cache still holds " + _cache.size() + " / " + _evictionCache.size() + " blocks"); Set cachedStreams = _cache.keySet().stream().map(BlockKey::getStreamId).collect(Collectors.toSet()); - Set evictedStreams = _evictionCache.keySet().stream().map(BlockKey::getStreamId).collect(Collectors.toSet()); + Set evictedStreams = _evictionCache.keySet().stream().map(BlockKey::getStreamId) + .collect(Collectors.toSet()); cachedStreams.addAll(evictedStreams); - System.out.println("[WARN] Affected stream IDs: " + cachedStreams + ", Pinned: " + _cache.values().stream().mapToInt( - e -> e.isPinned() ? 1 : 0).sum()); + System.out.println("[WARN] Affected stream IDs: " + cachedStreams + ", Pinned: " + + _cache.values().stream().mapToInt(e -> e.isPinned() ? 1 : 0).sum()); } _cache.clear(); _evictionCache.clear(); @@ -575,7 +578,8 @@ private void runMaintenanceLoop() { do { _maintenanceRequested.set(false); onCacheSizeChangedInternal(_maintenanceNeedsIncr.getAndSet(false)); - } while(_maintenanceRequested.get()); + } + while(_maintenanceRequested.get()); } finally { _maintenanceRunning.set(false); @@ -591,7 +595,8 @@ private void onCacheSizeChangedInternal(boolean incr) { if(incr) onCacheSizeIncremented(); else - while(onCacheSizeDecremented()) {} + while(onCacheSizeDecremented()) { + } while(processPendingHandovers()) { onCacheSizeIncremented(); } @@ -601,18 +606,23 @@ private void onCacheSizeChangedInternal(boolean incr) { } private synchronized void sanityCheck() { - if (_cacheSize > _hardLimit * 1.1) { - if (!_warnThrottling) { + if(_cacheSize > _hardLimit * 1.1) { + if(!_warnThrottling) { _warnThrottling = true; - System.out.println("[WARN] Cache hard limit exceeded by over 10%: " + String.format("%.2f", _cacheSize/1000000.0) + "MB (-" + String.format("%.2f", _bytesUpForEviction/1000000.0) + "MB) > " + String.format("%.2f", _hardLimit/1000000.0) + "MB"); + System.out.println( + "[WARN] Cache hard limit exceeded by over 10%: " + String.format("%.2f", _cacheSize / 1000000.0) + + "MB (-" + String.format("%.2f", _bytesUpForEviction / 1000000.0) + "MB) > " + + String.format("%.2f", _hardLimit / 1000000.0) + "MB"); } } - else if (_warnThrottling && _cacheSize < _hardLimit) { + else if(_warnThrottling && _cacheSize < _hardLimit) { _warnThrottling = false; - System.out.println("[INFO] Cache within limit: " + String.format("%.2f", _cacheSize/1000000.0) + "MB (-" + String.format("%.2f", _bytesUpForEviction/1000000.0) + "MB) <= " + String.format("%.2f", _hardLimit/1000000.0) + "MB"); + System.out.println("[INFO] Cache within limit: " + String.format("%.2f", _cacheSize / 1000000.0) + "MB (-" + + String.format("%.2f", _bytesUpForEviction / 1000000.0) + "MB) <= " + + String.format("%.2f", _hardLimit / 1000000.0) + "MB"); } - if (!SANITY_CHECKS) + if(!SANITY_CHECKS) return; int pinned = 0; @@ -625,16 +635,16 @@ else if (_warnThrottling && _cacheSize < _hardLimit) { long actualPinnedEvictingBytes = 0; long actualWarmPinnedBytes = 0; long actualReadingReservedBytes = 0; - for (BlockEntry entry : _cache.values()) { - if (entry.isPinned()) { + for(BlockEntry entry : _cache.values()) { + if(entry.isPinned()) { pinned++; actualPinnedBytes += entry.getSize(); if(entry.getState() == BlockState.WARM) actualWarmPinnedBytes += entry.getSize(); } - if (entry.getState().isBackedByDisk()) + if(entry.getState().isBackedByDisk()) backedByDisk++; - if (entry.getState() == BlockState.EVICTING) { + if(entry.getState() == BlockState.EVICTING) { evicting++; upForEviction += entry.getSize(); if(entry.isPinned()) @@ -642,43 +652,44 @@ else if (_warnThrottling && _cacheSize < _hardLimit) { } if(entry.getState() == BlockState.READING) actualReadingReservedBytes += entry.getSize(); - if (!entry.getState().isAvailable()) + if(!entry.getState().isAvailable()) throw new IllegalStateException(); total++; actualCacheSize += entry.getSize(); } - for (BlockEntry entry : _evictionCache.values()) { - if (entry.getState().isAvailable()) + for(BlockEntry entry : _evictionCache.values()) { + if(entry.getState().isAvailable()) throw new IllegalStateException("Invalid eviction state: " + entry.getState()); - if (entry.getState() == BlockState.EVICTING && entry.isPinned()) + if(entry.getState() == BlockState.EVICTING && entry.isPinned()) actualPinnedEvictingBytes += entry.getSize(); - if (entry.getState() == BlockState.READING) + if(entry.getState() == BlockState.READING) actualCacheSize += entry.getSize(); - if (entry.getState() == BlockState.READING) + if(entry.getState() == BlockState.READING) actualReadingReservedBytes += entry.getSize(); - if (entry.isPinned()) { + if(entry.isPinned()) { actualPinnedBytes += entry.getSize(); if(entry.getState() == BlockState.WARM) actualWarmPinnedBytes += entry.getSize(); } } - if (actualCacheSize != _cacheSize) + if(actualCacheSize != _cacheSize) throw new IllegalStateException(actualCacheSize + " != " + _cacheSize); - if (upForEviction != _bytesUpForEviction) + if(upForEviction != _bytesUpForEviction) throw new IllegalStateException(upForEviction + " != " + _bytesUpForEviction); - if (actualPinnedBytes != _pinnedBytes) + if(actualPinnedBytes != _pinnedBytes) throw new IllegalStateException(actualPinnedBytes + " != " + _pinnedBytes); - if (actualPinnedEvictingBytes != _pinnedEvictingBytes) + if(actualPinnedEvictingBytes != _pinnedEvictingBytes) throw new IllegalStateException(actualPinnedEvictingBytes + " != " + _pinnedEvictingBytes); - if (_pinnedEvictingBytes > _bytesUpForEviction) + if(_pinnedEvictingBytes > _bytesUpForEviction) throw new IllegalStateException(_pinnedEvictingBytes + " > " + _bytesUpForEviction); if(actualWarmPinnedBytes != _warmPinnedBytes) throw new IllegalStateException(actualWarmPinnedBytes + " != " + _warmPinnedBytes); - if (actualReadingReservedBytes != _readingReservedBytes) + if(actualReadingReservedBytes != _readingReservedBytes) throw new IllegalStateException(actualReadingReservedBytes + " != " + _readingReservedBytes); System.out.println("=========="); - System.out.println("Limit: " + _evictionLimit/1000 + "KB"); - System.out.println("Memory: (" + _cacheSize/1000 + "KB - " + _bytesUpForEviction/1000 + "KB) / " + _hardLimit/1000 + "KB"); + System.out.println("Limit: " + _evictionLimit / 1000 + "KB"); + System.out.println("Memory: (" + _cacheSize / 1000 + "KB - " + _bytesUpForEviction / 1000 + "KB) / " + + _hardLimit / 1000 + "KB"); System.out.println("Pinned: " + pinned + " / " + total); System.out.println("Disk backed: " + backedByDisk + " / " + total); System.out.println("Evicting: " + evicting + " / " + total); @@ -695,10 +706,11 @@ private void onCacheSizeIncremented() { if(pressure <= _evictionLimit) return; // Nothing to do - long overshoot = Math.max((long)(0.1 * _evictionLimit), 10000000); + long overshoot = Math.max((long) (0.1 * _evictionLimit), 10000000); long lowLimit = _evictionLimit - _readBuffer - overshoot; - //System.out.println("[CACHE] Claiming " + (pressure + overshoot - _evictionLimit)/1000 + "kB (last claim was " + (System.currentTimeMillis() - _lastEvictRun) + "ms ago)"); + // System.out.println("[CACHE] Claiming " + (pressure + overshoot - _evictionLimit)/1000 + "kB (last claim + // was " + (System.currentTimeMillis() - _lastEvictRun) + "ms ago)"); // Scan for values that can be evicted Collection entries = _cache.values(); @@ -713,8 +725,8 @@ private void onCacheSizeIncremented() { break; synchronized(entry) { - //if(entry.isPinned()) - // continue; + // if(entry.isPinned()) + // continue; if(!allowRetainHint && entry.getRetainHintCount() > 0) continue; if(entry.getState() == BlockState.COLD || entry.getState() == BlockState.EVICTING) @@ -748,12 +760,12 @@ private void onCacheSizeIncremented() { _lastEvictRun = System.currentTimeMillis(); } - for (BlockEntry entry : upForEvictionNeedsWrite) + for(BlockEntry entry : upForEvictionNeedsWrite) evict(entry, true); - for (BlockEntry entry : upForEvictionNoWrite) + for(BlockEntry entry : upForEvictionNoWrite) evict(entry, false); - if (cacheSizeDelta != 0) + if(cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); } @@ -813,7 +825,7 @@ private boolean onCacheSizeDecremented() { throw new IllegalStateException(); req.setPinned(idx); } - else if (entry.getState() == BlockState.READING) { + else if(entry.getState() == BlockState.READING) { req.schedule(idx); registerWaiter(entry.getKey(), req, idx); reading = true; @@ -836,7 +848,7 @@ else if (entry.getState() == BlockState.READING) { if(allReserved) { _deferredReadRequests.poll(); _deferredReadCountHint = _deferredReadRequests.size(); - if (!toRead.isEmpty()) + if(!toRead.isEmpty()) _processingReadRequests.add(req); } @@ -943,17 +955,17 @@ private void onEvicted(final BlockEntry entry) { if(tmp != null && tmp != entry) throw new IllegalStateException(); tmp = _evictionCache.put(entry.getKey(), entry); - if (tmp != null) + if(tmp != null) throw new IllegalStateException(); sanityCheck(); } - if (cacheSizeDelta != 0) + if(cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); } private void clearRetainHints(DeferredReadRequest request) { - for (int i = 0; i < request.getEntries().size(); i++) { - if (!request.isRetainHinted(i)) + for(int i = 0; i < request.getEntries().size(); i++) { + if(!request.isRetainHinted(i)) continue; BlockEntry entry = request.getEntries().get(i); synchronized(entry) { @@ -963,12 +975,12 @@ private void clearRetainHints(DeferredReadRequest request) { } /** - * Cleanly transitions state of a BlockEntry and handles accounting. - * Requires both the scheduler object and the entry to be locked: + * Cleanly transitions state of a BlockEntry and handles accounting. Requires both the scheduler object and the + * entry to be locked: */ private long transitionMemState(BlockEntry entry, BlockState newState) { BlockState oldState = entry.getState(); - if (oldState == newState) + if(oldState == newState) return 0; long sz = entry.getSize(); @@ -976,7 +988,7 @@ private long transitionMemState(BlockEntry entry, BlockState newState) { boolean pinned = entry.isPinned(); // Remove old contribution - switch (oldState) { + switch(oldState) { case REMOVED: throw new IllegalStateException(); case HOT: @@ -1000,7 +1012,7 @@ private long transitionMemState(BlockEntry entry, BlockState newState) { } // Add new contribution - switch (newState) { + switch(newState) { case REMOVED: case COLD: break; @@ -1056,6 +1068,7 @@ private int pinEntryWithAccounting(BlockEntry entry) { /** * Requires scheduler lock and entry lock. + * * @return true if this call transitioned pin count to zero. */ private boolean unpinEntryWithAccounting(BlockEntry entry) { diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java b/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java index 1f731fc3aa5..f04534481c9 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java @@ -66,8 +66,8 @@ protected boolean isHashCol(int colID) { } /** - * Domain size of a dummycoded source column: the hash domain K from the meta cell for - * feature-hashed columns, otherwise the column's {@code numDistinct} (0 when unset). + * Domain size of a dummycoded source column: the hash domain K from the meta cell for feature-hashed columns, + * otherwise the column's {@code numDistinct} (0 when unset). * * @param meta transform meta frame * @param colID 1-based column id of the dummycoded source column @@ -144,13 +144,13 @@ public FrameBlock decode(final MatrixBlock in, final FrameBlock out, final int k try { final List> tasks = new ArrayList<>(); int blz = Math.max((in.getNumRows() + k) / k, 1000); - - for(int i = 0; i < in.getNumRows(); i += blz){ + + for(int i = 0; i < in.getNumRows(); i += blz) { final int start = i; final int end = Math.min(in.getNumRows(), i + blz); tasks.add(pool.submit(() -> decode(in, out, start, end))); } - + for(Future f : tasks) f.get(); return out; diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java index a286c03dce8..01a502154cd 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java @@ -72,10 +72,10 @@ public void decode(MatrixBlock in, FrameBlock out, int rl, int ru) { final double val = in.get(i, _srcCols[j] - 1); if(!Double.isNaN(val)){ final int key = (int) Math.round(val); - if(key == 0){ + if(key == 0) { a.set(i, _binMins[j][key]); } - else{ + else { double bmin = _binMins[j][key - 1]; double bmax = _binMaxs[j][key - 1]; double oval = bmin + (bmax - bmin) / 2 // bin center diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java index ee1a33c49fd..8aa0b60d990 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java @@ -34,7 +34,7 @@ /** * Simple atomic decoder for dummycoded columns. This decoder builds internally inverted column mappings from the given * frame meta data. - * + * */ public class DecoderDummycode extends Decoder { private static final long serialVersionUID = 4758831042891032129L; diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java index 8f6c45d63e8..fc123657032 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java @@ -64,15 +64,15 @@ public static Decoder createDecoder(String spec, String[] colnames, ValueType[] try { //parse transform specification JSONObject jSpec = new JSONObject(spec); - - //create decoders 'bin', 'recode', 'hash', 'dummy', and 'pass-through' + + // create decoders 'bin', 'recode', 'hash', 'dummy', and 'pass-through' List binIDs = TfMetaUtils.parseBinningColIDs(jSpec, colnames, minCol, maxCol); - List rcIDs = Arrays.asList(ArrayUtils.toObject( - TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.RECODE.toString(), minCol, maxCol))); - List hcIDs = Arrays.asList(ArrayUtils.toObject( - TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.HASH.toString(), minCol, maxCol))); - List dcIDs = Arrays.asList(ArrayUtils.toObject( - TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.DUMMYCODE.toString(), minCol, maxCol))); + List rcIDs = Arrays.asList(ArrayUtils + .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.RECODE.toString(), minCol, maxCol))); + List hcIDs = Arrays.asList(ArrayUtils + .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.HASH.toString(), minCol, maxCol))); + List dcIDs = Arrays.asList(ArrayUtils + .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.DUMMYCODE.toString(), minCol, maxCol))); // only specially treat the columns with both recode and dictionary rcIDs = unionDistinct(rcIDs, dcIDs); // hashing is a lossy, one-way transform with no inverse recode map, so hash columns @@ -106,29 +106,18 @@ public static Decoder createDecoder(String spec, String[] colnames, ValueType[] // collect all the decoders in one list. List ldecoders = new ArrayList<>(); - - if( !binIDs.isEmpty() ) { - ldecoders.add(new DecoderBin(schema, - ArrayUtils.toPrimitive(binIDs.toArray(new Integer[0])), + + if(!binIDs.isEmpty()) { + ldecoders.add(new DecoderBin(schema, ArrayUtils.toPrimitive(binIDs.toArray(new Integer[0])), ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); } if( !dcIDs.isEmpty() ) { ldecoders.add(new DecoderDummycode(schema, ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); } - if( !rcIDs.isEmpty() ) { - // recode on output (after dummycode rebuilds the categorical columns) when dummycoding is present - ldecoders.add(new DecoderRecode(schema, !dcIDs.isEmpty(), - ArrayUtils.toPrimitive(rcIDs.toArray(new Integer[0])))); - } - if( !ptIDs.isEmpty() ) { - ldecoders.add(new DecoderPassThrough(schema, - ArrayUtils.toPrimitive(ptIDs.toArray(new Integer[0])), - ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); - } - - //create composite decoder of all created decoders - //and initialize with given meta data (recode, dummy, bin) + + // create composite decoder of all created decoders + // and initialize with given meta data (recode, dummy, bin) decoder = new DecoderComposite(schema, ldecoders); decoder.setColnames(colnames); decoder.initMetaData(meta); @@ -147,7 +136,7 @@ else if( decoder instanceof DecoderRecode ) return DecoderType.Recode.ordinal(); else if( decoder instanceof DecoderPassThrough ) return DecoderType.PassThrough.ordinal(); - else if( decoder instanceof DecoderBin ) + else if(decoder instanceof DecoderBin) return DecoderType.Bin.ordinal(); throw new DMLRuntimeException("Unsupported decoder type: " + decoder.getClass().getCanonicalName()); @@ -158,10 +147,14 @@ public static Decoder createInstance(int type) { // create instance switch(dtype) { - case Bin: return new DecoderBin(); - case Dummycode: return new DecoderDummycode(null, null); - case PassThrough: return new DecoderPassThrough(null, null, null); - case Recode: return new DecoderRecode(null, false, null); + case Bin: + return new DecoderBin(); + case Dummycode: + return new DecoderDummycode(null, null); + case PassThrough: + return new DecoderPassThrough(null, null, null); + case Recode: + return new DecoderRecode(null, false, null); default: throw new DMLRuntimeException("Unsupported Encoder Type used: " + dtype); } diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java index 11dd2c7faa5..d8d624122a0 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java @@ -126,20 +126,20 @@ public void initMetaData(FrameBlock meta) { _rcMaps = new HashMap[_colList.length]; for( int j=0; j<_colList.length; j++ ) { HashMap map = new HashMap<>(); - for( int i=0; i= 0} - * both the minimum and maximum fraction digits are pinned to {@code decimal}, so values are - * printed with exactly that many decimals; otherwise the {@link DecimalFormat} defaults apply. + * Creates a non-grouping {@link DecimalFormat} for printing values. When {@code decimal >= 0} both the minimum and + * maximum fraction digits are pinned to {@code decimal}, so values are printed with exactly that many decimals; + * otherwise the {@link DecimalFormat} defaults apply. + * * @param decimal number of decimal places to print, -1 for default * @return a configured {@link DecimalFormat} */ private static DecimalFormat createDecimalFormat(int decimal) { DecimalFormat df = new DecimalFormat(); df.setGroupingUsed(false); - if (decimal >= 0) { + if(decimal >= 0) { df.setMinimumFractionDigits(decimal); df.setMaximumFractionDigits(decimal); } diff --git a/src/main/java/org/apache/sysds/utils/DoubleParser.java b/src/main/java/org/apache/sysds/utils/DoubleParser.java index c0122f8061f..2252b7270c8 100644 --- a/src/main/java/org/apache/sysds/utils/DoubleParser.java +++ b/src/main/java/org/apache/sysds/utils/DoubleParser.java @@ -200,7 +200,7 @@ public static double parseFloatingPointLiteral(String str, int offset, int endIn // : is the first character after numbers. // 0 is the first number. // we use the last position, since this is not allowed to be other values than a number. - if(str.charAt(endIndex - 1) > '9' || str.charAt(endIndex - 1) < '0') + if(str.charAt(endIndex - 1) > '9' || str.charAt(endIndex - 1) < '0') return Double.parseDouble(str); final double val = parseDecFloatLiteral(str, index, offset, endIndex); diff --git a/src/main/java/org/apache/sysds/utils/SettingsChecker.java b/src/main/java/org/apache/sysds/utils/SettingsChecker.java index c5f14a7043b..10d1a42b246 100644 --- a/src/main/java/org/apache/sysds/utils/SettingsChecker.java +++ b/src/main/java/org/apache/sysds/utils/SettingsChecker.java @@ -106,25 +106,24 @@ private static long maxMemMachineOSX() { } private static long maxMemMachineWin() { - //try modern powershell, otherwise wmic, log errors as warning but avoid crashes + // try modern powershell, otherwise wmic, log errors as warning but avoid crashes long tmp = maxMemMachineWin(true); - if( tmp < 0 ) + if(tmp < 0) tmp = maxMemMachineWin(false); return tmp; } - + private static long maxMemMachineWin(boolean modern) { int startIx = modern ? 3 : 1; - String command = modern ? - "powershell Get-CimInstance -ClassName Win32_PhysicalMemory | Select-Object Capacity" : - "wmic memorychip get capacity"; //in bytes + String command = modern ? "powershell Get-CimInstance -ClassName Win32_PhysicalMemory | Select-Object Capacity" : "wmic memorychip get capacity"; // in + // bytes try { Runtime rt = Runtime.getRuntime(); Process pr = rt.exec(command); String[] memStr = new String(pr.getInputStream().readAllBytes(), StandardCharsets.UTF_8).split("\n"); //skip header, and aggregate DIMM capacities long capacity = 0; - for( int i=startIx; i 0 ) capacity += Long.parseLong(tmp); diff --git a/src/test/java/org/apache/sysds/performance/Main.java b/src/test/java/org/apache/sysds/performance/Main.java index 0622e789baa..a941e90f724 100644 --- a/src/test/java/org/apache/sysds/performance/Main.java +++ b/src/test/java/org/apache/sysds/performance/Main.java @@ -243,10 +243,9 @@ private static void run17(String[] args) throws Exception { } /** - * Repeatedly read the same on-disk Delta frame table (written once as setup). - * Args: {@code 18 [mode] [targetFileSizeMB]} - * where mode is one of serial|parallel|both (default parallel) and an omitted - * target file size uses the adaptive default sizing. + * Repeatedly read the same on-disk Delta frame table (written once as setup). Args: + * {@code 18 [mode] [targetFileSizeMB]} where mode is one of serial|parallel|both (default parallel) + * and an omitted target file size uses the adaptive default sizing. */ private static void run18(String[] args) throws Exception { int rows = Integer.parseInt(args[1]); diff --git a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java index 36ea11b3e2f..4aac0685698 100644 --- a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java @@ -117,8 +117,8 @@ public abstract class AutomatedTestBase { public static final double GPU_TOLERANCE = 1e-9; /** - * Default deadline (ms) for federated worker/monitoring readiness waits and a few legacy - * {@code sleep()} calls. {@link FederatedWorkerUtils} enforces its own minimum floor. + * Default deadline (ms) for federated worker/monitoring readiness waits and a few legacy {@code sleep()} calls. + * {@link FederatedWorkerUtils} enforces its own minimum floor. */ public static final int FED_WORKER_WAIT = 3000; @@ -1761,9 +1761,9 @@ private static Process spawnLocalFedWorker(int port, String[] addArgs) { /** * Start a new JVM for a federated monitoring backend at the port. * - *

Returns once the backend's TCP port accepts connections (Netty's bind has completed), or - * throws a {@link RuntimeException} once the {@link FederatedWorkerUtils} readiness floor - * elapses. + *

+ * Returns once the backend's TCP port accepts connections (Netty's bind has completed), or throws a + * {@link RuntimeException} once the {@link FederatedWorkerUtils} readiness floor elapses. * * @param port Port to use for the JVM * @param addArgs Extra CLI args to append, or null @@ -1776,10 +1776,10 @@ protected Process startLocalFedMonitoring(int port, String[] addArgs) { /** * Start a new JVM for a federated monitoring backend at the port. * - *

Returns once the backend's TCP port accepts connections, or throws a - * {@link RuntimeException} after {@code timeoutMs} elapses. The monitoring server opens the - * port after Netty's {@code bind().sync()} returns; a successful TCP connect therefore signals - * that the HTTP listener is ready to accept requests. + *

+ * Returns once the backend's TCP port accepts connections, or throws a {@link RuntimeException} after + * {@code timeoutMs} elapses. The monitoring server opens the port after Netty's {@code bind().sync()} returns; a + * successful TCP connect therefore signals that the HTTP listener is ready to accept requests. * * @param port Port to use for the JVM * @param addArgs Extra CLI args to append, or null @@ -1798,8 +1798,9 @@ private static Process spawnLocalFedMonitoring(int port, String[] addArgs) { String separator = System.getProperty("file.separator"); String classpath = System.getProperty("java.class.path"); String path = System.getProperty("java.home") + separator + "bin" + separator + "java"; - String[] args = ArrayUtils.addAll(new String[] {path, "-cp", classpath, DMLScript.class.getName(), - "-fedMonitoring", Integer.toString(port)}, addArgs); + String[] args = ArrayUtils.addAll( + new String[] {path, "-cp", classpath, DMLScript.class.getName(), "-fedMonitoring", Integer.toString(port)}, + addArgs); try { return new ProcessBuilder(args).start(); } diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java index f14a614b583..0c7cd2046e5 100644 --- a/src/test/java/org/apache/sysds/test/TestUtils.java +++ b/src/test/java/org/apache/sysds/test/TestUtils.java @@ -2941,10 +2941,9 @@ public static void writeTestScalar(String file, double value) { } } - /** * Write scalar to file - * + * * @param file File to write to * @param value Value to write */ @@ -3519,9 +3518,9 @@ public static void shutdownThread(Thread t) { // Bounded join: workers are daemon threads, so even if one ignores the interrupt // we must not block cleanup (and the JVM) indefinitely waiting for it. t.join(THREAD_SHUTDOWN_JOIN_MS); - if( t.isAlive() ) - LOG.warn("Federated worker thread " + t.getName() - + " did not stop within " + THREAD_SHUTDOWN_JOIN_MS + "ms; leaving it as a daemon."); + if(t.isAlive()) + LOG.warn("Federated worker thread " + t.getName() + " did not stop within " + + THREAD_SHUTDOWN_JOIN_MS + "ms; leaving it as a daemon."); } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java b/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java index 07ec9752928..d76c741d870 100644 --- a/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java +++ b/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java @@ -67,10 +67,10 @@ public void setUp() { /** * Compile a DML script string into a runtime {@link Program} without executing it. * - * @param dmlScript the DML source - * @param args named command-line arguments ($name -> value), may be null - * @param mode the global execution mode (e.g. {@link ExecMode#HYBRID}) - * @param localMaxMem the local memory budget in bytes used for memory-based exec-type decisions + * @param dmlScript the DML source + * @param args named command-line arguments ($name -> value), may be null + * @param mode the global execution mode (e.g. {@link ExecMode#HYBRID}) + * @param localMaxMem the local memory budget in bytes used for memory-based exec-type decisions * @return the compiled runtime program */ protected Program compile(String dmlScript, Map args, ExecMode mode, long localMaxMem) { @@ -145,8 +145,7 @@ else if(pb instanceof FunctionProgramBlock) { /** All instructions whose opcode equals {@code opcode} (exact match). */ protected List getByOpcode(Program prog, String opcode) { - return getInstructions(prog).stream().filter(i -> opcode.equals(i.getOpcode())) - .collect(Collectors.toList()); + return getInstructions(prog).stream().filter(i -> opcode.equals(i.getOpcode())).collect(Collectors.toList()); } protected static boolean isSpark(Instruction inst) { @@ -169,13 +168,13 @@ protected void assertCP(Program prog, String opcode) { private void assertExecType(Program prog, String opcode, boolean expectSpark) { List matches = getByOpcode(prog, opcode); - Assert.assertFalse("Expected at least one '" + opcode + "' instruction but found none.\n" - + Explain.explain(prog), matches.isEmpty()); + Assert.assertFalse( + "Expected at least one '" + opcode + "' instruction but found none.\n" + Explain.explain(prog), + matches.isEmpty()); for(Instruction inst : matches) { boolean spark = isSpark(inst); - Assert.assertEquals("Instruction '" + opcode + "' expected exec type " - + (expectSpark ? "SPARK" : "CP") + " but was " + (spark ? "SPARK" : "CP") + ".\n" - + Explain.explain(prog), expectSpark, spark); + Assert.assertEquals("Instruction '" + opcode + "' expected exec type " + (expectSpark ? "SPARK" : "CP") + + " but was " + (spark ? "SPARK" : "CP") + ".\n" + Explain.explain(prog), expectSpark, spark); } } diff --git a/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java b/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java index 0b3889db908..7b45120d8f1 100644 --- a/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java +++ b/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java @@ -33,14 +33,13 @@ */ public class SparkTransitiveExecTypeCompileTest extends CompilerTestBase { - private static final String DML_HEADER = - "X = rand(rows=20000000, cols=8, seed=1);\n" + // ~1.2GB -> rand and colSums run on Spark - "v = colSums(X);\n"; // 1x8 Spark-resident vector (opcode uack+) + private static final String DML_HEADER = "X = rand(rows=20000000, cols=8, seed=1);\n" + // ~1.2GB -> rand and + // colSums run on Spark + "v = colSums(X);\n"; // 1x8 Spark-resident vector (opcode uack+) @Test public void singleConsumerUnaryPulledIntoSpark() { - String dml = DML_HEADER + - "r = round(v);\n" + // sole consumer of the Spark-resident vector -> pulled into Spark + String dml = DML_HEADER + "r = round(v);\n" + // sole consumer of the Spark-resident vector -> pulled into Spark "print(sum(r));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); @@ -50,62 +49,58 @@ public void singleConsumerUnaryPulledIntoSpark() { @Test public void multiConsumerUnaryStaysCP() { - String dml = DML_HEADER + - "a = round(v);\n" + // v now has two consumers (round + abs) ... - "b = abs(v);\n" + - "print(sum(a) + sum(b));\n"; + String dml = DML_HEADER + "a = round(v);\n" + // v now has two consumers (round + abs) ... + "b = abs(v);\n" + "print(sum(a) + sum(b));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uack+"); // input still has a Spark output ... - assertCP(prog, "round"); // ... but the multi-parent guard keeps both unaries in CP + assertCP(prog, "round"); // ... but the multi-parent guard keeps both unaries in CP assertCP(prog, "abs"); } // A tall, Spark-resident column vector that is still small enough (40 KB) to be CP by memory // estimate: rowSums over a very wide matrix runs on Spark, but its 1-column result fits in CP. - private static final String TALL_VECTOR_HEADER = - "X = rand(rows=5000, cols=200000, seed=1);\n" + // ~8GB -> rand and rowSums run on Spark - "c = rowSums(X);\n"; // 5000x1 Spark-resident vector (opcode uark+) + private static final String TALL_VECTOR_HEADER = "X = rand(rows=5000, cols=200000, seed=1);\n" + // ~8GB -> rand and + // rowSums run + // on Spark + "c = rowSums(X);\n"; // 5000x1 Spark-resident vector (opcode uark+) @Test public void cumulativeUnaryStaysCP() { - String dml = TALL_VECTOR_HEADER + - "r = cumsum(c);\n" + // sole consumer of the Spark-resident vector, CP by estimate ... + String dml = TALL_VECTOR_HEADER + "r = cumsum(c);\n" + // sole consumer of the Spark-resident vector, CP by + // estimate ... "print(as.scalar(r[2500,1]));\n"; // ... consume via indexing (avoids the sum(cumsum) rewrite) Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); - assertSpark(prog, "uark+"); // input genuinely has a Spark output + assertSpark(prog, "uark+"); // input genuinely has a Spark output assertCP(prog, "ucumk+"); // ... but cumulative ops are excluded from the transitive pull } @Test public void singleConsumerBinaryPulledIntoSpark() { - String dml = TALL_VECTOR_HEADER + - "r = c + 2.0;\n" + // matrix-scalar on the Spark-resident vector, sole consumer -> pulled into Spark + String dml = TALL_VECTOR_HEADER + "r = c + 2.0;\n" + // matrix-scalar on the Spark-resident vector, sole + // consumer -> pulled into Spark "print(as.scalar(r[2500,1]));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uark+"); // input genuinely has a Spark output (multi-block column vector) - assertSpark(prog, "+"); // matrix-scalar binary pulled into Spark (CP by estimate, single consumer) + assertSpark(prog, "+"); // matrix-scalar binary pulled into Spark (CP by estimate, single consumer) } @Test public void multiConsumerBinaryStaysCP() { - String dml = TALL_VECTOR_HEADER + - "a = c + 2.0;\n" + // c now has two consumers (+ and *) ... - "b = c * 3.0;\n" + - "print(as.scalar(a[2500,1]) + as.scalar(b[2500,1]));\n"; + String dml = TALL_VECTOR_HEADER + "a = c + 2.0;\n" + // c now has two consumers (+ and *) ... + "b = c * 3.0;\n" + "print(as.scalar(a[2500,1]) + as.scalar(b[2500,1]));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uark+"); // input still has a Spark output ... - assertCP(prog, "+"); // ... but the multi-parent guard keeps both binaries in CP + assertCP(prog, "+"); // ... but the multi-parent guard keeps both binaries in CP assertCP(prog, "*"); } @Test public void transitiveDisabledUnaryStaysCP() { - String dml = DML_HEADER + - "r = round(v);\n" + // pullable unary, but flag is off + String dml = DML_HEADER + "r = round(v);\n" + // pullable unary, but flag is off "print(sum(r));\n"; Program prog = compileWithTransitive(dml, false); @@ -115,8 +110,7 @@ public void transitiveDisabledUnaryStaysCP() { @Test public void transitiveDisabledBinaryStaysCP() { - String dml = TALL_VECTOR_HEADER + - "r = c + 2.0;\n" + // pullable matrix-scalar, but flag is off + String dml = TALL_VECTOR_HEADER + "r = c + 2.0;\n" + // pullable matrix-scalar, but flag is off "print(as.scalar(r[2500,1]));\n"; Program prog = compileWithTransitive(dml, false); diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java index 083a29f965b..22f867819c3 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java @@ -43,7 +43,8 @@ /** * Tests the {@code order} (sort) reorg operation on compressed matrices. A single column held in a single column group * is sorted ascending while staying compressed (via {@link org.apache.sysds.runtime.compress.lib.CLALibSort}); every - * other configuration falls back to a decompressed reorg. In all cases the result must match the uncompressed reference. + * other configuration falls back to a decompressed reorg. In all cases the result must match the uncompressed + * reference. */ public class CompressedSortTest { @@ -263,8 +264,7 @@ private void runCompressed(MatrixBlock mb, CompressionType ct) { assertEquals("Expected a single column group", 1, cmb.getColGroups().size()); MatrixBlock actual = cmb.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); - assertTrue("Expected the sorted result to stay compressed for " + ct, - actual instanceof CompressedMatrixBlock); + assertTrue("Expected the sorted result to stay compressed for " + ct, actual instanceof CompressedMatrixBlock); MatrixBlock expected = mb.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); TestUtils.compareMatrices(expected, CompressedMatrixBlock.getUncompressed(actual, "sort"), 0.0, "sort " + ct); diff --git a/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java index 833128ad9f0..49cbbdd248d 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java @@ -62,8 +62,8 @@ public static void setup() { } /** - * Build a compressed matrix backed by a single DDC column group spanning all {@code nCol} columns. This guarantees a - * single (non-uncompressed) column group, which is what triggers the mm-chain fast path for wide enough matrices. + * Build a compressed matrix backed by a single DDC column group spanning all {@code nCol} columns. This guarantees + * a single (non-uncompressed) column group, which is what triggers the mm-chain fast path for wide enough matrices. */ private static CompressedMatrixBlock singleDDC(int nRow, int nCol, int nVal, int seed) { Random r = new Random(seed); diff --git a/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java b/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java index 549010a78cb..c6afd5a3543 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java @@ -43,8 +43,8 @@ /** * Drive the solve opcode through {@link BinaryMatrixMatrixCPInstruction} with compressed inputs to cover the - * commons-math matrix-matrix branch that decompresses compressed left/right operands before solving. The - * script-level solve tests only ever see uncompressed inputs, so this branch is otherwise unreached. + * commons-math matrix-matrix branch that decompresses compressed left/right operands before solving. The script-level + * solve tests only ever see uncompressed inputs, so this branch is otherwise unreached. */ public class CompressedBinaryMatrixMatrixSolveTest { @@ -72,8 +72,8 @@ public void solveCompressedLeftCompressedRight() { assertTrue("A must compress to exercise the compressed-left path", aC instanceof CompressedMatrixBlock); CompressedMatrixBlock bC = CompressedMatrixBlockFactory.createConstant(n, 2, 1.0); - MatrixBlock expected = LibCommonsMath.matrixMatrixOperations( - CompressedMatrixBlock.getUncompressed(aC), CompressedMatrixBlock.getUncompressed(bC), SOLVE); + MatrixBlock expected = LibCommonsMath.matrixMatrixOperations(CompressedMatrixBlock.getUncompressed(aC), + CompressedMatrixBlock.getUncompressed(bC), SOLVE); MatrixBlock actual = runSolve(aC, bC); TestUtils.compareMatricesBitAvgDistance(expected, actual, 0, 0, SOLVE); @@ -94,8 +94,8 @@ public void solveCompressedLeftDenseRight() { assertTrue("A must compress to exercise the compressed-left path", aC instanceof CompressedMatrixBlock); MatrixBlock b = TestUtils.round(TestUtils.generateTestMatrixBlock(n, 1, -5, 5, 1.0, 7)); - MatrixBlock expected = LibCommonsMath.matrixMatrixOperations( - CompressedMatrixBlock.getUncompressed(aC), b, SOLVE); + MatrixBlock expected = LibCommonsMath.matrixMatrixOperations(CompressedMatrixBlock.getUncompressed(aC), b, + SOLVE); MatrixBlock actual = runSolve(aC, b); TestUtils.compareMatricesBitAvgDistance(expected, actual, 0, 0, SOLVE); @@ -119,7 +119,8 @@ private static BinaryMatrixMatrixCPInstruction solveInstruction() { } private static MatrixObject matrixObject(String name, MatrixBlock mb) { - MatrixCharacteristics mc = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), 1000, mb.getNonZeros()); + MatrixCharacteristics mc = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), 1000, + mb.getNonZeros()); MatrixObject mo = new MatrixObject(ValueType.FP64, "/dev/null/" + name, new MetaDataFormat(mc, FileFormat.BINARY), mb); return mo; diff --git a/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java b/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java index 8907c82f1d1..0d2f37e319c 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java @@ -50,7 +50,9 @@ public class OffsetClassInitConcurrencyTest { private static final String[] INIT_TARGETS = {PKG + "AOffset", PKG + "OffsetEmpty", PKG + "OffsetChar", PKG + "OffsetByte", PKG + "OffsetSingle", PKG + "OffsetTwo"}; - /** Whether a class-init cycle deadlocks depends on thread timing, so repeat to make a regression reliable to catch. */ + /** + * Whether a class-init cycle deadlocks depends on thread timing, so repeat to make a regression reliable to catch. + */ private static final int ROUNDS = 20; /** A real init deadlock never resolves; a healthy round finishes in milliseconds, so this bound is generous. */ diff --git a/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java b/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java index 3493da7d2b1..f7dffa9021c 100644 --- a/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java +++ b/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java @@ -35,12 +35,10 @@ public class SparkContextReferenceCountTest { /** - * Two DML executions sharing the JVM-wide singleton spark context (as happens - * with surefire parallel tests, threadCount>1). When the first execution - * finishes and calls close(), the shared context must stay alive because the - * second execution still has in-flight work. Before reference counting, - * close() stopped the context unconditionally, which cancelled the second - * execution's spark job and wedged it until the test watchdog. + * Two DML executions sharing the JVM-wide singleton spark context (as happens with surefire parallel tests, + * threadCount>1). When the first execution finishes and calls close(), the shared context must stay alive + * because the second execution still has in-flight work. Before reference counting, close() stopped the context + * unconditionally, which cancelled the second execution's spark job and wedged it until the test watchdog. */ @Test public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { @@ -64,16 +62,13 @@ public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { // context that B still uses SparkExecutionContext.exitSparkExecution(); ecA.close(); - assertFalse("shared context must stay alive while another execution is active", - sc.sc().isStopped()); - assertEquals("B's job must still run on the live context", - 10L, rdd.reduce(Integer::sum).longValue()); + assertFalse("shared context must stay alive while another execution is active", sc.sc().isStopped()); + assertEquals("B's job must still run on the live context", 10L, rdd.reduce(Integer::sum).longValue()); // B finishes last: releasing the final registration lets close() stop it SparkExecutionContext.exitSparkExecution(); ecB.close(); - assertTrue("shared context must be stopped once the last execution closes", - sc.sc().isStopped()); + assertTrue("shared context must be stopped once the last execution closes", sc.sc().isStopped()); } finally { // drain any remaining registrations and stop the context so a failed @@ -87,10 +82,9 @@ public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { } /** - * An unpaired close() (a caller that borrows the shared context but never - * registered via enterSparkExecution()) must not stop a context another - * execution still uses. This fails on the old unconditional-stop code, which - * tore the context down out from under the active execution. + * An unpaired close() (a caller that borrows the shared context but never registered via enterSparkExecution()) + * must not stop a context another execution still uses. This fails on the old unconditional-stop code, which tore + * the context down out from under the active execution. */ @Test public void unpairedCloseDoesNotStopAContextStillInUse() { @@ -106,14 +100,12 @@ public void unpairedCloseDoesNotStopAContextStillInUse() { // borrows the shared context): close() must not stop a context in use unregistered = ExecutionContextFactory.createSparkExecutionContext(); unregistered.close(); - assertFalse("unpaired close() must not stop a context still in use", - sc.sc().isStopped()); + assertFalse("unpaired close() must not stop a context still in use", sc.sc().isStopped()); // the registered execution finishing stops the context as the last user SparkExecutionContext.exitSparkExecution(); active.close(); - assertTrue("context must stop once the last registered execution closes", - sc.sc().isStopped()); + assertTrue("context must stop once the last registered execution closes", sc.sc().isStopped()); } finally { SparkExecutionContext.exitSparkExecution(); diff --git a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java index 2c854b4a81b..459936fa24c 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java @@ -78,17 +78,18 @@ public MatrixBlock getMatrixBlock(long id) { } /** - * Poll the federated worker until the matrix at {@code id} is observed as a - * {@link CompressedMatrixBlock}, or {@link #COMPRESS_TIMEOUT_MS} elapses. + * Poll the federated worker until the matrix at {@code id} is observed as a {@link CompressedMatrixBlock}, or + * {@link #COMPRESS_TIMEOUT_MS} elapses. * - *

Federated workers compress asynchronously after a PUT/READ_VAR (see - * {@code CompressedMatrixBlockFactory.compressAsync}), so a {@code getMatrixBlock} fired right - * after the operation can race against the in-flight compression and return the uncompressed - * block. Tests that need to observe the compressed form should poll instead of sleeping a fixed - * amount. + *

+ * Federated workers compress asynchronously after a PUT/READ_VAR (see + * {@code CompressedMatrixBlockFactory.compressAsync}), so a {@code getMatrixBlock} fired right after the operation + * can race against the in-flight compression and return the uncompressed block. Tests that need to observe the + * compressed form should poll instead of sleeping a fixed amount. * - *

On timeout this returns the most recent (uncompressed) read so the caller can produce a - * meaningful assertion failure naming the variable. + *

+ * On timeout this returns the most recent (uncompressed) read so the caller can produce a meaningful assertion + * failure naming the variable. * * @param id federated variable id * @return the matrix block, compressed if compression finished in time, otherwise the latest read diff --git a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java index 2b5ff327ef3..d3e4485bdf4 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java @@ -68,13 +68,11 @@ public void verifySameOrAlsoCompressedAsLocalCompress() { // federated. Compression on the worker is async; poll only when we expect compression to // match the local result, otherwise a single read is enough. final long id = putMatrixBlock(mb); - final MatrixBlock mbr = (mbcLocal instanceof CompressedMatrixBlock) - ? awaitCompressed(id) - : getMatrixBlock(id); + final MatrixBlock mbr = (mbcLocal instanceof CompressedMatrixBlock) ? awaitCompressed(id) : getMatrixBlock(id); if(mbcLocal instanceof CompressedMatrixBlock && !(mbr instanceof CompressedMatrixBlock)) - fail("Invalid result, the federated site did not compress the matrix block within " - + COMPRESS_TIMEOUT_MS + "ms"); + fail("Invalid result, the federated site did not compress the matrix block within " + COMPRESS_TIMEOUT_MS + + "ms"); TestUtils.compareMatricesBitAvgDistance(mbcLocal, mbr, 0, 0, "Not equivalent matrix block returned from federated site"); diff --git a/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java b/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java index 60587bf51a2..0de3b6db640 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java @@ -42,7 +42,7 @@ public void test100x100() { @Test public void testDecimalClampsFractionDigits() { - FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); + FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); f.ensureAllocatedColumns(1); f.set(0, 0, 5.244058388023880); // decimal=2 must print exactly two fraction digits, not DecimalFormat's default max of 3 @@ -53,10 +53,10 @@ public void testDecimalClampsFractionDigits() { @Test public void testDecimalPadsAndRounds() { - FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); + FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); f.ensureAllocatedColumns(2); - f.set(0, 0, 22.0); // integer-valued: padded up to the requested digits - f.set(1, 0, 5.244058388023880); // rounded at the last requested digit + f.set(0, 0, 22.0); // integer-valued: padded up to the requested digits + f.set(1, 0, 5.244058388023880); // rounded at the last requested digit String out = DataConverter.toString(f, false, " ", "\n", 2, 1, 4); assertTrue("expected 22.0000 padded: " + out, out.contains("22.0000\n")); assertTrue("expected 5.2441 rounded: " + out, out.contains("5.2441\n")); @@ -64,10 +64,10 @@ public void testDecimalPadsAndRounds() { @Test public void testNegativeDecimalUsesDefaultFormatting() { - FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); + FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); f.ensureAllocatedColumns(2); - f.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained - f.set(1, 0, 5.244058388023880); // default cap of three fraction digits + f.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained + f.set(1, 0, 5.244058388023880); // default cap of three fraction digits // decimal < 0 leaves DecimalFormat unconstrained (no min/max fraction digits set) String out = DataConverter.toString(f, false, " ", "\n", 2, 1, -1); assertTrue("expected unpadded 22: " + out, out.contains("22\n")); diff --git a/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java b/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java index 43a53879f17..7d67c04983b 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java @@ -142,8 +142,7 @@ public void safeCastWarnsOnlyOnce() { // the fallback warning must be logged exactly once across both conversions final List log = LoggingUtils.reinsert(appender); long warnings = log.stream() - .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")) - .count(); + .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")).count(); assertEquals(1, warnings); } @@ -153,8 +152,7 @@ public void strictThrowsWhenWarnCastDisabled() { setWarnCast(false); FrameBlock fb = mixedFrame(); - Exception e = assertThrows(DMLRuntimeException.class, - () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 1)); + Exception e = assertThrows(DMLRuntimeException.class, () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 1)); assertTrue(e.getMessage().contains("Failed to convert FrameBlock to MatrixBlock")); } @@ -164,8 +162,7 @@ public void strictThrowsParallelWhenWarnCastDisabled() { setWarnCast(false); FrameBlock fb = mixedFrame(); - Exception e = assertThrows(DMLRuntimeException.class, - () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 4)); + Exception e = assertThrows(DMLRuntimeException.class, () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 4)); assertTrue(e.getMessage().contains("Failed to convert FrameBlock to MatrixBlock")); } @@ -185,8 +182,7 @@ public void warnCastValidFrameConvertsWithoutFallback() { final List log = LoggingUtils.reinsert(appender); long warnings = log.stream() - .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")) - .count(); + .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")).count(); assertEquals(0, warnings); } diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java index ccba674707b..982941bb190 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java @@ -74,8 +74,8 @@ private void runDecode(String spec, int nCol, int nCat) { try { FrameBlock data = categoricalFrame(ROWS, nCol, nCat, 17); - MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), - data.getNumColumns(), null); + MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), data.getNumColumns(), + null); MatrixBlock encoded = encoder.encode(data, 1); Decoder decoder = buildDecoder(data, spec, encoder); @@ -122,8 +122,8 @@ public void singleThreadEqualsParallelManyCategories() { public void decoderIsComposite() { FrameBlock data = categoricalFrame(100, 2, 3, 1); String spec = "{recode:[C1], dummycode:[C2]}"; - MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), - data.getNumColumns(), null); + MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), data.getNumColumns(), + null); encoder.encode(data, 1); Decoder decoder = buildDecoder(data, spec, encoder); if(!(decoder instanceof DecoderComposite)) diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java index d9c540f54c5..412686c1e83 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java @@ -47,9 +47,9 @@ import org.junit.Test; /** - * Unit tests that drive the get_categorical_mask instruction directly to exercise the defensive code - * paths (distinct-count prefix in the metadata frame, default column metadata, non id-based specs and - * the unsupported opcode guard) that the script-level transform tests cannot reach. + * Unit tests that drive the get_categorical_mask instruction directly to exercise the defensive code paths + * (distinct-count prefix in the metadata frame, default column metadata, non id-based specs and the unsupported opcode + * guard) that the script-level transform tests cannot reach. */ public class GetCategoricalMaskInstructionTest { protected static final Log LOG = LogFactory.getLog(GetCategoricalMaskInstructionTest.class.getName()); @@ -210,7 +210,8 @@ public void imputeAndOmitAreAccepted() { // impute and omit do not change the output column count or categorical flag, so a spec that // only adds them on top of a recoded column must still succeed and mark that column categorical FrameBlock meta = new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}); - MatrixBlock res = run(meta, "{\"ids\": true, \"recode\": [1], \"impute\": [{\"id\": 1, \"method\": \"global_mode\"}], \"omit\": [1]}"); + MatrixBlock res = run(meta, + "{\"ids\": true, \"recode\": [1], \"impute\": [{\"id\": 1, \"method\": \"global_mode\"}], \"omit\": [1]}"); assertEquals(1, res.getNumRows()); assertEquals(1, res.getNumColumns()); @@ -230,8 +231,7 @@ public void unsupportedOpcodeThrows() { // any frame-scalar binary opcode other than get_categorical_mask must be rejected ExecutionContext ec = ExecutionContextFactory.createContext(); ec.setAutoCreateVars(true); - ec.setVariable("F", frameObject(new FrameBlock(new ValueType[] {ValueType.STRING}, - new String[][] {{"a"}}))); + ec.setVariable("F", frameObject(new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}))); assertThrowsMessage("Unsupported operation", () -> maskInstruction("+").processInstruction(ec)); } @@ -263,9 +263,9 @@ private static void assertMask(MatrixBlock res, double[] expected) { } /** - * Build a single-row metadata frame of nCol string columns. A positive distinct[i] is written to - * that column's metadata as the recode distinct count (the path real transformencode uses), while - * a zero leaves the column with default metadata (a continuous / non-dummycoded column). + * Build a single-row metadata frame of nCol string columns. A positive distinct[i] is written to that column's + * metadata as the recode distinct count (the path real transformencode uses), while a zero leaves the column with + * default metadata (a continuous / non-dummycoded column). */ private static FrameBlock metaWithDistinct(int nCol, int[] distinct) { ValueType[] schema = new ValueType[nCol]; @@ -290,7 +290,8 @@ private static MatrixBlock run(FrameBlock meta, String spec) { private static BinaryFrameScalarCPInstruction maskInstruction(String opcode) { String in1 = InstructionUtils.concatOperandParts("F", DataType.FRAME.name(), ValueType.STRING.name(), "false"); - String in2 = InstructionUtils.concatOperandParts("spec", DataType.SCALAR.name(), ValueType.STRING.name(), "true"); + String in2 = InstructionUtils.concatOperandParts("spec", DataType.SCALAR.name(), ValueType.STRING.name(), + "true"); String out = InstructionUtils.concatOperandParts("out", DataType.MATRIX.name(), ValueType.FP64.name(), "false"); String str = InstructionUtils.concatOperands("CP", opcode, in1, in2, out); return (BinaryFrameScalarCPInstruction) BinaryCPInstruction.parseInstruction(str); diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java index b2d31f43b83..645c4dd09bd 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -44,10 +44,10 @@ import org.junit.Test; /** - * Exact inverse correctness tests for the transform decoders. Recode and dummycode are lossless category encodings, so a - * decode of the encoded matrix must reconstruct the original categorical frame. These tests assert exact reconstruction - * for the dense path, the sparse path, and the parallel path so that the dummycode sparse binary search and the parallel - * block split are validated against ground truth rather than only against each other. + * Exact inverse correctness tests for the transform decoders. Recode and dummycode are lossless category encodings, so + * a decode of the encoded matrix must reconstruct the original categorical frame. These tests assert exact + * reconstruction for the dense path, the sparse path, and the parallel path so that the dummycode sparse binary search + * and the parallel block split are validated against ground truth rather than only against each other. */ public class TransformDecodeRoundTripTest { protected static final Log LOG = LogFactory.getLog(TransformDecodeRoundTripTest.class.getName()); @@ -59,9 +59,9 @@ public void setUp() { } private static FrameBlock categoricalFrame() { - final String[] values = new String[] { - "apple", "banana", "apple", "cherry", "banana", "date", "apple", "cherry", "date", "banana", "elderberry", - "apple", "fig", "banana", "cherry", "apple", "date", "fig", "elderberry", "banana"}; + final String[] values = new String[] {"apple", "banana", "apple", "cherry", "banana", "date", "apple", "cherry", + "date", "banana", "elderberry", "apple", "fig", "banana", "cherry", "apple", "date", "fig", "elderberry", + "banana"}; final FrameBlock f = new FrameBlock(new ValueType[] {ValueType.STRING}); f.ensureAllocatedColumns(values.length); for(int i = 0; i < values.length; i++) @@ -117,8 +117,8 @@ public void binWithDummycodeOnOtherColumnConsistency() { /** * Dummycode on an earlier column (1) shifts the bin column (2) to the right in the encoded matrix. The bin decoder - * must walk the dummycode domain sizes to recover the bin column's true source position. This drives the - * non-magic offset branch of the bin source-column mapping. + * must walk the dummycode domain sizes to recover the bin column's true source position. This drives the non-magic + * offset branch of the bin source-column mapping. */ @Test public void binAfterDummycodeOnEarlierColumnConsistency() { @@ -128,9 +128,9 @@ public void binAfterDummycodeOnEarlierColumnConsistency() { } /** - * Same right-shift as above, but the earlier column is feature-hashed before being dummycoded. The hash domain - * size K is stored as a plain integer in the single meta cell, so the bin source-column mapping reads it (instead - * of numDistinct) to compute the offset. + * Same right-shift as above, but the earlier column is feature-hashed before being dummycoded. The hash domain size + * K is stored as a plain integer in the single meta cell, so the bin source-column mapping reads it (instead of + * numDistinct) to compute the offset. */ @Test public void binAfterHashDummycodeOnEarlierColumnConsistency() { @@ -211,10 +211,10 @@ public void binDecodeZeroCodeUsesFirstBinBoundary() { } /** - * Spark broadcasts the decoder to executors via Java serialization without re-running initMetaData, so the - * decoder must round-trip all of its decode state through writeExternal/readExternal. Decode with a freshly - * deserialized decoder and assert it matches the in-memory decode. Covers plain bin and bin-with-dummycode - * (the latter exercises the serialized _srcCols/_dcCols source-column mapping). + * Spark broadcasts the decoder to executors via Java serialization without re-running initMetaData, so the decoder + * must round-trip all of its decode state through writeExternal/readExternal. Decode with a freshly deserialized + * decoder and assert it matches the in-memory decode. Covers plain bin and bin-with-dummycode (the latter exercises + * the serialized _srcCols/_dcCols source-column mapping). */ @Test public void binDecoderSurvivesSerialization() { @@ -481,8 +481,7 @@ public void parallelDecodeWrapsWorkerException() { fail("expected the parallel decode wrapper to propagate the worker failure"); } catch(DMLRuntimeException expected) { - assertNotNull("parallel decode wrapper must retain the worker exception as cause", - expected.getCause()); + assertNotNull("parallel decode wrapper must retain the worker exception as cause", expected.getCause()); } } catch(Exception e) { diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java index 54bd1679716..5745a35d506 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java index 8d7ad14539a..64012c06bbf 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java @@ -57,17 +57,15 @@ import io.delta.kernel.types.TimestampType; /** - * Targeted tests for the error/defensive branches of the native Delta matrix - * read/write code that the round-trip and interop tests do not reach: malformed - * per-file statistics, unsupported column types, unsupported stream operations, + * Targeted tests for the error/defensive branches of the native Delta matrix read/write code that the round-trip and + * interop tests do not reach: malformed per-file statistics, unsupported column types, unsupported stream operations, * bad table paths, and the non-dense writer input path. * - *

A few of these branches guard against inputs that the SystemDS writer and - * the Delta Kernel scan API never produce in a normal round trip (e.g. a - * statistics JSON without {@code numRecords}, or a column type code outside the - * supported set). They are exercised here by mocking the Delta Kernel data - * objects and invoking the (package-private) helpers reflectively, rather than - * widening their production visibility purely for testing. + *

+ * A few of these branches guard against inputs that the SystemDS writer and the Delta Kernel scan API never produce in + * a normal round trip (e.g. a statistics JSON without {@code numRecords}, or a column type code outside the supported + * set). They are exercised here by mocking the Delta Kernel data objects and invoking the (package-private) helpers + * reflectively, rather than widening their production visibility purely for testing. */ public class DeltaMatrixCoverageTest { @@ -89,8 +87,8 @@ public void qualifyRejectsUnknownFilesystemScheme() { @Test public void typeCodeReturnsNegativeForUnsupportedTypes() { - //non-numeric / unsupported Delta types must map to the sentinel -1 so the - //reader can reject them with a clear message rather than mis-decoding. + // non-numeric / unsupported Delta types must map to the sentinel -1 so the + // reader can reject them with a clear message rather than mis-decoding. assertEquals(-1, DeltaKernelUtils.typeCode(DateType.DATE)); assertEquals(-1, DeltaKernelUtils.typeCode(TimestampType.TIMESTAMP)); assertEquals(-1, DeltaKernelUtils.typeCode(BinaryType.BINARY)); @@ -112,17 +110,17 @@ public void writerRejectsStreamWrite() throws Exception { @Test public void numRecordsHandlesAbsentNullAndMalformedStats() throws Exception { - //no "stats" field at all -> -1 + // no "stats" field at all -> -1 assertEquals(-1, numRecords(addFileRow(new StructType().add("path", StringType.STRING), false, null))); - //stats column present but null-at -> -1 + // stats column present but null-at -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), true, null))); - //stats string explicitly null -> -1 + // stats string explicitly null -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, null))); - //malformed JSON -> JsonProcessingException -> -1 + // malformed JSON -> JsonProcessingException -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, "{not valid json"))); - //valid JSON but no numRecords field -> -1 + // valid JSON but no numRecords field -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, "{\"minValues\":{}}"))); - //well-formed stats -> the parsed count + // well-formed stats -> the parsed count assertEquals(1234L, numRecords(addFileRow(statsSchema(), false, "{\"numRecords\":1234}"))); } @@ -131,8 +129,8 @@ public void getDoubleValueRejectsUnknownTypeCode() throws Exception { Method m = ReaderDelta.class.getDeclaredMethod("getDoubleValue", ColumnVector.class, int.class, int.class); m.setAccessible(true); try { - //type code outside the supported T_* set; the switch default must throw - //before touching the (null) vector. + // type code outside the supported T_* set; the switch default must throw + // before touching the (null) vector. m.invoke(null, (ColumnVector) null, 0, 999); fail("expected a DMLRuntimeException for an unsupported type code"); } @@ -156,9 +154,9 @@ public void numericTypeCodeRejectsNonNumericType() throws Exception { @Test public void parallelReadWrapsFileFailure() throws Exception { - //a per-file decode failure in the parallel reader must surface as a single - //clear IOException (the awaitFileTasks catch), not a raw executor error. - //Provoke it by deleting one data file after the table (and its log) exist. + // a per-file decode failure in the parallel reader must surface as a single + // clear IOException (the awaitFileTasks catch), not a raw executor error. + // Provoke it by deleting one data file after the table (and its log) exist. MatrixBlock in = TestUtils.generateTestMatrixBlock(100_000, 8, -10, 10, 1.0, 13); in.recomputeNonZeros(); DMLConfig conf = new DMLConfig(); @@ -167,15 +165,14 @@ public void parallelReadWrapsFileFailure() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_fail_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); - //delete one parquet data file; the transaction log still references it, - //so the scan enumerates it but the decode task fails. + // delete one parquet data file; the transaction log still references it, + // so the scan enumerates it but the decode task fails. File victim; - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { - victim = s.filter(p -> p.toString().endsWith(".parquet")) - .findFirst().map(Path::toFile).orElse(null); + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { + victim = s.filter(p -> p.toString().endsWith(".parquet")).findFirst().map(Path::toFile).orElse(null); } assertTrue("expected at least one data file to delete", victim != null && victim.delete()); @@ -200,8 +197,8 @@ public void parallelReadWrapsFileFailure() throws Exception { @Test public void sparseFormatMatrixRoundTrips() throws Exception { - //a sparse-backed MatrixBlock takes the writer's non-contiguous path (no - //direct double[] view), exercising MatrixColumnVector.get via MatrixBlock. + // a sparse-backed MatrixBlock takes the writer's non-contiguous path (no + // direct double[] view), exercising MatrixColumnVector.get via MatrixBlock. MatrixBlock in = TestUtils.generateTestMatrixBlock(2000, 7, -5, 5, 0.05, 13); in.recomputeNonZeros(); in.examSparsity(); @@ -210,8 +207,8 @@ public void sparseFormatMatrixRoundTrips() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_sparse_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", in.getNumRows(), out.getNumRows()); assertEquals("cols", in.getNumColumns(), out.getNumColumns()); @@ -224,15 +221,15 @@ public void sparseFormatMatrixRoundTrips() throws Exception { @Test public void fillDenseHandlesNonContiguousBlock() throws Exception { - //the dense fill normally hits the contiguous fast path; force a multi-block - //(non-contiguous) dense block so the row-by-row fallback is exercised. Such - //blocks only arise for matrices beyond a single contiguous array, so we - //shrink the per-block allocation cap to provoke it on a tiny matrix. + // the dense fill normally hits the contiguous fast path; force a multi-block + // (non-contiguous) dense block so the row-by-row fallback is exercised. Such + // blocks only arise for matrices beyond a single contiguous array, so we + // shrink the per-block allocation cap to provoke it on a tiny matrix. int rows = 5, cols = 4; int savedMaxAlloc = DenseBlockLDRB.MAX_ALLOC; DenseBlock db; try { - DenseBlockLDRB.MAX_ALLOC = 2 * cols; //~2 rows per block -> multiple blocks + DenseBlockLDRB.MAX_ALLOC = 2 * cols; // ~2 rows per block -> multiple blocks db = new DenseBlockLFP64(new int[] {rows, cols}); } finally { @@ -240,14 +237,14 @@ public void fillDenseHandlesNonContiguousBlock() throws Exception { } assertTrue("expected a non-contiguous (multi-block) dense block", !db.isContiguous()); - //two row-major batches (3 rows + 2 rows) covering all 5 rows + // two row-major batches (3 rows + 2 rows) covering all 5 rows double[] b0 = new double[3 * cols]; double[] b1 = new double[2 * cols]; - for( int r = 0; r < 3; r++ ) - for( int c = 0; c < cols; c++ ) + for(int r = 0; r < 3; r++) + for(int c = 0; c < cols; c++) b0[r * cols + c] = cell(r, c); - for( int r = 0; r < 2; r++ ) - for( int c = 0; c < cols; c++ ) + for(int r = 0; r < 2; r++) + for(int c = 0; c < cols; c++) b1[r * cols + c] = cell(3 + r, c); java.util.ArrayList batches = new java.util.ArrayList<>(); batches.add(b0); @@ -258,8 +255,8 @@ public void fillDenseHandlesNonContiguousBlock() throws Exception { m.setAccessible(true); m.invoke(null, ret, batches); - for( int r = 0; r < rows; r++ ) - for( int c = 0; c < cols; c++ ) + for(int r = 0; r < rows; r++) + for(int c = 0; c < cols; c++) assertEquals("r" + r + " c" + c, cell(r, c), ret.getDenseBlock().get(r, c), 0.0); } @@ -276,8 +273,8 @@ private static StructType statsSchema() { } /** - * Build a mocked scan-file row whose AddFile child has the given schema, null - * flag and (when not null) stats string, matching what {@code numRecords} reads. + * Build a mocked scan-file row whose AddFile child has the given schema, null flag and (when not null) stats + * string, matching what {@code numRecords} reads. */ private static Row addFileRow(StructType addSchema, boolean statsNull, String statsValue) { Row outer = mock(Row.class); @@ -285,12 +282,12 @@ private static Row addFileRow(StructType addSchema, boolean statsNull, String st when(outer.getStruct(InternalScanFileUtils.ADD_FILE_ORDINAL)).thenReturn(add); when(add.getSchema()).thenReturn(addSchema); int statsOrd = addSchema.fieldNames().indexOf("stats"); - if( statsOrd >= 0 ) { + if(statsOrd >= 0) { when(add.isNullAt(statsOrd)).thenReturn(statsNull); - if( !statsNull ) + if(!statsNull) when(add.getString(statsOrd)).thenReturn(statsValue); } - return outer; //the scan-file row numRecords consumes (its AddFile child is 'add') + return outer; // the scan-file row numRecords consumes (its AddFile child is 'add') } private static long numRecords(Row scanFileRow) throws Exception { diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java index 54a3bcf6334..65c4ec21c0f 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java @@ -63,20 +63,19 @@ import io.delta.kernel.utils.CloseableIterator; /** - * Direct (no DML) round-trip tests for the native Delta Kernel based matrix - * reader/writer. Each test writes a MatrixBlock to a fresh local Delta table - * directory and reads it back, asserting dimensions and values match. + * Direct (no DML) round-trip tests for the native Delta Kernel based matrix reader/writer. Each test writes a + * MatrixBlock to a fresh local Delta table directory and reads it back, asserting dimensions and values match. */ public class DeltaMatrixReadWriteTest { - //small writer target file size (bytes) used to force a multi-file table - //layout cheaply, instead of brute-forcing huge row counts. + // small writer target file size (bytes) used to force a multi-file table + // layout cheaply, instead of brute-forcing huge row counts. private static final long SMALL_TARGET_FILE_SIZE = 256L * 1024; private static final int ROWS_MULTI_FILE = 100_000; private static MatrixBlock writeThenRead(MatrixBlock in) throws Exception { Path dir = Files.createTempDirectory("sysds_delta_"); - //WriterDelta creates the table at the given (empty) directory + // WriterDelta creates the table at the given (empty) directory String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { WriterDelta writer = new WriterDelta(); @@ -92,9 +91,9 @@ private static MatrixBlock writeThenRead(MatrixBlock in) throws Exception { @Test public void parallelReadMatchesSerialMultiFile() throws Exception { - //force a multi-file table cheaply via a small writer target file size - //(rather than a huge row count), so the parallel per-file path is - //actually exercised rather than falling back to serial. + // force a multi-file table cheaply via a small writer target file size + // (rather than a huge row count), so the parallel per-file path is + // actually exercised rather than falling back to serial. MatrixBlock in = TestUtils.generateTestMatrixBlock(ROWS_MULTI_FILE, 8, -10, 10, 1.0, 13); in.recomputeNonZeros(); @@ -104,21 +103,18 @@ public void parallelReadMatchesSerialMultiFile() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_par_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); - //sanity: confirm the table really is split across multiple files + // sanity: confirm the table really is split across multiple files long files; - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } - assertTrue("expected a multi-file Delta table to exercise the parallel path, got " + files, - files > 1); + assertTrue("expected a multi-file Delta table to exercise the parallel path, got " + files, files > 1); - MatrixBlock serial = new ReaderDelta() - .readMatrixFromHDFS(tablePath, -1, -1, -1, -1); - MatrixBlock parallel = new ReaderDeltaParallel() - .readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + MatrixBlock serial = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + MatrixBlock parallel = new ReaderDeltaParallel().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", serial.getNumRows(), parallel.getNumRows()); assertEquals("cols", serial.getNumColumns(), parallel.getNumColumns()); @@ -135,8 +131,8 @@ public void parallelReadMatchesSerialMultiFile() throws Exception { public void roundTripDenseSmall() throws Exception { MatrixBlock in = new MatrixBlock(3, 4, false); double v = 1.0; - for( int i=0; i<3; i++ ) - for( int j=0; j<4; j++ ) + for(int i = 0; i < 3; i++) + for(int j = 0; j < 4; j++) in.set(i, j, v++); in.recomputeNonZeros(); @@ -157,7 +153,7 @@ public void roundTripDenseRandom() throws Exception { @Test public void roundTripSparseRandom() throws Exception { - //values written are dense parquet, but exercise a sparse-ish input + // values written are dense parquet, but exercise a sparse-ish input MatrixBlock in = TestUtils.generateTestMatrixBlock(1200, 9, -5, 5, 0.1, 13); in.recomputeNonZeros(); MatrixBlock out = writeThenRead(in); @@ -168,7 +164,7 @@ public void roundTripSparseRandom() throws Exception { @Test public void roundTripMultiBatch() throws Exception { - //more rows than the writer batch size (4096) to exercise chunking + // more rows than the writer batch size (4096) to exercise chunking MatrixBlock in = TestUtils.generateTestMatrixBlock(10000, 5, 0, 100, 1.0, 1); MatrixBlock out = writeThenRead(in); assertEquals("rows", 10000, out.getNumRows()); @@ -182,8 +178,9 @@ public void readDiscoversUnknownDimensions() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); - //pass -1 dimensions: the reader must discover them from the table + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); + // pass -1 dimensions: the reader must discover them from the table MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 123, out.getNumRows()); assertEquals("cols", 6, out.getNumColumns()); @@ -212,22 +209,21 @@ public void emptyMatrixRoundTrip() throws Exception { @Test public void readNonDoubleNumericColumns() throws Exception { - //tables produced by external tools (or the frame writer) can carry - //long/int/boolean columns; the matrix reader must coerce them to double + // tables produced by external tools (or the frame writer) can carry + // long/int/boolean columns; the matrix reader must coerce them to double Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] longVals = {1, -2, 1_000_000_000L, 0}; - double[] intVals = {7, -8, 123456, 0}; + double[] intVals = {7, -8, 123456, 0}; double[] boolVals = {1, 0, 1, 0}; - writeTypedColumns(tablePath, - new DataType[] {LongType.LONG, IntegerType.INTEGER, BooleanType.BOOLEAN}, + writeTypedColumns(tablePath, new DataType[] {LongType.LONG, IntegerType.INTEGER, BooleanType.BOOLEAN}, new double[][] {longVals, intVals, boolVals}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 3, out.getNumColumns()); - for( int r=0; r<4; r++ ) { + for(int r = 0; r < 4; r++) { assertEquals("long col r" + r, longVals[r], out.get(r, 0), 0.0); assertEquals("int col r" + r, intVals[r], out.get(r, 1), 0.0); assertEquals("bool col r" + r, boolVals[r], out.get(r, 2), 0.0); @@ -240,14 +236,14 @@ public void readNonDoubleNumericColumns() throws Exception { @Test public void rewriteSamePathReplacesData() throws Exception { - //writing to a path that already holds a Delta table must fully replace it + // writing to a path that already holds a Delta table must fully replace it Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { MatrixBlock first = TestUtils.generateTestMatrixBlock(50, 8, 0, 100, 1.0, 1); new WriterDelta().writeMatrixToHDFS(first, tablePath, 50, 8, -1, first.getNonZeros()); - //second write has different dimensions and values + // second write has different dimensions and values MatrixBlock second = TestUtils.generateTestMatrixBlock(20, 3, -5, 5, 1.0, 2); new WriterDelta().writeMatrixToHDFS(second, tablePath, 20, 3, -1, second.getNonZeros()); @@ -263,10 +259,10 @@ public void rewriteSamePathReplacesData() throws Exception { @Test public void parallelBufferedPathMatchesSerial() throws Exception { - //the direct fast path is always taken for SystemDS-written tables (exact - //row stats, no deletion vectors); force the buffered fallback to exercise - //its per-file decode + serial concatenation and assert it matches serial. - //force a multi-file table cheaply via a small writer target file size. + // the direct fast path is always taken for SystemDS-written tables (exact + // row stats, no deletion vectors); force the buffered fallback to exercise + // its per-file decode + serial concatenation and assert it matches serial. + // force a multi-file table cheaply via a small writer target file size. MatrixBlock in = TestUtils.generateTestMatrixBlock(ROWS_MULTI_FILE, 8, -10, 10, 1.0, 23); in.recomputeNonZeros(); @@ -276,19 +272,22 @@ public void parallelBufferedPathMatchesSerial() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_buf_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); long files; - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } assertTrue("expected a multi-file Delta table, got " + files, files > 1); MatrixBlock serial = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); - //subclass that always declines the direct path -> readBuffered() + // subclass that always declines the direct path -> readBuffered() MatrixBlock buffered = new ReaderDeltaParallel() { - @Override protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { return false; } + @Override + protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { + return false; + } }.readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", serial.getNumRows(), buffered.getNumRows()); @@ -304,30 +303,30 @@ public void parallelBufferedPathMatchesSerial() throws Exception { @Test public void writerTargetFileSizeConfigProducesMoreFiles() throws Exception { - //a smaller configured target file size must make the writer roll more - //data files for the same matrix (the lever the parallel reader relies on). + // a smaller configured target file size must make the writer roll more + // data files for the same matrix (the lever the parallel reader relies on). MatrixBlock in = TestUtils.generateTestMatrixBlock(400_000, 16, -10, 10, 1.0, 7); in.recomputeNonZeros(); - //isolate the override in a fresh thread-local config (restored in finally) + // isolate the override in a fresh thread-local config (restored in finally) DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(1L * 1024 * 1024)); ConfigurationManager.setLocalConfig(conf); Path dir = Files.createTempDirectory("sysds_delta_cfg_"); try { - assertEquals("config getter reflects the override", - 1L * 1024 * 1024, ConfigurationManager.getDeltaWriterTargetFileSize()); + assertEquals("config getter reflects the override", 1L * 1024 * 1024, + ConfigurationManager.getDeltaWriterTargetFileSize()); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); long files; - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } assertTrue("expected >1 data file with a 1MB target, got " + files, files > 1); - //data still round-trips correctly with the custom layout + // data still round-trips correctly with the custom layout MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); TestUtils.compareMatrices(in, out, 1e-12, "small-target-roundtrip"); } @@ -339,21 +338,20 @@ public void writerTargetFileSizeConfigProducesMoreFiles() throws Exception { @Test public void readerBatchSizeConfigRoundTrips() throws Exception { - //a non-default reader batch size must not change the result (more, smaller - //batches exercise the per-batch extract/concatenate loop more often). + // a non-default reader batch size must not change the result (more, smaller + // batches exercise the per-batch extract/concatenate loop more often). MatrixBlock in = TestUtils.generateTestMatrixBlock(5000, 7, -10, 10, 1.0, 11); - //isolate the override in a fresh thread-local config (restored in finally) + // isolate the override in a fresh thread-local config (restored in finally) DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_READER_BATCH_SIZE, "128"); ConfigurationManager.setLocalConfig(conf); Path dir = Files.createTempDirectory("sysds_delta_bs_"); try { - assertEquals("config getter reflects the override", - 128, ConfigurationManager.getDeltaReaderBatchSize()); + assertEquals("config getter reflects the override", 128, ConfigurationManager.getDeltaReaderBatchSize()); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); TestUtils.compareMatrices(in, out, 1e-12, "small-batch-roundtrip"); } @@ -365,14 +363,13 @@ public void readerBatchSizeConfigRoundTrips() throws Exception { @Test public void factoryRoutesDeltaToParallelWhenEnabled() { - //the factory must pick the parallel reader iff parallel CP read is enabled + // the factory must pick the parallel reader iff parallel CP read is enabled CompilerConfig cc = ConfigurationManager.getCompilerConfig(); try { cc.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, true); ConfigurationManager.setLocalConfig(cc); MatrixReader par = MatrixReaderFactory.createMatrixReader(FileFormat.DELTA); - assertTrue("expected ReaderDeltaParallel when parallel read enabled", - par instanceof ReaderDeltaParallel); + assertTrue("expected ReaderDeltaParallel when parallel read enabled", par instanceof ReaderDeltaParallel); cc.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, false); ConfigurationManager.setLocalConfig(cc); @@ -387,20 +384,18 @@ public void factoryRoutesDeltaToParallelWhenEnabled() { @Test public void readFloatColumnsCoercedToDouble() throws Exception { - //float columns must be widened to double on read (exact-representable values) + // float columns must be widened to double on read (exact-representable values) Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] f0 = {1.5, -2.25, 0.0, 1024.5}; double[] f1 = {-0.5, 3.75, 100.125, -7.0}; - writeTypedColumns(tablePath, - new DataType[] {FloatType.FLOAT, FloatType.FLOAT}, - new double[][] {f0, f1}); + writeTypedColumns(tablePath, new DataType[] {FloatType.FLOAT, FloatType.FLOAT}, new double[][] {f0, f1}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 2, out.getNumColumns()); - for( int r=0; r<4; r++ ) { + for(int r = 0; r < 4; r++) { assertEquals("f0 r" + r, f0[r], out.get(r, 0), 0.0); assertEquals("f1 r" + r, f1[r], out.get(r, 1), 0.0); } @@ -412,21 +407,20 @@ public void readFloatColumnsCoercedToDouble() throws Exception { @Test public void readShortByteColumnsCoercedToDouble() throws Exception { - //short/byte columns must be coerced to double on read, exercising the - //T_SHORT / T_BYTE branches of ReaderDelta.getDoubleValue. + // short/byte columns must be coerced to double on read, exercising the + // T_SHORT / T_BYTE branches of ReaderDelta.getDoubleValue. Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] shortVals = {1, -2, 30000, 0}; - double[] byteVals = {7, -8, 120, 0}; - writeTypedColumns(tablePath, - new DataType[] {ShortType.SHORT, ByteType.BYTE}, + double[] byteVals = {7, -8, 120, 0}; + writeTypedColumns(tablePath, new DataType[] {ShortType.SHORT, ByteType.BYTE}, new double[][] {shortVals, byteVals}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 2, out.getNumColumns()); - for( int r=0; r<4; r++ ) { + for(int r = 0; r < 4; r++) { assertEquals("short col r" + r, shortVals[r], out.get(r, 0), 0.0); assertEquals("byte col r" + r, byteVals[r], out.get(r, 1), 0.0); } @@ -438,8 +432,8 @@ public void readShortByteColumnsCoercedToDouble() throws Exception { @Test public void writerRejectsDimensionMismatch() throws Exception { - //WriterDelta validates that the passed rlen/clen match the MatrixBlock - //and rejects a mismatch with an IOException. + // WriterDelta validates that the passed rlen/clen match the MatrixBlock + // and rejects a mismatch with an IOException. MatrixBlock in = TestUtils.generateTestMatrixBlock(10, 4, -1, 1, 1.0, 5); in.recomputeNonZeros(); Path dir = Files.createTempDirectory("sysds_delta_"); @@ -459,18 +453,18 @@ public void writerRejectsDimensionMismatch() throws Exception { @Test public void readNullCellsBecomeZero() throws Exception { - //nullable numeric columns with null cells must read back as 0.0 + // nullable numeric columns with null cells must read back as 0.0 Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - double[] vals = {3.0, 7.0, 9.0, 11.0}; - boolean[] nulls = {false, true, false, true}; + double[] vals = {3.0, 7.0, 9.0, 11.0}; + boolean[] nulls = {false, true, false, true}; writeNullableDoubleColumn(tablePath, vals, nulls); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 1, out.getNumColumns()); - for( int r=0; r<4; r++ ) + for(int r = 0; r < 4; r++) assertEquals("r" + r, nulls[r] ? 0.0 : vals[r], out.get(r, 0), 0.0); } finally { @@ -480,7 +474,7 @@ public void readNullCellsBecomeZero() throws Exception { @Test public void readStringColumnRejected() throws Exception { - //string columns cannot back an all-double matrix -> reader must reject them + // string columns cannot back an all-double matrix -> reader must reject them Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { @@ -503,9 +497,10 @@ public void readStringColumnRejected() throws Exception { private static void writeTypedColumns(String tablePath, DataType[] types, double[][] vals) throws Exception { Engine engine = DeltaKernelUtils.createEngine(); StructType schema = new StructType(); - for( int c=0; c singleton(FilteredColumnarBatch fcb) { return new CloseableIterator() { private boolean _done = false; - @Override public boolean hasNext() { return !_done; } - @Override public FilteredColumnarBatch next() { - if( _done ) throw new NoSuchElementException(); + + @Override + public boolean hasNext() { + return !_done; + } + + @Override + public FilteredColumnarBatch next() { + if(_done) + throw new NoSuchElementException(); _done = true; return fcb; } - @Override public void close() {} + + @Override + public void close() { + } }; } - /** Minimal in-memory columnar batch backed by per-column double[] values, with - * an optional per-column null mask ({@code nulls==null} => no nulls). */ + /** + * Minimal in-memory columnar batch backed by per-column double[] values, with an optional per-column null mask + * ({@code nulls==null} => no nulls). + */ private static class TypedBatch implements ColumnarBatch { private final StructType _schema; private final DataType[] _types; private final double[][] _vals; private final boolean[][] _nulls; + TypedBatch(StructType schema, DataType[] types, double[][] vals, boolean[][] nulls) { - _schema = schema; _types = types; _vals = vals; _nulls = nulls; + _schema = schema; + _types = types; + _vals = vals; + _nulls = nulls; } - @Override public StructType getSchema() { return _schema; } - @Override public int getSize() { return _vals[0].length; } - @Override public ColumnVector getColumnVector(int ordinal) { - return new TypedVector(_types[ordinal], _vals[ordinal], - _nulls == null ? null : _nulls[ordinal]); + + @Override + public StructType getSchema() { + return _schema; + } + + @Override + public int getSize() { + return _vals[0].length; + } + + @Override + public ColumnVector getColumnVector(int ordinal) { + return new TypedVector(_types[ordinal], _vals[ordinal], _nulls == null ? null : _nulls[ordinal]); } } @@ -568,28 +599,98 @@ private static class TypedVector implements ColumnVector { private final DataType _type; private final double[] _vals; private final boolean[] _nulls; - TypedVector(DataType type, double[] vals, boolean[] nulls) { _type = type; _vals = vals; _nulls = nulls; } - @Override public DataType getDataType() { return _type; } - @Override public int getSize() { return _vals.length; } - @Override public boolean isNullAt(int rowId) { return _nulls != null && _nulls[rowId]; } - @Override public double getDouble(int rowId) { return _vals[rowId]; } - @Override public float getFloat(int rowId) { return (float) _vals[rowId]; } - @Override public long getLong(int rowId) { return (long) _vals[rowId]; } - @Override public int getInt(int rowId) { return (int) _vals[rowId]; } - @Override public short getShort(int rowId) { return (short) _vals[rowId]; } - @Override public byte getByte(int rowId) { return (byte) _vals[rowId]; } - @Override public boolean getBoolean(int rowId) { return _vals[rowId] != 0; } - @Override public void close() {} + + TypedVector(DataType type, double[] vals, boolean[] nulls) { + _type = type; + _vals = vals; + _nulls = nulls; + } + + @Override + public DataType getDataType() { + return _type; + } + + @Override + public int getSize() { + return _vals.length; + } + + @Override + public boolean isNullAt(int rowId) { + return _nulls != null && _nulls[rowId]; + } + + @Override + public double getDouble(int rowId) { + return _vals[rowId]; + } + + @Override + public float getFloat(int rowId) { + return (float) _vals[rowId]; + } + + @Override + public long getLong(int rowId) { + return (long) _vals[rowId]; + } + + @Override + public int getInt(int rowId) { + return (int) _vals[rowId]; + } + + @Override + public short getShort(int rowId) { + return (short) _vals[rowId]; + } + + @Override + public byte getByte(int rowId) { + return (byte) _vals[rowId]; + } + + @Override + public boolean getBoolean(int rowId) { + return _vals[rowId] != 0; + } + + @Override + public void close() { + } } /** Column view exposing a String[] as a Delta string column. */ private static class StringVector implements ColumnVector { private final String[] _vals; - StringVector(String[] vals) { _vals = vals; } - @Override public DataType getDataType() { return StringType.STRING; } - @Override public int getSize() { return _vals.length; } - @Override public boolean isNullAt(int rowId) { return _vals[rowId] == null; } - @Override public String getString(int rowId) { return _vals[rowId]; } - @Override public void close() {} + + StringVector(String[] vals) { + _vals = vals; + } + + @Override + public DataType getDataType() { + return StringType.STRING; + } + + @Override + public int getSize() { + return _vals.length; + } + + @Override + public boolean isNullAt(int rowId) { + return _vals[rowId] == null; + } + + @Override + public String getString(int rowId) { + return _vals[rowId]; + } + + @Override + public void close() { + } } } diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java index 2d79b79f2dd..45194d12b5a 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java @@ -49,24 +49,22 @@ import org.junit.Test; /** - * Cross-engine interoperability tests for the native (Delta Kernel based) matrix - * reader/writer against the reference Delta implementation (Delta's Spark - * connector, {@code delta-spark}, pulled in test-only). + * Cross-engine interoperability tests for the native (Delta Kernel based) matrix reader/writer against the reference + * Delta implementation (Delta's Spark connector, {@code delta-spark}, pulled in test-only). * - *

The other Delta matrix tests round-trip exclusively through SystemDS' own - * Kernel-based read/write paths, so they cannot catch a table that SystemDS - * writes in a way other Delta engines reject (or vice versa). These tests close - * that gap by routing data through two independent engines: + *

+ * The other Delta matrix tests round-trip exclusively through SystemDS' own Kernel-based read/write paths, so they + * cannot catch a table that SystemDS writes in a way other Delta engines reject (or vice versa). These tests close that + * gap by routing data through two independent engines: *

    - *
  • SystemDS writes -> Spark/Delta reads (our output is spec-compliant), and
    • - *
  • Spark/Delta writes -> SystemDS reads, including a multi-file layout and a - * table with deletion vectors / a second commit that the SystemDS writer - * never produces itself.
    • + *
  • SystemDS writes -> Spark/Delta reads (our output is spec-compliant), and
    • + *
  • Spark/Delta writes -> SystemDS reads, including a multi-file layout and a table with deletion vectors / a + * second commit that the SystemDS writer never produces itself.
    • *
* - *

Row order is never assumed: every table carries a unique id in column 0 and - * comparisons are keyed by that id, since neither engine guarantees row order - * across files. + *

+ * Row order is never assumed: every table carries a unique id in column 0 and comparisons are keyed by that id, since + * neither engine guarantees row order across files. */ @net.jcip.annotations.NotThreadSafe public class DeltaMatrixSparkInteropTest { @@ -75,23 +73,19 @@ public class DeltaMatrixSparkInteropTest { @BeforeClass public static void startSpark() { - //each test class runs in its own fork (surefire reuseForks=false), so this - //is the only SparkSession in the JVM and gets the Delta extensions injected. + // each test class runs in its own fork (surefire reuseForks=false), so this + // is the only SparkSession in the JVM and gets the Delta extensions injected. SparkSession.clearActiveSession(); SparkSession.clearDefaultSession(); - spark = SparkSession.builder() - .appName("sysds-delta-interop") - .master("local[2]") - .config("spark.ui.enabled", "false") - .config("spark.sql.shuffle.partitions", "2") + spark = SparkSession.builder().appName("sysds-delta-interop").master("local[2]") + .config("spark.ui.enabled", "false").config("spark.sql.shuffle.partitions", "2") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") - .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") - .getOrCreate(); + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog").getOrCreate(); } @AfterClass public static void stopSpark() { - if( spark != null ) + if(spark != null) spark.stop(); SparkSession.clearActiveSession(); SparkSession.clearDefaultSession(); @@ -100,13 +94,13 @@ public static void stopSpark() { @Test public void systemdsWriteSparkReadMultiFile() throws Exception { - //SystemDS writes a (forced) multi-file Delta table; the reference Delta - //engine (Spark) must read every data file back with matching values. + // SystemDS writes a (forced) multi-file Delta table; the reference Delta + // engine (Spark) must read every data file back with matching values. int rows = 500, cols = 5; MatrixBlock in = indexedMatrix(rows, cols); - //small target file size -> multiple parquet data files (exercise that an - //external reader stitches all of our data files, not just the first). + // small target file size -> multiple parquet data files (exercise that an + // external reader stitches all of our data files, not just the first). DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(16L * 1024)); ConfigurationManager.setLocalConfig(conf); @@ -122,10 +116,10 @@ public void systemdsWriteSparkReadMultiFile() throws Exception { List read = df.collectAsList(); assertEquals(rows, read.size()); - for( Row r : read ) { + for(Row r : read) { int id = (int) Math.round(r.getDouble(0)); assertTrue("id in range: " + id, id >= 0 && id < rows); - for( int c = 0; c < cols; c++ ) + for(int c = 0; c < cols; c++) assertEquals("r" + id + " c" + c, in.get(id, c), r.getDouble(c), 1e-9); } } @@ -137,10 +131,10 @@ public void systemdsWriteSparkReadMultiFile() throws Exception { @Test public void sparkWriteSystemdsReadMultiFile() throws Exception { - //the reference Delta engine writes a multi-file table; both the serial and - //parallel SystemDS readers must reconstruct it (coercing long ids to double). + // the reference Delta engine writes a multi-file table; both the serial and + // parallel SystemDS readers must reconstruct it (coercing long ids to double). int rows = 600, cols = 4; - Dataset df = indexedDataFrame(rows, cols).repartition(3); //-> multiple data files + Dataset df = indexedDataFrame(rows, cols).repartition(3); // -> multiple data files Path dir = Files.createTempDirectory("sysds_delta_p2s_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { @@ -148,10 +142,10 @@ public void sparkWriteSystemdsReadMultiFile() throws Exception { assertTrue("spark should have written a multi-file table", countParquet(tablePath) > 1); Map expected = expectedById(rows, cols); - assertMatchesById(new ReaderDelta() - .readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, "serial"); - assertMatchesById(new ReaderDeltaParallel() - .readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, "parallel"); + assertMatchesById(new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, + "serial"); + assertMatchesById(new ReaderDeltaParallel().readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, + "parallel"); } finally { FileUtils.deleteQuietly(dir.toFile()); @@ -160,15 +154,15 @@ public void sparkWriteSystemdsReadMultiFile() throws Exception { @Test public void sparkDeletionVectorsSystemdsRead() throws Exception { - //a Delta table with deletion vectors + a second commit (the DELETE) is a - //layout the SystemDS writer never emits; the readers must honor the DV and - //return only the surviving rows. This exercises the hasDeletionVector path. + // a Delta table with deletion vectors + a second commit (the DELETE) is a + // layout the SystemDS writer never emits; the readers must honor the DV and + // return only the surviving rows. This exercises the hasDeletionVector path. int rows = 400, cols = 3, deleteBelow = 50; Path dir = Files.createTempDirectory("sysds_delta_dv_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - //enable deletion vectors for tables created in this block, then delete a - //row range so Delta records a DV rather than rewriting the data files. + // enable deletion vectors for tables created in this block, then delete a + // row range so Delta records a DV rather than rewriting the data files. spark.conf().set(DV_DEFAULT, "true"); indexedDataFrame(rows, cols).write().format("delta").save(tablePath); spark.sql("DELETE FROM delta.`" + tablePath + "` WHERE c0 < " + deleteBelow); @@ -185,21 +179,20 @@ public void sparkDeletionVectorsSystemdsRead() throws Exception { assertMatchesById(parallel, expected, cols, "parallel-dv"); } finally { - //fresh fork per test class, so simply clearing the override is enough + // fresh fork per test class, so simply clearing the override is enough spark.conf().unset(DV_DEFAULT); FileUtils.deleteQuietly(dir.toFile()); } } - private static final String DV_DEFAULT = - "spark.databricks.delta.properties.defaults.enableDeletionVectors"; + private static final String DV_DEFAULT = "spark.databricks.delta.properties.defaults.enableDeletionVectors"; /** Matrix whose column 0 is the row index and remaining columns are exact doubles. */ private static MatrixBlock indexedMatrix(int rows, int cols) { MatrixBlock mb = new MatrixBlock(rows, cols, false); - for( int r = 0; r < rows; r++ ) { + for(int r = 0; r < rows; r++) { mb.set(r, 0, r); - for( int c = 1; c < cols; c++ ) + for(int c = 1; c < cols; c++) mb.set(r, c, value(r, c)); } mb.recomputeNonZeros(); @@ -209,15 +202,15 @@ private static MatrixBlock indexedMatrix(int rows, int cols) { /** Spark DataFrame mirroring {@link #indexedMatrix} with columns c0..c(cols-1) as doubles. */ private static Dataset indexedDataFrame(int rows, int cols) { StructField[] fields = new StructField[cols]; - for( int c = 0; c < cols; c++ ) + for(int c = 0; c < cols; c++) fields[c] = DataTypes.createStructField("c" + c, DataTypes.DoubleType, false); StructType schema = DataTypes.createStructType(fields); List data = new ArrayList<>(rows); - for( int r = 0; r < rows; r++ ) { + for(int r = 0; r < rows; r++) { Object[] vals = new Object[cols]; vals[0] = (double) r; - for( int c = 1; c < cols; c++ ) + for(int c = 1; c < cols; c++) vals[c] = value(r, c); data.add(RowFactory.create(vals)); } @@ -231,10 +224,10 @@ private static double value(int row, int col) { private static Map expectedById(int rows, int cols) { Map exp = new HashMap<>(rows); - for( int r = 0; r < rows; r++ ) { + for(int r = 0; r < rows; r++) { double[] row = new double[cols]; row[0] = r; - for( int c = 1; c < cols; c++ ) + for(int c = 1; c < cols; c++) row[c] = value(r, c); exp.put(r, row); } @@ -246,25 +239,25 @@ private static void assertMatchesById(MatrixBlock out, Map ex assertEquals(tag + " rows", expected.size(), out.getNumRows()); assertEquals(tag + " cols", cols, out.getNumColumns()); boolean[] seen = new boolean[expected.size() == 0 ? 0 : maxId(expected) + 1]; - for( int r = 0; r < out.getNumRows(); r++ ) { + for(int r = 0; r < out.getNumRows(); r++) { int id = (int) Math.round(out.get(r, 0)); double[] exp = expected.get(id); assertTrue(tag + ": unexpected/duplicate id " + id, exp != null && id < seen.length && !seen[id]); seen[id] = true; - for( int c = 0; c < cols; c++ ) + for(int c = 0; c < cols; c++) assertEquals(tag + " id" + id + " c" + c, exp[c], out.get(r, c), 1e-9); } } private static int maxId(Map expected) { int m = 0; - for( int id : expected.keySet() ) + for(int id : expected.keySet()) m = Math.max(m, id); return m; } private static long countParquet(String tablePath) throws Exception { - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { return s.filter(p -> p.toString().endsWith(".parquet")).count(); } } diff --git a/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java b/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java index 472b61d8cd6..ae19b291882 100644 --- a/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java +++ b/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -26,11 +26,10 @@ /** * Tests the single-column (unweighted) branch of {@link MatrixBlock#pickValue(double, boolean)} and - * {@link MatrixBlock#median()}. The values are assumed to be sorted in ascending order, mirroring the contract used - * by the quantile pick instructions. The unweighted branch uses the same ceil-based rank as the two-column weighted - * branch (with an implicit weight of 1 per value), so a single column yields the same quantile as the equivalent - * (value, weight) representation. The two-column (weighted) branch is exercised separately through the compressed - * sort tests. + * {@link MatrixBlock#median()}. The values are assumed to be sorted in ascending order, mirroring the contract used by + * the quantile pick instructions. The unweighted branch uses the same ceil-based rank as the two-column weighted branch + * (with an implicit weight of 1 per value), so a single column yields the same quantile as the equivalent (value, + * weight) representation. The two-column (weighted) branch is exercised separately through the compressed sort tests. */ public class QuantilePickTest { diff --git a/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java b/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java index 5c9ed821e78..05aca41ebc7 100644 --- a/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java +++ b/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java @@ -30,7 +30,7 @@ public class TensorToStringTest { @Test public void testDecimalClampsFractionDigits() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 1}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 1}); tb.allocateBlock(); tb.set(0, 0, 5.244058388023880); // decimal=2 must print exactly two fraction digits, not DecimalFormat's default max of 3 @@ -41,10 +41,10 @@ public void testDecimalClampsFractionDigits() { @Test public void testDecimalPadsAndRounds() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 2}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 2}); tb.allocateBlock(); - tb.set(0, 0, 22.0); // integer-valued: padded up to the requested digits - tb.set(0, 1, 5.244058388023880); // rounded at the last requested digit + tb.set(0, 0, 22.0); // integer-valued: padded up to the requested digits + tb.set(0, 1, 5.244058388023880); // rounded at the last requested digit String out = DataConverter.toString(tb, false, " ", "\n", "[", "]", 1, 2, 4); assertTrue("expected 22.0000 padded: " + out, out.contains("22.0000")); assertTrue("expected 5.2441 rounded: " + out, out.contains("5.2441")); @@ -52,10 +52,10 @@ public void testDecimalPadsAndRounds() { @Test public void testNegativeDecimalUsesDefaultFormatting() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 2}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 2}); tb.allocateBlock(); - tb.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained - tb.set(0, 1, 5.244058388023880); // default cap of three fraction digits + tb.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained + tb.set(0, 1, 5.244058388023880); // default cap of three fraction digits // decimal < 0 leaves DecimalFormat unconstrained (no min/max fraction digits set) String out = DataConverter.toString(tb, false, " ", "\n", "[", "]", 1, 2, -1); assertTrue("expected unpadded 22: " + out, out.contains("22")); diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java index 810e2614e7c..426de81263f 100644 --- a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java +++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java @@ -52,18 +52,12 @@ public class QuantileTest extends AutomatedTestBase public void setUp() { TestUtils.clearAssertionInformation(); - addTestConfiguration(TEST_NAME1, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"})); - addTestConfiguration(TEST_NAME2, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"})); - addTestConfiguration(TEST_NAME3, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] {"R"})); - addTestConfiguration(TEST_NAME4, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] {"R"})); - addTestConfiguration(TEST_NAME5, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] {"R"})); - addTestConfiguration(TEST_NAME6, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] {"R"})); + addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"})); + addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"})); + addTestConfiguration(TEST_NAME3, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] {"R"})); + addTestConfiguration(TEST_NAME4, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] {"R"})); + addTestConfiguration(TEST_NAME5, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] {"R"})); + addTestConfiguration(TEST_NAME6, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] {"R"})); } @Test diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java index e70aedcabe4..35cfb2982fc 100644 --- a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java +++ b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java @@ -59,7 +59,8 @@ private void runSTEPGlmTest(ExecType instType) { // runTest executes the script; fails if the DML script invokes stop() runTest(true, false, null, -1); - } finally { + } + finally { rtplatform = platformOld; } } diff --git a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java index 5de429a3c53..d8cfc4d9fd7 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java @@ -91,10 +91,12 @@ public void testBackendPerformance() throws InterruptedException { taskFutures.forEach(res -> { try { Assert.assertEquals("Stats parsed correctly", res.get().statusCode(), 200); - } catch (InterruptedException e) { + } + catch(InterruptedException e) { Thread.currentThread().interrupt(); Assert.fail("Interrupted while fetching statistics: " + e.getMessage()); - } catch (ExecutionException e) { + } + catch(ExecutionException e) { Assert.fail("Failed to fetch statistics: " + e.getMessage()); } }); diff --git a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java index f8acdd07930..1cab99ee1ab 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java @@ -375,9 +375,8 @@ public void federatedLogicalTest(String testname, Type op_type, ExecMode execMod int port2 = single_fed_worker ? 0 : getRandomAvailablePort(); int port3 = single_fed_worker ? 0 : getRandomAvailablePort(); int port4 = single_fed_worker ? 0 : getRandomAvailablePort(); - Process[] workers = startLocalFedWorkers(single_fed_worker - ? new int[] {port1} - : new int[] {port1, port2, port3, port4}); + Process[] workers = startLocalFedWorkers( + single_fed_worker ? new int[] {port1} : new int[] {port1, port2, port3, port4}); try { if(!isAlive(workers)) diff --git a/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java b/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java index dbbf199f8fa..9d2bb583a4f 100644 --- a/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java +++ b/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java @@ -72,27 +72,26 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod } if(et == ExecType.SPARK) { - rtplatform = ExecMode.SPARK; + rtplatform = ExecMode.SPARK; } else { // rtplatform = (et==ExecType.MR)? ExecMode.HADOOP : ExecMode.SINGLE_NODE; - rtplatform = ExecMode.HYBRID; + rtplatform = ExecMode.HYBRID; } if( rtplatform == ExecMode.SPARK ) DMLScript.USE_LOCAL_SPARK_CONFIG = true; config.addVariable("rows", rows); - config.addVariable("cols", cols); + config.addVariable("cols", cols); - long rowstart=816, rowend=1229, colstart=967, colend=1009; - // long rowstart=2, rowend=4, colstart=9, colend=10; + long rowstart = 816, rowend = 1229, colstart = 967, colend = 1009; + // long rowstart=2, rowend=4, colstart=9, colend=10; /* - Random rand=new Random(System.currentTimeMillis()); - rowstart=(long)(rand.nextDouble()*((double)rows))+1; - rowend=(long)(rand.nextDouble()*((double)(rows-rowstart+1)))+rowstart; - colstart=(long)(rand.nextDouble()*((double)cols))+1; - colend=(long)(rand.nextDouble()*((double)(cols-colstart+1)))+colstart; - */ + * Random rand=new Random(System.currentTimeMillis()); rowstart=(long)(rand.nextDouble()*((double)rows))+1; + * rowend=(long)(rand.nextDouble()*((double)(rows-rowstart+1)))+rowstart; + * colstart=(long)(rand.nextDouble()*((double)cols))+1; + * colend=(long)(rand.nextDouble()*((double)(cols-colstart+1)))+colstart; + */ config.addVariable("rowstart", rowstart); config.addVariable("rowend", rowend); config.addVariable("colstart", colstart); @@ -119,17 +118,20 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod double sparsity=1.0;//rand.nextDouble(); double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, System.currentTimeMillis()); writeInputMatrix("A", A, true); - - sparsity=0.1;//rand.nextDouble(); - double[][] B = getRandomMatrix((int)(rowend-rowstart+1), (int)(colend-colstart+1), min, max, sparsity, System.currentTimeMillis()); + + sparsity = 0.1;// rand.nextDouble(); + double[][] B = getRandomMatrix((int) (rowend - rowstart + 1), (int) (colend - colstart + 1), min, max, + sparsity, System.currentTimeMillis()); writeInputMatrix("B", B, true); - - sparsity=0.5;//rand.nextDouble(); - double[][] C = getRandomMatrix((int)(rowend), (int)(cols-colstart+1), min, max, sparsity, System.currentTimeMillis()); + + sparsity = 0.5;// rand.nextDouble(); + double[][] C = getRandomMatrix((int) (rowend), (int) (cols - colstart + 1), min, max, sparsity, + System.currentTimeMillis()); writeInputMatrix("C", C, true); - - sparsity=0.01;//rand.nextDouble(); - double[][] D = getRandomMatrix(rows, (int)(colend-colstart+1), min, max, sparsity, System.currentTimeMillis()); + + sparsity = 0.01;// rand.nextDouble(); + double[][] D = getRandomMatrix(rows, (int) (colend - colstart + 1), min, max, sparsity, + System.currentTimeMillis()); writeInputMatrix("D", D, true); /* @@ -138,9 +140,9 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod * While loop iteration - 10 jobs * Final output write - 1 job */ - //boolean exceptionExpected = false; - //int expectedNumberOfJobs = 12; - //runTest(exceptionExpected, null, expectedNumberOfJobs); + // boolean exceptionExpected = false; + // int expectedNumberOfJobs = 12; + // runTest(exceptionExpected, null, expectedNumberOfJobs); boolean exceptionExpected = false; int expectedNumberOfJobs = -1; runTest(true, exceptionExpected, null, expectedNumberOfJobs); diff --git a/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java b/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java index a1b51ee9d81..fad1cc123c8 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java @@ -81,8 +81,9 @@ public void testDoubleScalarWrite() { fullDMLScriptName = HOME + "ScalarComputeWrite.dml"; runTest(true, false, null, -1); - double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1,1)).doubleValue(); - Assert.assertEquals("Computation test for Doubles failed: Values not equal: " + double_scalar + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar); + double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)).doubleValue(); + Assert.assertEquals("Computation test for Doubles failed: Values not equal: " + double_scalar + + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar); } @Test @@ -122,8 +123,7 @@ public void testIntScalarRead() { programArgs = new String[]{"-args", String.valueOf(int_scalar), output("a.scalar")}; runTest(true, false, null, -1); - int int_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)) - .get(new CellIndex(1,1)).intValue(); + int int_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)).intValue(); Assert.assertEquals("Values not equal: " + int_scalar + Opcodes.NOTEQUAL.toString() + int_out_scalar, int_scalar, int_out_scalar); @@ -143,8 +143,8 @@ public void testDoubleScalarRead() { programArgs = new String[]{ "-args", String.valueOf(double_scalar), output("a.scalar") }; runTest(true, false, null, -1); - double double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)) - .get(new CellIndex(1,1)).doubleValue(); + double double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)) + .doubleValue(); Assert.assertEquals("Values not equal: " + double_scalar + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar, 0); diff --git a/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java b/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java index a4013c3672d..20ae7aa63ea 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java @@ -35,14 +35,14 @@ /** * End-to-end DML test of the native Delta read/write path. * - *

The write and the read are run as two separate SystemDS executions - * on purpose. If they shared a single script/process, SystemDS would reuse the - * still-materialized in-memory matrix for the subsequent read and never invoke - * {@link org.apache.sysds.runtime.io.ReaderDelta} at all (verified: the cache - * reports 0 HDFS hits in that case). Splitting the executions forces a genuine - * read from disk, and we additionally assert via {@link CacheStatistics} that - * the read run actually performed HDFS reads (the Delta table + the text - * reference) rather than serving the matrix from cache.

+ *

+ * The write and the read are run as two separate SystemDS executions on purpose. If they shared a single + * script/process, SystemDS would reuse the still-materialized in-memory matrix for the subsequent read and never invoke + * {@link org.apache.sysds.runtime.io.ReaderDelta} at all (verified: the cache reports 0 HDFS hits in that case). + * Splitting the executions forces a genuine read from disk, and we additionally assert via {@link CacheStatistics} that + * the read run actually performed HDFS reads (the Delta table + the text reference) rather than serving the matrix from + * cache. + *

*/ public class DeltaReadWriteTest extends AutomatedTestBase { @@ -54,10 +54,8 @@ public class DeltaReadWriteTest extends AutomatedTestBase { @Override public void setUp() { TestUtils.clearAssertionInformation(); - addTestConfiguration(WRITE_NAME, - new TestConfiguration(TEST_CLASS_DIR, WRITE_NAME, new String[] { "ref" })); - addTestConfiguration(READ_NAME, - new TestConfiguration(TEST_CLASS_DIR, READ_NAME, new String[] { "R" })); + addTestConfiguration(WRITE_NAME, new TestConfiguration(TEST_CLASS_DIR, WRITE_NAME, new String[] {"ref"})); + addTestConfiguration(READ_NAME, new TestConfiguration(TEST_CLASS_DIR, READ_NAME, new String[] {"R"})); } @Test @@ -84,17 +82,16 @@ private void runDeltaRoundTrip(int rows, int cols, double sparsity) { String deltaPath = output("deltaTable"); String refPath = output("ref"); fullDMLScriptName = HOME + WRITE_NAME + ".dml"; - programArgs = new String[] { "-stats", "-args", - String.valueOf(rows), String.valueOf(cols), String.valueOf(sparsity), - deltaPath, refPath }; + programArgs = new String[] {"-stats", "-args", String.valueOf(rows), String.valueOf(cols), + String.valueOf(sparsity), deltaPath, refPath}; runTest(true, false, null, -1); // the write run must have materialized two matrices to disk (the Delta // table under test + the text reference); WriterDelta genuinely hitting // HDFS is what produces these write-side cache statistics. long hdfsWrites = CacheStatistics.getHDFSWrites(); - assertTrue("expected >= 2 HDFS writes in the write run (delta + reference), got " - + hdfsWrites, hdfsWrites >= 2); + assertTrue("expected >= 2 HDFS writes in the write run (delta + reference), got " + hdfsWrites, + hdfsWrites >= 2); // and a real Delta table (transaction log) must have been created assertTrue("missing Delta transaction log under " + deltaPath, new File(deltaPath, "_delta_log").isDirectory()); @@ -102,19 +99,18 @@ private void runDeltaRoundTrip(int rows, int cols, double sparsity) { // ---- phase 2: fresh execution reads the Delta table and compares ---- getAndLoadTestConfiguration(READ_NAME); fullDMLScriptName = HOME + READ_NAME + ".dml"; - programArgs = new String[] { "-stats", "-args", - deltaPath, refPath, output("R") }; + programArgs = new String[] {"-stats", "-args", deltaPath, refPath, output("R")}; runTest(true, false, null, -1); // the read run must have materialized two matrices from disk (the Delta // table under test + the text reference); a cached/short-circuited read // would report fewer HDFS hits and fail here. long hdfsReads = CacheStatistics.getHDFSHits(); - assertTrue("expected >= 2 HDFS reads in the read run (delta + reference), got " - + hdfsReads, hdfsReads >= 2); + assertTrue("expected >= 2 HDFS reads in the read run (delta + reference), got " + hdfsReads, + hdfsReads >= 2); HashMap R = readDMLMatrixFromOutputDir("R"); - //text-cell output omits exact zeros, so a missing cell means 0.0 + // text-cell output omits exact zeros, so a missing cell means 0.0 double diff = R.getOrDefault(new CellIndex(1, 1), 0.0); double nrow = R.getOrDefault(new CellIndex(1, 2), 0.0); double ncol = R.getOrDefault(new CellIndex(1, 3), 0.0); diff --git a/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java index cc1412b1606..a844321c249 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java @@ -49,10 +49,9 @@ public class FrameParquetSchemaTest extends AutomatedTestBase { @Override public void setUp() { - addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[]{"Rout"})); + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"Rout"})); } - /** * Test for sequential writer and reader * diff --git a/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java b/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java index dfb3d8a19de..6e6e4665f5e 100644 --- a/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java @@ -42,12 +42,8 @@ */ @net.jcip.annotations.NotThreadSafe public class JMLConnectionTest extends AutomatedTestBase { - public static final String META = "{\"data_type\": \"matrix\",\n" + - " \"value_type\": \"double\", \n" + - " \"rows\": 1,\n" + - " \"cols\": 1,\n" + - " \"nnz\": 1,\n" + - " \"format\": \"csv\"}"; + public static final String META = "{\"data_type\": \"matrix\",\n" + " \"value_type\": \"double\", \n" + + " \"rows\": 1,\n" + " \"cols\": 1,\n" + " \"nnz\": 1,\n" + " \"format\": \"csv\"}"; private final static String TEST_NAME = "JMLConnectionTest"; private final static String TEST_DIR = "functions/jmlc/"; @@ -99,12 +95,14 @@ public void testConnectionInvalidInName() throws DMLException { conn.gatherMemStats(false); Assert.assertFalse(DMLScript.STATISTICS); - try (conn) { - conn.prepareScript("printx('hello')", new String[]{"$inScalar1", null}, new String[]{null}); + try(conn) { + conn.prepareScript("printx('hello')", new String[] {"$inScalar1", null}, new String[] {null}); throw new AssertionError("Test should have thrown a LanguageException"); - } catch (LanguageException e) { + } + catch(LanguageException e) { Assert.assertTrue(e.getMessage().startsWith("Invalid variable names")); - } finally { + } + finally { DMLScript.STATISTICS = oldStat; DMLScript.JMLC_MEM_STATISTICS = oldJMLCStat; } @@ -112,21 +110,24 @@ public void testConnectionInvalidInName() throws DMLException { @Test public void testConnectionParseLanguageException() { - try (Connection conn = new Connection()) { - conn.prepareScript("printx('hello')", new String[]{}, new String[]{}); + try(Connection conn = new Connection()) { + conn.prepareScript("printx('hello')", new String[] {}, new String[] {}); throw new AssertionError("Test should have thrown a DMLException"); - } catch (DMLException e) { + } + catch(DMLException e) { Throwable cause = e.getCause(); - Assert.assertTrue(cause.getMessage().startsWith("ERROR: [line 1:0] -> printx('hello') -- function printx is undefined in namespace .builtinNS")); + Assert.assertTrue(cause.getMessage().startsWith( + "ERROR: [line 1:0] -> printx('hello') -- function printx is undefined in namespace .builtinNS")); } } @Test public void testConnectionParseException() { - try (Connection conn = new Connection()) { - conn.prepareScript("print('hello'", new String[]{}, new String[]{}); + try(Connection conn = new Connection()) { + conn.prepareScript("print('hello'", new String[] {}, new String[] {}); throw new AssertionError("Test should have thrown a ParseException"); - } catch (Exception e) { + } + catch(Exception e) { Assert.assertEquals("ParseException", e.getClass().getSimpleName()); } } @@ -144,10 +145,11 @@ public void testConnectionClose() { @Test public void testReadScriptHDFS() { - try (Connection conn = new Connection()) { + try(Connection conn = new Connection()) { conn.readScript("hdfs://localhost:9000/Test"); - } catch (IOException e) { - Assert.assertEquals("ConnectException",e.getClass().getSimpleName()); + } + catch(IOException e) { + Assert.assertEquals("ConnectException", e.getClass().getSimpleName()); } } diff --git a/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java b/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java index 4852220861e..2178884ef5b 100644 --- a/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java @@ -111,17 +111,16 @@ public void federatedReuse(String test) { // Run reference dml script with normal matrix. Reuse of ba+*. fullDMLScriptName = HOME + test + "Reference.dml"; - programArgs = new String[] {"-stats", "-lineage", "reuse_full", - "-nvargs", "X1=" + input("X1"), "X2=" + input("X2"), "Y1=" + input("Y1"), - "Y2=" + input("Y2"), "Z=" + expected("Z")}; + programArgs = new String[] {"-stats", "-lineage", "reuse_full", "-nvargs", "X1=" + input("X1"), + "X2=" + input("X2"), "Y1=" + input("Y1"), "Y2=" + input("Y2"), "Z=" + expected("Z")}; runTest(true, false, null, -1); long mmCount = Statistics.getCPHeavyHitterCount(Opcodes.MMULT.toString()); // Run actual dml script with federated matrix // The fed workers reuse ba+* fullDMLScriptName = HOME + test + ".dml"; - programArgs = new String[] {"-stats","-lineage", "reuse_full", - "-nvargs", "X1=" + TestUtils.federatedAddress(port1, input("X1")), + programArgs = new String[] {"-stats", "-lineage", "reuse_full", "-nvargs", + "X1=" + TestUtils.federatedAddress(port1, input("X1")), "X2=" + TestUtils.federatedAddress(port2, input("X2")), "Y1=" + TestUtils.federatedAddress(port1, input("Y1")), "Y2=" + TestUtils.federatedAddress(port2, input("Y2")), "r=" + rows, "c=" + cols, "Z=" + output("Z")}; @@ -129,12 +128,12 @@ public void federatedReuse(String test) { long mmCount_fed = Statistics.getCPHeavyHitterCount(Opcodes.MMULT.toString()); long fedMMCount = Statistics.getCPHeavyHitterCount("fed_ba+*"); - // compare results + // compare results compareResults(1e-9); // compare matrix multiplication count - // #federated execution of ba+* = #threads times #non-federated execution of ba+* (after reuse) - Assert.assertTrue("Violated reuse count: "+mmCount_fed+" == "+mmCount*2, - mmCount_fed == mmCount * 2); // #threads = 2 + // #federated execution of ba+* = #threads times #non-federated execution of ba+* (after reuse) + Assert.assertTrue("Violated reuse count: " + mmCount_fed + " == " + mmCount * 2, + mmCount_fed == mmCount * 2); // #threads = 2 switch(test) { case TEST_NAME1: // If the o/p is federated, fed_ba+* will be called everytime diff --git a/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java b/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java index eca3628a89b..c86eb0f4941 100644 --- a/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java @@ -121,9 +121,8 @@ private void runTriUDFReuse(ExecMode execMode) { // Run reference dml script with normal matrix fullDMLScriptName = HOME + TEST_NAME + "Reference.dml"; - programArgs = new String[] {"-lineage", "reuse_full", "-stats", "100", "-args", - input("X1"), input("X2"), input("X3"), input("X4"), - Boolean.toString(rowPartitioned).toUpperCase(), expected("S")}; + programArgs = new String[] {"-lineage", "reuse_full", "-stats", "100", "-args", input("X1"), input("X2"), + input("X3"), input("X4"), Boolean.toString(rowPartitioned).toUpperCase(), expected("S")}; runTest(null); // Run actual dml script with federated matrix diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java index 18ca2fbc454..3ffdfe1d30b 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java @@ -272,82 +272,79 @@ protected void toStringTestHelper(ExecMode platform, String testName, String exp } @Test - public void testPrintWithDecimal(){ + public void testPrintWithDecimal() { String testName = "ToString12"; String decimalPoints = "2"; String value = "22"; String expectedOutput = "22.00\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - @Test - public void testPrintWithDecimal2(){ + public void testPrintWithDecimal2() { String testName = "ToString12"; String decimalPoints = "2"; String value = "5.244058388023880"; String expectedOutput = "5.24\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - @Test - public void testPrintWithDecimal3(){ + public void testPrintWithDecimal3() { String testName = "ToString12"; String decimalPoints = "10"; String value = "5.244058388023880"; String expectedOutput = "5.2440583880\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - @Test - public void testPrintWithDecimal4(){ + public void testPrintWithDecimal4() { String testName = "ToString12"; String decimalPoints = "4"; String value = "5.244058388023880"; String expectedOutput = "5.2441\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - @Test - public void testPrintWithDecimal5(){ + public void testPrintWithDecimal5() { String testName = "ToString12"; String decimalPoints = "10"; String value = "0.000000008023880"; String expectedOutput = "0.0000000080\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - protected void toStringTestHelper2(ExecMode platform, String testName, String expectedOutput, String decimalPoints, String value) { + protected void toStringTestHelper2(ExecMode platform, String testName, String expectedOutput, String decimalPoints, + String value) { ExecMode platformOld = rtplatform; - + rtplatform = platform; boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - if (rtplatform == ExecMode.SPARK) + if(rtplatform == ExecMode.SPARK) DMLScript.USE_LOCAL_SPARK_CONFIG = true; try { // Create and load test configuration getAndLoadTestConfiguration(testName); String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testName + ".dml"; - programArgs = new String[]{"-args", output(OUTPUT_NAME), value, decimalPoints}; + programArgs = new String[] {"-args", output(OUTPUT_NAME), value, decimalPoints}; // Run DML and R scripts runTest(true, false, null, -1); diff --git a/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java index 770c5b7c5bf..26143dc16ee 100644 --- a/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java @@ -76,10 +76,9 @@ public ReshapeTest(int rlen, int clen, int rows, int cols, boolean rowWise) { @Parameterized.Parameters(name = "{0}x{1} {2}x{3} rowWise {4}") public static Iterable getParams() { - int[][][] dims = { - {{1000, 1000}, {1, 1000000}}, // single row/col - {{3000, 4000}, {1500, 8000}}, // partialBlocks - {{2400, 1400}, {800, 4200}} // fullBlocks + int[][][] dims = {{{1000, 1000}, {1, 1000000}}, // single row/col + {{3000, 4000}, {1500, 8000}}, // partialBlocks + {{2400, 1400}, {800, 4200}} // fullBlocks }; ArrayList params = new ArrayList<>(); @@ -117,7 +116,8 @@ public void runTestMatrixReshapeOOC() { double[][] X = getRandomMatrix(rlen, clen, 0, 1, 1, 7); MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(Types.FileFormat.BINARY); - writer.writeMatrixToHDFS(DataConverter.convertToMatrixBlock(X), input(INPUT_NAME), rlen, clen, 1000, rlen * clen); + writer.writeMatrixToHDFS(DataConverter.convertToMatrixBlock(X), input(INPUT_NAME), rlen, clen, 1000, + rlen * clen); HDFSTool.writeMetaDataFile(input(INPUT_NAME + ".mtd"), Types.ValueType.FP64, new MatrixCharacteristics(rlen, clen, blen, rlen * clen), Types.FileFormat.BINARY); @@ -143,8 +143,8 @@ public void runTestMatrixReshapeOOC() { runTest(true, false, null, -1); // compare results - MatrixBlock actual = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME), - Types.FileFormat.BINARY, rows, cols, blen); + MatrixBlock actual = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME), Types.FileFormat.BINARY, rows, + cols, blen); MatrixBlock expected = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME + "_target"), Types.FileFormat.BINARY, rows, cols, blen); diff --git a/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java index 5f42db7d733..49a52587cde 100644 --- a/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java @@ -335,9 +335,9 @@ private void runTestMatrixReshape( ReshapeType type, boolean rowwise, boolean sp String.valueOf(trows), String.valueOf(tcols), output("Y") }; fullRScriptName = HOME + TEST_NAME + ".R"; - rCmd = "Rscript" + " " + fullRScriptName + " " + - inputDir() + " " + trows + " " + tcols + " " + expectedDir(); - + rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + trows + " " + tcols + " " + + expectedDir(); + double[][] X = getRandomMatrix(rows, cols, 0, 1, sparsity, 7); writeInputMatrix("X", X, true); diff --git a/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java index dcdafddcd47..69d6958f8a6 100644 --- a/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java @@ -94,9 +94,9 @@ private void runVectorReshape(boolean sparse, ExecType et) String.valueOf(rows2), String.valueOf(cols2), output("R") }; fullRScriptName = HOME + TEST_NAME + ".R"; - rCmd = "Rscript" + " " + fullRScriptName + " " + - inputDir() + " " + rows2 + " " + cols2 + " " + expectedDir(); - + rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + rows2 + " " + cols2 + " " + + expectedDir(); + double sparsity = sparse ? sparsitySparse : sparsityDense; double[][] X = getRandomMatrix(rows1, cols1, 0, 1, sparsity, 7); writeInputMatrixWithMTD("X", X, true); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java index 60b491b8141..b16554045e4 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java @@ -151,8 +151,8 @@ private void runTestMatrixChainDP(String testName) { overwriteCurrentConfig(dmlConfig); } catch(FileNotFoundException fnfe) { - Assert.fail("Could not find DML config file: " + - getCurConfigFile().getPath() + " . " + fnfe.getMessage()); + Assert.fail( + "Could not find DML config file: " + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); } catch(IOException ioe) { Assert.fail("Could not overwrite the DML configuration file. " + ioe.getMessage()); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java index bf9acd9e52a..96c479e206d 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java @@ -123,8 +123,8 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { overwriteCurrentConfig(dmlConfig); } catch(FileNotFoundException fnfe) { - Assert.fail("Could not find DML config file: " + - getCurConfigFile().getPath() + " . " + fnfe.getMessage()); + Assert.fail( + "Could not find DML config file: " + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); } catch(IOException ioe) { Assert.fail("Could not overwrite the DML configuration file. " + ioe.getMessage()); @@ -132,8 +132,7 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + TEST_NAME + ".dml"; - programArgs = new String[] {"-explain", "hops", "-stats", - "-args", input("X"), input("Y"), output("R")}; + programArgs = new String[] {"-explain", "hops", "-stats", "-args", input("X"), input("Y"), output("R")}; fullRScriptName = HOME + TEST_NAME + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java index e8e885f905f..15b80e49618 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java @@ -74,7 +74,7 @@ public void testRewriteQuantizationFusedCompressionNoRewrite() { /** * Unified method to test both scalar and matrix scale factors. - * + * * @param testname Test name * @param rewrites Whether to enable fusion rewrites * @param isScalar Whether the scale factor is a scalar or a matrix diff --git a/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java b/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java index 30681f373e4..39266f5f3d3 100644 --- a/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java +++ b/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -106,7 +106,8 @@ public void testHash2() throws Exception { @Test public void testHash3() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.INT64,ValueType.UINT8}, 32); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, + new ValueType[] {ValueType.UINT8, ValueType.INT64, ValueType.UINT8}, 32); MatrixBlock expected = new MatrixBlock(1, 7, new double[] {1, 1, 1, 0, 1, 1, 1}); String spec = "{\"ids\": true, \"dummycode\": [1,3], \"hash\": [1,3], \"K\": 3}"; @@ -114,11 +115,11 @@ public void testHash3() throws Exception { } - @Test public void testHybrid1() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.INT64,ValueType.UINT8, ValueType.BOOLEAN}, 32); - MatrixBlock expected = new MatrixBlock(1, 9, new double[] {1, 1, 1, 0, 1, 1, 1,1,1}); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, + new ValueType[] {ValueType.UINT8, ValueType.INT64, ValueType.UINT8, ValueType.BOOLEAN}, 32); + MatrixBlock expected = new MatrixBlock(1, 9, new double[] {1, 1, 1, 0, 1, 1, 1, 1, 1}); String spec = "{\"ids\": true, \"dummycode\": [1,3,4], \"hash\": [1,3], \"K\": 3}"; runTransformTest(fb, spec, expected); @@ -127,8 +128,9 @@ public void testHybrid1() throws Exception { @Test public void testHybrid2() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.BOOLEAN,ValueType.UINT8, ValueType.BOOLEAN}, 32); - MatrixBlock expected = new MatrixBlock(1, 10, new double[] {1, 1, 1, 1,1, 1, 1, 1,1,1}); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, + new ValueType[] {ValueType.UINT8, ValueType.BOOLEAN, ValueType.UINT8, ValueType.BOOLEAN}, 32); + MatrixBlock expected = new MatrixBlock(1, 10, new double[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}); String spec = "{\"ids\": true, \"dummycode\": [1,2,3,4], \"hash\": [1,3], \"K\": 3}"; runTransformTest(fb, spec, expected); @@ -139,7 +141,7 @@ private void runTransformTest(FrameBlock fb, String spec, MatrixBlock expected) try { getAndLoadTestConfiguration(TEST_NAME1); - + String inF = input("F-In"); String inS = input("spec"); diff --git a/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java b/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java index 8c4ba6ae8ad..cd28649dc42 100644 --- a/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java +++ b/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java @@ -283,7 +283,8 @@ private String[][] readTwoColumnStringCSV(String s) { out[1][i] = in.getString(i, 1); } return out; - } catch (IOException e) { + } + catch(IOException e) { throw new RuntimeException(e); } } diff --git a/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java b/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java index d3d71d820d6..ae13cbd510f 100644 --- a/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java +++ b/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java @@ -92,7 +92,7 @@ private void runVectorizationTest( String testName, boolean rewrites ) runTest(true, false, null, -1); runRScript(true); - //compare results + // compare results HashMap dmlfile = readDMLMatrixFromOutputDir("R"); HashMap rfile = readRMatrixFromExpectedDir("R"); TestUtils.compareMatrices(dmlfile, rfile, 1e-14, "DML", "R");