Skip to contents

Model comparison

Source: R/functions.R
cv_multiple.Rd

Compares predictive methods for multi-task learning (cv_multiple) or transfer learning (cv_transfer) by \(k\)-fold cross-validation.

Usage

cv_multiple(
  y,
  X,
  family,
  alpha = 1,
  nfolds = 10,
  method = c("wrap_separate", "wrap_mgaussian", "sparselink", "wrap_spls"),
  alpha.init = 0.95,
  type = "exp",
  cands = NULL
)

cv_transfer(
  y,
  X,
  family,
  alpha = 1,
  nfolds = 10,
  method = c("wrap_separate", "wrap_glmtrans", "sparselink", "wrap_xrnet"),
  alpha.init = 0.95,
  type = "exp",
  cands = NULL
)

Arguments

y

\(n \times q\) matrix (multi-task learning) or list of \(n_k\)-dimensional vectors (transfer learning)

family

character "gaussian" or "binomial"

alpha

elastic net mixing parameter of final regressions, default: 1 (lasso)

nfolds

number of internal cross-validation folds, default: 10 (10-fold cross-validation)

alpha.init

elastic net mixing parameter for initial regressions, default: 0.95 (lasso-like elastic net)

type

default "exp" scales weights with \(w_{ext}^{v_{ext}}+w_{int}^{v_{int}}\) (see internal function construct_penfacs for details)

cands

candidate values for both scaling parameters, default: NULL ({0, 0.2, 0.4, 0.6, 0.8, 1})

Value

Returns a list with slots deviance, auc (only relevant if family="binomial"), and refit.

Examples

#--- multi-task learning ---
# \donttest{
family <- "gaussian"
data <- sim_data_multi(family=family)
metric <- cv_multiple(y=data$y_train,X=data$X_train,family=family)
#> fold 1
#> method: wrap_separate
#> method: wrap_mgaussian
#> method: sparselink
#> mode: multi-target learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_spls
#> fold 2
#> method: wrap_separate
#> method: wrap_mgaussian
#> method: sparselink
#> mode: multi-target learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_spls
#> fold 3
#> method: wrap_separate
#> method: wrap_mgaussian
#> method: sparselink
#> mode: multi-target learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_spls
#> fold 4
#> method: wrap_separate
#> method: wrap_mgaussian
#> method: sparselink
#> mode: multi-target learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_spls
#> fold 5
#> method: wrap_separate
#> method: wrap_mgaussian
#> method: sparselink
#> mode: multi-target learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_spls
#> fold 6
#> method: wrap_separate
#> method: wrap_mgaussian
#> method: sparselink
#> mode: multi-target learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_spls
#> fold 7
#> method: wrap_separate
#> method: wrap_mgaussian
#> method: sparselink
#> mode: multi-target learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_spls
#> fold 8
#> method: wrap_separate
#> method: wrap_mgaussian
#> method: sparselink
#> mode: multi-target learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_spls
#> fold 9
#> method: wrap_separate
#> method: wrap_mgaussian
#> method: sparselink
#> mode: multi-target learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_spls
#> fold 10
#> method: wrap_separate
#> method: wrap_mgaussian
#> method: sparselink
#> mode: multi-target learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_spls
#> refit on all folds
#> method: wrap_separate
#> method: wrap_mgaussian
#> method: sparselink
#> mode: multi-target learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_spls
metric$deviance# }
#>      wrap_separate wrap_mgaussian sparselink wrap_spls
#> [1,]      2.662426       2.178143   2.712249  3.536392
#> [2,]      4.393651       3.372834   2.950362  4.439447
#> [3,]      2.007589       1.446400   1.385281  1.889839

#--- transfer learning ---
# \donttest{
family <- "gaussian"
data <- sim_data_trans(family=family)
metric <- cv_transfer(y=data$y_train,X=data$X_train,family=family)
#> fold 1
#> method: wrap_separate
#> method: wrap_glmtrans
#> Registered S3 methods overwritten by 'caret':
#>   method         from
#>   predict.splsda spls
#>   print.splsda   spls
#> method: sparselink
#> mode: transfer learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_xrnet
#> fold 2
#> method: wrap_separate
#> method: wrap_glmtrans
#> Warning: Option grouped=FALSE enforced in cv.glmnet, since < 3 observations per fold
#> method: sparselink
#> mode: transfer learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_xrnet
#> fold 3
#> method: wrap_separate
#> method: wrap_glmtrans
#> method: sparselink
#> mode: transfer learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_xrnet
#> fold 4
#> method: wrap_separate
#> method: wrap_glmtrans
#> Warning: Option grouped=FALSE enforced in cv.glmnet, since < 3 observations per fold
#> Warning: Option grouped=FALSE enforced in cv.glmnet, since < 3 observations per fold
#> method: sparselink
#> mode: transfer learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_xrnet
#> fold 5
#> method: wrap_separate
#> method: wrap_glmtrans
#> method: sparselink
#> mode: transfer learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_xrnet
#> fold 6
#> method: wrap_separate
#> method: wrap_glmtrans
#> Warning: Option grouped=FALSE enforced in cv.glmnet, since < 3 observations per fold
#> method: sparselink
#> mode: transfer learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_xrnet
#> fold 7
#> method: wrap_separate
#> method: wrap_glmtrans
#> method: sparselink
#> mode: transfer learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_xrnet
#> fold 8
#> method: wrap_separate
#> method: wrap_glmtrans
#> Warning: Option grouped=FALSE enforced in cv.glmnet, since < 3 observations per fold
#> method: sparselink
#> mode: transfer learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_xrnet
#> fold 9
#> method: wrap_separate
#> method: wrap_glmtrans
#> method: sparselink
#> mode: transfer learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_xrnet
#> fold 10
#> method: wrap_separate
#> method: wrap_glmtrans
#> method: sparselink
#> mode: transfer learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_xrnet
#> refit on all folds
#> method: wrap_separate
#> method: wrap_glmtrans
#> method: sparselink
#> mode: transfer learning, alpha.init=0.95 (elastic net), alpha=1 (lasso)
#> method: wrap_xrnet
metric$deviance# }
#>      wrap_separate wrap_glmtrans sparselink wrap_xrnet
#> [1,]     10.924619      3.906340   4.473771   4.836102
#> [2,]      2.263707      2.471004   1.538593   2.240509
#> [3,]      1.789421      1.986591   1.463384   1.845089