From d6168b0851a52adeaed8099e41a960d79acedae9 Mon Sep 17 00:00:00 2001 From: Johannes Ranke Date: Mon, 14 Nov 2022 21:05:01 +0100 Subject: Start benchmark vignette for saem fits --- vignettes/web_only/saem_benchmarks.R | 121 ++ vignettes/web_only/saem_benchmarks.html | 1953 +++++++++++++++++++++++++++++++ vignettes/web_only/saem_benchmarks.rda | Bin 0 -> 367 bytes vignettes/web_only/saem_benchmarks.rmd | 207 ++++ 4 files changed, 2281 insertions(+) create mode 100644 vignettes/web_only/saem_benchmarks.R create mode 100644 vignettes/web_only/saem_benchmarks.html create mode 100644 vignettes/web_only/saem_benchmarks.rda create mode 100644 vignettes/web_only/saem_benchmarks.rmd (limited to 'vignettes') diff --git a/vignettes/web_only/saem_benchmarks.R b/vignettes/web_only/saem_benchmarks.R new file mode 100644 index 00000000..210e79e3 --- /dev/null +++ b/vignettes/web_only/saem_benchmarks.R @@ -0,0 +1,121 @@ +## ---- include = FALSE--------------------------------------------------------- +library("knitr") # For the kable() function +opts_chunk$set(tidy = FALSE, cache = FALSE) +library("mkin") + +## ----include = FALSE---------------------------------------------------------- +cpu_model <- benchmarkme::get_cpu()$model_name +# Abbreviate CPU identifiers +cpu_model <- gsub("AMD ", "", cpu_model) +cpu_model <- gsub("Intel\\(R\\) Core\\(TM\\) ", "", cpu_model) +cpu_model <- gsub(" Eight-Core Processor", "", cpu_model) +cpu_model <- gsub(" CPU @ 2.50GHz", "", cpu_model) + +operating_system <- Sys.info()[["sysname"]] +mkin_version <- as.character(packageVersion("mkin")) +saemix_version <- as.character(packageVersion("saemix")) +R_version <- paste0(R.version$major, ".", R.version$minor) +system_string <- paste0(operating_system, ", ", cpu_model, ", mkin ", mkin_version, ", saemix ", saemix_version, ", R ", R_version) + +benchmark_path = normalizePath("~/git/mkin/vignettes/web_only/saem_benchmarks.rda") +load(benchmark_path) + +# Initialization 14 November 2022 +#saem_benchmarks <- data.frame() + +saem_benchmarks[system_string, c("CPU", "OS", "mkin", "saemix", "R")] <- + c(cpu_model, operating_system, mkin_version, saemix_version, R_version) + +## ----setup-------------------------------------------------------------------- +n_cores <- parallel::detectCores() + +## ----dimethenamid_data-------------------------------------------------------- +dmta_ds <- lapply(1:7, function(i) { + ds_i <- dimethenamid_2018$ds[[i]]$data + ds_i[ds_i$name == "DMTAP", "name"] <- "DMTA" + ds_i$time <- ds_i$time * dimethenamid_2018$f_time_norm[i] + ds_i +}) +names(dmta_ds) <- sapply(dimethenamid_2018$ds, function(ds) ds$title) +dmta_ds[["Elliot"]] <- rbind(dmta_ds[["Elliot 1"]], dmta_ds[["Elliot 2"]]) +dmta_ds[["Elliot 1"]] <- NULL +dmta_ds[["Elliot 2"]] <- NULL + +## ----parent_only-------------------------------------------------------------- +parent_mods <- c("SFO", "DFOP", "SFORB", "HS") +parent_sep_const <- mmkin(parent_mods, dmta_ds, quiet = TRUE, cores = n_cores) +parent_sep_tc <- update(parent_sep_const, error_model = "tc") + +t1 <- system.time(sfo_const <- saem(parent_sep_const["SFO", ]))[["elapsed"]] +t2 <- system.time(dfop_const <- saem(parent_sep_const["DFOP", ]))[["elapsed"]] +t3 <- system.time(sforb_const <- saem(parent_sep_const["SFORB", ]))[["elapsed"]] +t4 <- system.time(hs_const <- saem(parent_sep_const["HS", ]))[["elapsed"]] +t5 <- system.time(sfo_tc <- saem(parent_sep_tc["SFO", ]))[["elapsed"]] +t6 <- system.time(dfop_tc <- saem(parent_sep_tc["DFOP", ]))[["elapsed"]] +t7 <- system.time(sforb_tc <- saem(parent_sep_tc["SFORB", ]))[["elapsed"]] +t8 <- system.time(hs_tc <- saem(parent_sep_tc["HS", ]))[["elapsed"]] + +## ----------------------------------------------------------------------------- +anova( + sfo_const, dfop_const, sforb_const, hs_const, + sfo_tc, dfop_tc, sforb_tc, hs_tc) |> kable(, digits = 1) + +## ----------------------------------------------------------------------------- +illparms(dfop_tc) +illparms(sforb_tc) + +## ----one_metabolite, message = FALSE------------------------------------------ +one_met_mods <- list( + DFOP_SFO = mkinmod( + DMTA = mkinsub("DFOP", "M23"), + M23 = mkinsub("SFO")), + SFORB_SFO = mkinmod( + DMTA = mkinsub("SFORB", "M23"), + M23 = mkinsub("SFO"))) + +one_met_sep_const <- mmkin(one_met_mods, dmta_ds, error_model = "const", + cores = n_cores, quiet = TRUE) +one_met_sep_tc <- mmkin(one_met_mods, dmta_ds, error_model = "tc", + cores = n_cores, quiet = TRUE) + +t9 <- system.time(dfop_sfo_tc <- saem(one_met_sep_tc["DFOP_SFO", ], + no_random_effect = "log_k2"))[["elapsed"]] +t10 <- system.time(sforb_sfo_tc <- saem(one_met_sep_tc["SFORB_SFO", ], + no_random_effect = "log_k_DMTA_bound_free"))[["elapsed"]] + +## ----------------------------------------------------------------------------- +illparms(sforb_sfo_tc) + +## ----three_metabolites, message = FALSE--------------------------------------- +three_met_mods <- list( + SFORB_SFO3_plus = mkinmod( + DMTA = mkinsub("SFORB", c("M23", "M27", "M31")), + M23 = mkinsub("SFO"), + M27 = mkinsub("SFO"), + M31 = mkinsub("SFO", "M27", sink = FALSE))) + +three_met_sep_tc <- mmkin(three_met_mods, dmta_ds, error_model = "tc", + cores = n_cores, quiet = TRUE) + +t11 <- system.time(sforb_sfo3_plus_const <- saem(three_met_sep_tc["SFORB_SFO3_plus", ], + no_random_effect = "log_k_DMTA_bound_free"))[["elapsed"]] + +## ----results, include = FALSE------------------------------------------------- +saem_benchmarks[system_string, paste0("t", 1:11)] <- + c(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11) +save(saem_benchmarks, file = benchmark_path) +# Hide rownames from kable for results section +rownames(saem_benchmarks) <- NULL + +## ---- echo = FALSE------------------------------------------------------------ +kable(saem_benchmarks[, c(1:4, 6:9)]) + +## ---- echo = FALSE------------------------------------------------------------ +kable(saem_benchmarks[, c(1:4, 10:13)]) + +## ---- echo = FALSE------------------------------------------------------------ +kable(saem_benchmarks[, c(1:4, 14:15)]) + +## ---- echo = FALSE------------------------------------------------------------ +kable(saem_benchmarks[, c(1:4, 16)]) + diff --git a/vignettes/web_only/saem_benchmarks.html b/vignettes/web_only/saem_benchmarks.html new file mode 100644 index 00000000..42d6f71f --- /dev/null +++ b/vignettes/web_only/saem_benchmarks.html @@ -0,0 +1,1953 @@ + + + + + + + + + + + + + + +Benchmark timings for saem.mmkin + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+
+
+
+
+ +
+ + + + + + + +

Each system is characterized by operating system type, CPU type, mkin version, saemix version and R version. A compiler was available, so if no analytical solution was available, compiled ODE models are used.

+

Every fit is only performed once, so the accuracy of the benchmarks is limited.

+

For the initial mmkin fits, we use all available cores.

+
n_cores <- parallel::detectCores()
+
+

Test data

+

Please refer to the vignette dimethenamid_2018 for an explanation of the following preprocessing.

+
dmta_ds <- lapply(1:7, function(i) {
+  ds_i <- dimethenamid_2018$ds[[i]]$data
+  ds_i[ds_i$name == "DMTAP", "name"] <-  "DMTA"
+  ds_i$time <- ds_i$time * dimethenamid_2018$f_time_norm[i]
+  ds_i
+})
+names(dmta_ds) <- sapply(dimethenamid_2018$ds, function(ds) ds$title)
+dmta_ds[["Elliot"]] <- rbind(dmta_ds[["Elliot 1"]], dmta_ds[["Elliot 2"]])
+dmta_ds[["Elliot 1"]] <- NULL
+dmta_ds[["Elliot 2"]] <- NULL
+
+
+

Test cases

+
+

Parent only

+
parent_mods <- c("SFO", "DFOP", "SFORB", "HS")
+parent_sep_const <- mmkin(parent_mods, dmta_ds, quiet = TRUE, cores = n_cores)
+parent_sep_tc <- update(parent_sep_const, error_model = "tc")
+
+t1 <- system.time(sfo_const <- saem(parent_sep_const["SFO", ]))[["elapsed"]]
+t2 <- system.time(dfop_const <- saem(parent_sep_const["DFOP", ]))[["elapsed"]]
+t3 <- system.time(sforb_const <- saem(parent_sep_const["SFORB", ]))[["elapsed"]]
+t4 <- system.time(hs_const <- saem(parent_sep_const["HS", ]))[["elapsed"]]
+t5 <- system.time(sfo_tc <- saem(parent_sep_tc["SFO", ]))[["elapsed"]]
+t6 <- system.time(dfop_tc <- saem(parent_sep_tc["DFOP", ]))[["elapsed"]]
+t7 <- system.time(sforb_tc <- saem(parent_sep_tc["SFORB", ]))[["elapsed"]]
+t8 <- system.time(hs_tc <- saem(parent_sep_tc["HS", ]))[["elapsed"]]
+
anova(
+  sfo_const, dfop_const, sforb_const, hs_const,
+  sfo_tc, dfop_tc, sforb_tc, hs_tc) |> kable(, digits = 1)
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
nparAICBICLik
sfo_const5796.3795.3-393.2
sfo_tc6798.3797.1-393.2
dfop_const9709.4707.5-345.7
sforb_const9710.0708.1-346.0
hs_const9713.7711.8-347.8
dfop_tc10669.8667.7-324.9
sforb_tc10662.8660.7-321.4
hs_tc10667.3665.2-323.6
+

The above model comparison suggests to use the SFORB model with two-component error. For comparison, we keep the DFOP model with two-component error, as it competes with SFORB for biphasic curves.

+
illparms(dfop_tc)
+
## [1] "sd(log_k2)"
+
illparms(sforb_tc)
+
## [1] "sd(log_k_DMTA_bound_free)"
+

For these two models, random effects for the transformed parameters k2 and k_DMTA_bound_free could not be quantified.

+
+
+

One metabolite

+

We remove parameters that were found to be ill-defined in the parent only fits.

+
one_met_mods <- list(
+  DFOP_SFO = mkinmod(
+    DMTA = mkinsub("DFOP", "M23"),
+    M23 = mkinsub("SFO")),
+  SFORB_SFO = mkinmod(
+    DMTA = mkinsub("SFORB", "M23"),
+    M23 = mkinsub("SFO")))
+
+one_met_sep_const <- mmkin(one_met_mods, dmta_ds, error_model = "const",
+  cores = n_cores, quiet = TRUE)
+one_met_sep_tc <- mmkin(one_met_mods, dmta_ds, error_model = "tc",
+  cores = n_cores, quiet = TRUE)
+
+t9 <- system.time(dfop_sfo_tc <- saem(one_met_sep_tc["DFOP_SFO", ],
+    no_random_effect = "log_k2"))[["elapsed"]]
+t10 <- system.time(sforb_sfo_tc <- saem(one_met_sep_tc["SFORB_SFO", ],
+    no_random_effect = "log_k_DMTA_bound_free"))[["elapsed"]]
+
+
+

Three metabolites

+

For the case of three metabolites, we only keep the SFORB model in order to limit the time for compiling this vignette, and as fitting in parallel may disturb the benchmark. Again, we do not include random effects that were ill-defined in previous fits of subsets of the degradation model.

+
illparms(sforb_sfo_tc)
+
three_met_mods <- list(
+  SFORB_SFO3_plus = mkinmod(
+    DMTA = mkinsub("SFORB", c("M23", "M27", "M31")),
+    M23 = mkinsub("SFO"),
+    M27 = mkinsub("SFO"),
+    M31 = mkinsub("SFO", "M27", sink = FALSE)))
+
+three_met_sep_tc <- mmkin(three_met_mods, dmta_ds, error_model = "tc",
+  cores = n_cores, quiet = TRUE)
+
+t11 <- system.time(sforb_sfo3_plus_const <- saem(three_met_sep_tc["SFORB_SFO3_plus", ],
+    no_random_effect = "log_k_DMTA_bound_free"))[["elapsed"]]
+
+
+
+

Results

+

Benchmarks for all available error models are shown. They are intended for improving mkin, not for comparing CPUs or operating systems. All trademarks belong to their respective owners.

+
+

Parent only

+

Constant variance for SFO, DFOP, SFORB and HS.

+ + + + + + + + + + + + + + + + + + + + + + + + + +
CPUOSmkinsaemixt1t2t3t4
Ryzen 7 1700Linux1.2.03.23.0255.2375.4595.218
+

Two-component error fits for SFO, DFOP, SFORB and HS.

+ + + + + + + + + + + + + + + + + + + + + + + + + +
CPUOSmkinsaemixt5t6t7t8
Ryzen 7 1700Linux1.2.03.25.6457.7267.9528.104
+
+
+

One metabolite

+

Two-component error for DFOP-SFO and SFORB-SFO.

+ + + + + + + + + + + + + + + + + + + + + +
CPUOSmkinsaemixt9t10
Ryzen 7 1700Linux1.2.03.224.84816.153
+
+
+

Three metabolites

+

Two-component error for SFORB-SFO3-plus

+ + + + + + + + + + + + + + + + + + + +
CPUOSmkinsaemixt11
Ryzen 7 1700Linux1.2.03.21323.128
+
+
+ + + +
+
+ +
+ + + + + + + + + + + + + + + + + diff --git a/vignettes/web_only/saem_benchmarks.rda b/vignettes/web_only/saem_benchmarks.rda new file mode 100644 index 00000000..506d7591 Binary files /dev/null and b/vignettes/web_only/saem_benchmarks.rda differ diff --git a/vignettes/web_only/saem_benchmarks.rmd b/vignettes/web_only/saem_benchmarks.rmd new file mode 100644 index 00000000..116713de --- /dev/null +++ b/vignettes/web_only/saem_benchmarks.rmd @@ -0,0 +1,207 @@ +--- +title: "Benchmark timings for saem.mmkin" +author: "Johannes Ranke" +date: Last change 14 November 2022 (rebuilt `r Sys.Date()`) +output: + html_document: + toc: true + toc_float: true + code_folding: show + fig_retina: null +vignette: > + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +library("knitr") # For the kable() function +opts_chunk$set(tidy = FALSE, cache = FALSE) +library("mkin") +``` + +Each system is characterized by operating system type, CPU type, +mkin version, saemix version and R version. A compiler was available, so if no +analytical solution was available, compiled ODE models are used. + +Every fit is only performed once, so the accuracy of the benchmarks is limited. + +```{r include = FALSE} +cpu_model <- benchmarkme::get_cpu()$model_name +# Abbreviate CPU identifiers +cpu_model <- gsub("AMD ", "", cpu_model) +cpu_model <- gsub("Intel\\(R\\) Core\\(TM\\) ", "", cpu_model) +cpu_model <- gsub(" Eight-Core Processor", "", cpu_model) +cpu_model <- gsub(" CPU @ 2.50GHz", "", cpu_model) + +operating_system <- Sys.info()[["sysname"]] +mkin_version <- as.character(packageVersion("mkin")) +saemix_version <- as.character(packageVersion("saemix")) +R_version <- paste0(R.version$major, ".", R.version$minor) +system_string <- paste0(operating_system, ", ", cpu_model, ", mkin ", mkin_version, ", saemix ", saemix_version, ", R ", R_version) + +benchmark_path = normalizePath("~/git/mkin/vignettes/web_only/saem_benchmarks.rda") +load(benchmark_path) + +# Initialization 14 November 2022 +#saem_benchmarks <- data.frame() + +saem_benchmarks[system_string, c("CPU", "OS", "mkin", "saemix", "R")] <- + c(cpu_model, operating_system, mkin_version, saemix_version, R_version) +``` + +For the initial mmkin fits, we use all available cores. + +```{r setup} +n_cores <- parallel::detectCores() +``` + +## Test data + +Please refer to the vignette `dimethenamid_2018` for an explanation +of the following preprocessing. + +```{r dimethenamid_data} +dmta_ds <- lapply(1:7, function(i) { + ds_i <- dimethenamid_2018$ds[[i]]$data + ds_i[ds_i$name == "DMTAP", "name"] <- "DMTA" + ds_i$time <- ds_i$time * dimethenamid_2018$f_time_norm[i] + ds_i +}) +names(dmta_ds) <- sapply(dimethenamid_2018$ds, function(ds) ds$title) +dmta_ds[["Elliot"]] <- rbind(dmta_ds[["Elliot 1"]], dmta_ds[["Elliot 2"]]) +dmta_ds[["Elliot 1"]] <- NULL +dmta_ds[["Elliot 2"]] <- NULL +``` + +## Test cases + +### Parent only + +```{r parent_only} +parent_mods <- c("SFO", "DFOP", "SFORB", "HS") +parent_sep_const <- mmkin(parent_mods, dmta_ds, quiet = TRUE, cores = n_cores) +parent_sep_tc <- update(parent_sep_const, error_model = "tc") + +t1 <- system.time(sfo_const <- saem(parent_sep_const["SFO", ]))[["elapsed"]] +t2 <- system.time(dfop_const <- saem(parent_sep_const["DFOP", ]))[["elapsed"]] +t3 <- system.time(sforb_const <- saem(parent_sep_const["SFORB", ]))[["elapsed"]] +t4 <- system.time(hs_const <- saem(parent_sep_const["HS", ]))[["elapsed"]] +t5 <- system.time(sfo_tc <- saem(parent_sep_tc["SFO", ]))[["elapsed"]] +t6 <- system.time(dfop_tc <- saem(parent_sep_tc["DFOP", ]))[["elapsed"]] +t7 <- system.time(sforb_tc <- saem(parent_sep_tc["SFORB", ]))[["elapsed"]] +t8 <- system.time(hs_tc <- saem(parent_sep_tc["HS", ]))[["elapsed"]] +``` + +```{r} +anova( + sfo_const, dfop_const, sforb_const, hs_const, + sfo_tc, dfop_tc, sforb_tc, hs_tc) |> kable(, digits = 1) +``` + +The above model comparison suggests to use the SFORB model with two-component +error. For comparison, we keep the DFOP model with two-component error, +as it competes with SFORB for biphasic curves. + +```{r} +illparms(dfop_tc) +illparms(sforb_tc) +``` + +For these two models, random effects for the transformed parameters `k2` and +`k_DMTA_bound_free` could not be quantified. + + +### One metabolite + +We remove parameters that were found to be ill-defined in the parent only fits. + +```{r one_metabolite, message = FALSE} +one_met_mods <- list( + DFOP_SFO = mkinmod( + DMTA = mkinsub("DFOP", "M23"), + M23 = mkinsub("SFO")), + SFORB_SFO = mkinmod( + DMTA = mkinsub("SFORB", "M23"), + M23 = mkinsub("SFO"))) + +one_met_sep_const <- mmkin(one_met_mods, dmta_ds, error_model = "const", + cores = n_cores, quiet = TRUE) +one_met_sep_tc <- mmkin(one_met_mods, dmta_ds, error_model = "tc", + cores = n_cores, quiet = TRUE) + +t9 <- system.time(dfop_sfo_tc <- saem(one_met_sep_tc["DFOP_SFO", ], + no_random_effect = "log_k2"))[["elapsed"]] +t10 <- system.time(sforb_sfo_tc <- saem(one_met_sep_tc["SFORB_SFO", ], + no_random_effect = "log_k_DMTA_bound_free"))[["elapsed"]] +``` + +### Three metabolites + +For the case of three metabolites, we only keep the SFORB model in order +to limit the time for compiling this vignette, and as fitting in parallel +may disturb the benchmark. Again, we do not include random effects that +were ill-defined in previous fits of subsets of the degradation model. + +```{r} +illparms(sforb_sfo_tc) +``` + +```{r three_metabolites, message = FALSE} +three_met_mods <- list( + SFORB_SFO3_plus = mkinmod( + DMTA = mkinsub("SFORB", c("M23", "M27", "M31")), + M23 = mkinsub("SFO"), + M27 = mkinsub("SFO"), + M31 = mkinsub("SFO", "M27", sink = FALSE))) + +three_met_sep_tc <- mmkin(three_met_mods, dmta_ds, error_model = "tc", + cores = n_cores, quiet = TRUE) + +t11 <- system.time(sforb_sfo3_plus_const <- saem(three_met_sep_tc["SFORB_SFO3_plus", ], + no_random_effect = "log_k_DMTA_bound_free"))[["elapsed"]] +``` + +```{r results, include = FALSE} +saem_benchmarks[system_string, paste0("t", 1:11)] <- + c(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11) +save(saem_benchmarks, file = benchmark_path) +# Hide rownames from kable for results section +rownames(saem_benchmarks) <- NULL +``` + +## Results + +Benchmarks for all available error models are shown. They are intended for +improving mkin, not for comparing CPUs or operating systems. All trademarks +belong to their respective owners. + +### Parent only + +Constant variance for SFO, DFOP, SFORB and HS. + +```{r, echo = FALSE} +kable(saem_benchmarks[, c(1:4, 6:9)]) +``` + +Two-component error fits for SFO, DFOP, SFORB and HS. + +```{r, echo = FALSE} +kable(saem_benchmarks[, c(1:4, 10:13)]) +``` + +### One metabolite + +Two-component error for DFOP-SFO and SFORB-SFO. + +```{r, echo = FALSE} +kable(saem_benchmarks[, c(1:4, 14:15)]) +``` + +### Three metabolites + +Two-component error for SFORB-SFO3-plus + +```{r, echo = FALSE} +kable(saem_benchmarks[, c(1:4, 16)]) +``` + -- cgit v1.2.1