From 7008a4ee100830a29a923a05274fd4ca216cd1ea Mon Sep 17 00:00:00 2001 From: Johannes Ranke Date: Fri, 1 Jul 2022 09:08:49 +0200 Subject: Add R version to data in benchmark vignette --- vignettes/web_only/benchmarks.R | 112 ++ vignettes/web_only/benchmarks.html | 2190 ++++++++++++++++++++++++++++++++ vignettes/web_only/benchmarks.rmd | 53 +- vignettes/web_only/mkin_benchmarks.rda | Bin 1269 -> 1300 bytes 4 files changed, 2341 insertions(+), 14 deletions(-) create mode 100644 vignettes/web_only/benchmarks.R create mode 100644 vignettes/web_only/benchmarks.html (limited to 'vignettes/web_only') diff --git a/vignettes/web_only/benchmarks.R b/vignettes/web_only/benchmarks.R new file mode 100644 index 00000000..c9ba5be3 --- /dev/null +++ b/vignettes/web_only/benchmarks.R @@ -0,0 +1,112 @@ +## ---- include = FALSE--------------------------------------------------------- +library(knitr) +opts_chunk$set(tidy = FALSE, cache = FALSE) +library("mkin") + +## ----include = FALSE---------------------------------------------------------- +cpu_model <- benchmarkme::get_cpu()$model_name +# Abbreviate CPU identifiers +cpu_model <- gsub("AMD ", "", cpu_model) +cpu_model <- gsub("Intel\\(R\\) Core\\(TM\\) ", "", cpu_model) +cpu_model <- gsub(" Eight-Core Processor", "", cpu_model) +cpu_model <- gsub(" CPU @ 2.50GHz", "", cpu_model) + +operating_system <- Sys.info()[["sysname"]] +mkin_version <- as.character(packageVersion("mkin")) +R_version <- paste0(R.version$major, ".", R.version$minor) +system_string <- paste0(operating_system, ", ", cpu_model, ", mkin ", mkin_version, ", R ", R_version) + +benchmark_path = normalizePath("~/git/mkin/vignettes/web_only/mkin_benchmarks.rda") +load(benchmark_path) + +# Used for reformatting the data on 2022-06-30 +# mkin_benchmarks[, "R"] <- NA +# mkin_benchmarks <- mkin_benchmarks[c(2, 1, 15, 3, 4:14)] +# mkin_benchmarks[, "CPU"] <- gsub("AMD.*", "Ryzen 7 1700", mkin_benchmarks[, "CPU"]) +# mkin_benchmarks[, "CPU"] <- gsub("Intel.*", "i7-4710MQ", mkin_benchmarks[, "CPU"]) +# rownames(mkin_benchmarks) <- gsub("AMD Ryzen 7 1700 Eight-Core Processor", "Ryzen 7 1700", rownames(mkin_benchmarks)) +# rownames(mkin_benchmarks) <- gsub("Intel\\(R\\) Core\\(TM\\) i7-4710MQ CPU @ 2.50GHz", "i7-4710MQ", rownames(mkin_benchmarks)) +# rownames(mkin_benchmarks) <- gsub(" version", "", rownames(mkin_benchmarks)) + +mkin_benchmarks[system_string, c("CPU", "OS", "mkin", "R")] <- + c(cpu_model, operating_system, mkin_version, R_version) + +if (mkin_version > "0.9.48.1") { + mmkin_bench <- function(models, datasets, error_model = "const") { + mmkin(models, datasets, error_model = error_model, cores = 1, quiet = TRUE) + } +} else { + mmkin_bench <- function(models, datasets, error_model = NULL) { + mmkin(models, datasets, reweight.method = error_model, cores = 1, quiet = TRUE) + } +} + +## ----parent_only, warning = FALSE--------------------------------------------- +FOCUS_C <- FOCUS_2006_C +FOCUS_D <- subset(FOCUS_2006_D, value != 0) +parent_datasets <- list(FOCUS_C, FOCUS_D) + +t1 <- system.time(mmkin_bench(c("SFO", "FOMC", "DFOP", "HS"), parent_datasets))[["elapsed"]] +t2 <- system.time(mmkin_bench(c("SFO", "FOMC", "DFOP", "HS"), parent_datasets, + error_model = "tc"))[["elapsed"]] + +## ----one_metabolite, message = FALSE------------------------------------------ +SFO_SFO <- mkinmod( + parent = mkinsub("SFO", "m1"), + m1 = mkinsub("SFO")) +FOMC_SFO <- mkinmod( + parent = mkinsub("FOMC", "m1"), + m1 = mkinsub("SFO")) +DFOP_SFO <- mkinmod( + parent = mkinsub("FOMC", "m1"), + m1 = mkinsub("SFO")) +t3 <- system.time(mmkin_bench(list(SFO_SFO, FOMC_SFO, DFOP_SFO), list(FOCUS_D)))[["elapsed"]] +t4 <- system.time(mmkin_bench(list(SFO_SFO, FOMC_SFO, DFOP_SFO), list(FOCUS_D), + error_model = "tc"))[["elapsed"]] +t5 <- system.time(mmkin_bench(list(SFO_SFO, FOMC_SFO, DFOP_SFO), list(FOCUS_D), + error_model = "obs"))[["elapsed"]] + +## ----two_metabolites, message = FALSE----------------------------------------- +m_synth_SFO_lin <- mkinmod(parent = mkinsub("SFO", "M1"), + M1 = mkinsub("SFO", "M2"), + M2 = mkinsub("SFO"), + use_of_ff = "max", quiet = TRUE) + +m_synth_DFOP_par <- mkinmod(parent = mkinsub("DFOP", c("M1", "M2")), + M1 = mkinsub("SFO"), + M2 = mkinsub("SFO"), + use_of_ff = "max", quiet = TRUE) + +SFO_lin_a <- synthetic_data_for_UBA_2014[[1]]$data + +DFOP_par_c <- synthetic_data_for_UBA_2014[[12]]$data + +t6 <- system.time(mmkin_bench(list(m_synth_SFO_lin), list(SFO_lin_a)))[["elapsed"]] +t7 <- system.time(mmkin_bench(list(m_synth_DFOP_par), list(DFOP_par_c)))[["elapsed"]] + +t8 <- system.time(mmkin_bench(list(m_synth_SFO_lin), list(SFO_lin_a), + error_model = "tc"))[["elapsed"]] +t9 <- system.time(mmkin_bench(list(m_synth_DFOP_par), list(DFOP_par_c), + error_model = "tc"))[["elapsed"]] + +t10 <- system.time(mmkin_bench(list(m_synth_SFO_lin), list(SFO_lin_a), + error_model = "obs"))[["elapsed"]] +t11 <- system.time(mmkin_bench(list(m_synth_DFOP_par), list(DFOP_par_c), + error_model = "obs"))[["elapsed"]] + +## ----results------------------------------------------------------------------ +mkin_benchmarks[system_string, paste0("t", 1:11)] <- + c(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11) +save(mkin_benchmarks, file = benchmark_path) +# Hide rownames from kable for results section +rownames(mkin_benchmarks) <- NULL + +## ---- echo = FALSE------------------------------------------------------------ +kable(mkin_benchmarks[, c(1:4, 5:6)]) + +## ---- echo = FALSE------------------------------------------------------------ +kable(mkin_benchmarks[, c(1:4, 7:9)]) + +## ---- echo = FALSE------------------------------------------------------------ +kable(mkin_benchmarks[, c(1:4, 10:15)]) + diff --git a/vignettes/web_only/benchmarks.html b/vignettes/web_only/benchmarks.html new file mode 100644 index 00000000..f6164796 --- /dev/null +++ b/vignettes/web_only/benchmarks.html @@ -0,0 +1,2190 @@ + + + + + + + + + + + + + + +Benchmark timings for mkin + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+
+
+
+
+ +
+ + + + + + + +

Each system is characterized by the operating system type, the CPU type, the mkin version, and, as in June 2022 the current R version lead to worse performance, the R version. A compiler was available, so if no analytical solution was available, compiled ODE models are used.

+

Every fit is only performed once, so the accuracy of the benchmarks is limited.

+
+

Test cases

+

Parent only:

+
FOCUS_C <- FOCUS_2006_C
+FOCUS_D <- subset(FOCUS_2006_D, value != 0)
+parent_datasets <- list(FOCUS_C, FOCUS_D)
+
+t1 <- system.time(mmkin_bench(c("SFO", "FOMC", "DFOP", "HS"), parent_datasets))[["elapsed"]]
+t2 <- system.time(mmkin_bench(c("SFO", "FOMC", "DFOP", "HS"), parent_datasets,
+    error_model = "tc"))[["elapsed"]]
+

One metabolite:

+
SFO_SFO <- mkinmod(
+  parent = mkinsub("SFO", "m1"),
+  m1 = mkinsub("SFO"))
+FOMC_SFO <- mkinmod(
+  parent = mkinsub("FOMC", "m1"),
+  m1 = mkinsub("SFO"))
+DFOP_SFO <- mkinmod(
+  parent = mkinsub("FOMC", "m1"),
+  m1 = mkinsub("SFO"))
+t3 <- system.time(mmkin_bench(list(SFO_SFO, FOMC_SFO, DFOP_SFO), list(FOCUS_D)))[["elapsed"]]
+t4 <- system.time(mmkin_bench(list(SFO_SFO, FOMC_SFO, DFOP_SFO), list(FOCUS_D),
+    error_model = "tc"))[["elapsed"]]
+t5 <- system.time(mmkin_bench(list(SFO_SFO, FOMC_SFO, DFOP_SFO), list(FOCUS_D),
+    error_model = "obs"))[["elapsed"]]
+

Two metabolites, synthetic data:

+
m_synth_SFO_lin <- mkinmod(parent = mkinsub("SFO", "M1"),
+                           M1 = mkinsub("SFO", "M2"),
+                           M2 = mkinsub("SFO"),
+                           use_of_ff = "max", quiet = TRUE)
+
+m_synth_DFOP_par <- mkinmod(parent = mkinsub("DFOP", c("M1", "M2")),
+                           M1 = mkinsub("SFO"),
+                           M2 = mkinsub("SFO"),
+                           use_of_ff = "max", quiet = TRUE)
+
+SFO_lin_a <- synthetic_data_for_UBA_2014[[1]]$data
+
+DFOP_par_c <- synthetic_data_for_UBA_2014[[12]]$data
+
+t6 <- system.time(mmkin_bench(list(m_synth_SFO_lin), list(SFO_lin_a)))[["elapsed"]]
+t7 <- system.time(mmkin_bench(list(m_synth_DFOP_par), list(DFOP_par_c)))[["elapsed"]]
+
+t8 <- system.time(mmkin_bench(list(m_synth_SFO_lin), list(SFO_lin_a),
+    error_model = "tc"))[["elapsed"]]
+t9 <- system.time(mmkin_bench(list(m_synth_DFOP_par), list(DFOP_par_c),
+    error_model = "tc"))[["elapsed"]]
+
+t10 <- system.time(mmkin_bench(list(m_synth_SFO_lin), list(SFO_lin_a),
+    error_model = "obs"))[["elapsed"]]
+t11 <- system.time(mmkin_bench(list(m_synth_DFOP_par), list(DFOP_par_c),
+    error_model = "obs"))[["elapsed"]]
+
mkin_benchmarks[system_string, paste0("t", 1:11)] <-
+  c(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11)
+save(mkin_benchmarks, file = benchmark_path)
+# Hide rownames from kable for results section
+rownames(mkin_benchmarks) <- NULL
+
+
+

Results

+

Benchmarks for all available error models are shown. They are intended for improving mkin, not for comparing CPUs or operating systems. All trademarks belong to their respective owners.

+
+

Parent only

+

Constant variance (t1) and two-component error model (t2) for four models fitted to two datasets, i.e. eight fits for each test.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OSCPURmkint1t2
LinuxRyzen 7 1700NA0.9.48.13.61011.019
LinuxRyzen 7 1700NA0.9.49.18.18422.889
LinuxRyzen 7 1700NA0.9.49.27.06412.558
LinuxRyzen 7 1700NA0.9.49.37.29621.239
LinuxRyzen 7 1700NA0.9.49.45.93620.545
LinuxRyzen 7 1700NA0.9.50.21.7143.971
LinuxRyzen 7 1700NA0.9.50.31.7524.156
LinuxRyzen 7 1700NA0.9.50.41.7863.729
LinuxRyzen 7 1700NA1.0.31.8813.504
LinuxRyzen 7 1700NA1.0.41.8673.450
LinuxRyzen 7 1700NA1.1.01.7913.466
Linuxi7-4710MQNA1.1.01.6213.123
LinuxRyzen 7 17004.1.31.1.01.7913.289
+
+
+

One metabolite

+

Constant variance (t3), two-component error model (t4), and variance by variable (t5) for three models fitted to one dataset, i.e. three fits for each test.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OSCPURmkint3t4t5
LinuxRyzen 7 1700NA0.9.48.13.76414.3479.495
LinuxRyzen 7 1700NA0.9.49.14.64913.7896.395
LinuxRyzen 7 1700NA0.9.49.24.7868.4615.675
LinuxRyzen 7 1700NA0.9.49.34.51013.8057.386
LinuxRyzen 7 1700NA0.9.49.44.44615.3356.002
LinuxRyzen 7 1700NA0.9.50.21.4026.1742.764
LinuxRyzen 7 1700NA0.9.50.31.4306.6152.878
LinuxRyzen 7 1700NA0.9.50.41.3977.2512.810
LinuxRyzen 7 1700NA1.0.31.4306.3442.798
LinuxRyzen 7 1700NA1.0.41.4156.3642.820
LinuxRyzen 7 1700NA1.1.03.90121.5748.564
Linuxi7-4710MQNA1.1.03.31918.2097.773
LinuxRyzen 7 17004.1.31.1.01.3106.2792.681
+
+
+

Two metabolites

+

Constant variance (t6 and t7), two-component error model (t8 and t9), and variance by variable (t10 and t11) for one model fitted to one dataset, i.e. one fit for each test.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OSCPURmkint6t7t8t9t10t11
LinuxRyzen 7 1700NA0.9.48.12.6234.5877.52516.6218.57631.267
LinuxRyzen 7 1700NA0.9.49.12.5424.1284.6328.1713.6765.636
LinuxRyzen 7 1700NA0.9.49.22.7234.4784.8627.6183.5795.574
LinuxRyzen 7 1700NA0.9.49.32.6434.3747.02011.1245.3887.365
LinuxRyzen 7 1700NA0.9.49.42.6354.2594.7377.7633.4275.626
LinuxRyzen 7 1700NA0.9.50.20.7771.2361.3322.8722.0692.987
LinuxRyzen 7 1700NA0.9.50.30.8581.2641.3332.9842.1133.073
LinuxRyzen 7 1700NA0.9.50.40.7831.2821.4863.8151.9583.105
LinuxRyzen 7 1700NA1.0.30.7631.2441.4573.0541.9232.839
LinuxRyzen 7 1700NA1.0.40.7851.2521.4663.0911.9362.826
LinuxRyzen 7 1700NA1.1.03.0884.3245.11410.9886.8009.860
Linuxi7-4710MQNA1.1.02.3483.5794.25411.3155.9818.368
LinuxRyzen 7 17004.1.31.1.00.7441.2271.2883.5531.8952.738
+
+
+ + + +
+
+ +
+ + + + + + + + + + + + + + + + + diff --git a/vignettes/web_only/benchmarks.rmd b/vignettes/web_only/benchmarks.rmd index 9697f96b..9ac87631 100644 --- a/vignettes/web_only/benchmarks.rmd +++ b/vignettes/web_only/benchmarks.rmd @@ -1,7 +1,7 @@ --- title: "Benchmark timings for mkin" author: "Johannes Ranke" -date: Last change 13 May 2020 (rebuilt `r Sys.Date()`) +date: Last change 30 June 2022 (rebuilt `r Sys.Date()`) output: html_document: toc: true @@ -19,19 +19,40 @@ opts_chunk$set(tidy = FALSE, cache = FALSE) library("mkin") ``` -Each system is characterized by its CPU type, the operating system type and the -mkin version. Currently only values for one system are available. A compiler -was available, so if no analytical solution was available, compiled ODE models are -used. +Each system is characterized by the operating system type, the CPU type, the +mkin version, and, as in June 2022 the current R version lead to worse +performance, the R version. A compiler was available, so if no analytical +solution was available, compiled ODE models are used. + +Every fit is only performed once, so the accuracy of the benchmarks is limited. ```{r include = FALSE} cpu_model <- benchmarkme::get_cpu()$model_name +# Abbreviate CPU identifiers +cpu_model <- gsub("AMD ", "", cpu_model) +cpu_model <- gsub("Intel\\(R\\) Core\\(TM\\) ", "", cpu_model) +cpu_model <- gsub(" Eight-Core Processor", "", cpu_model) +cpu_model <- gsub(" CPU @ 2.50GHz", "", cpu_model) + operating_system <- Sys.info()[["sysname"]] mkin_version <- as.character(packageVersion("mkin")) -system_string <- paste0(operating_system, ", ", cpu_model, ", mkin version ", mkin_version) -load("~/git/mkin/vignettes/web_only/mkin_benchmarks.rda") -mkin_benchmarks[system_string, c("CPU", "OS", "mkin")] <- - c(cpu_model, operating_system, mkin_version) +R_version <- paste0(R.version$major, ".", R.version$minor) +system_string <- paste0(operating_system, ", ", cpu_model, ", mkin ", mkin_version, ", R ", R_version) + +benchmark_path = normalizePath("~/git/mkin/vignettes/web_only/mkin_benchmarks.rda") +load(benchmark_path) + +# Used for reformatting the data on 2022-06-30 +# mkin_benchmarks[, "R"] <- NA +# mkin_benchmarks <- mkin_benchmarks[c(2, 1, 15, 3, 4:14)] +# mkin_benchmarks[, "CPU"] <- gsub("AMD.*", "Ryzen 7 1700", mkin_benchmarks[, "CPU"]) +# mkin_benchmarks[, "CPU"] <- gsub("Intel.*", "i7-4710MQ", mkin_benchmarks[, "CPU"]) +# rownames(mkin_benchmarks) <- gsub("AMD Ryzen 7 1700 Eight-Core Processor", "Ryzen 7 1700", rownames(mkin_benchmarks)) +# rownames(mkin_benchmarks) <- gsub("Intel\\(R\\) Core\\(TM\\) i7-4710MQ CPU @ 2.50GHz", "i7-4710MQ", rownames(mkin_benchmarks)) +# rownames(mkin_benchmarks) <- gsub(" version", "", rownames(mkin_benchmarks)) + +mkin_benchmarks[system_string, c("CPU", "OS", "mkin", "R")] <- + c(cpu_model, operating_system, mkin_version, R_version) if (mkin_version > "0.9.48.1") { mmkin_bench <- function(models, datasets, error_model = "const") { @@ -111,12 +132,16 @@ t11 <- system.time(mmkin_bench(list(m_synth_DFOP_par), list(DFOP_par_c), ```{r results} mkin_benchmarks[system_string, paste0("t", 1:11)] <- c(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11) -save(mkin_benchmarks, file = "~/git/mkin/vignettes/web_only/mkin_benchmarks.rda") +save(mkin_benchmarks, file = benchmark_path) +# Hide rownames from kable for results section +rownames(mkin_benchmarks) <- NULL ``` ## Results -Benchmarks for all available error models are shown. +Benchmarks for all available error models are shown. They are intended for +improving mkin, not for comparing CPUs or operating systems. All trademarks +belong to their respective owners. ### Parent only @@ -124,7 +149,7 @@ Constant variance (t1) and two-component error model (t2) for four models fitted to two datasets, i.e. eight fits for each test. ```{r, echo = FALSE} -kable(mkin_benchmarks[, 4:5]) +kable(mkin_benchmarks[, c(1:4, 5:6)]) ``` ### One metabolite @@ -133,7 +158,7 @@ Constant variance (t3), two-component error model (t4), and variance by variable for three models fitted to one dataset, i.e. three fits for each test. ```{r, echo = FALSE} -kable(mkin_benchmarks[, 6:8]) +kable(mkin_benchmarks[, c(1:4, 7:9)]) ``` ### Two metabolites @@ -143,5 +168,5 @@ variance by variable (t10 and t11) for one model fitted to one dataset, i.e. one fit for each test. ```{r, echo = FALSE} -kable(mkin_benchmarks[, 9:14]) +kable(mkin_benchmarks[, c(1:4, 10:15)]) ``` diff --git a/vignettes/web_only/mkin_benchmarks.rda b/vignettes/web_only/mkin_benchmarks.rda index 93adfb61..73f59b1a 100644 Binary files a/vignettes/web_only/mkin_benchmarks.rda and b/vignettes/web_only/mkin_benchmarks.rda differ -- cgit v1.2.1