1 files changed, 157 insertions, 0 deletions
diff --git a/vignettes/web_only/benchmarks.rmd b/vignettes/web_only/benchmarks.rmd
new file mode 100644
index 00000000..7ae12451
--- /dev/null
+++ b/vignettes/web_only/benchmarks.rmd
@@ -0,0 +1,157 @@
+---
+title: "Benchmark timings for mkin"
+author: "Johannes Ranke"
+output:
+  html_document:
+    toc: true
+    toc_float: true
+    code_folding: show
+    fig_retina: null
+date: "`r Sys.Date()`"
+vignette: >
+  %\VignetteEngine{knitr::rmarkdown}
+  %\VignetteEncoding{UTF-8}
+---
+
+```{r, include = FALSE}
+library(knitr)
+opts_chunk$set(tidy = FALSE, cache = FALSE)
+library("mkin")
+```
+
+Each system is characterized by its CPU type, the operating system type and the
+mkin version. Currently only values for one system are available. A compiler
+was available, so if no analytical solution was available, compiled ODE models are
+used.
+
+```{r include = FALSE}
+cpu_model <- benchmarkme::get_cpu()$model_name
+operating_system <- Sys.info()[["sysname"]]
+mkin_version <- as.character(packageVersion("mkin"))
+system_string <- paste0(operating_system, ", ", cpu_model, ", mkin version ", mkin_version)
+load("~/git/mkin/vignettes/web_only/mkin_benchmarks.rda")
+mkin_benchmarks[system_string, c("CPU", "OS", "mkin")] <-
+  c(cpu_model, operating_system, mkin_version)
+
+if (mkin_version > "0.9.48.1") {
+  mmkin_bench <- function(models, datasets, error_model = "const") {
+    mmkin(models, datasets, error_model = error_model, cores = 1, quiet = TRUE)
+  }
+} else {
+  mmkin_bench <- function(models, datasets, error_model = NULL) {
+    mmkin(models, datasets, reweight.method = error_model, cores = 1, quiet = TRUE)
+  }
+}
+```
+
+## Test cases
+
+Parent only:
+
+```{r parent_only, warning = FALSE}
+FOCUS_C <- FOCUS_2006_C
+FOCUS_D <- subset(FOCUS_2006_D, value != 0)
+parent_datasets <- list(FOCUS_C, FOCUS_D)
+
+t1 <- system.time(mmkin_bench(c("SFO", "FOMC", "DFOP", "HS"), parent_datasets))[["elapsed"]]
+t2 <- system.time(mmkin_bench(c("SFO", "FOMC", "DFOP", "HS"), parent_datasets,
+    error_model = "tc"))[["elapsed"]]
+```
+
+One metabolite:
+
+```{r one_metabolite, message = FALSE}
+SFO_SFO <- mkinmod(
+  parent = mkinsub("SFO", "m1"),
+  m1 = mkinsub("SFO"))
+FOMC_SFO <- mkinmod(
+  parent = mkinsub("FOMC", "m1"),
+  m1 = mkinsub("SFO"))
+DFOP_SFO <- mkinmod(
+  parent = mkinsub("FOMC", "m1"),
+  m1 = mkinsub("SFO"))
+t3 <- system.time(mmkin_bench(list(SFO_SFO, FOMC_SFO, DFOP_SFO), list(FOCUS_D)))[["elapsed"]]
+t4 <- system.time(mmkin_bench(list(SFO_SFO, FOMC_SFO, DFOP_SFO), list(FOCUS_D),
+    error_model = "tc"))[["elapsed"]]
+t5 <- system.time(mmkin_bench(list(SFO_SFO, FOMC_SFO, DFOP_SFO), list(FOCUS_D),
+    error_model = "obs"))[["elapsed"]]
+```
+
+Two metabolites, synthetic data:
+
+```{r two_metabolites, message = FALSE}
+m_synth_SFO_lin <- mkinmod(parent = mkinsub("SFO", "M1"),
+                           M1 = mkinsub("SFO", "M2"),
+                           M2 = mkinsub("SFO"),
+                           use_of_ff = "max", quiet = TRUE)
+
+m_synth_DFOP_par <- mkinmod(parent = mkinsub("DFOP", c("M1", "M2")),
+                           M1 = mkinsub("SFO"),
+                           M2 = mkinsub("SFO"),
+                           use_of_ff = "max", quiet = TRUE)
+
+SFO_lin_a <- synthetic_data_for_UBA_2014[[1]]$data
+
+DFOP_par_c <- synthetic_data_for_UBA_2014[[12]]$data
+
+t6 <- system.time(mmkin_bench(list(m_synth_SFO_lin), list(SFO_lin_a)))[["elapsed"]]
+t7 <- system.time(mmkin_bench(list(m_synth_DFOP_par), list(DFOP_par_c)))[["elapsed"]]
+
+t8 <- system.time(mmkin_bench(list(m_synth_SFO_lin), list(SFO_lin_a),
+    error_model = "tc"))[["elapsed"]]
+t9 <- system.time(mmkin_bench(list(m_synth_DFOP_par), list(DFOP_par_c),
+    error_model = "tc"))[["elapsed"]]
+
+t10 <- system.time(mmkin_bench(list(m_synth_SFO_lin), list(SFO_lin_a),
+    error_model = "obs"))[["elapsed"]]
+t11 <- system.time(mmkin_bench(list(m_synth_DFOP_par), list(DFOP_par_c),
+    error_model = "obs"))[["elapsed"]]
+```
+
+```{r results}
+mkin_benchmarks[system_string, paste0("t", 1:11)] <-
+  c(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11)
+save(mkin_benchmarks, file = "~/git/mkin/vignettes/web_only/mkin_benchmarks.rda")
+```
+
+## Results
+
+Currently, we only have benchmark information on one system, therefore only the mkin
+version is shown with the results below. Timings are in seconds, shorter is better.
+All results were obtained by serial, i.e. not using multiple computing cores.
+
+```{r, include = FALSE}
+dimnames(mkin_benchmarks) <- list(as.character(mkin_benchmarks$mkin),
+  c("CPU", "OS", "mkin version", paste0("t", 1:11, " [s]")))
+```
+
+
+Benchmarks for all available error models are shown.
+
+### Parent only
+
+Constant variance (t1) and two-component error model (t2) for four models
+fitted to two datasets, i.e. eight fits for each test.
+
+```{r, echo = FALSE}
+kable(mkin_benchmarks[, 4:5], rownames = "mkin version")
+```
+
+### One metabolite
+
+Constant variance (t3), two-component error model (t4), and variance by variable (t5)
+for three models fitted to one dataset, i.e. three fits for each test.
+
+```{r, echo = FALSE}
+kable(mkin_benchmarks[, 6:8], rownames = "mkin version")
+```
+
+### Two metabolites
+
+Constant variance (t6 and t7), two-component error model (t8 and t9), and
+variance by variable (t10 and t11) for one model fitted to one dataset, i.e.
+one fit for each test.
+
+```{r, echo = FALSE}
+kable(mkin_benchmarks[, 9:14], rownames = "mkin version")
+```