---
title: "Benchmark timings for mkin"
author: "Johannes Ranke"
date: Last change 13 May 2020 (rebuilt `r Sys.Date()`)
output:
html_document:
toc: true
toc_float: true
code_folding: show
fig_retina: null
vignette: >
%\VignetteEngine{knitr::rmarkdown}
%\VignetteEncoding{UTF-8}
---
```{r, include = FALSE}
library(knitr)
opts_chunk$set(tidy = FALSE, cache = FALSE)
library("mkin")
```
Each system is characterized by its CPU type, the operating system type and the
mkin version. Currently only values for one system are available. A compiler
was available, so if no analytical solution was available, compiled ODE models are
used.
```{r include = FALSE}
cpu_model <- benchmarkme::get_cpu()$model_name
operating_system <- Sys.info()[["sysname"]]
mkin_version <- as.character(packageVersion("mkin"))
system_string <- paste0(operating_system, ", ", cpu_model, ", mkin version ", mkin_version)
load("~/git/mkin/vignettes/web_only/mkin_benchmarks.rda")
mkin_benchmarks[system_string, c("CPU", "OS", "mkin")] <-
c(cpu_model, operating_system, mkin_version)
if (mkin_version > "0.9.48.1") {
mmkin_bench <- function(models, datasets, error_model = "const") {
mmkin(models, datasets, error_model = error_model, cores = 1, quiet = TRUE)
}
} else {
mmkin_bench <- function(models, datasets, error_model = NULL) {
mmkin(models, datasets, reweight.method = error_model, cores = 1, quiet = TRUE)
}
}
```
## Test cases
Parent only:
```{r parent_only, warning = FALSE}
FOCUS_C <- FOCUS_2006_C
FOCUS_D <- subset(FOCUS_2006_D, value != 0)
parent_datasets <- list(FOCUS_C, FOCUS_D)
t1 <- system.time(mmkin_bench(c("SFO", "FOMC", "DFOP", "HS"), parent_datasets))[["elapsed"]]
t2 <- system.time(mmkin_bench(c("SFO", "FOMC", "DFOP", "HS"), parent_datasets,
error_model = "tc"))[["elapsed"]]
```
One metabolite:
```{r one_metabolite, message = FALSE}
SFO_SFO <- mkinmod(
parent = mkinsub("SFO", "m1"),
m1 = mkinsub("SFO"))
FOMC_SFO <- mkinmod(
parent = mkinsub("FOMC", "m1"),
m1 = mkinsub("SFO"))
DFOP_SFO <- mkinmod(
parent = mkinsub("FOMC", "m1"),
m1 = mkinsub("SFO"))
t3 <- system.time(mmkin_bench(list(SFO_SFO, FOMC_SFO, DFOP_SFO), list(FOCUS_D)))[["elapsed"]]
t4 <- system.time(mmkin_bench(list(SFO_SFO, FOMC_SFO, DFOP_SFO), list(FOCUS_D),
error_model = "tc"))[["elapsed"]]
t5 <- system.time(mmkin_bench(list(SFO_SFO, FOMC_SFO, DFOP_SFO), list(FOCUS_D),
error_model = "obs"))[["elapsed"]]
```
Two metabolites, synthetic data:
```{r two_metabolites, message = FALSE}
m_synth_SFO_lin <- mkinmod(parent = mkinsub("SFO", "M1"),
M1 = mkinsub("SFO", "M2"),
M2 = mkinsub("SFO"),
use_of_ff = "max", quiet = TRUE)
m_synth_DFOP_par <- mkinmod(parent = mkinsub("DFOP", c("M1", "M2")),
M1 = mkinsub("SFO"),
M2 = mkinsub("SFO"),
use_of_ff = "max", quiet = TRUE)
SFO_lin_a <- synthetic_data_for_UBA_2014[[1]]$data
DFOP_par_c <- synthetic_data_for_UBA_2014[[12]]$data
t6 <- system.time(mmkin_bench(list(m_synth_SFO_lin), list(SFO_lin_a)))[["elapsed"]]
t7 <- system.time(mmkin_bench(list(m_synth_DFOP_par), list(DFOP_par_c)))[["elapsed"]]
t8 <- system.time(mmkin_bench(list(m_synth_SFO_lin), list(SFO_lin_a),
error_model = "tc"))[["elapsed"]]
t9 <- system.time(mmkin_bench(list(m_synth_DFOP_par), list(DFOP_par_c),
error_model = "tc"))[["elapsed"]]
t10 <- system.time(mmkin_bench(list(m_synth_SFO_lin), list(SFO_lin_a),
error_model = "obs"))[["elapsed"]]
t11 <- system.time(mmkin_bench(list(m_synth_DFOP_par), list(DFOP_par_c),
error_model = "obs"))[["elapsed"]]
```
```{r results}
mkin_benchmarks[system_string, paste0("t", 1:11)] <-
c(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11)
save(mkin_benchmarks, file = "~/git/mkin/vignettes/web_only/mkin_benchmarks.rda")
```
## Results
Currently, we only have benchmark information on one system, therefore only the mkin
version is shown with the results below. Timings are in seconds, shorter is better.
All results were obtained by serial, i.e. not using multiple computing cores.
```{r, include = FALSE}
dimnames(mkin_benchmarks) <- list(as.character(mkin_benchmarks$mkin),
c("CPU", "OS", "mkin version", paste0("t", 1:11, " [s]")))
```
Benchmarks for all available error models are shown.
### Parent only
Constant variance (t1) and two-component error model (t2) for four models
fitted to two datasets, i.e. eight fits for each test.
```{r, echo = FALSE}
kable(mkin_benchmarks[, 4:5], rownames = "mkin version")
```
### One metabolite
Constant variance (t3), two-component error model (t4), and variance by variable (t5)
for three models fitted to one dataset, i.e. three fits for each test.
```{r, echo = FALSE}
kable(mkin_benchmarks[, 6:8], rownames = "mkin version")
```
### Two metabolites
Constant variance (t6 and t7), two-component error model (t8 and t9), and
variance by variable (t10 and t11) for one model fitted to one dataset, i.e.
one fit for each test.
```{r, echo = FALSE}
kable(mkin_benchmarks[, 9:14], rownames = "mkin version")
```
