From 218a9c55bd80fb708b15fa7196422f759bfe4b27 Mon Sep 17 00:00:00 2001 From: Johannes Ranke Date: Wed, 13 May 2020 16:20:23 +0200 Subject: Further formatting improvement of benchmark vignette Also, use .rmd extension instead of .Rmd for vignettes. --- vignettes/web_only/benchmarks.rmd | 157 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 vignettes/web_only/benchmarks.rmd (limited to 'vignettes/web_only/benchmarks.rmd') diff --git a/vignettes/web_only/benchmarks.rmd b/vignettes/web_only/benchmarks.rmd new file mode 100644 index 00000000..7ae12451 --- /dev/null +++ b/vignettes/web_only/benchmarks.rmd @@ -0,0 +1,157 @@ +--- +title: "Benchmark timings for mkin" +author: "Johannes Ranke" +output: + html_document: + toc: true + toc_float: true + code_folding: show + fig_retina: null +date: "`r Sys.Date()`" +vignette: > + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +library(knitr) +opts_chunk$set(tidy = FALSE, cache = FALSE) +library("mkin") +``` + +Each system is characterized by its CPU type, the operating system type and the +mkin version. Currently only values for one system are available. A compiler +was available, so if no analytical solution was available, compiled ODE models are +used. + +```{r include = FALSE} +cpu_model <- benchmarkme::get_cpu()$model_name +operating_system <- Sys.info()[["sysname"]] +mkin_version <- as.character(packageVersion("mkin")) +system_string <- paste0(operating_system, ", ", cpu_model, ", mkin version ", mkin_version) +load("~/git/mkin/vignettes/web_only/mkin_benchmarks.rda") +mkin_benchmarks[system_string, c("CPU", "OS", "mkin")] <- + c(cpu_model, operating_system, mkin_version) + +if (mkin_version > "0.9.48.1") { + mmkin_bench <- function(models, datasets, error_model = "const") { + mmkin(models, datasets, error_model = error_model, cores = 1, quiet = TRUE) + } +} else { + mmkin_bench <- function(models, datasets, error_model = NULL) { + mmkin(models, datasets, reweight.method = error_model, cores = 1, quiet = TRUE) + } +} +``` + +## Test cases + +Parent only: + +```{r parent_only, warning = FALSE} +FOCUS_C <- FOCUS_2006_C +FOCUS_D <- subset(FOCUS_2006_D, value != 0) +parent_datasets <- list(FOCUS_C, FOCUS_D) + +t1 <- system.time(mmkin_bench(c("SFO", "FOMC", "DFOP", "HS"), parent_datasets))[["elapsed"]] +t2 <- system.time(mmkin_bench(c("SFO", "FOMC", "DFOP", "HS"), parent_datasets, + error_model = "tc"))[["elapsed"]] +``` + +One metabolite: + +```{r one_metabolite, message = FALSE} +SFO_SFO <- mkinmod( + parent = mkinsub("SFO", "m1"), + m1 = mkinsub("SFO")) +FOMC_SFO <- mkinmod( + parent = mkinsub("FOMC", "m1"), + m1 = mkinsub("SFO")) +DFOP_SFO <- mkinmod( + parent = mkinsub("FOMC", "m1"), + m1 = mkinsub("SFO")) +t3 <- system.time(mmkin_bench(list(SFO_SFO, FOMC_SFO, DFOP_SFO), list(FOCUS_D)))[["elapsed"]] +t4 <- system.time(mmkin_bench(list(SFO_SFO, FOMC_SFO, DFOP_SFO), list(FOCUS_D), + error_model = "tc"))[["elapsed"]] +t5 <- system.time(mmkin_bench(list(SFO_SFO, FOMC_SFO, DFOP_SFO), list(FOCUS_D), + error_model = "obs"))[["elapsed"]] +``` + +Two metabolites, synthetic data: + +```{r two_metabolites, message = FALSE} +m_synth_SFO_lin <- mkinmod(parent = mkinsub("SFO", "M1"), + M1 = mkinsub("SFO", "M2"), + M2 = mkinsub("SFO"), + use_of_ff = "max", quiet = TRUE) + +m_synth_DFOP_par <- mkinmod(parent = mkinsub("DFOP", c("M1", "M2")), + M1 = mkinsub("SFO"), + M2 = mkinsub("SFO"), + use_of_ff = "max", quiet = TRUE) + +SFO_lin_a <- synthetic_data_for_UBA_2014[[1]]$data + +DFOP_par_c <- synthetic_data_for_UBA_2014[[12]]$data + +t6 <- system.time(mmkin_bench(list(m_synth_SFO_lin), list(SFO_lin_a)))[["elapsed"]] +t7 <- system.time(mmkin_bench(list(m_synth_DFOP_par), list(DFOP_par_c)))[["elapsed"]] + +t8 <- system.time(mmkin_bench(list(m_synth_SFO_lin), list(SFO_lin_a), + error_model = "tc"))[["elapsed"]] +t9 <- system.time(mmkin_bench(list(m_synth_DFOP_par), list(DFOP_par_c), + error_model = "tc"))[["elapsed"]] + +t10 <- system.time(mmkin_bench(list(m_synth_SFO_lin), list(SFO_lin_a), + error_model = "obs"))[["elapsed"]] +t11 <- system.time(mmkin_bench(list(m_synth_DFOP_par), list(DFOP_par_c), + error_model = "obs"))[["elapsed"]] +``` + +```{r results} +mkin_benchmarks[system_string, paste0("t", 1:11)] <- + c(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11) +save(mkin_benchmarks, file = "~/git/mkin/vignettes/web_only/mkin_benchmarks.rda") +``` + +## Results + +Currently, we only have benchmark information on one system, therefore only the mkin +version is shown with the results below. Timings are in seconds, shorter is better. +All results were obtained by serial, i.e. not using multiple computing cores. + +```{r, include = FALSE} +dimnames(mkin_benchmarks) <- list(as.character(mkin_benchmarks$mkin), + c("CPU", "OS", "mkin version", paste0("t", 1:11, " [s]"))) +``` + + +Benchmarks for all available error models are shown. + +### Parent only + +Constant variance (t1) and two-component error model (t2) for four models +fitted to two datasets, i.e. eight fits for each test. + +```{r, echo = FALSE} +kable(mkin_benchmarks[, 4:5], rownames = "mkin version") +``` + +### One metabolite + +Constant variance (t3), two-component error model (t4), and variance by variable (t5) +for three models fitted to one dataset, i.e. three fits for each test. + +```{r, echo = FALSE} +kable(mkin_benchmarks[, 6:8], rownames = "mkin version") +``` + +### Two metabolites + +Constant variance (t6 and t7), two-component error model (t8 and t9), and +variance by variable (t10 and t11) for one model fitted to one dataset, i.e. +one fit for each test. + +```{r, echo = FALSE} +kable(mkin_benchmarks[, 9:14], rownames = "mkin version") +``` -- cgit v1.2.1