From 92bb4db92cce501f67842d7445b9f4d771eb634c Mon Sep 17 00:00:00 2001 From: statasarus Date: Tue, 7 Nov 2023 09:54:31 +0000 Subject: [PATCH] Add mmrm and wilcoxs --- Comp/r-sas_mmrm.qmd | 637 ++++++++++++++++-- R/anova_cache/html/__packages | 12 - ...k-2_a959c7d69d77a7d483cef0c7910ca827.RData | Bin 2813 -> 0 bytes ...unk-2_a959c7d69d77a7d483cef0c7910ca827.rdb | Bin 3837 -> 0 bytes ...unk-2_a959c7d69d77a7d483cef0c7910ca827.rdx | Bin 148 -> 0 bytes ...k-3_03313b7a8524dd7a7440d66f44058036.RData | Bin 3082 -> 0 bytes ...unk-3_03313b7a8524dd7a7440d66f44058036.rdb | 0 ...unk-3_03313b7a8524dd7a7440d66f44058036.rdx | Bin 125 -> 0 bytes ...k-4_2c39991c34fb054f157959a554b2175c.RData | Bin 3233 -> 0 bytes ...unk-4_2c39991c34fb054f157959a554b2175c.rdb | 0 ...unk-4_2c39991c34fb054f157959a554b2175c.rdx | Bin 125 -> 0 bytes ...k-5_1e3227cddc2c5d4b92f6b3238e69038a.RData | Bin 3067 -> 0 bytes ...unk-5_1e3227cddc2c5d4b92f6b3238e69038a.rdb | 0 ...unk-5_1e3227cddc2c5d4b92f6b3238e69038a.rdx | Bin 125 -> 0 bytes ...k-6_87216dbd4f33263d00c668206b754202.RData | Bin 3132 -> 0 bytes ...unk-6_87216dbd4f33263d00c668206b754202.rdb | 0 ...unk-6_87216dbd4f33263d00c668206b754202.rdx | Bin 125 -> 0 bytes ...k-7_a6ade8d6f53133a4cb343d600ba67101.RData | Bin 3096 -> 0 bytes ...unk-7_a6ade8d6f53133a4cb343d600ba67101.rdb | 0 ...unk-7_a6ade8d6f53133a4cb343d600ba67101.rdx | Bin 125 -> 0 bytes ...k-8_fa270a2f00df15b12e7fd336f04352a4.RData | Bin 3090 -> 0 bytes ...unk-8_fa270a2f00df15b12e7fd336f04352a4.rdb | 0 ...unk-8_fa270a2f00df15b12e7fd336f04352a4.rdx | Bin 125 -> 0 bytes ...k-9_172a7fa3eda2baf75813b4ecc2a279ce.RData | Bin 3145 -> 0 bytes ...unk-9_172a7fa3eda2baf75813b4ecc2a279ce.rdb | 0 ...unk-9_172a7fa3eda2baf75813b4ecc2a279ce.rdx | Bin 125 -> 0 bytes R/nonpara_wilcoxon_ranksum.qmd | 79 +++ data/stat_method_tbl.csv | 6 +- renv.lock | 137 +++- renv/activate.R | 67 +- 30 files changed, 838 insertions(+), 100 deletions(-) delete mode 100644 R/anova_cache/html/__packages delete mode 100644 R/anova_cache/html/unnamed-chunk-2_a959c7d69d77a7d483cef0c7910ca827.RData delete mode 100644 R/anova_cache/html/unnamed-chunk-2_a959c7d69d77a7d483cef0c7910ca827.rdb delete mode 100644 R/anova_cache/html/unnamed-chunk-2_a959c7d69d77a7d483cef0c7910ca827.rdx delete mode 100644 R/anova_cache/html/unnamed-chunk-3_03313b7a8524dd7a7440d66f44058036.RData delete mode 100644 R/anova_cache/html/unnamed-chunk-3_03313b7a8524dd7a7440d66f44058036.rdb delete mode 100644 R/anova_cache/html/unnamed-chunk-3_03313b7a8524dd7a7440d66f44058036.rdx delete mode 100644 R/anova_cache/html/unnamed-chunk-4_2c39991c34fb054f157959a554b2175c.RData delete mode 100644 R/anova_cache/html/unnamed-chunk-4_2c39991c34fb054f157959a554b2175c.rdb delete mode 100644 R/anova_cache/html/unnamed-chunk-4_2c39991c34fb054f157959a554b2175c.rdx delete mode 100644 R/anova_cache/html/unnamed-chunk-5_1e3227cddc2c5d4b92f6b3238e69038a.RData delete mode 100644 R/anova_cache/html/unnamed-chunk-5_1e3227cddc2c5d4b92f6b3238e69038a.rdb delete mode 100644 R/anova_cache/html/unnamed-chunk-5_1e3227cddc2c5d4b92f6b3238e69038a.rdx delete mode 100644 R/anova_cache/html/unnamed-chunk-6_87216dbd4f33263d00c668206b754202.RData delete mode 100644 R/anova_cache/html/unnamed-chunk-6_87216dbd4f33263d00c668206b754202.rdb delete mode 100644 R/anova_cache/html/unnamed-chunk-6_87216dbd4f33263d00c668206b754202.rdx delete mode 100644 R/anova_cache/html/unnamed-chunk-7_a6ade8d6f53133a4cb343d600ba67101.RData delete mode 100644 R/anova_cache/html/unnamed-chunk-7_a6ade8d6f53133a4cb343d600ba67101.rdb delete mode 100644 R/anova_cache/html/unnamed-chunk-7_a6ade8d6f53133a4cb343d600ba67101.rdx delete mode 100644 R/anova_cache/html/unnamed-chunk-8_fa270a2f00df15b12e7fd336f04352a4.RData delete mode 100644 R/anova_cache/html/unnamed-chunk-8_fa270a2f00df15b12e7fd336f04352a4.rdb delete mode 100644 R/anova_cache/html/unnamed-chunk-8_fa270a2f00df15b12e7fd336f04352a4.rdx delete mode 100644 R/anova_cache/html/unnamed-chunk-9_172a7fa3eda2baf75813b4ecc2a279ce.RData delete mode 100644 R/anova_cache/html/unnamed-chunk-9_172a7fa3eda2baf75813b4ecc2a279ce.rdb delete mode 100644 R/anova_cache/html/unnamed-chunk-9_172a7fa3eda2baf75813b4ecc2a279ce.rdx create mode 100644 R/nonpara_wilcoxon_ranksum.qmd diff --git a/Comp/r-sas_mmrm.qmd b/Comp/r-sas_mmrm.qmd index c7221da56..db2d0458e 100644 --- a/Comp/r-sas_mmrm.qmd +++ b/Comp/r-sas_mmrm.qmd @@ -2,75 +2,614 @@ title: "R vs SAS MMRM" --- -```{r} -#| echo: false -plots <- readRDS("../images/mixed/plots.rds") +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, comment = "#>", out.width = "100%", + dpi = 150, fig.path = "../images/mmrm/" +) ``` -### Data +```{r review-setup} +#| message: FALSE +#| warning: FALSE +#| include: FALSE -The data used for this comparison was the lab ADaM dataset adlbh.xpt from the Phuse Pilot Study. Results were generated for each lab parameter and time point in the dataset using three different covariance structures, i.e. unstructured, compound symmetry and autoregressive of first order (AR(1)). +library(dplyr) +library(purrr) +library(microbenchmark) +library(stringr) +library(mmrm) +library(knitr) +library(emmeans) +library(ggplot2) +library(details) +``` -### Comparison between SAS and R +```{r load-data} +#| message: FALSE +#| warning: FALSE +#| include: FALSE -With results available for SAS and R model fits, we turn our attention to generating some visual comparisons of the results. Note that here we adopt a Bland-Altman type plot which plots the difference on the y-axis and the average on the x-axis. This offers a way to inspect any bias or relationships with the size of effect and the associated bias. +load('../data/cached_mmrm_results.rda') +``` -For the extracted LS-means +```{r set-seed} +#| message: FALSE +#| warning: FALSE +#| include: FALSE -```{r} -#| echo: false -#| warning: false -#| message: false -#| out.width: "100%" -plots$lsmeans_bland_alt_emmean$`Unstructured (KR)` -plots$lsmeans_bland_alt_emmean$`Unstructured (satterthwaite)` -plots$lsmeans_bland_alt_emmean$`Heterogeneous Compound Symmetry` -plots$lsmeans_bland_alt_emmean$`Heterogeneous First Order Autoregressive` +set.seed(5123) ``` -and corresponding SEs +# Introduction -```{r} -#| echo: false -#| warning: false -#| message: false -plots$lsmeans_bland_alt_se$`Unstructured (KR)` -plots$lsmeans_bland_alt_se$`Unstructured (satterthwaite)` -plots$lsmeans_bland_alt_se$`Heterogeneous Compound Symmetry` -plots$lsmeans_bland_alt_se$`Heterogeneous First Order Autoregressive` +In this vignette we briefly compare the `mmrm::mmrm`, SAS's `PROC GLIMMIX`, `nlme::gls`, `lme4::lmer`, and `glmmTMB::glmmTMB` functions for fitting mixed models for repeated measures (MMRMs). A primary difference in these implementations lies in the covariance structures that are supported "out of the box". In particular, `PROC GLIMMIX` and `mmrm` are the only procedures which provide support for many of the most common MMRM covariance structures. Most covariance structures can be implemented in `gls`, though users are required to define them manually. `lmer` and `glmmTMB` are more limited. We find that `mmmrm` converges more quickly than other R implementations while also producing estimates that are virtually identical to `PROC GLIMMIX`'s. + +# Datasets + +Two datasets are used to illustrate model fitting with the `mmrm`, `lme4`, `nlme`, `glmmTMB` R packages as well as `PROC GLIMMIX`. These data are also used to compare these implementations' operating characteristics. + +## FEV Data + +The FEV dataset contains measurements of FEV1 (forced expired volume in one second), a measure of how quickly the lungs can be emptied. Low levels of FEV1 may indicate chronic obstructive pulmonary disease (COPD). It is summarized below. + +``` + Stratified by ARMCD + Overall PBO TRT + n 800 420 380 + USUBJID (%) + PT[1-200] 200 105 (52.5) 95 (47.5) + AVISIT + VIS1 200 105 95 + VIS2 200 105 95 + VIS3 200 105 95 + VIS4 200 105 95 + RACE (%) + Asian 280 (35.0) 152 (36.2) 128 (33.7) + Black or African American 300 (37.5) 184 (43.8) 116 (30.5) + White 220 (27.5) 84 (20.0) 136 (35.8) + SEX = Female (%) 424 (53.0) 220 (52.4) 204 (53.7) + FEV1_BL (mean (SD)) 40.19 (9.12) 40.46 (8.84) 39.90 (9.42) + FEV1 (mean (SD)) 42.30 (9.32) 40.24 (8.67) 44.45 (9.51) + WEIGHT (mean (SD)) 0.52 (0.23) 0.52 (0.23) 0.51 (0.23) + VISITN (mean (SD)) 2.50 (1.12) 2.50 (1.12) 2.50 (1.12) + VISITN2 (mean (SD)) -0.02 (1.03) 0.01 (1.07) -0.04 (0.98) +``` + +## BCVA Data + +The BCVA dataset contains data from a randomized longitudinal ophthalmology trial evaluating the change in baseline corrected visual acuity (BCVA) over the course of 10 visits. BCVA corresponds to the number of letters read from a visual acuity chart. A summary of the data is given below: + +``` + Stratified by ARMCD + Overall CTL TRT + n 8605 4123 4482 + USUBJID (%) + PT[1-1000] 1000 494 (49.4) 506 (50.6) + AVISIT + VIS1 983 482 501 + VIS2 980 481 499 + VIS3 960 471 489 + VIS4 946 458 488 + VIS5 925 454 471 + VIS6 868 410 458 + VIS7 816 388 428 + VIS8 791 371 420 + VIS9 719 327 392 + VIS10 617 281 336 + RACE (%) + Asian 297 (29.7) 151 (30.6) 146 (28.9) + Black or African American 317 (31.7) 149 (30.1) 168 (33.2) + White 386 (38.6) 194 (39.3) 192 (37.9) + BCVA_BL (mean (SD)) 75.12 (9.93) 74.90 (9.76) 75.40 (10.1) + BCVA_CHG (mean (SD)) + VIS1 5.59 (1.31) 5.32 (1.23) 5.86 (1.33) + VIS10 9.18 (2.91) 7.49 (2.58) 10.60 (2.36) +``` + +# Model Implementations {.tabset} + +Listed below are some of the most commonly used covariance structures used when fitting MMRMs. We indicate which matrices are available "out of the box" for each implementation considered in this vignette. Note that this table is not exhaustive; `PROC GLIMMIX` and `glmmTMB` support additional spatial covariance structures. + +| Covariance structures | `mmrm` | `PROC GLIMMIX` | `gls` | `lmer` | `glmmTMB` | +|:---------------------------------:|:------:|:--------------:|:-----:|:------:|:---------:| +| Ante-dependence (heterogeneous) | X | X | | | | +| Ante-dependence (homogeneous) | X | | | | | +| Auto-regressive (heterogeneous) | X | X | X | | | +| Auto-regressive (homogeneous) | X | X | X | | X | +| Compound symmetry (heterogeneous) | X | X | X | | X | +| Compound symmetry (homogeneous) | X | X | X | | | +| Spatial exponential | X | X | X | | X | +| Toeplitz (heterogeneous) | X | X | | | X | +| Toeplitz (homogeneous) | X | X | | | | +| Unstructured | X | X | X | X | X | + +Code for fitting MMRMs to the FEV data using each of the considered functions and covariance structures are provided below. Fixed effects for the visit number, treatment assignment and the interaction between the two are modeled. + +## Ante-dependence (heterogeneous) + +### `PROC GLIMMIX` + +```{=html} +
PROC GLIMMIX DATA = fev_data;
+CLASS AVISIT(ref = 'VIS1') ARMCD(ref = 'PBO') USUBJID;
+MODEL FEV1 = AVISIT|ARMCD / ddfm=satterthwaite solution chisq;
+RANDOM AVISIT / subject=USUBJID type=ANTE(1);
+
+``` +### `mmrm` + +```{=html} +
mmrm(
+  formula = FEV1 ~ ARMCD * AVISIT + adh(VISITN | USUBJID),
+  data = fev_data
+)
+
+``` +## Ante-dependence (homogeneous) + +### `mmrm` + +```{=html} +
mmrm(
+  formula =FEV1 ~ ARMCD * AVISIT + ad(VISITN | USUBJID),
+  data = fev_data
+)
+
+``` +## Auto-regressive (heterogeneous) + +### `PROC GLIMMIX` + +```{=html} +
PROC GLIMMIX DATA = fev_data;
+CLASS AVISIT(ref = 'VIS1') ARMCD(ref = 'PBO') USUBJID;
+MODEL FEV1 = AVISIT|ARMCD / ddfm=satterthwaite solution chisq;
+RANDOM AVISIT / subject=USUBJID type=ARH(1);
+
+``` +### `mmrm` + +```{=html} +
mmrm(
+  formula = FEV1 ~ ARMCD * AVISIT + ar1h(VISITN | USUBJID),
+  data = fev_data
+)
+
+``` +### `gls` + +```{=html} +
gls(
+  formula = FEV1 ~ ARMCD * AVISIT,
+  data = fev_data,
+  correlation = corCAR1(form = ~AVISIT | USUBJID),
+  weights = varIdent(form = ~1|AVISIT),
+  na.action = na.omit
+)
+
+``` +## Auto-regressive (homogeneous) + +### `PROC GLIMMIX` + +```{=html} +
PROC GLIMMIX DATA = fev_data;
+CLASS AVISIT(ref = 'VIS1') ARMCD(ref = 'PBO') USUBJID;
+MODEL FEV1 =  ARMCD|AVISIT / ddfm=satterthwaite solution chisq;
+RANDOM AVISIT / subject=USUBJID type=AR(1);
+
+``` +### `mmrm` + +```{=html} +
mmrm(
+  formula = FEV1 ~ ARMCD * AVISIT + ar1(VISITN | USUBJID),
+  data = fev_data
+)
+
+``` +### `gls` + +```{=html} +
gls(
+  formula = FEV1 ~ ARMCD * AVISIT,
+  data = fev_data,
+  correlation = corCAR1(form = ~AVISIT | USUBJID),
+  na.action = na.omit
+)
+
+``` +### `glmmTMB` + +```{=html} +
glmmTMB(
+  FEV1 ~ ARMCD * AVISIT + ar1(0 + AVISIT | USUBJID),
+  dispformula = ~ 0,
+  data = fev_data
+)
+
+``` +## Compound symmetry (heterogeneous) + +### `PROC GLIMMIX` + +```{=html} +
PROC GLIMMIX DATA = fev_data;
+CLASS AVISIT(ref = 'VIS1') ARMCD(ref = 'PBO') USUBJID;
+MODEL FEV1 = AVISIT|ARMCD / ddfm=satterthwaite solution chisq;
+RANDOM AVISIT / subject=USUBJID type=CSH;
+
+``` +### `mmrm` + +```{=html} +
mmrm(
+  formula = FEV1 ~ ARMCD * AVISIT + csh(VISITN | USUBJID),
+  data = fev_data
+)
+
+``` +### `gls` + +```{=html} +
gls(
+  formula = FEV1 ~ ARMCD * AVISIT,
+  data = fev_data,
+  correlation = corCompSymm(form = ~AVISIT | USUBJID),
+  weights = varIdent(form = ~1|AVISIT),
+  na.action = na.omit
+)
+
+``` +### `glmmTMB` + +```{=html} +
glmmTMB(
+  FEV1 ~ ARMCD * AVISIT + cs(0 + AVISIT | USUBJID),
+  dispformula = ~ 0,
+  data = fev_data
+)
+
+``` +## Compound symmetry (homogeneous) + +### `PROC GLIMMIX` + +```{=html} +
PROC GLIMMIX DATA = fev_data;
+CLASS AVISIT(ref = 'VIS1') ARMCD(ref = 'PBO') USUBJID;
+MODEL FEV1 = AVISIT|ARMCD / ddfm=satterthwaite solution chisq;
+RANDOM AVISIT / subject=USUBJID type=CS;
+
+``` +### `mmrm` + +```{=html} +
mmrm(
+  formula = FEV1 ~ ARMCD * AVISIT + cs(VISITN | USUBJID),
+  data = fev_data
+)
+
+``` +### `gls` + +```{=html} +
gls(
+  formula = FEV1 ~ ARMCD * AVISIT,
+  data = fev_data,
+  correlation = corCompSymm(form = ~AVISIT | USUBJID),
+  na.action = na.omit
+)
+
+``` +## Spatial exponential + +### `PROC GLIMMIX` + +```{=html} +
PROC GLIMMIX DATA = fev_data;
+CLASS AVISIT(ref = 'VIS1') ARMCD(ref = 'PBO') USUBJID;
+MODEL FEV1 = AVISIT|ARMCD / ddfm=satterthwaite solution chisq;
+RANDOM / subject=USUBJID type=sp(exp)(visitn) rcorr;
+
+``` +### `mmrm` + +```{=html} +
mmrm(
+  formula = FEV1 ~ ARMCD * AVISIT + sp_exp(VISITN | USUBJID),
+  data = fev_data
+)
+
+``` +### `gls` + +```{=html} +
gls(
+  formula = FEV1 ~ ARMCD * AVISIT,
+  data = fev_data,
+  correlation = corExp(form = ~AVISIT | USUBJID),
+  weights = varIdent(form = ~1|AVISIT),
+  na.action = na.omit
+)
+
+``` +### `glmmTMB` + +```{=html} +
# NOTE: requires use of coordinates
+glmmTMB(
+  FEV1 ~ ARMCD * AVISIT + exp(0 + AVISIT | USUBJID),
+  dispformula = ~ 0,
+  data = fev_data
+)
+
+``` +## Toeplitz (heterogeneous) + +### `PROC GLIMMIX` + +```{=html} +
PROC GLIMMIX DATA = fev_data;
+CLASS AVISIT(ref = 'VIS1') ARMCD(ref = 'PBO') USUBJID;
+MODEL FEV1 = AVISIT|ARMCD / ddfm=satterthwaite solution chisq;
+RANDOM AVISIT / subject=USUBJID type=TOEPH;
+
+``` +### `mmrm` + +```{=html} +
mmrm(
+  formula = FEV1 ~ ARMCD * AVISIT + toeph(AVISIT | USUBJID),
+  data = fev_data
+)
+
+``` +### `glmmTMB` + +```{=html} +
 glmmTMB(
+  FEV1 ~ ARMCD * AVISIT + toep(0 + AVISIT | USUBJID),
+  dispformula = ~ 0,
+  data = fev_data
+)
+
+``` +## Toeplitz (homogeneous) + +### `PROC GLIMMIX` + +```{=html} +
PROC GLIMMIX DATA = fev_data;
+CLASS AVISIT(ref = 'VIS1') ARMCD(ref = 'PBO') USUBJID;
+MODEL FEV1 = AVISIT|ARMCD / ddfm=satterthwaite solution chisq;
+RANDOM AVISIT / subject=USUBJID type=TOEP;
+
+``` +### `mmrm` + +```{=html} +
mmrm(
+  formula = FEV1 ~ ARMCD * AVISIT + toep(AVISIT | USUBJID),
+  data = fev_data
+)
+
+``` +## Unstructured + +### `PROC GLIMMIX` + +```{=html} +
PROC GLIMMIX DATA = fev_data;
+CLASS AVISIT(ref = 'VIS1') ARMCD(ref = 'PBO') USUBJID;
+MODEL FEV1 = ARMCD|AVISIT / ddfm=satterthwaite solution chisq;
+RANDOM AVISIT / subject=USUBJID type=un;
+
+``` +### `mmrm` + +```{=html} +
mmrm(
+  formula = FEV1 ~ ARMCD * AVISIT + us(AVISIT | USUBJID),
+  data = fev_data
+)
+
+``` +### `gls` + +```{=html} +
gls(
+  formula = FEV1 ~  ARMCD * AVISIT,
+  data = fev_data,
+  correlation = corSymm(form = ~AVISIT | USUBJID),
+  weights = varIdent(form = ~1|AVISIT),
+  na.action = na.omit
+)
+
``` +### `lmer` -For the derived contrasts +```{=html} +
lmer(
+  FEV1 ~ ARMCD * AVISIT + (0 + AVISIT | USUBJID),
+  data = fev_data,
+  control = lmerControl(check.nobs.vs.nRE = "ignore"),
+  na.action = na.omit
+)
+
+``` +### `glmmTMB` + +```{=html} +
glmmTMB(
+  FEV1 ~ ARMCD * AVISIT + us(0 + AVISIT | USUBJID),
+  dispformula = ~ 0,
+  data = fev_data
+)
+
+``` +# Benchmarking + +Next, the MMRM fitting procedures are compared using the FEV and BCVA datasets. FEV1 measurements are modeled as a function of race, treatment arm, visit number, and the interaction between the treatment arm and the visit number. Change in BCVA is assumed to be a function of race, baseline BCVA, treatment arm, visit number, and the treatment--visit interaction. In both datasets, repeated measures are modeled using an unstructured covariance matrix. The implementations' convergence times are evaluated first, followed by a comparison of their estimates. Finally, we fit these procedures on simulated BCVA-like data to assess the impact of missingness on convergence rates. + +## Convergence Times + +### FEV Data + +The `mmrm`, `PROC GLIMMIX`, `gls`, `lmer`, and `glmmTMB` functions are applied to the FEV dataset 10 times. The convergence times are recorded for each replicate and are reported in the table below. ```{r} -#| echo: false -#| warning: false -#| message: false -plots$contrasts_bland_alt_estimate$`Unstructured (KR)` -plots$contrasts_bland_alt_estimate$`Unstructured (satterthwaite)` -plots$contrasts_bland_alt_estimate$`Heterogeneous Compound Symmetry` -plots$contrasts_bland_alt_estimate$`Heterogeneous First Order Autoregressive` +#| message: FALSE +#| warning: FALSE +#| echo: FALSE + +# format table in markdown +cached_mmrm_results$conv_time_fev |> + arrange(median) |> + transmute( + Implementation = expression, + Median = median, + `First Quartile` = lower, + `Third Quartile` = upper + ) |> + knitr::kable( + caption = "Comparison of convergence times: milliseconds", digits = 2 + ) ``` -and corresponding 95%CI widths +It is clear from these results that `mmrm` converges significantly faster than other R functions. Though not demonstrated here, this is generally true regardless of the sample size and covariance structure used. `mmrm` is faster than `PROC GLIMMIX`. + +### BCVA Data + +The MMRM implementations are now applied to the BCVA dataset 10 times. The convergence times are presented below. ```{r} -#| echo: false -#| warning: false -#| message: false -plots$contrasts_bland_alt_ci$`Unstructured (KR)` -plots$contrasts_bland_alt_ci$`Unstructured (satterthwaite)` -plots$contrasts_bland_alt_ci$`Heterogeneous Compound Symmetry` -plots$contrasts_bland_alt_ci$`Heterogeneous First Order Autoregressive` +#| message: FALSE +#| warning: FALSE +#| echo: FALSE +# format table in markdown +cached_mmrm_results$conv_time_bcva |> + arrange(median) |> + transmute( + Implementation = expression, + Median = median, + `First Quartile` = lower, + `Third Quartile` = upper + ) |> + knitr::kable( + caption = "Comparison of convergence times: seconds", digits = 2 + ) ``` -### Analysis of SAS and R Comparison +We again find that `mmrm` produces the fastest convergence times on average. + +## Marginal Treatment Effect Estimates Comparison + +We next estimate the marginal mean treatment effects for each visit in the FEV and BCVA datasets using the MMRM fitting procedures. All R implementations' estimates are reported relative to `PROC GLIMMIX`'s estimates. Convergence status is also reported. + +### FEV Data + +```{r review-treatment-fev} +#| message: FALSE +#| warning: FALSE +#| echo: FALSE +# plot estimates +ggplot( + cached_mmrm_results$rel_diff_ests_tbl_fev, + aes(x = parameter, y = rel_diff, color = estimator, shape = converged) +) + + geom_point(position = position_dodge(width = 0.5)) + + geom_hline(yintercept = 0, linetype = 2, alpha = 0.5) + + scale_color_discrete(name = "Procedure") + + scale_shape_discrete(name = "Convergence") + + ylab("Relative Difference") + + xlab("Marginal Treatment Effect") + + ggtitle("Average Treatment Effect Estimates Relative to SAS Estimates") + + theme_classic() +``` + +The R procedures' estimates are very similar to those output by `PROC GLIMMIX`, though `mmrm` and `gls` generate the estimates that are closest to those produced when using SAS. All methods converge using their default optimization arguments. + +### BCVA Data + +```{r review-treatment-bcva} +#| message: FALSE +#| warning: FALSE +#| echo: FALSE +# plot estimates +ggplot( + cached_mmrm_results$rel_diff_ests_tbl_bcva, + aes(x = parameter, y = rel_diff, color = estimator, shape = converged) +) + + geom_point(position = position_dodge(width = 0.5)) + + geom_hline(yintercept = 0, linetype = 2, alpha = 0.5) + + scale_color_discrete(name = "Procedure") + + scale_shape_discrete(name = "Convergence") + + ylab("Relative Difference") + + xlab("Marginal Treatment Effect") + + ggtitle("Average Treatment Effect Estimates Relative to SAS Estimates") + + theme_classic() + +# excluding glmmTMB +cached_mmrm_results$rel_diff_ests_tbl_bcva |> + dplyr::filter(estimator != "glmmTMB") |> + ggplot( + aes(x = parameter, y = rel_diff, color = estimator, shape = converged) + ) + + geom_point(position = position_dodge(width = 0.5)) + + geom_hline(yintercept = 0, linetype = 2, alpha = 0.5) + + scale_color_discrete(name = "Procedure") + + scale_shape_discrete(name = "Convergence") + + ylab("Relative Difference") + + xlab("Marginal Treatment Effect") + + ggtitle( + "Average Treatment Effect Estimates Relative to SAS Estimates + (Excluding glmmTMB)" + ) + + theme_classic() +``` + +`mmrm`, `gls` and `lmer` produce estimates that are virtually identical to `PROC GLIMMIX`'s, while `glmmTMB` does not. This is likely explained by `glmmTMB`'s failure to converge. Note too that `lmer` fails to converge. + +## Impact of Missing Data on Convergence Rates + +The results of the previous benchmark suggest that the amount of patients missing from later time points affect certain implementations' capacity to converge. We investigate this further by simulating data using a data-generating process similar to that of the BCVA datasets, though with various rates of patient dropout. + +Ten datasets of 200 patients are generated each of the following levels of missingness: none, mild, moderate, and high. In all scenarios, observations are missing at random. The number patients observed at each visit is obtained for one replicated dataset at each level of missingness is presented in the table below. + +```{r review-missingness-table, warning=FALSE, message=FALSE, echo=FALSE} +## construct the table +cached_mmrm_results$df_missingness |> + kable(caption = "Number of patients per visit") +``` + +The convergence rates of all implementations for stratified by missingness level is presented in the plot below. + +```{r review-convergence-rate-missingness} +#| message: FALSE +#| warning: FALSE +#| echo: FALSE +## plot the convergence rates +cached_mmrm_results$conv_rate |> + mutate( + missingness = factor( + missingness, + levels = c("none", "mild", "moderate", "high") + ) + ) |> + ggplot(aes(x = method, y = convergence_rate)) + + geom_point() + + facet_grid(rows = vars(missingness)) + + xlab("Method") + + ylab("Convergence Rate (10 Replicates)") + + ggtitle("Convergence Rates by Missingness Levels") + + scale_y_continuous(labels = scales::percent_format(accuracy = 1)) + + theme_bw() +``` -Using SAS PROC MIXED and R functions such as gls, lmer, mod_grid, and mod_emm, results were broadly aligned. Results not being exact can be attributed to many factors such as rounding precision, data handling, and many other internal processing nuances. However, Bland-Altman type plots showed small but randomly distributed differences across a broad range of parameters from the input data. Apart from a small subset of the parameters, there were no trends observed which would have suggested systemic differences between the languages. These analyses were based on a single set of data so more research must be done. However, based on comparing R documentation with SAS documentation, as well as the results displayed above in this paper, it is evident that the R and the SAS methods cover do produce similarly valid results for the options which were tested. +`mmrm`, `gls`, and `PROC GLIMMIX` are resilient to missingness, only exhibiting some convergence problems in the scenarios with the most missingness. These implementations converged in all the other scenarios' replicates. `glmmTMB`, on the other hand, has convergence issues in the no-, mild-, and high-missingness datasets, with the worst convergence rate occurring in the datasets with the most dropout. Finally, `lmer` is unreliable in all scenarios, suggesting that it's convergence issues stem from something other than the missing observations. -### Future work +Note that the default optimization schemes are used for each method; these schemes can be modified to potentially improve convergence rates. -- Run SAS code by also removing assessments at `avisitn=0` from the response variable, and using `trtp` (or `trtpn`) and `avisit` (or `avisitn`) -- Investigating the differences -- Implement `lmer` equivalent to MMRM with compound symmetry -- Comparisons for other models, i.e. only random, random and repeated, no repeated +A more comprehensive simulation study using data-generating processes similar to the one used here is outlined in the [`simulations/missing-data-benchmarks`](https://github.com/openpharma/mmrm/tree/main/simulations/missing-data-benchmarks) subdirectory. In addition to assessing the effect of missing data on software convergence rates, we also evaluate these methods' fit times and empirical bias, variance, 95% coverage rates, type I error rates and type II error rates. `mmrm` is found to be the most most robust software for fitting MMRMs in scenarios where a large proportion of patients are missing from the last time points. Additionally, `mmrm` has the fastest average fit times regardless of the amount of missingness. All implementations considered produce similar empirical biases, variances, 95% coverage rates, type I error rates and type II error rates. diff --git a/R/anova_cache/html/__packages b/R/anova_cache/html/__packages deleted file mode 100644 index e9619db91..000000000 --- a/R/anova_cache/html/__packages +++ /dev/null @@ -1,12 +0,0 @@ -tidyverse -ggplot2 -tibble -tidyr -readr -purrr -dplyr -stringr -forcats -emmeans -broom -knitr diff --git a/R/anova_cache/html/unnamed-chunk-2_a959c7d69d77a7d483cef0c7910ca827.RData b/R/anova_cache/html/unnamed-chunk-2_a959c7d69d77a7d483cef0c7910ca827.RData deleted file mode 100644 index 974e1cf991dde4fee7f877ea1bf499f18fcb6ae3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2813 zcmVXiwFP!000001ErQ}I1~yShR3ckwmBoD82irH8pgg1LY9f_F=ob+eP;{? zMJJ_6l08e=*HpGaq0WeEWGUM;_O-}P(s91){P?c#@AvDypXYk7=Y4*@C=(|&pc4=P z05AcV85safET;%yaj-X4)C8P5r-A{%3^;Q-K}sle5H>haDIABxo@!12fH4dJkSY6T z=dY_xKn^|6bsqN1!Q4LJr+!7XZgV^es=7t~iYqHkOMLI%b8)qMEoYl4tf}qc{`8xC zAz(ESW1161FKN9Xt=2$%S;_!m$+FP3;%7>n;uLvtC=n++3Oq=99$B{F@e%F~7&C8w zMJzQv>217Wjy9M)peD#ZQ+h-l9&ZOvu9Zt8TvU^+&acMhiZlwGi8SZC1u6@)n@yIL z>&)oJ!QUM9)93&jfeb#@xRK8$f^0=Qm9Fd`uEtkf9||45opQ#;5PYv}{)tY6y3cY+ zXD8^szB+ixEidp==>pWjEpd)C#@fV%CLyNItKI$GJd||dkjM)hxyIkgRF$^v;@YvT zcio{k1(YdnGxch$gY50o0FK`>>MQQ%S6VIh z3%n~1;u-WG2|b`Jkd3&^n08dy?(WCe41J@3XKzr4MylJ1MtpzhbTtqgk6#iX@95@A zd*Zh|4-Iz$v9N?M3D$NUt8H8g9|k1#TJVlMdW6Hv?cHnblMh2_LbZ!byvf=(Kf4o0 z5RMEsO?pw2Los_za;_$n={7+KfW01M^m2^k`~#WHBmUVH7WSqY+{cDzPFpiZ&@7f}VWmoNX6gQYoy3=43c{P?I@1nuvW>FWDY+Yi99#`R3xuyDU> zRSP2tHFWcT#gV3W&$po5!$8Upze%Veq!)26*3<{WmGQC9Yc$w2`pO6kiZMp8_&9f{%+4sHlU5&T*Qccj3C2ODiQ?{z3%UuAHiEy;b8` zJh#*zg~-M9uHy~%hbS{Pm-i{Ws%b=the6}aAI@;p94b{^$ON+;HtdR>gY%XcJHb^L2wk+9+gDAoyq8c+}Ll(&98G8;n_o2?M{t zMhKp{f{T=&6pL)*<9LnYLR{vM)G?l0O?2&@Cwe)Yhh$+KWk69B#H|OK)MDp@fXrd8 z=O-Hh;I)`5bb=6O@M5qiozGN8Y*27P#`atj2zgSfGc$OwW&HGb(+A{e(rbC%iTQgJ zB3H+D(OZI zpKGbZ{OD^~xxsJPOy3QOb(E3jN~P&US|bw)%`9--P&kt%R4Hx*mdyjyUc~AYv##2H zxWruQ$*^xOX{!h7XiS*uS0$p*wM3$tkC1&3rEK{i!0X7bQN;E=;FW@1yGjJ#x6I7I zuvt%eB=a2; zKf%6C^=og!w^0u;HS43F)3WZbD12HRrg$1@l~_roEJ58N)yXtU|8gPm!8%NurlXsf z7iP@q5i*y`D>1zDL6gEdoj1oBNv#s)C8L(TvSJHfChyKvU}neiRJJlBCg ztk$Kf==cKt((-WbN^tScoo}k)WWDJxVcaF#oBT@#CfZZ>PVdWP-}fib~=sS@7KPh-I%tR~chJnKzRMs) zG>_Xi-?mSsHthgWH$!`sKe5Y^S%jlMQlqFFp^6G)E|i^F3SXw(c&z6lTPw)fJb^Bu zGU4^MSFf@xIok@@kWUDGysO&rm==;N;%d^k8UBSgKM{}HK*2=nILLENv-So7wMGUZ z2gSt=$z-wiX)59NXRpV(sO9Ju>>xy=?wF6;dCNC#k0@__bLXMcvs=hlnZciD2P@mw zR#`VM)~Kz->OYis5!y<>7Gl7J8mhV%1rWORL~)%(>h95gPP_69`<_-t1=O0Qm{OpH ze#favFOROQ>mp7@y3sNWx@$I%HtVXGaOVupzWxGtJTkm1)wta?a}Q!3>|Lr9Ez&|H z*Cdf>&Z(?dwEN_W%Xht4w>_FvF0?Y6NMw5}<&`zh6L3+jRmpscVVmN$i@BKY)pUHP zJi#i6vziUQ@Uwge-|g$` zEloNGra(3Wy;at2 z9(XnI?24G>e1f2)p<^%D&PYhWw9}@a-Z4HibNrwVd-}>QBrJqn^sY;?pkem*OZ|W z2e3Cx^7p6jgi9E^PMRc^^G;=pv%A8tAs-YPMLHBzsAtWIcm2Viq$@U&UM*%?VBQ&L z5<1poH)rA=Fpm-v01sUJxfJd(*TgAYxaVFCg)EzHNcv^wD33gSy*zn+m2 zIw&{_?SVmKd~wRY5rF~9kwHP|Kpa*P;~N>|uc+#Q*3!_zz_Ay#uy8mUj#bxG!{EGO z7`T=S41?BGg`cMI{8wT<0)c?WDPeE{0Z?CbxUa7C|9wF^9tiaG^o&yqgZ?L15fhBX z#RCHZJpxZ50IH)14G5Hpg(g6;VUa#ic_`K|9ET3a{X?gQ+`m!*iuLyRmu3G61fEts PZG7@K^(tEx918#dsR((q diff --git a/R/anova_cache/html/unnamed-chunk-2_a959c7d69d77a7d483cef0c7910ca827.rdb b/R/anova_cache/html/unnamed-chunk-2_a959c7d69d77a7d483cef0c7910ca827.rdb deleted file mode 100644 index f8496e225e51014017af4198591420140ce9190b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3837 zcmV4ob8$mTolzBz-J!_@(}P5@-P)20Rk#4@^I-4BB*HHd6WSuaxQfnj`7Fk3uy zI(@*~G#m&|Ep^-_hNn@78s4PK-HCzq)Nu}Xp>>=XvfH0LfD0V6FSpTon1Sa-^ZL`i~NO`bSV4@ZSI9upEZa+Oq`tnp;($y6z`{R|a< zLq&uXE15zqm#F1_ES;&`gKj2-Rf{h5@3K_XWvRXe|1Mkb@3IAdvg8qRa|rH9yDb6t zp*aJ0DWHx#Vua59#9&(ANc5xCm+_wTYq(8N&ciS_q~9XkXgCv^WXn)eA`x||O#(s~_ip!J@FkY+brpsP>#(ssRy_oq1x zA4J=`4If6cfbiB`CwRK+gfCs3kNW)@Zb5SFgLn@0Ls#!Op=(Ep4?yh0Lnn}~eh<`!BlYS?p)$M=DU`FE zNa;(83n{KtD3L-rwLd8XNEt|qJ1K)m8BEFxqLoRj}#Fp z0i*2l|vDBzNxz&KhzL z4{aXUeD30>1UNrfIxatz2wV7tWpr>h?oc+drq8rdgvheV=VAF2+;45swf@%y4p*-V--Fibe^A&AWl!W1(#$q zx*f)yooCg-K9%5C`C`?B=;63W*3RZxE3V+8$mq7=M^wZ>^(&4WQd~j9tr?D2Ad=;^jzjy)e9N&2D4&MbIuv>Ghoo9!8=tvBo z^uDd|(QU%c&wE6#_6ow^aZ*3~RX+H@`q@VsE^dZ5dmlJ;kADuwqWcc=+i(eddDQAe ze$W7@om{o=)6x*4f33PXe&l>Qu@J_@dY>`{Atzq`?o*9`>YIH=e`Al zKdlUZ_4?ixeCR<3$+doC3EQ^P`O+0FxI5B)LT}{ty;On=tiqP%ra0iP9~S-~jHn?7 z1ijGvVZD={!EU`)ENWgsI3&O9*}i``(VM0(;gE1Z8u`|zguiQYOm*IDcyq7C2hLaL z5COCvPk4`NDEvZPX^=7wbiDk35B|DR>@Pke>P8P{G_loByt?&G6yj9G7-rprI2myo zF2uvRGiUd<5zTn~wCv zm4lee(iyJD2O_(4#1I9De0&%^K?Wd>Vo`(`hC`WbXN*Ug5j56WJmk)r0qk_79yLHlkohvfz)WgPRKXaz^udgkN z-2Jo&k5^>;60=+MN0AWRme1I8$^%wsu2`0LITlp!;rD3V_z!S8YGbIwiE`X_%G`^u zWafb_S2CvF9sfG4+_9i{%xou9Wa@FUD0R@n47(OC!t$o_t!r8XP0`$T^ep}4jh^AT zJK2PX&wIy;3eIrR)ctzyOIi{uZhe)Duq@emUl1lVm*%!hSn&V$UFE;CLDeBGkrCP( zY#15aR@l7(Zr^_8+VNh8;luEh>>d?;aI0&#q}|uH!!I>ky0?tqVUj0G9N{iOUoL{9 zFI}kp#`!iUH-?_Om^#$Q6yd?D3hjCz(sQ?d#Z_Q90Mx1bfER&Hs4e94^Ab-8{GD zF-J^t^V#B<7Zzo35qz`5&!xY$tGR?bK792KZ>=XMuh99q2L4#x|KuE}{oq#4@wqLM z+>(qaYCQG_*I&1PP-RyH3nsLt``-T!)(4zeH z@Z4svW1yw5dP>}A2hh^G<818mO0YIPf3IZj9e8Q~oD;t3yI|w`*w>FQvjD$l{2W$a zDh8+Ozw>|z&2X1>PjP1dC7jGTDtfnYO}HZC+{#9;igrMPB*OGO%arRzv-nf?T&E~R+ql(dcEhr%q2LI^>yCazSp_P zyiagDdgu4AWjKhpYy(3V=f5r97qXuB^Q4)uu4(78`LQ=Z)4L^;q?jBwewA}*V@?6s zc+59K)lCdmof&pqU7-bcq-n2h3fKaO`@H*atiH|3ni=5oa>*Gk!sT!8t6K8PB$KRi zT0Op58*VoD$3Duv{;;_d)7!%K*SuH9MNo7fqu zg8e`Jed&U_-dtqnPk5UCy&?iPW^KKAjla5N4ZZ&LhZhEpI&;iEA6)v-|7V{m+d%cS zbwywQk`5aumV`#v>;@Ub4qEmJJ_dj2cR0Ycz6P}V)K-@&-U0`QL}+k_4^2{?I5x3| zlbDNeg?C(gTY05Prtgzab6lEjF2UJBYw*}Q8#dv4?$h>&?)KtjO{PsoX>*ZD7B>|4 zvWv^)^C!G8bKR-AH}8UF3{fj8Q4ldP!-YOh|ql#5W4W`mt*ylOV9Dqj9Tk1O-j_LBx(&AfP5wZ9Skbi z(-nL*D!D|HAWzhAUN-t4n$PUZpt^+4Em6m?jz3T6%~GJ%x%Q~mBHcuraC!|3set{--?2nb+L}T=CRi~D!Dp2DUr6FRmX~*3l>yg)Scqns6AWXh{RNd zDk%{q{zLZjl2vjU87?SNdTSpeI&7%G45x-)D$w|bzO+nCNsy})Qf4mJ#Su$0&Tw-B z0i}oy%TlA%G&t{+V(7Yqsq0AS4F}Zg)NSC2&_y^Ni8uz)3(+5O6e6CkV+czRKswNo zutIt;i$RED#6ZL_M8+O3Ryp$=z_OEh4r33~E{^DcI6_Czg7i>U|Ar#EBf1&aC;DPW z{nWR^)MxAnVwEGm5g5{a5l16>7!&Oz*4wME&-fqNrT1rV4 zQiQ)xjMv6Wd1}W1Xx^0F_1Z(9%XH3g`P3XAJn4Pxd(H9rw)383 zFf^Wsbkl);Inax0%*-*pkM@6lihqvnyT_Z)otWNqF@^VKt{L>+xr?&LnW0^%x@b0b zeT+Bdrv1O0YK>P(s-!eu=1b3??5jA;%JPj@kuG7oDpBG~zFtLA;-BNkZKqMP z2b0dyBzb(iLaHFwxN3bI$<+#3ibSbTKeCEks=u!LmY_Sgi&tnga+z{lm;q%dqiykhOD3x7}5f%waGP%-(E9f7!*rWdd9sh`6YZjI% diff --git a/R/anova_cache/html/unnamed-chunk-2_a959c7d69d77a7d483cef0c7910ca827.rdx b/R/anova_cache/html/unnamed-chunk-2_a959c7d69d77a7d483cef0c7910ca827.rdx deleted file mode 100644 index 40aa6461a96c37adc77d51f2fa3408ef9cf0c051..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 148 zcmb2|=3oE==I#ec2?+^l35iK532CfGk`d0%cS>|6BxbZ66cU&v*iqRyqmz;6$3G*6 z(uSjv5*CiLJ_!{EcQ^7dsyOz|x%1=BpD9;l6n@KJQDEzEJN7?dE02IBo8Ai5o{Udt z0ybNm7V0(7MK^xX11dhYgNc3PhlT0S9*ODNR zAOi%_y7Fz!Uk9qd5~hArGUEH_(gDzGl-x4!AsJoNuG}EK@$stna*v-#N!1a?F6_~r zL@$#6r1acUaA=FPNe}P8d}pgBx`J~%UjQa@&Tf~3yiojGNu^sWYBAa`z{^K&hFux) zeP-wn>b9-B%gML-(p-7e7H`(G%#YK)0l&!ad|e0anJm!6dZCUuY?+A3P^whe5N0cP z9C9VtWiUZory;e8Z1`aHQC~mENg-8EG^XpN)h4mr`J#j3PY=Wv9_|S3JdwD;$sBt6 z%J5YamVUt4<%R~xnce!(_uiSod-F#CH}Ci%ZntQa6rPKHy9IsfYjwwwtt%W^aMvOE z2BG5QIj@8DbEb#gniC;uDrBem+HE4G3s-sB5iioZlT%Nr9TWa5;mN>tv}n6{T`x%D ztCs$_kL~_Dc1zj*h0C!5yDuu9Z5`oRNbM1tM~a;$Fo$y3|k9O6fjrQ#Y-Nc9$__uk5R9nyl+NAKIr_ zHYy~WV0`SQ52p+3F5pyU8s5_pxmcxh(5kh!b`uOFejQ?QyPNCz$vQzs_2oeX<3TmF zMJ!r4dD8rj4_on&>u7&=^iBCa4?GqgDWSmP1!JFe5Xo*KoeGj)K0Pza1s=6%HMV>z zdjukh#@v5%)XwLCLm7grSkWZ^j?C>{*iwV@VL*1A|DcA(YL1e<9QkLr6~#u~EX9an z9&IETc1UE6Mhn5$5Y||6|t6K){>ZT&O=(=47C(>MBj)uaQ5KcVQ9Dn2E&R+j)5F+4rdy>L zIxMz)d0T)++Lv_m0PB@b$Lh|6^6m0BgUqE3joU@_t_MX}L0H0v&D#53z}gk)pJQH! zzx|M(mn`H&+>>f0Ag?eT5-nUP#mzbrWnC*LaStbj-6x@LVmUMsf3SI&LvY&yJ4bZa zf`qqXkDoQ-=Xowu(>kSYewn61CnNXwvlWT$26W|qIU6mN_Dv&N&YPD!}a-|OZd;CC^ri-j1 z|Gmm59V=vcuQebjF@(FB5(IY>VZY0^ir8J(W|=7S*gAd6{)w(@a-BWYz_~mxr=zBw zl~nD=8}`1>yJVrIIowc|m0@opeDNKl!0anxpyjCA6wbnEyvXKFa-|R#FRZXukuz{k zu}IYds+|ccGa#AdiB32_-7B2$C$MCz?raLFuZ(;92*ttS%Qzf#fTAm{^~%^Xm9R>w zRC0a-x~uC_hh)ipNJ|T54Ek-i7CvdUW5MqT$98Z|5L9F-I?*n*Mqb&>y7vrpHf#9UCUo4mQKBRnSAcdxbE}O&Q?E*otGUn65j(p zu#$wn)<C1%|4S|@tF)uN#Bs6Bw4l2`KLy$qP>|zl41N}Wmz6>jBqY0`*y<8 znKO6G?Bh*b+rzo`+51(r;~t8iH>!ac_Ag~KH2|nvvMMJI?F$=5{;Ac?&BF#IT;zYX zoS-H(K=dUV&ZWiH|Lr#g{^T&FQQRM$y*s~vDP083n?Lyh#pIdxzG6sUo|}<>Z)Rou z*41M;o{L=(Gz&RC=xqbf8sY(}lV(8vWcXc=S|cO^NFw<7wK1su(~F71W+qzScjPIdR{0 z3eXx}kti#An8_pCTF*N96d;aS?zn1gA#x`@tI2Efg$Fl&&YoS-rV0x%5wMtaY6dO5 zcjDd7Wjg!0VN`b1_u29T#~|1U8Sf8)wa@dN>Ojig=B~&;Nn>H>*sFgeh4ZIF^>n+v zTIUB_<I=zGI%rin!+a&1 z6(3EW#vzo-C3r(sgRW-OG7Gbi<-EL#1fEJ=FQ0wlCE-#AZY-jP)DFXxugOV!&ITqg za<)4jn?LVyb&I*P6W(|5(zfzaXNhyphur=G~WS)>JyPI#XUOMBZ$+@!l&# z_f_*#8kKX61E*oObpL#Kgi;NMS9*lo=b0qB-?&95uV8^7I_F!3+QEi_ibpC(M<-e z^@rrEq(~9BX&I+a7j7+NSU+CcZ93LF*wKOE4mAtd@LG!B2r*o8`}Po%;$-h9N?-U( zcXr+*opsFa-uSaFowZijOt3$4%IPejLS{i}Px^BhP7~<9CAL9L4Pl(zd+nsb#YF{mX5<1)8ri^GzNi|JTj#8J53ev{vN3BE_vg(!P zzD{;d-hs}Se=tt)FVc~BiVGY_(ALn%WA0X)zdHS*B(o;Iv0YGsS|VWJe~#XKOF31< zus0|F;~r&hR6*^|+~!@AWd)6I&EXQ@W`_FL=O5X7i7pL&>LkgNMKzv2x_@^B;P5!^ z$|l|GqUo^*Kgl22rSc}FM8ziCwjstUw7be>$jXO0j8jyAwi$eW&-5LtlH8WF=u-l~ z#%!k5DQW3Dx~|&Os&HeWRLhde+m9~zeb5GznjVC|nWKZ|Hs|`kCtKYGg1r^ry-66?PQMK-3uL^j zt=_sprpt{UHn{4Z7ja8Y|TN)8lygb$^#5Z9tj zFI(s~zentpVtN*DiELBLK=1v}zn%pgPosz9eTjHtAbCe1E10??j7Gx;lSz8Sz%W{n z9@-ayMkCP#L%b0NrB5Om;tlon5hMcx|Mh?|LZA)SRb;*^f;j7nf-zVu7Knip$y6#3 zh-U`w()_=7(2NCx{rvo5;0)lqt{#z2BFBQM!M?%k2^CN`RR_QT5J1J#h-5A8pLGZ& zoX+y4(rE!?h88)T%!pzIQfL8M+S=dLz~Fz4>e0hkAz>`w*FulUBCTh!-`A=F<^YRA zAW+Fz0Eq&cF9N9b4S;X20sMZ2cj^8ei=68TRA1BL*V9^gO;`bqRqNEG}BGXl)_TK?Lmk3sxo zI{}OTFhj#142b|p|6BxbZ66cUi)h?vp9qRh4e8Sk7_lb_T&M6IQ{CAi@xlbwUtfQ9%e1b#b=VH-rdQ1c?wt1hQT*!;mC$FfAkq8Ahkm z1i2IhvW^LXXkS|1^XI-QsHBO{gsjA#5E(!Cm4FLno&&Oa=p7dV^+!jm-pfCJA}w7< z89BF0ZyYmE|2?4VhN68-lyyc#@5NhNv@jK%oB2Wr(KEI?>=lF)-byLoSX7VIc>!HG zbUpmikoPmZFQn6^?k*?a`fFq5VH>h(*8)FY=Nj@nzx{O`ylcEbi|BzqWWRYVHdDD$ zaecUr{88AYAgBIB9o>esJ9PYmFOPb9Ar6XZ@?xekpS(19*_eg(8@brf06Vo43xDsK6|_5l2ypRC7~pn_RY{Y%#J8IYoY2nmN~fELc;?c(glAFWCqBoOwtZ-Z_aElBb(GX` z%;kSEzFonoT)N3dJnpqY&Z`~uSu~psqJiZ{+q+LUi<9JK(1279ffyvZui{g8j=?iw*Bk}bqrB!nC`Kce|l?C zT=ex)oH*{$21@j+pvR%fX5ioCb%(e!8N7FtK*sBiwCS>!Q(YgQnCU#Ey`fv$5;NLv zSo6JP@M<0#O60J2hxFk|a7UAiikm=gQgT;t-oYaqpYU&#W@NQz(Slk$6UUEvs|GW1 zuIo0oreqvc9X&M@SE0m>+_9!CD&>=_4d-F#97iVagQfSa_AI}`q|*uBF}8H z+>GpEJ5`I}gW?MpxB6+Of6XxSvs!H5Tiu>mzD?nJpqUKbs8vk=YG9-#j3x4$X=~35 zM5|)RU$L(v-hRl>OA&UU?n<*1Qc#=>jS(r7=4Ky?wyKqvyhoBI?vd0mwip;o*xxwF zp}1^DoT0jE!y=lAM^78_^V}C0>Fv_jzfLmX<59o%vX!W<1|cfF^48j_ts9559XD0M ztiI+O_q8reTU=S1@`JfrHrj1=6ZsfH%rthK^2na&L9aYVvQF5C59#Sa%zmG3RwBnOx6k<;;@I@htb-I;h>f*MFVU9> z?%%FSq`6MvhOC;(8ZzBP*ti(B^ibtfi2xq;aE!BW>7M)c&x`=k4U)WE%O=NRC*^+A zZEF6;C*Iq?>ie?v+g;-HwSa>VE2$2aZ1x+85q>~jRZJ~^fm%B8;$_O2Q=@v%hufQd z%n29mHIv>0UWk&!p5{j*xt!AzpjMCZj)W|xg_L*bK(d^A`|MN0X0h(90jY3)v5Fjz zG(tHOopUqs^QlvJ%j^=2om(Tgb~(SQ>cl^kIBQq~!}os93DX4NE-7l9c#Jn<5cRuu zCpV86n0TK5&0>s^+yKK$Hk?V1tN+_)0{Y2*LbJFxCTC}UL6}StJa6{+2Xq+Er27?9 z=Hkqh!h2Ioqqoj(g9%*X;yTmN)0d`=@7u;AWUj{FPd)avH*Lo(dtJOKA32T5@1Vf7 zYvMFuX2ID!PVNJ=_`TzI{Zf0dQ3v)lOKv<@_f@l(n~jW_h^i@Zo@YKye^3<*h@%5r z)6187$Fs)nJ5K=GgNu@7MGv!hbQ`N_2d@I~h{g8HR_3C&GP3V@jK6T>Cd}BeD_YbL ze#S!P;|`6Gx%Updn-{_yeOz#=+v|I5`2Hg>Vx+9+2mjjV`3`ju6;Cr~)bF&B@H6Z$ zza&TSClBiDb$T?<_BYF?JH3wf85X|UVaMApz+ z5xDXtc^UU<|CD*ow!KGY&$?YZYIQd~mN=Gm2Pl#%s`0_cUZhN1^JRz+R*ejaXoI@Z#mNM)uh%73<&FKpF>`61dQ#$o! zs=Qd3zR7gMy;peGFJ>n+D`#%^okZA#_~s)cm1{V>(nH)H_hhkOjhb}x3g#$cGu~CG zEw@B0)zA4Nvo2K+vgr}GiWBAanNw+t~KOQYGT}p?6pDVFBqm)$!-( z+ZeUay&GC&^->^9y`lN4snXP~+C~|ZgM{gqcnk zdUjEhS$`_h3!HXd2Mie<#@|D@LQn}%4+gPm1Q1D0NlL6VXpyv5+J z_hH@xRZ?4X=e_@xG+FF|h%VevDBbi~_3cL|@*a4DaZMNKYr5B7YpY}Z-{Z}$LP4HN z@7^Si>SWvm7KAe2)mCp=FWcb~65jvCH81jpyy(5H5+T;u10IKMQwDL;Jx$7|H#bJ< zDwpglbc+~BWr0^>OfH)1HNFQ4(qZn!o1%X2kk{lcoLH4GSss8ls{;VLz z_VD0fauA)SPxTKE4%Ekbqp27IfqrLI0Tghr2pGJ?E=1OWrkFwp>5 z0kmjso#ky%=ywPDA>pjha2D`mq94Yh38J`_y=s7&0HshEbRvL41C8ea925Wm7M&Rc zd>`~MRzMJ$MF#{ZlFnp?Fa;5r6(CrquI>m$Mvxid^kvqV>d!O*ejLU$3d7`oMr#Qa zKmppjg9Xx5dMHb0^$wbV#H{VYBGH89*#K#X6-*!!TpcI_5DK6H8Z+Dv{iCct3W-^R z4Uiav4xj~u(aB+S^bfG0`oApRaMiQmge#u$YdTQCV2%B5Ti*~#`0w+g zkXUd<7iEC|m&LF4iuj|t0TQ>SOIXzvKn8fs@`MAw>l&{1>IYUI1tR}x2OtR1YsWBH zDS;+@?=xYwS3l{3NDKMIlk(&z_f?1mag%)W4ol=$}1fR>g2@#t29>xLOWeeJ3$1=U!o#zc0b3 TxO@qIAHMzv0uk8|f(`%xv3_sR diff --git a/R/anova_cache/html/unnamed-chunk-4_2c39991c34fb054f157959a554b2175c.rdb b/R/anova_cache/html/unnamed-chunk-4_2c39991c34fb054f157959a554b2175c.rdb deleted file mode 100644 index e69de29bb..000000000 diff --git a/R/anova_cache/html/unnamed-chunk-4_2c39991c34fb054f157959a554b2175c.rdx b/R/anova_cache/html/unnamed-chunk-4_2c39991c34fb054f157959a554b2175c.rdx deleted file mode 100644 index c28f3f9dbb3f1f2ce024b914f82eddfa9c83ac95..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 125 zcmb2|=3oE==I#ec2?+^l35iK532CfGk`d0%cS>|6BxbZ66cUi)h?vp9qRh4e8Sk7_lb_T&M6HI6iH|a zJwOBnR}>>Qx`0YYKvY@)1$04BgCG_J>CzOW35t4yclYkj{p-%$cjle*Jm2|#@AJ($ z=Z}N8a6-ci)}FE^m<(@n64md(u5kw5G=qm1N>p(k$dh*a=Hs*FYGrms zT1%Y)-Jm*6B&%z*X0($rkG}SdjsYBGGNc6K2VYt25h_?K^Avt|ETQy7fB3-JlwA&{ z;0rgVZW_{c17?d_TR|6%b-^FJa;S%kr=c!hNt3K0!8$Pl3-@jx>ip08{*(K+n3BN3 zKZS-cy9(SG@%Zs}!edv+fdXF3uC|ol8O#`0h5;h9VknPy<3SE zLig5ODRPpRuR;>P9L))k$sY zn?;N2K;eai{fd6{Ju?gsW7Mfzdq+mwA#s*gT`9I48_h?n*f@k_Jv-lodCmJkb3G|&c6Xe(%5V9WGwE&HOTt(1hlD(yrbr^wm`Ybd-ta%S&T%uajUQ4d z2TuuY799vs$^4OJ8eq9KU|&CwT&*d6H`r7hW6&q4eLFbX0z~IOVbVAH64ECV`YrxV z)Vq(xg=u^a#6uYtJkl~tVR8JWVyxVgv6f9zA`kImxFaIUhUSy=NuHfkOh1=>kjq3@ zHBeMH?#v}Dr_gGWcj1maJ+AG z#r)RxasbHHqSJPt8~>*$T(+U(vPbR)8+hvlWFVN)@uC~(D>6FHkgBxfJmdr_v|po_ z`5%@)XI~@9exm|H3c+1XQrl0Znmu<|F!oKB_==N<6Wne7V&1-xiYJ_X|;ST0>8|5P5$_0 z*)l~luzC)lN{?h%C^+x<>@a_^FV7ciWk(}WOKsx29wZY_sA4iv0kX~^-8W`8DSlsr zYULcC0v>2NH6!R!A2Tzl5fi?emi*@|bk==OGOZ_8c>!hSg7bZ1+vIJ1_n&u!GhCO^ z)0SOTt=Vq;40IerY`XTDa1fh#D$d!z@<@l>a|5WrZV`6AMVI4@liY-fredJs`44ul z$G>i$cz|2E6BG`x6dj~X){=a@#!?Ub?Z8R&5zIy^4?kLmsmQhfcuy z2M6~30-B?*3tB;^Wz?DzI19o>OL%<@Yt*+_0h0wIZun)W!h`W z+@v`25*{vBEyA9xn{YOvRGFECZ59^RB(oKoM>veLul%lO<7cBANPQ4=^|q9_`$}Ni z22<1i^x74-J7+8(WQM+)=qqblm=|2ut3=Jk8eh@$kX=bX9%jOa?=Qa)1&}>+Q+rWB z<^0z#SvDBDA9=6I?uV~2 zu3t^htJJQxjbDIRhx!-8qU9Qx?8=j@QTJ5AqXu0Xg(d5Lf~!7ti2aTH7D`wBVL3PI zrx@g@#`0t-?TBT?s+nwJ`+Ry_t0u!PM6^N(GyVQHh8C2WSD$#5tVwBl;nUhHp_K;M z9t$g0NEag>P&3F{D&1ciVfo~XvC-_vM1Mb;HQC8y#qKJ9%NOBPFnWlXnkd8i{qeLs>5%cva}oJ)vDnN^pa;kz41?cfH$d zTgAFCtsd~^aYA=UbgJ!kPeL|zziqwrIq=F@uROdhJ;NwnK2}*YHdq~}8@m-#N^g;u z`nfQ$&t@Co_vM? zW+cD((;<0QY)O-TLFd7Rs*<*MrZ5p;XN2<47oQl1iB7Gf$|TWqWex7$THpHv@B}<- zYme4#!K?(&&(bFkD!fgvP_W9gZjHAHAF6YjwD6`(;bmpOy?Wn1(0nHAL=WU|cvnCn zvsO#WL77=PgSQ^emPy}#taQwkmN>uk5|kly=Vm+)v&2R}$Pj;ISn;K$)>U8)u4Rm8 zsoVt;Te`bctm})yyH8GpBj8rUhGDqBiM^fb0mqi_3*D|fR4>{0Zx|Kmt2kEb7B!hphu?}b zDl*gR`~cS%qq&#wi|JL$MjigIe?2o8AtW@4;6o%31IappbSgzBG9-jRC6ly?fsrA> z+GrmH8HGe*h$IpbNko%${q&LkdVVM*3QN}0hoi6r?i7i?C-Pe2a5!i@j7X+Xpn(Kh z;6c^@J3%!82=w*!jfX`*f6dhsObX+fk&0)_VPd|fV&hGXF{EJE)Wgn)C8y#xJO_NM=oJFjM^xgXSze*wsR JHL9}=000xm@J#>! diff --git a/R/anova_cache/html/unnamed-chunk-5_1e3227cddc2c5d4b92f6b3238e69038a.rdb b/R/anova_cache/html/unnamed-chunk-5_1e3227cddc2c5d4b92f6b3238e69038a.rdb deleted file mode 100644 index e69de29bb..000000000 diff --git a/R/anova_cache/html/unnamed-chunk-5_1e3227cddc2c5d4b92f6b3238e69038a.rdx b/R/anova_cache/html/unnamed-chunk-5_1e3227cddc2c5d4b92f6b3238e69038a.rdx deleted file mode 100644 index c28f3f9dbb3f1f2ce024b914f82eddfa9c83ac95..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 125 zcmb2|=3oE==I#ec2?+^l35iK532CfGk`d0%cS>|6BxbZ66cUi)h?vp9qRh4e8Sk7_lb_T&M6#tHHmV{j9RX1h2sY4*f*J&|AV`;{AWiUc^#$H-^Jd=6n|a^NH)pMV_FC)P zv(Jx1vhhNLy}%$4NDL${DgqLd5JHfIhr68-0VG@z5)qI%Xp`^>L6FGd)CdNGMWa!L zTowcpWr9F@SJwCZdY}$0V;(T0AiXa_J{Wo}^kS9oh=L)?>|&VF`8vRKM{S+!fMT8a9vz$=Gu zMqe58e}NAI_1ZVx=N8+2Z>>FIPqye=5hUy1KwJ=Xzioo{O_%5reNcy;woN7Gs@1A& zindoe2D!p;8&1_XXvw-q!$11=WMB~FqLQT~nb`B%W{cFt#j=Ca&krP({@fMSeLQWG zizW2bmC>uFY)tUP<(3x6>D?IUN8dcg-r_O1hi}RVuUE28j?5#z--bTWT$?XwaL`t3fnpD~KsU13eM9|e!(Zsb< z`pNWe4YzjfHV5`}&>pd@c`RVna*;s=lHVjdyERWW${9ZI&^B*K@KiA8tnO>NH{H~C zE^0uvYFtb))#T`FKW-1vQ^cjtJf^QJZn@6jpiO&!;}#f5`Z~nwb}!HSi*2fc=Iet1 z^HDvtO)5b=W7_hLA4m0&`}kl%!cFBpkGz(isG-2pB@@3H02v;U-72!*zq}w^gg

IsM{nfUPC5eL5mPE`O;wdS7k2O6(`XowkK=8>h)aT=VzAskE!oyHV(v`u3L5j zKY6^S&*ln6%>6MVXd2Yj{IbecfR~ou7hV)~bjvfrt;(Fd?mQ}^!#8#MxW7g?6YD9u zr86TZN@Mc$LQ;(?FV^|G$Hh%lL$8R5+~!(?58iK&`%YB9IJ#LP`26X>-3P4~meWb; zZhE7?*);2R7oAGlt{KSQGS2bYjXrX1YyZGKSc0uxTbiT5VdtX-LIP5;o?mFez2LW_ zI&Sae<@qpG4zOf^a=q?UyE#E#=chv3@yxqxv(o2Cd!&4xr75B^xLW7J--^Gm&+{^I zOzhPthmK0ET;36^ll?u%GT3&t+qu3wwR)%W%`i)Oyh*2|(e_d|ZCV|-d5M3HA?BL8&iUjo^TF3%!n(sa4_~%}17rqS zje8f^WkGw@PdL{o^55z}P*Q-Wg&Kr(6L2)MucP!fby%kqdmRS4%hT4v3&5*1JlVx`AGHS(mWO0?X zn!=&8s%4s1Q2jhm6^?3JBst~!e6M(MfXG*SZC7(hb8YhbCnzq7T*c+0gH_$b+pkQl z&;!4P)~dNa1Kl@tYeKS>K4oV!n8N`(ZN*R6>{<#q%(Wk05CxT4OHOsltk(xH6Hkp%ak%6$)=UYNioHp}n}ZQ5MN-PDFHc4~&0 zp8V+aX6W16&-aP*H$tO8wz55Jh5UEY6N1pDx`al-3Z-)9)$5G2rzZ_xj(4{QSQ%e- z(ncoMyGr*pxP}V%W#;qjM|FvCDO6ZzcJ7xt(_C=O!fCpA%SMz|qAY=kUj8*Xc6w>&z*rf@i?rZ6e{DnFZeU|g>r}P-X zY?P56-u+?iy z^4{|y8Oz+A&PNx|dEGc>dp|qk^>Ald)G`l>uyi;TP!EYZYwTH*1JFY!R%4KYKtXDn(ogeH_2kvHOnP;lUYs)7{f|po`U?Oj3i&43u4xLxxiIb##hYyR{at&c3S1Ye>1SXU<-VTg!kpRrW+#gSVgI z?_I$pGKsgk#qhdhPSU|I%7@J~-ep#3*cI5fB-%vv*13(?_|Zp6sw&V9+;1OQ{v&m= zI|`TmD&VjQyIJkf?3`Ua*B(riDc^pqb-&=%8%J|M`#*-3YYYxD2Z?VchGU)2xqQYZCu zZo^kZaz8ZIZ{MWQ;}H=({LQl{_Lh>wgB{Wlc9=sxM;tOnv2p`#YG<~!#u=zp94PgQ z8OdY=*AmPxTN$=~1dQca-sRikI<#`pd;jBK&k8{fkBA}rQ^=GM+O7~bgT5;|Je}e(cxi6IDY~jg~U+m z(dqCIGAqPP_y11NO#*`h0s;~dO!$Vc5ha33O9Imw{tO|Y!?n$|!EpFSjGq1paCbL6X z>`==3Y9u0tOpm5SS%ig?UwV|VF-QV}fJWojDIk;~amGT3L=cRz z#>T+99*INZQ2_FfeBz%b!d*CF;{iMZg~OrOMMyk?fXCxE^Z*WzLSfhSXk#n^vyo4t zu|lcQWIF5L`52)d3lIPV0g2lXApzmo>!Lr(+47$d%-Oizzt4{qij2`1;e=Q$>c5Eo WsTW@DdJ8}4@Bae$CI!Sb4FCZ8jX!?? diff --git a/R/anova_cache/html/unnamed-chunk-6_87216dbd4f33263d00c668206b754202.rdb b/R/anova_cache/html/unnamed-chunk-6_87216dbd4f33263d00c668206b754202.rdb deleted file mode 100644 index e69de29bb..000000000 diff --git a/R/anova_cache/html/unnamed-chunk-6_87216dbd4f33263d00c668206b754202.rdx b/R/anova_cache/html/unnamed-chunk-6_87216dbd4f33263d00c668206b754202.rdx deleted file mode 100644 index c28f3f9dbb3f1f2ce024b914f82eddfa9c83ac95..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 125 zcmb2|=3oE==I#ec2?+^l35iK532CfGk`d0%cS>|6BxbZ66cUi)h?vp9qRh4e8Sk7_lb_T&M6uS~duX%1iGt1p$s3AO7EJV+RRMtdy(fa1OysAr>~f) z&Y{=Hzxxi{5wmKKG|dPf$-lQx0bNbMTfhe6O1IFn66J^=ulmK8$;-Y z!igKYRCVv^{N`rJ#e?e5_pVt1hYBVE2iN#1W{y{_}*z74uCxcj(h zGe>FivdgiiW&INlEs2mcX|l~q!+x&v)f>#5h@Q0G%V!K)IN+9sJs0%ja5NS^bbW6L`~v_Z#Cu{yiU8tFzNzv0W_ggTSUD;3PxDp9J=kjO7gGJ5N9VzV6heiZ_b`A72pETe>hx|d?<*HQ4<0jKUQ5C!*(pwZ zHEB}l%DWJ|PsWofK1Fjmh(2|5&(QD#STxSGHPMn~q4H!|D;g^|WDXmmS*i(}sMtbqG-t5@Bl~0&u0e$F5ncd$m`_gEczibgSjz<6(_t?xo#?0t z32(!mxunI)b6)pP>k_#AZNVQp7kOlaCPD1L1WApEm?}zlh)*io?x}^~z7^<>cC0TM z-`rgEhB%tESnhS={1}eS)U{o7$zEfEZ@z|g`OzM~ZUcMp4Uf`9%B@(BSU%DZu2M_9 z4oRJ}t`=tY$wN>)2uA}+2>v$0QqR1W*xT4{oGA1hmoaboLe)OG(GrTWt<1~qtnZ*E z)p;-{Tpu&98!0MGG*@J2TIzCMeHU6}@EtMQdP;U4Z=^L-V%ncv!@(eMifvZsj;2eL z$QVJDvOpCWl5QUNtnJG~oCO|ipUvfL^&w3)af8oLbUdMgPDgu7*ax;1POtlWe(|l5 zw0!}3sA|`Uq>6kT!~8KDNvbDlz7s?GV@`Z)!V#zr~yCxTrCS zYprO`bmF9GMAHN&YhLpDGKr_6?Y+tmKel?M190u;W9FK)+D_R?jv1)P_~@R0Z}oQc z%jSuP*ri*(As`%oH&rORpLd$&+gKajz*;Ak&-c7ZPQN&#`g*FX&BI7L-%395J>Ui_ zOBil@Hl0hqG>=di*42y83N_|;4<1hvmhD=3sny0klr_#D#wwK(X5yzk)1z|kCVakl z@nMB!ysmvmIKwjMh_q7NQ{Ky3^$^XG&p8x%0P2t|Lytqd!zPfwEA}$-uzm?wS>KIk z{gav@>U_=VX|YXzdd!1ASK%d#rjw|h+$whlb-to z#q;RgL+_Mf^~mE#+xWzUOtLv{$;Pb+aoYI64V)3zy^QP!E^|FjjQC|sT6Mb&%v+buXwIeuwEEtL zdG{K{*26(V`asjLIm>4nf{hSz{ovE^y1=FpB;{&okNll99hOe}@>^0kYav8Uwb!L> zWvopk&8{!XV~Xb<#MwMAfa`Lsp9q%h%%6}UQ^7IJ+R)pUJ0I8*YNNzEeVt#EdH zGw6kwKuV>V ziuLK0%TBk>;2x$0y&3B$X_%YkUc{86XQB>XR&kM7N;w{Ez=7{9y$}wPICDd7o=g7x zm(LlE2S$0=iF~R$<)) zjU0ZjG(kiybWx^aDwFtNHYK`Qg=Q7VU&f<3`L0Tn;+vLJ7k7oM;@|Mvy}4aTH5s%y z5?mmiB0$`)sFSf!ystPE_x$rg{pq2x&Q1-+cng~;v$eFJBh>ENuo*EaPWW-6;FUjA zmsXrIsHZI+&AjSX-fV*{26!RoZ7z9M3$03;WxR&r6@cD5JiBCN5jx33H_sXG8r)-9 z!MHG~(4~=fDa8GhRQ(jGC^`NpKP9Ys)J9}6wMk0k z`&`%DJ?Kj12b~135@k`FIG@o3C3*Qg%0bzc8w)*US@rSvJJ`ki%h)hp=|L@bq*A#w zhjI%(nn^LDiW;=9wdl=N6x|;*g!6%0Lgl`{{zyASv}+!gBk`Xrsdsi${j<{>Prx%a z#8s=fGh&Z@5ZJJ}=9AgsFTWdFN;^sermk@)tW4^wpxR2NJ@6TOxD+6HD zrVDbuX&DE)Z$6qX5xx6V_NXHzZg!yulFD=IMhqLh$U@s!7k_v_rU$2b1ze478ev-~ zb$~?`?=BW-{VYBB(T;E!+N@hYfbcS~wo=${+w|vLn36Q zjZ|CSBeVr5&ZT=J+hsG+hyL}iX9On%289#ci3Fk#`G5~K!2dv4U?3rYOj0BIga!Jk zX}S|I1QJ<`gz?fqBhhGrI?)rYjwWFc2u}h=6Nx}>QH1^`vfsgCu|N!*NcQ&!d+}ltLv?eIv2hpDC$i z3RQ7S27o9a4j_ewc>^lIUsqIq2+BL5D)8eRH7b}4=mE%`kVK{ueErF!Ey*75fK&RP zJl-`=@~Yb!$5U z(|+^|_*EGIptp=Vh6Zdu5*&rrrU4oVEd*Q(sYwI0;p!S&TN-c;EwUOygQmQ53js%K zA-7>WUH=6`?Z5zaq!wHYt^Q9K60W6<*8^94FCXSUgZ}6 diff --git a/R/anova_cache/html/unnamed-chunk-7_a6ade8d6f53133a4cb343d600ba67101.rdb b/R/anova_cache/html/unnamed-chunk-7_a6ade8d6f53133a4cb343d600ba67101.rdb deleted file mode 100644 index e69de29bb..000000000 diff --git a/R/anova_cache/html/unnamed-chunk-7_a6ade8d6f53133a4cb343d600ba67101.rdx b/R/anova_cache/html/unnamed-chunk-7_a6ade8d6f53133a4cb343d600ba67101.rdx deleted file mode 100644 index c28f3f9dbb3f1f2ce024b914f82eddfa9c83ac95..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 125 zcmb2|=3oE==I#ec2?+^l35iK532CfGk`d0%cS>|6BxbZ66cUi)h?vp9qRh4e8Sk7_lb_T&M6@my?vS!9U zm?%0aO_Ho-DZ7x$HYn6lR3lQ>?AeNJQBHkY&*}UA{GRK2pL=`m-+jOL^ZxPTOdL^Q zM=%HkVgs?WvVhn)c2p3Dy`8CwCTPdAW3YhOLA!P&1dhW85`qHYWFnEUWAlPQtRWzf zeBswKe;us>^XPdj3UD6^;`fH$@V#2$Iw_!x)V=DbGB;QAM(FVqKE6iJ*>p4I1=I%d zPv4PF>fb zsCm!kH#b8r>Z?KDxMl?$ESLuDUE?O{{hT#?csgcyFY5f4+Mbj9wrGOjzGK48Y^6!7 z&Mr->ddKbC6CkOQMC-MN{T$`%*BLqCFH-xHQqD`CX8$Yx$@onaXE%4_2#Dv4yxN?b z`H^~y4>_db%@`K_D`J;Arx`|kW^8M!T<1T;l=r;tgif4d_VkrC(u{=u&}pfnRd3&= z!XA&A!#AYPcx)N21rQLiuVN3|HqAHiDL?I!(W{Gc5YVG;9cpY{XdJqHVpObRhD|VD z`}9jUS|7%N#kxi>Y^Wz4FG(Pz&wgiegV% zFsyf@iXF3?8Ow>fEo}C{as81v63ksR`$-XzWFOos!u$FCGtH~OqYn9&j`tOhKm^X{ zw!u>tZbz*u5OlGsR^it~`pEj;I-FYw((kP7F4AJ>8^V0s3n8@4Z>RjrNc6&y? z3V0lxXo&b5&w7%+n!$MO>E}Ptm$Fjva(U?86GMfEgh8T-1tz2Cjw#cPUqi}ycbTBt zy=fI_0@Ow?Ut|x0o{%^cn0Ml|=o99h@{FwBEJ8q+Yy84lcgesIbq7|_?xc(pl5-bV zW2(gH;fHV9U)@Deb_|-$Y^qj#?KE)Kb++=^>D?UO=@&iqU5r;Z5^;&P@>8Eon&f)( zF2wAU@}!8)P@VNrr*7;S8Eu6{VNE*{ESVNc4^}m!R0V4pxdyZ=Zu=`E4$kGT_=!;w z>xxL5n^x7kqxfb2$i>biw`?zQU&fhnIX_7dL?+RsF9!~=KQqsA)V7T7mn?-&ac$=B z_m)lloMGsV-ReDD+Z$i0Bz)V?kY7W)n^WbcU$_Z`!hYPKd-MgYTO{aH^g!6~+k(6# zHft}l6cZLZD zNt2o9wCSJP%XjZ|P%&Qw6BrzsrDlgeBIxTOrrv`%kT%a@Fy}4q1V|&>f z;$YHlxz~~XT^J@)$7b0%dxHVK@f_CcM{RrF3HIO_9j6MFTQTo5eI#42P)bM#CC(kL z5@Za>LXcbt2Lo{k?l!_w*SyB7zp=|WLEteqW6|=7vRzW6B@}H_nU~vB*G);R^absV9a3cQ1u@=nN_r7zq&Zh&I+#?=M#r;@Y**!ur-_wF89^1Y zKow|$P9EpH&C`SI1s*IP%w=r!AWhY=!;g?O9KM1^LwSqY1$GwBZu)zE^sN@Rc>=np zY}=?x5qg`N8W1w!p@d~WXL4ZO<0Q>|VwDwCV$3<;&9_b5)^hrCi#x?(S$!JYQPG^~ z$WB#{qVi2wKjrpic%6!}Bb6U&vwEftaO~z`>Bia_=t_bk)5u_mg4x#?BD_~KzK_bM#o zbnLpr=$1K0Bo$&Ga$nZ0gJ_I>$RW!DQ2Qh)S}e*PHl_Nfd_O%8;}?I0`NeqNKd~92 z#?zdZ8q@T*$0GQ>)uL?aSX7RFK@pk11e&*Y?k$qc&>MLb!k@poEd0j6M0?oIaVn0E z*i%l{MCY312}K^m#(%>tbXehJo1(TJ8fR*Zp(% zypu=OB90yH=ps}sV$z0 zl80FgqB(ZO+N}t2+W5eAtPw|jMs}<7!V5=w+^QwDs!Iyyt;1rpVBHQ{e`C$KdzEbC zVXrQEplQ^c=`#z#gbTR7^=WusVBH9ka5c13{gW^onnwNjM`9Rr>4b`MzjNo>M5j=y z?Lef*3|BqG$vk#UT6NI#VY^;QVSKt3xGI}^;@Ud0>Dp*;rnrkq^)mUDV0K&-aS4YI zujFA&)=bzL_*WPi1aIc$RmC$T8%LPbvoAfbW#VST>j>R2^~!A_ey0_mqz#(V;nQoE z9dDh%-b)R7IniCxurSZLj4nsbMe1Kxau!=jJ{D}ihU+Q45C#%Eb6sVTL-zc~4;hUI z#<@4NQi_4Y9C@8NTKLDjnqx)bC6z6-p-|V+6oc0C<&VpirEJ7K2D|US(s1}_cwV-8 zwPpMQ%shxx01p?hqcO@)(np;VIge;}DCQNddvdP2*QoBRXE%{fC&9A{Yp1Bhu=>(? zA(fD2sfw9Quh#kGsAeUqRUmH}m&Ww#n;K-_)STMbbfS`f!*loME&=5v(Dqnxfn+kD z*M52JjHTjz#Ua?oAN2KRM<#lD)ajG$ET)W((m}Qm+pD8yUWu`SZ3+D8e<`o5Ic88! zTil;})~C4L30n>zsV-Vy@~jeA7dOjz4#UX-{dc%_NlPQNlSXcwGu}14$FhQcVOqg^ z;9*Q>V0fbC_M@20fPI#=!snnXV_g!6mgE$@WQj-_-bgpv??OFv zt@5pQJgG!c*gDo{JYGRoHjk_?y>@-+MOju|TuV2rh<_Oinv@pQen%pOLt`Yj;GLNS zJ+i1l>uS61LPb%_upyiW+#VwH<@r16K`-0pQ5gd7xsp03H|4*3ym5FOeM?mNCTB*B z%X{IIx{`y*Ws;^j=FQP2C;Dq_Cr#Y^r*L8-&@S|+H)Qw88s7c68*XI)Y}Ry1#y2(N zK;Mn~vn9fJA4(r}Aji%xy?~@}-MSvlLMyV+^3}l|8j^Z}RZa(2VVcHRmP+kmk;S`< z`8qyG4!^U-AA&aP)D0m>28XTW_S-c5z0m2v65uNKdN6)YA>%Hv$&&fHp?2Rcfj;}7 z(20)@dEs}2IPUM~4l-3c=6uQ`X-b`Mv_t&T-u4JZ@v@`Ej$xC@6vT}vy?i6(_BRME zKC)Bk-iR*gOw_^u`qwjp;{$`j@a|rCFCXFoA4-7#fzZG}d;pQ4;^h+>=%=FTPQoKK z5O^dBfgq4n)jd^_L=6%Fg+h}MYAAIiUTudW@DGvo4hDk(qTyade}BLSPxjH3`@bv5 z#el&c9v;!~5a1hE#Vd$Fi~$o!?gU>l5lzix!@KGPo|00^cA>%bA}-?F@kWGa}7QTf-R{!}o(YyWJ&hcN%j!tB7l%?tR| z847_$!PQVGDxi+gM8GvwHNMu}F#$C+70`ks)zql~8m^&ER6(dy6@O$Q;3y6BSJ;oK z|9nj~RZX}iO6_+T@&`;6uBr7+ppiQ;RVwfchWHNK=?wFexD(YtXuy$Z^zShAFBk&7 g<3Pc+(EkNfMIxzR@8;{X^8tVU52~4s10xLp0K%3M)c^nh diff --git a/R/anova_cache/html/unnamed-chunk-8_fa270a2f00df15b12e7fd336f04352a4.rdb b/R/anova_cache/html/unnamed-chunk-8_fa270a2f00df15b12e7fd336f04352a4.rdb deleted file mode 100644 index e69de29bb..000000000 diff --git a/R/anova_cache/html/unnamed-chunk-8_fa270a2f00df15b12e7fd336f04352a4.rdx b/R/anova_cache/html/unnamed-chunk-8_fa270a2f00df15b12e7fd336f04352a4.rdx deleted file mode 100644 index c28f3f9dbb3f1f2ce024b914f82eddfa9c83ac95..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 125 zcmb2|=3oE==I#ec2?+^l35iK532CfGk`d0%cS>|6BxbZ66cUi)h?vp9qRh4e8Sk7_lb_T&M6}`*ybHkDvy1io|(anv1A(wg^4T?W9G4B zZD#C)iJ~{9Nt?B#vJ0t%(MG+BYD8K@_H0GADAkwVwzuD(zjIyBeeUJlpZh-NT<5wT zvZX5y?h1#&VEizFReUgh!DSUD=>4jnBf4Yv*{vUo&6KZ{T^(*M zeGqXz*s(uRL$e|6CWOEH@qSM)%w9H4S}3;TndMsHGqXjzL>}#oE8NqzxBXDkYI`%} z(er~BjF`H9Bj*|#5XUy_BHwsq1#itC0-QY(1~{EURpJy5@zr|Vk+0Ql`!+1HrQjXA zWg7U4Q)b+E)z6shac)dPq$@)9v$Y!qOXn_db0VLncc!ErQ9dB>PvXPAOE{rck-Bb} z=vQ^!QBUg~*KI!J_!ch2@ohdWcf5IsYc9Tpe-3Ag z8x`!zrE4tIgI;U&ywXAMMYGvp8W{gPe!FA+SgpABqZSpDniv;J6V~Fkx|`#5T_^YU z$dwKAOC=f}c;?COAiD6`SD8d~wMETWY3{OY?yg;nf{9#0m|yGUxVcx)?!G)0E>bY^NfS(Q4r`Yc`||!V;S6xUN&QCC z`?C8mno#VWm-}rzciNSK9Jz{{GOr;{_uTp#vL_v}>Ev5w98ql;a^KE7zOg7S`bw#u zu-^SOKGDyEAA}{FfuAX>_Hkx1xUYQzgI;u`O_x2J>U#IkOyeHyC1hzs%xJr6#q*TF zE4i;Jk zO2%Hr(PJ}l6>^-&?U$U-tfpzZhK^*`S8Bd?dvVBPr2O%LHG+Ppj`?ifWid0KOip%G zAN*uluhyP-G;V{E4|DA>%Y8F$|HXCPJvUJ?B&((*Tb_;9-5Ej*Myi^dYsx*%@ZKsgpTr-^mAWLLQ+Mnd(kT+(prjipJEljJSJ?hlisuqkZilOZnz9$RO*8rsHo zDi$LLg%{3k^ixg$l40gYT5R85-JV#kC37XvOagD%Dx`BMFwzpi6xd_h+Vd3EDjWJK z_C>_2xA}P~{Pxr>X_kC4vXfyk0)^t7?0wOsT4~YSWO3p)Q57SLfw6>Lje~3-=k=%) zR2OwbL^JW=aRM*TZ6PSVUHr%z3f#ic1fgo|aP?Rr;%cM-%)BZn#X?0GKy;uBPRAnVSPX1KR#PajLV)Q)$X=dXD4 zG_%NetHR;!6;j+6stBww=wd35AYTD(jjgMwopmi1NsAm5wsgSCo-38KC-{?&afn@lNVvvGcMP9e?bM;3y7 zJ_c0EJ3NHl(sr!FFs0w7rw7yfy|qXJhb=eFdGBLe_s^_?6tP zOz91gn#vk7T?JTrF)Z<+%10srTXldm<+ZH3)p~ha83~fOJk#V#njWP=lDi)jK(P#K6SU zyss8xLCFmWUD1XU>2dX+y(i%B?Iu)luT$AqSbcu5_Q!;N% zEe&5exeg|9h>NRC!;YVyHo9XIi;}n$e<$_Om)y=HvE_ zu(>z(+-qkT4&Kgsiks?tta<(;2x6q9$6No}C;9euFa-}YC(Pfpk?<3&kAEdc@Fw@_ zXm`3d&-OPF3ZX( z#&fCcgcxX&49b^_atEsVolJwu%uT}<^71MYxr%k&Jl3ISK4&w@Bat<NfR>%p0T?0^0@insW>&55x$)1(8MNH`ra_9zAKMH%q5JPL&t)L+eb} z+GLJQu&lG zI_rG(APb7PUYsbcL!VMA8_uNO97~O9&|=wzh?NNAhhAU8GXm0cs^d>VT0yl>yc$|0 zwNqeAyV_GUg&PX#qz4~1n~Ze#x3%eU1{(RSxJ|_``RR^ldbUuLR0P*%Q#1eH2hxNi)#*vOtxLRAD0=t!M0lFFmk%LMFG5#nr4!!5UnB>9jHOnjb4l@ zWY#N4e;schzmA+Oe`}cNTcjytAMf9nsG+Kw$JneqdtvfvNmfn5jn-ANK_z_pz9&K( zuPUSo;=6P6-)&LgL>JT=oM|*3FDtn5$_yxNj%y`5E#1D2jagJd~bWQ+c%Hw>?VDzS~Z z^PVLDYQ$<%B_KUxQ^&>IBSkXT?kVqdVZ@J3K1HMnU%n8_$1boj2rwdV>r#43(mn;R zAlCQtO%^+&q6^m)iZ^{weD%(evJKf_RMQ3enr^pK+vrgLdA!+$FW5uw^~=Ojjf`u+ z0$=9q+UgCfB|Dr$!}~wFl>pq6Cg;qx8{dEi;taRq^-(R#nYgWg{Og&cDIuW|6fY`;>JM%5X9fpt3J(dP1Vc0( zs(*M$ppJnT29KrSeJMDIM#1_}eDQh&49-UvqEfLGEZ%?$EmI_a5&2O>A`ys1Q=yw7JVPY|U3b4B34=1SXe_^%Vu2@Pk4g)?=QtA+(pqT}E!lCLjBWdi>W z*7teGju6%ljm82jtwL4=?MK+^hXRDL!i>~0Ynop_?6n2WudFj0x$#&S{I|s0zfnf zLRuim()_+L5REq=Ec3qI0`P0USOWt*S{IM~FAw*f2LJ|Wa77`YK?6t!)ccKx!(&!> z->>uM`RQWx&;n`&!EZ`>(#Or_WkN5++62*gfG*(~#zdZfl jc%c5W3x_t)|AU9YVp%I!zw#}=k1ziN@XGIUf(-xwKinqr diff --git a/R/anova_cache/html/unnamed-chunk-9_172a7fa3eda2baf75813b4ecc2a279ce.rdb b/R/anova_cache/html/unnamed-chunk-9_172a7fa3eda2baf75813b4ecc2a279ce.rdb deleted file mode 100644 index e69de29bb..000000000 diff --git a/R/anova_cache/html/unnamed-chunk-9_172a7fa3eda2baf75813b4ecc2a279ce.rdx b/R/anova_cache/html/unnamed-chunk-9_172a7fa3eda2baf75813b4ecc2a279ce.rdx deleted file mode 100644 index c28f3f9dbb3f1f2ce024b914f82eddfa9c83ac95..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 125 zcmb2|=3oE==I#ec2?+^l35iK532CfGk`d0%cS>|6BxbZ66cUi)h?vp9qRh4e8Sk7_lb_T&M6 Reading from a connection which does not supply a ‘gzip’ magic + # > Reading from a connection which does not supply a 'gzip' magic # > header is equivalent to reading from the original connection conn <- gzcon(file(bundle, open = "rb", raw = TRUE)) on.exit(close(conn)) @@ -767,10 +782,12 @@ local({ renv_bootstrap_validate_version <- function(version, description = NULL) { # resolve description file - description <- description %||% { - path <- getNamespaceInfo("renv", "path") - packageDescription("renv", lib.loc = dirname(path)) - } + # + # avoid passing lib.loc to `packageDescription()` below, since R will + # use the loaded version of the package by default anyhow. note that + # this function should only be called after 'renv' is loaded + # https://github.com/rstudio/renv/issues/1625 + description <- description %||% packageDescription("renv") # check whether requested version 'version' matches loaded version of renv sha <- attr(version, "sha", exact = TRUE) @@ -841,7 +858,7 @@ local({ hooks <- getHook("renv::autoload") for (hook in hooks) if (is.function(hook)) - tryCatch(hook(), error = warning) + tryCatch(hook(), error = warnify) # load the project renv::load(project) @@ -982,10 +999,15 @@ local({ } - renv_bootstrap_version_friendly <- function(version, sha = NULL) { + renv_bootstrap_version_friendly <- function(version, shafmt = NULL, sha = NULL) { sha <- sha %||% attr(version, "sha", exact = TRUE) - parts <- c(version, sprintf("[sha: %s]", substring(sha, 1L, 7L))) - paste(parts, collapse = " ") + parts <- c(version, sprintf(shafmt %||% " [sha: %s]", substring(sha, 1L, 7L))) + paste(parts, collapse = "") + } + + renv_bootstrap_exec <- function(project, libpath, version) { + if (!renv_bootstrap_load(project, libpath, version)) + renv_bootstrap_run(version, libpath) } renv_bootstrap_run <- function(version, libpath) { @@ -1012,11 +1034,6 @@ local({ } - - renv_bootstrap_in_rstudio <- function() { - commandArgs()[[1]] == "RStudio" - } - renv_json_read <- function(file = NULL, text = NULL) { jlerr <- NULL @@ -1155,26 +1172,8 @@ local({ # construct full libpath libpath <- file.path(root, prefix) - # attempt to load - if (renv_bootstrap_load(project, libpath, version)) - return(TRUE) - - if (renv_bootstrap_in_rstudio()) { - setHook("rstudio.sessionInit", function(...) { - renv_bootstrap_run(version, libpath) - - # Work around buglet in RStudio if hook uses readline - tryCatch( - { - tools <- as.environment("tools:rstudio") - tools$.rs.api.sendToConsole("", echo = FALSE, focus = FALSE) - }, - error = function(cnd) {} - ) - }) - } else { - renv_bootstrap_run(version, libpath) - } + # run bootstrap code + renv_bootstrap_exec(project, libpath, version) invisible()