From 087e09f352e9c66b262d24330962dadf66e4302d Mon Sep 17 00:00:00 2001 From: Chitu Okoli Date: Thu, 8 Feb 2024 23:12:59 +0100 Subject: [PATCH] Replaced gridextra with patchwork. Relegated ggplot2 from Depends to Import. --- DESCRIPTION | 8 ++--- R/ale_core.R | 26 ++++++++++------- R/model_bootstrap.R | 3 +- R/stats.R | 7 +++-- README.Rmd | 13 +++++---- README.md | 16 +++++----- man/ale.Rd | 12 ++++---- man/ale_ixn.Rd | 10 ++++--- man/create_p_funs.Rd | 7 +++-- man/figures/README-gam-and-ale-1.png | Bin 20880 -> 20474 bytes man/model_bootstrap.Rd | 3 +- vignettes/ale-intro.Rmd | 25 +++++++++------- vignettes/ale-small-datasets.Rmd | 27 ++++++++--------- vignettes/ale-statistics.Rmd | 5 ++-- vignettes/ale-x-datatypes.Rmd | 16 +++++----- vignettes/articles/ale-ALEPlot.Rmd | 42 +++++++++++++++++---------- 16 files changed, 125 insertions(+), 95 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 63d63af..0741484 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: ale Title: Interpretable Machine Learning and Statistical Inference with Accumulated Local Effects (ALE) -Version: 0.2.0.20240207 +Version: 0.2.0.20240208 Authors@R: c( person("Chitu", "Okoli", , "Chitu.Okoli@skema.edu", role = c("aut", "cre"), comment = c(ORCID = "0000-0001-5574-7572")), @@ -14,9 +14,9 @@ Roxygen: list(markdown = TRUE) RoxygenNote: 7.3.1 Suggests: ALEPlot, - gridExtra, knitr, mgcv, + patchwork, readr, rmarkdown, testthat (>= 3.0.0) @@ -28,6 +28,7 @@ Imports: ellipsis, furrr, future, + ggplot2, glue, grDevices, insight, @@ -42,8 +43,7 @@ Imports: univariateML, yaImpute Depends: - R (>= 3.5.0), - ggplot2 + R (>= 3.5.0) Remotes: tidyverse/ggplot2#5592 URL: https://github.com/tripartio/ale, https://tripartio.github.io/ale/ diff --git a/R/ale_core.R b/R/ale_core.R index 865188b..404a391 100644 --- a/R/ale_core.R +++ b/R/ale_core.R @@ -273,10 +273,9 @@ #' arguments to understand how these are determined. #' #' @examples -# Sample 1000 rows from the diamonds dataset (for a simple example) -#' diamonds +# Sample 1000 rows from the ggplot2::diamonds dataset (for a simple example) #' set.seed(0) -#' diamonds_sample <- diamonds[sample(nrow(diamonds), 1000), ] +#' diamonds_sample <- ggplot2::diamonds[sample(nrow(ggplot2::diamonds), 1000), ] #' #' # Split the dataset into training and test sets #' # https://stackoverflow.com/a/54892459/2449926 @@ -309,7 +308,8 @@ #' ) #' #' # Plot the ALE data -#' gridExtra::grid.arrange(grobs = ale_gam_diamonds$plots, ncol = 2) +#' ale_gam_diamonds$plots |> +#' patchwork::wrap_plots() #' #' # Bootstrapped ALE #' # This can be slow, since bootstrapping runs the algorithm boot_it times @@ -322,7 +322,8 @@ #' ) #' #' # Bootstrapped ALEs print with confidence intervals -#' gridExtra::grid.arrange(grobs = ale_gam_diamonds_boot$plots, ncol = 2) +#' ale_gam_diamonds_boot$plots |> +#' patchwork::wrap_plots() #' #' #' # If the predict function you want is non-standard, you may define a @@ -339,7 +340,8 @@ #' ) #' #' # Plot the ALE data -#' gridExtra::grid.arrange(grobs = ale_gam_diamonds_custom$plots, ncol = 2) +#' ale_gam_diamonds_custom$plots |> +#' patchwork::wrap_plots() #' #' } #' @@ -449,10 +451,9 @@ ale <- function ( #' #' @examples #' -# Sample 1000 rows from the diamonds dataset (for a simple example) -#' diamonds +# Sample 1000 rows from the ggplot2::diamonds dataset (for a simple example) #' set.seed(0) -#' diamonds_sample <- diamonds[sample(nrow(diamonds), 1000), ] +#' diamonds_sample <- ggplot2::diamonds[sample(nrow(ggplot2::diamonds), 1000), ] #' #' # Split the dataset into training and test sets #' # https://stackoverflow.com/a/54892459/2449926 @@ -484,8 +485,11 @@ ale <- function ( #' #' # Print interaction plots #' ale_ixn_gam_diamonds$plots |> -#' purrr::walk(\(.x1) { # extract list of x1 ALE outputs -#' gridExtra::grid.arrange(grobs = .x1, ncol = 2) # plot all x1 plots +#' # extract list of x1 ALE outputs +#' purrr::walk(\(.x1) { +#' # plot all x2 plots in each .x1 element +#' patchwork::wrap_plots(.x1) |> +#' print() #' }) #' } #' diff --git a/R/model_bootstrap.R b/R/model_bootstrap.R index bc86947..45baae2 100644 --- a/R/model_bootstrap.R +++ b/R/model_bootstrap.R @@ -150,7 +150,8 @@ #' mb_gam$model_coefs #' #' # Plot ALE -#' gridExtra::grid.arrange(grobs = mb_gam$ale$plots, ncol = 2) +#' mb_gam$ale$plots |> +#' patchwork::wrap_plots() #' } #' #' diff --git a/R/stats.R b/R/stats.R index 10ac0e8..262f678 100644 --- a/R/stats.R +++ b/R/stats.R @@ -121,9 +121,9 @@ #' #' @examples #' \donttest{ -#' # Sample 1000 rows from the diamonds dataset (for a simple example)diamonds +#' # Sample 1000 rows from the ggplot2::diamonds dataset (for a simple example) #' set.seed(0) -#' diamonds_sample <- diamonds[sample(nrow(diamonds), 1000), ] +#' diamonds_sample <- ggplot2::diamonds[sample(nrow(ggplot2::diamonds), 1000), ] #' #' # Split the dataset into training and test sets #' # https://stackoverflow.com/a/54892459/2449926 @@ -168,7 +168,8 @@ #' ) #' #' # Plot the ALE data. The horizontal bands in the plots use the p-values. -#' gridExtra::grid.arrange(grobs = ale_gam_diamonds$plots, ncol = 2) +#' ale_gam_diamonds$plots |> +#' patchwork::wrap_plots() #' #' } #' diff --git a/README.Rmd b/README.Rmd index d091a4e..5f1c6c0 100644 --- a/README.Rmd +++ b/README.Rmd @@ -74,10 +74,9 @@ Here is a simple example that demonstrates the usage of the model. First, we tra ```{r gam-and-ale, fig.width=7, fig.height=11} library(ale) -# Sample 1000 rows from the diamonds dataset (for a simple example). -# diamonds is included with ggplot2, which is imported by the ale package. +# Sample 1000 rows from the ggplot2::diamonds dataset (for a simple example). set.seed(0) -diamonds_sample <- diamonds[sample(nrow(diamonds), 1000), ] +diamonds_sample <- ggplot2::diamonds[sample(nrow(ggplot2::diamonds), 1000), ] # Split the dataset into training and test sets # https://stackoverflow.com/a/54892459/2449926 @@ -97,17 +96,19 @@ gam_diamonds <- mgcv::gam( data = diamonds_train ) -# Create ALE data and plot it +# Create ALE data ale_gam_diamonds <- ale( diamonds_test, gam_diamonds, model_packages = 'mgcv' # required for parallel processing ) -gridExtra::grid.arrange(grobs = ale_gam_diamonds$plots, ncol = 2) +# Plot the ALE data +ale_gam_diamonds$plots |> + patchwork::wrap_plots(ncol = 2) ``` ## Getting help -If you find a bug, please report it on [GitHub](https://github.com/tripartio/ale/issues). If you have a question about how to use the package, you can post it on [Stack Overflow with the “ale” tag](https://stackoverflow.com/questions/tagged/ale). I will follow that tag, so I will try my best to respond quickly. However, be sure to always include a minimal reproducible example for your usage requests. If you cannot include your own dataset in the question, then use one of the built-in datasets to frame your help request: `var_cars`, `census`, or `diamonds`. +If you find a bug, please report it on [GitHub](https://github.com/tripartio/ale/issues). If you have a question about how to use the package, you can post it on [Stack Overflow with the “ale” tag](https://stackoverflow.com/questions/tagged/ale). I will follow that tag, so I will try my best to respond quickly. However, be sure to always include a minimal reproducible example for your usage requests. If you cannot include your own dataset in the question, then use one of the built-in datasets to frame your help request: `var_cars` or `census`. You may also use `ggplot2::diamonds` for a larger sample. diff --git a/README.md b/README.md index f37a7f4..a1f6a5b 100644 --- a/README.md +++ b/README.md @@ -107,12 +107,10 @@ the `ggplot` plot objects. ``` r library(ale) -#> Loading required package: ggplot2 -# Sample 1000 rows from the diamonds dataset (for a simple example). -# diamonds is included with ggplot2, which is imported by the ale package. +# Sample 1000 rows from the ggplot2::diamonds dataset (for a simple example). set.seed(0) -diamonds_sample <- diamonds[sample(nrow(diamonds), 1000), ] +diamonds_sample <- ggplot2::diamonds[sample(nrow(ggplot2::diamonds), 1000), ] # Split the dataset into training and test sets # https://stackoverflow.com/a/54892459/2449926 @@ -132,13 +130,15 @@ gam_diamonds <- mgcv::gam( data = diamonds_train ) -# Create ALE data and plot it +# Create ALE data ale_gam_diamonds <- ale( diamonds_test, gam_diamonds, model_packages = 'mgcv' # required for parallel processing ) -gridExtra::grid.arrange(grobs = ale_gam_diamonds$plots, ncol = 2) +# Plot the ALE data +ale_gam_diamonds$plots |> + patchwork::wrap_plots(ncol = 2) ``` @@ -153,5 +153,5 @@ tag](https://stackoverflow.com/questions/tagged/ale). I will follow that tag, so I will try my best to respond quickly. However, be sure to always include a minimal reproducible example for your usage requests. If you cannot include your own dataset in the question, then use one of -the built-in datasets to frame your help request: `var_cars`, `census`, -or `diamonds`. +the built-in datasets to frame your help request: `var_cars` or +`census`. You may also use `ggplot2::diamonds` for a larger sample. diff --git a/man/ale.Rd b/man/ale.Rd index 7ad2ef2..62ccdbf 100644 --- a/man/ale.Rd +++ b/man/ale.Rd @@ -339,9 +339,8 @@ to the \href{https://progressr.futureverse.org/articles/progressr-intro.html}{\c } \examples{ -diamonds set.seed(0) -diamonds_sample <- diamonds[sample(nrow(diamonds), 1000), ] +diamonds_sample <- ggplot2::diamonds[sample(nrow(ggplot2::diamonds), 1000), ] # Split the dataset into training and test sets # https://stackoverflow.com/a/54892459/2449926 @@ -374,7 +373,8 @@ ale_gam_diamonds <- ale( ) # Plot the ALE data -gridExtra::grid.arrange(grobs = ale_gam_diamonds$plots, ncol = 2) +ale_gam_diamonds$plots |> + patchwork::wrap_plots() # Bootstrapped ALE # This can be slow, since bootstrapping runs the algorithm boot_it times @@ -387,7 +387,8 @@ ale_gam_diamonds_boot <- ale( ) # Bootstrapped ALEs print with confidence intervals -gridExtra::grid.arrange(grobs = ale_gam_diamonds_boot$plots, ncol = 2) +ale_gam_diamonds_boot$plots |> + patchwork::wrap_plots() # If the predict function you want is non-standard, you may define a @@ -404,7 +405,8 @@ ale_gam_diamonds_custom <- ale( ) # Plot the ALE data -gridExtra::grid.arrange(grobs = ale_gam_diamonds_custom$plots, ncol = 2) +ale_gam_diamonds_custom$plots |> + patchwork::wrap_plots() } diff --git a/man/ale_ixn.Rd b/man/ale_ixn.Rd index 14fb664..f1d11af 100644 --- a/man/ale_ixn.Rd +++ b/man/ale_ixn.Rd @@ -110,9 +110,8 @@ number specified will be used, including the middle quantile. } \examples{ -diamonds set.seed(0) -diamonds_sample <- diamonds[sample(nrow(diamonds), 1000), ] +diamonds_sample <- ggplot2::diamonds[sample(nrow(ggplot2::diamonds), 1000), ] # Split the dataset into training and test sets # https://stackoverflow.com/a/54892459/2449926 @@ -144,8 +143,11 @@ ale_ixn_gam_diamonds <- ale_ixn( # Print interaction plots ale_ixn_gam_diamonds$plots |> - purrr::walk(\(.x1) { # extract list of x1 ALE outputs - gridExtra::grid.arrange(grobs = .x1, ncol = 2) # plot all x1 plots + # extract list of x1 ALE outputs + purrr::walk(\(.x1) { + # plot all x2 plots in each .x1 element + patchwork::wrap_plots(.x1) |> + print() }) } diff --git a/man/create_p_funs.Rd b/man/create_p_funs.Rd index c26f32a..0dd53ad 100644 --- a/man/create_p_funs.Rd +++ b/man/create_p_funs.Rd @@ -152,9 +152,9 @@ train the model, then the same dataset should be entered for both \examples{ \donttest{ -# Sample 1000 rows from the diamonds dataset (for a simple example)diamonds +# Sample 1000 rows from the ggplot2::diamonds dataset (for a simple example) set.seed(0) -diamonds_sample <- diamonds[sample(nrow(diamonds), 1000), ] +diamonds_sample <- ggplot2::diamonds[sample(nrow(ggplot2::diamonds), 1000), ] # Split the dataset into training and test sets # https://stackoverflow.com/a/54892459/2449926 @@ -199,7 +199,8 @@ ale_gam_diamonds <- ale( ) # Plot the ALE data. The horizontal bands in the plots use the p-values. -gridExtra::grid.arrange(grobs = ale_gam_diamonds$plots, ncol = 2) +ale_gam_diamonds$plots |> + patchwork::wrap_plots() } diff --git a/man/figures/README-gam-and-ale-1.png b/man/figures/README-gam-and-ale-1.png index cb1657ac1968e6e7bc2f4172f4690798ba861335..e7e4859043de2f44019d2f6ba7ee815e878e6776 100644 GIT binary patch literal 20474 zcmb@u2V4}(_Ac5pBmoDMs6+!90TB>I!T^#P9D<-o5D^%1&S?}80RbJ7Z=MyYAC@dm?!`MfT<|o z*8%`2766E^oFf9CP`zZM2OmTs)pZoWTL4f702e?x5pZdQKp;d!M5LsoWMpJ87>tUF zikh05iHV7om6e^HT|`7gR8&+#LPADH227(2C@TXl%83&Q!p|Ima6%w32pcDa69(Z_ zgury_AZ(r>5KnXvPt1#q2|rI`;}By{Ph-zG~@8miCAQ0-q`J#kt;y3~*r(ohoDPy&X4;=yx6 zqYKbDk=QtK+IZU7cskK|dfFL;=){2kojSn+6ghRC<>4smBFfYp_+4J19SP23u=0;efQ-_td2+GN>ZpPORMihEY0Ve zugIB{LiPBf51(q~73!ER!PBftcz@pi*i#cmn-hbe_vtUX{_7Lh+b8cYTB=m!Y4>xL z^T{Hb(V_xR7%*Ro)bJQYoXz`J7F-LDc|UEPh$O}Nj!K)mTbNt8nXfe%(>xkI`c=GJ z+ugPN?OXHK)vqR=6%T-y?Zoh_*(CpiBl9xe&{UwlAznwky;@Ejo1@2kq-+bbyMVtY z|0!-Dbl4^<24eV>!nnGb0T`fhJ`W)3oq_~_Tuv^)RpJsFip&mx@cip?{t^&U>%j`e z7P}A)oBShLpBbEt-Qf4}Z@WtGiQLA0kKGeyzh8gwj2+22AOSsq-Z}`FSJpyEIzFqc zouOu{nY8z-ij-~WNch)SFv#9HO$8wHFak2_$L8Azo zfp#rTSUf+d#wegu#mY4I8V7+p!g6mEyPXpQg`>5EsJaWNQlu{#+~y=CH|gMn$Eltr zcafPNH`@6^dyJi;Ia9@hL5S@mIeyXY96FpB{y9KwCm^O$9&q6R_}JcZs{s8APXP*e z5)~=d6yPBZoxcEt1v2SKUP6aMusB)QbLcN}Fg~{Pa0ssN6F`TSU_B3S1@M+x>(TdY zAV>k>RscE5u>hvZWJZoOn;<9%33K#Tb^LI$ zE+9u(8AwUsVI=9KtN~1P;wpq#5ah;FK!OyVW%SSLReQFt5WF&pQ~BLQm>XQE*MJR*C@&V1oRui0|o}F)%2eQnjM2J zVdl2xPXUTVBST;A$Q1~=IWgRjEUNaSmsc#cLBJxRK&7Nz(W_KX039{;LC9w@lsuR! z1DF-e3pbSBrbK_~9i!%R!q~z{v4TW=TtKb@sA2NwP%>*k=OftGt`kaIK#lvkbqA23 z0l;e*ZvO0Au^)_NHARx#YBKD2qaQon{bk5FQ_>y_DjfrK zRckGKvH+|q;@EjIL%#6IX=s1CLFx-c zY4%vC>H%=q(c^ckTlJRAoEDixA0Km?a$9{d;F2xRS!2@ai3Y7x$sQ%4Q_S9# zjaR|`A&UrIG70-+Ys^Nv|%>R&`?Wm#yuzsJdr0o&i&?p8}usM#|h zb-imyaJl>t)W<15BZ`#YK9r1SH=eFE@@hU9QlM{7Kj zxJr&545BZ}ci;UzpBQcmfe9}*2)CO0_1SwiT~3sAnyW30sqFsJn4n?Z=OjaErw6?d zoKOD_q((7;bA0x=o#xoEqC+VF?KJ0DMT$!0S0jO&s=yd7C!6q7M&%dTTtW(69rM&9 zH7$C>jAn?qcstfeJu z^(#7p%;7NVQzUfzS@-Eq!uIX}<2V1cOEOjn2>wy;G1l8Vv(^X2+V$LLFNgv=!@Njs zp0fY0aX-<2VW0L0CkC2o!4D()nXVx>NP(zX#>!o!zB%Ht9$3qH={T~c)JcdUj~pXhniY#lM!*E3U!-`P>` zyNQk7q5(e*g23Xa2%;3N+pPv*uipi5-rb^IP`;Z&CeSR_Pz`lFY(Cy98UPRRWY#|;)Hw39{U$>`CCPwZI7|S?#?rbUP9+DZ!e3xw|cG9y(yYQ6JqPaoOu^HfEtH zu*>E<>oz{g-Or@s>ts~l%e_LX;Vz4s7oO|Jo1i&PuKTlE6elz57dnv3f_ z1@Lhg(PH(hL>g?fh*ZzOiVoPI#9%1?*WY5HHJ&qwk9K<3&9~55x|K^e0}D5~7d8SY zpWS0%soI%p4pnLl7+8riy-kJkij8oBKe<&YUzb3hH=@#N!-C(Y=-}1f;gIrkkhPh6 z*+V_73hyhmY&TCI8cR=a08so8#MTFg7Ip@CElHvCV4 z5RGu#b4Vq4ZhU)3F;d{jeGFRR|}7hEEXm|Hzy7yeh}K+LVP|?VJuy5KVkY%^KoK{ zIt2IWX(+luS!TVpVf&A~6i+^mBJ?{wzg2GB{wy`S(YMxbtv>rbu~qw7&t3=m|Ky~} zlU$5t!w$uoEwW(ij>cpQ#L%}is7kw!wkOp+ZY}0D?lIE^>&a|UUd84N?SF{Md{@)- z!DKQ+<5#jAm(@9xW>$Jm(8~_WOQdB<)}upYQ3%QB5x0L3UG?HfIm+AKn^#+KIH=|} z)`tJE?){QAbx}^MQ04XZ-t+?j=dItdeT9}V)rdnjkNqB7g&u)RoW_fe`4}74`<=+mLf5t^wNzUIP2AjEGmXFZwDn;K#aQs+J&0_(6LSD^>|2W`Q zVJR0I;=tC8g@;{8yJv0Rxt`10CMr(}PwWdL_iBBof=5F@{Lv+`r7Zq%8YXjhG z%57h1Dhoks6kQ+0vOw_h!(!iim-3=+)v`vE&ak9kMqX`8O1oDa#y3E%&l60`5Mgz% zPYZ(AZM&@&3E?3or$ozX{QbyNfh$SzE{)mkHVlN!JGPcTqE@GS!m;)#U~Cn0f80%> zyJAG@QvJcx$%Lgy)Y{zT*AGtVd~3HIqm@f=aEX=>X9*JS3$N#Bq7#eIOvIwXX*Pko zUrHB)5;K+39)4zMOOV!PSzaTbvwfVF;*jEjcH5g1^Eoi?`{|k-7WtN5IkAPTY)W*5 za(>=&eEsboY#;f(Q>e!uAMieXX!E}AkWPiP1UX(opY#;D7O5BTcIjTfis*>=+IzkYnW`*gSU`+;Dh^z^rz!)^z|b8Ya(|@ zsbj?J`N@9F<62S=d7<3CrTojmd|g=ocPDdP%t!K0d&iC65BpXpcAM#%vn_LXmAEfJ zB`D5|JTnsrv{d9anC=okH;=l1+17}}s%AauPqTF1v*{z=s^E81xnvOZj;V3|dPLz( z%ToBb&J^5ArME8fbi1C^b1&h+c-7D}H)VmW%oON}AHU}zaplc>Av3P_9qWkgmPQwS zyxBGX%DT>zUyMilcX^r?j4x*Q*EE^u+)UP#dxf0+X$T~SYUG5 z+Q__5nV{z&99JNrBsRak%)H%kso(lW!0*oiByFUp^C_izA4z?$HIy+H%BFlxbO;ll zm^bthO?!L$V{>WR*S{Krw+oE>L#3gD0D0y!|H8fNYkjJrCsPIif`O8xCN~1G>;bWF z4FfQG{;Ki&H$`DQoKD9LlGtEtd{NX-u$2414rLei!2-Yoo7Qsf%ix}lm*qu`sszv3 z>IJhIIfWW%araoaZm-;vI-g?h{v8s*VPh}!RrrKyF0!ZA1GC(Ea-ft(X9-`(If~@I zgMMvB@@EZAtMs;5t-o0jxo{Ck(#fYRGod>7A1?~wrLpz_j<+VteMtpk6R232;r8`w zQ7hW0N*32EKp^z);vc#YuepplTh7)ewNl+9(-Sqjv30eA#6t6HQWf2Wh7|Q}f7JbM zFm~*JVG9LE_W36(^3Kbx#_1@U(=S4!AA!+_GhG+(+TjvK2(+T$Q7HeX&?mKXG(w}> zg|#6pr10_u(635sRL*PQbh2C|;mG}(0O%xKiyfCPo7GwF?Wa^W0CZ}vF^<EuOMr zYm(Aw)avkPIP-b-n9N~zHjC@iJRp$k0YK-Z|Nr%fhm4(dQ!1|MiTo}^@zldS2&A1@ z+#~5VQb<>w;N1aIgx4k!_RA(IRze-4v01d*eeI+)RmHz7z{Gb zhKJ3<;I1D4TyY%)M@0*W#fSp~CeXhnW&&Wht=3A$0-M8P`pEoLfHM7O1ti$a#nR&f zy&=2b&DpW@y1y6XWx)`wmtcsQ{*Ob+G412?Bjz4k# z6~X<0AiKQ;_O(oEzdIO9Qy;UEU`B`hl}dZ9cz|_zt*6lI)bCg2_SLtqnM7mgumc4O zF#Uq6oX3y(4&t{|p;!axVZ}Rrzh~ zC-8puVKLg4aQ{%7w%ZWPl)yuG4juAD4AO2m#s1e#LXNB_Cj@B-#m+MQWTFFG&=Xqh zza-Hg|7AT7Oh)r_3S{w`G%#?D0MJxUFIrpybV5OQEN3h{Z%yV;i#sK>>BimnN;Yrp?!SZ| zO^SS?{o+|S96;$C=;;y?0A7c9mi@Jfx+S?=bjMf<^rj&ph1w;gFa#e3;NEljg;l75 zFX;P@9Ne}Fsm=famDuH4$WqE^;s&}@&rk}|&^J$*m(5F#)tj({x1onRw>blXzsJ41 zw`q7?A@hMKHiUqeR2^gcQ>|V>@nv)bSYV?zZXPnG{+QQQq=%>`j^-b4j>Lg%jtq!a zM5@Z*;z5+*#&$r+6N1cFc3h^O=?NlqhR*y9~>g1-e*`_SAmcX2+Y}mz*w^m zJDC04%qEYJF9!0^l@kbiI7*$%0mNenxW$bV59E)#qUpW15z)fDnHyNuGxfxH>@Z~m z*eaDKgqEr2QjLoKn%jb$+@mp#`+&}9E+syi^MJY5=UfiTsBVteaGVJL_D%syvPqtS z`W{7x9wTFf-|^_w#+qRAdgDY#W=PCe~xmcPd)o%I+)FV2c>h*u`c(H3S03%Sg;_r5X?IAHWX zshr4YENpN%@ouGvjNJ~%bqR#xqpAur^7q%57hkPy%1ZwHJ6sUh*3-pdH!wt9QyF9^ zP2-h)=Pp2B`AjsDidJIr|JdXG^;&t;Knyj$sx=@5DZr-h-3#6k>9^P^XcD^AX-br% z81Ffh0(y=HVtW^;hZSbF{71+%3d4v?4}#J+TWE5buV#{{DQIPF)IPS;8;TGzNtPfP(gOD<>8+#zUdZeIOdEm@H?ArV5ulJhy+cH|Wqdxs9bMiyU|3i@$gVFJ}X`$aQ-lpbQ zwkxgZ!uJc&5mLW5yZvsP>if;k`AwB_NM{P0Q3!N&2>I_h+-kglk7f4_e*X8&RwPvp zrj0Goep`dICNf7Ur%RPY@YD6iTX0irWIij`IYo4`bfS%WaPZCFaoM-B^!zu(p>M9i zuWLW|{4LVxa3{_fiq|n;;5zx-d>8yI$5ztARmPMPJfPD&H>})^Pu3&6*=+BVbCZkm zJojH}HaUzJ@1aDuI%Q#FyoGJ1OhGvt4nr+LF}Psv>~$SJNpc>QI1vca*>ePhLH+ly zyWN^NO0pL`>Uh)hE?&qNT~W9lAagvl;LGm8E4N~r;dyE=B+&R(oCN&~VIq3a%F6*UE>;H?_JlkDtd+$j^gG+oS<#dm3+z(^F?aCqH&J?I>M3lUjPw?}v%fv{p z(Mq9MS#a?c~!+4=`TMrS`adJH3w5{VM=pAuglwX!lb+bv5a+8U^Z- zWUg%N!Ypfb`}KZJcrb6xfq}UgPb=Lhhnc#`Ym?ddn*!WxQ?xPQ1dMc`CX6X6MkK7R zYJcX5Ro=+Lx{Vy5W2C7vVQ}TzuFUCS`u3U)?E~*XwsV$mnXPFQmumX987Eq5;%;SW zzK+wm3^!GCfyGM&Uc{PMKGr@Oe{JEF3!30hG4!mzJ4kLl#F4NRl{vC|S;;{?7oGTME{CC*t8R7NKkMZ;#9a{!mrel;m7#~plh z#6wsD_MNj9kR<%o!T& zEc*>0`Zfd!#Pg`E`vOSbY8Yy<{D=eqN1{$7_%sr2++BkJP76Z~F2DC(mTd`2*0qwl ziB&C~h6QHZ#f*iwiQwfkbLX&ulvzmUB0kmennaDdM58oLQ6;t9=em|E+=92aVy!Rx zE(hjy0eDAfna8K7uUloCo%*rb!9VC8g?rvjH2GkZ$r<~>-sqRmAm(JqGq3Buy@to< z7d3XoSatIRk8dw|+Y}|O5tY4B88pu3(J?ktaw6{4G5xLIp4f8vIXCfB^}AjN%z9y{ zXKmy)PgR{%$R)%i_TvS!Y3ihpA6;4vYSkOsa4Cu+F_hpuM;-goW7hOR;AE!;qppfz zwJZb{JA5wuPu#+hESjudOnNK;GjG0uL4;S<7fUE zA9AdPCy5ec{J4TyGv~YM3|jTECx1l-KyT=XEntT5^v$^ScX0ix`TmpXcV_l28X&qi z17f8dhpEFnAV=%+Rl*RT$!sHv^@CaUB|9-Wndo1Y(*=9j6=7X|VZfFbyPto7UmE58 zc0lz!{0YT;hHdNED**NJ0If0ZHu{S&3-D2PKa&XQ369}w3T0dM%Y8ZdM(q1>GX|;C zg3{HwNTXS99mg6{+QE+6$I*%E0bDBz!OG4uJW4=W1HeOE%?nW675Mv+j;3N;rV)rv z*k$O7Ki5eXs}zgolwR1J^q9S@=c*F9K~j3; zE5R;zY1J5CfAf18xthI~wH=3PI$-$Y&}YZyWkP#2NDTOs3$S+5>OC`*z4Eo4raMYQrCv<3FLb%h<+An% zL5kUo!{f#_5wmY)oape1)5&;#t-MREFyyx%=ToTI!e714i0I3@uPR4lCRoDVH;WK8 zq-$5%wLf|If_J2)5XUOJD9Lc8zReB@PTe7)IA|6%x1z)tAm*MmVU{c?mbc3JM7fh?(Uf95r9 ziCGXS$(RgY&N(aEx!J7ggyD0zoLUck&z-J*Y}8>>;lpM=5AM#qPf5tv-5Ilj^bH!PRJnmO~mskY6qT+fmMhf3kMZ?2t}B@8jD1lFX^$y*Vu|Kq7T& z@0*G{6i-eMh^5}+ma<=(SC(Qr_FNP4bT=&W^|M7bxmUV*Cs%T>kNNvg6u#5*HcDSL z9I-m&P_m-Ox@*||wr7twRXUJQWI3%Kxtd+`@oP|v`}WVmG1nLS;dNP^;~W#4po3tO zGOL^q6}1Y$F)j#gN}M=X%4@hjeox)DAyh!D^sDv)^2=&Hm6$?4blZIB%|zXy>P$~d zO3Z@-&JdrkEN+pa8&9AU3ep^1wQ2&AK$0jYQ16TC$Qg3q*chL1&Iz<<#5&S8&gzV| zM4XiQI0LSt>xyRWUT(48$|zm z7crb9+F{6ghGJWfx=R6#8SMZ0eZ{;M2<+_0@%8{H%*rcTcP?iT!&5WtR*VZx}#{FaWtIB5cMc8T7A2jJru-Ozu1314=!u1aSp$O@r9Bbo>cLGkZ1fXo#<=r*k{1bhm!*nG#wie^wg zi?rZ}udlZmG|Xi$+th)e$_Y9UV3JRf96)Y26i#molwGC-9+pAy<~IPH9#VKhF@RKH z25i6lEimjZ_7Zij{3U7vg`@P!uwxmPUQKbe)fIO7Zmmr?7So*ibrc^uyAT4E( zK;@ADTg(!#i>Z!7DuDZlZ?V1@zxSvHQZAJD?@(XJmkOT@t5|Xf&?nbD92}lr!5OgL8p-pTy03^SN-U0Ala1yaG6N-0wCzI(FFmp2 z+_?7eNzkv=#L;*PmN$k0y2*>rA@Qz%hO)rHzU2Dpae}*{@@uU%$1{YW$l zRq*i&f_#$bi-p0)=_|b}^4>E}sxee>dES4JNVebT?6s8cee5>~w%91X`RM-W%TL$9 zd2@pO2}Yql!%BV_amhmd!bThqWpm^Q&MDBQ#H8`}w6spSi}c86C8((5WVN+iIj!)I zKtuux9|QW0(+QLhFe2tGB1piAmr6?HqC^h(A>V7~-V7JPE<(>7=mpX;5VZWOad2Hg z-md#(3L+g}t}P5R72Jn0QAS977Vzj_AO$4vxy*lVOx>;@3y$8)LAn%Q zJmZ?DYV2;6O1pAbbd^d-rgxF@j*C+rCNx~n6CJ&?| z<(GgO2o9uH@QQA*go41vJji;<%V5{ZZ(W(lK*dAm2D?PK|GQwHG%g(+oc*6{Y>+J*MKg# z(KBII%Sv?i1Rc&nGQWO)B(KmWyLy9cdvR*2r!eJ-E1y*Wo&GQ_ zG{zh^VmM`gTP2ij&DLw0yfnZxfVQQ%zR=(_@$D{0-JNfCZ>^ES;;Imdp^5Z~(~_}p zsqXXuh52*JG4(Bu>G+1L(GO<9zeaxJ#4HE9>!ViwJCP}i>@o*2sw8|rKYtA?A2%<) z7d@u!NO$VLKb% zY)QD)Xa9Z;PDo|ZK#d>$9aVYebohTWq zdF_dYss-Ji>+v76Kf2i;udsU@KRO+n4HDcvi17LSM=W)n-%rIr`~Fd${cxwX2MHY> znK^V9zGKtuS`6!|I1LW6{!TL)5%ozpaQ86c#Hv7PY1Yewt8zX?NSG7DR3jAEcJ4l_TLzEbb^8cX@v?L6*6 zS9tas95C_5p2g{D34OAT)4po*HAmza?7oA(oWm!=wx+`9UwLkTwzfjD%7I;l`^FSQ z=OV~s5PteN=;qVA3^5HCE1xbTfIa(2&O7ZT2&*&!-Eh6Fc>!Ob$v#yD`+i(upLVgZN-g#EG6`u}+|!<647(#HQeTuW-&Bcs;F_h9$h(Wh>L77( zg2b-Ju6pb3XPY#djCHo!ZF(b>?uwe(71N zzz_PZdtIL=0UbdJr3GKFX@u?FC%?bx9mjbHv!C2$Ke{WzetE7f^vZ{$&UX`A_bl8x zyKN@!HU_lB{}__>>v?m}-0hov!qv$UH;jx*BI}Gy@cjFAR*nu5_J(~E%F8(Td^{uT49sa(Y{Ydys-pYuDqNcme zE{5b=Cr8L=!}WbDE@z3lZb|>)FrxVJl{-+oBdv4yVa-`l~Ok(e|6!-r9 zZTHIBsIQso&Dpiy-fhbY6cn^Dw(}kDd6ZXKEwvvl&%foQeQhC>nrO;NOBztJt$H2H zuhu5-M*P_Gnv4fi5(ST%{I!S@Mu9gRX%A@04!yNVR2r_^QiLE9iw&CQmuG3RSoq`c z*%L5#2y5~JCzapMf}!QLi3qm)5l{++Ergkn7V-n`0c*E3eBhMFy>yygmG)xb#1Bn)Dk=3bmn~B9M<^W~*E5A8+!RS$ z{dHT_M65xaNmI)`2Ayxk81go~X_BA>k726!LiS+t>t9uA!=wW=H>X6YwbmvsnF;A2 z-+iS{_EMwg*bOhU)UomSf+1I0dvwR-JDB?TcYua*fEH(=d|+d+sN14dPMrefR&>5> z^PPkUZfV=pmx{v7`Lff`v@LEVBQvjc-qDpd$ph#cVp5CssLY`u<@+4YK{H~gcW-=u za8fdTw%KM9<-6_sWr{q~LMZFZsJ_A!6Ir(wh@!r~XfDm0{3LW)Hs*n-m5owcXn7M; zl;+MglE7falnM`7r#0n-rf0MJZ@;-KR0MZ})=6FS5U1UT2G+mcR!#6BAvu)Y{Su26 z)x3n{!U(OzoCoWLN14v8s{O2Vygavi6mG=cl<+3cWXXGncZ6S6ZUlH{%Kq~hCY|r9 z6`W~0IdVTHc%zfghxw+8l;2M$y8gTbsQNB_^f%P$eGKT70w`YyQohFb>&RvYF*szC zieRy;UyGGmTjdA6(Q&q&N^y(B^G~k+y1A3U2Iam5mv5}eWPpHkY;2p(gMs(BD-TYX zI7Rjn2C%ikk%Ng_1qo?XwZ3nB5`vp?({kL3e&xPimgVP=-)m@7O0GMu=Z`se{(xYe zsl>IzgAW!!pVXft@zVal&O+wEHTaL%!|(Ns-(L#c+_}ldi|YZs1$ynD*$FK{Pc$z2 zD{8zwvO3)EwYrWq;U|XE(}TY1hd~Uw=6j6@Aq(R7#^RN)#FIa|*6c!oXu98h2-WKw z&u8uox7a)O_zFO`cT)2w5j2j1zimAt?+YaWxL3Glp0G+cZEh|*4771ry1q=G7?qn4 zEQV}c_MG|B=ifC#0XMtPqUcvA(5RZQ^GC3zc%-#Y1qJ@?AJZKF;4$QVM$1KB`8$U6 ze|z19US+ag_&tzOqxp@#2Ar=S|EXZbhjfpDLG}Hv1-|&6PLV-|{Zc}VmFRs!>hglu z4+f9^i(+qc0pGtndfOG~M?MXO2*|u|f-h-8aOzh8TS-13nw%ILrX-*)Gu3$u+o!%^ z4S;inXEgdNz(b9cKhBS-6U4SJj6p=Vn9zKdqX0t7{4yR~zYu^P_REW(9kD6lph5EyL-A8^VA!MI02@pB+2bsOefS*k zSqw)N2E!UjgAQWx4QAr9W*wI^Q$Ue%e=rb9a2;dWxX;4G0ExYyxod^)wPT06$6c+p zyA$i+`VDYSv`sD+84#2E;M2BV%=`pMN;vwxYe6-NvC+<67Kvb_fQOkqX2wU?cSkb0 zcw{LV+ejCy^7n7Qc!IHA{6YbGcA07CF<sZCm%89G`C^q+7Al9I#*$Khw)~j6*a32K@kbpOE3aNCdMLGqN9>sR~H!3}Wq| z@t!bvU)uTJ?>^vIbZ`69&mAexpE1u9B*aoYN0CeDEVdT_{$tvi|4oLfVb_^lPJCl^ z5&28M;6Y@80&Ss!DS&H*{SJe)*L$+}h1jO)wCcAsdE7jZ`KsK2rom<&*VKzrOYOqwHM}M&&H`6OH7A zlN88OY7D9Vt64R}Bbb@Ka$rXrW_1`v{+!r%m*J6Ebj8ggd>C>S0eXHwB!-(oA*eo0sTbQZkfdiJ#)LAK1@GSPO(zwL0lm(UKfVBl zTXogtwS`}2TLgTkAP=3*?pfoPvx1IrOn~posy%Cbt+P{}fyQVrH!D&>uYv|Yn(TCh z`wan+vGAi^`*BxUL4I9{nmCM>s~LY~swEkA9```?GJcbm_k7u9vBqo`ncUa^;vMkv zu!^BnLkeuB5n2K0197Qq^wP(MTk2hJ-`eWGur3r7WIH~n{yaTlC^_lhs5cq$k^eK7 zePuD}>@sg_gjIDyun$Wadz9ae8{XrN=~pEEM}G#|>tu^)e=9J2;Y`*_%6$%Z=7wc> z6EhYQV}JORThgpf1sxTl44z>U;Fd&_}EBCg9@1>!~5Z%H#|4WMl0HdHvK9kO&E(og@236@O#j z$)nxy4Xtp(a+PZ_E7nuh-t8OdRtx57WaV~Q-XHd|zq?>B!-FhTIPV;G1|4=j6T4c(zh?wWiy@FyLz=bl`2C*U8HvD}4pqgH+Al)J*5)Irm{LIE8i=85-UwPM67fXc{Q zL)%}Ad#HC&F>K5{jLtTLTUV7BB7D-(-=jQW@tFunB1bM2R9`+HTIK+XLHWYh5)&gk zu0QtT#MUiFuxnq#!(;S;hpuJ`@q6vB)$d_V=8`Z?et7#C`WJ1&3z*5}#!rqb>qoOC zK}opQgO9pm(_m(4GSVg0T!!%K`p#E2*cRD6fkB{CDoV*x}_#I*BH}6z<`c;qGXaA2S zGP{|vRG-NHWUg^2)-}q9p}Sr%!zv@84L&)Eo*n-syty>BQJ55Gn29)Aw+&fXmn1_z zEYBtt7%2&5AC9TrBZZ24yjlGBIeS;4;rI=#8(^3|5YanZZylhpIe-&{lPM+pzB+nW zv$eS4T-wb5$^F&-U35cke84|aR5w6D@jWYpyMz;C>{QM)0EBeg%NfBy)N1i)*+L!Teq8s%0u5HMRi4q7OT6R;-77{&D zqPtZ+i;t6*>L-7Q-?wn(t100ut^CY=jWeT=fS8CvMKRaKc%t=g3F{UFPxx#Ke0oeU zd#mo7G+z!4SCMK;)Q@3CEi|}-?-iIlmmVQW&B_fygp;^$a`Ec+6mj#y#Utu?rSr5k z^Q%;>D6u(W!Uidunnf+S=dq0JMv@1t1d&$qv_dC|-)>=Z_TPLfyrDI>pMCUE<6(I! zsVxkuOC?=d(D9dRltzd-&7nG$t&7?U;>C>je&>=fQ zBgFmtwH|Syo(2I7#0-i$i7Zf%3*s8|s4?`x$KW0+xO1iIH^}0FL+x!y{3==35DSS8 z>dGCkPnNG|S|~>6KWbU5HUi3S(xI)$Z$$9OkW!;xV|>id!zrQs*Ho!5|AElD@)C$p3!LnJs#!-55%~RFe3MB@#7yK;KayGN8zs;hfSZFHM)`hF~(Cvm;0k}~*4}gyr7gh6I&?x||d!B(ecVL@H z-~GonSb+T4VHn*1RgO*RsQK-AF(d_$A6Mg{`zQc?!l8Zl9_)(M^KR6)!^|hi3{_C{ zFDT{Q!0mJM-f=g6nl_+4G*);n0khRHbYdQn+7cc}D@|!&H1o$p}T{Cd}snV-V z9wpjI@{M?Lq)C!Jp7bwpDCb@wn~eA&$Ss&Zv6iQ0S9W7<%f&4+Qh_ix`NzokLHgD9 z$P6E>-q05E(j%6E4wy6v)*Nb``Rn}qqT6Z7jDg-iJ`y3nfs_O*t|Ftwqbnj72_Ae8 zHdCL=>Z}s;5T1TUs}5otgsNjjPNfEawCfb{0J;1fiPi?o=BjPl<0&`Q3mlq-40Xy% zz7o2=H?xe^B<8+-SFmi_=^aA7jZ6l9jsy^iG{*QN;|E8LEwmzclcuKOXUj~I+;Uvm zFDv8(p_{wW_(PB_p%p5Xg83FPDrHr$ZLD5s;0^?-V78;e_ou^>KY#^SC2))J zxX(rDOpEgQzBsno>%omsvA!7j(ctSfvi+q=$0D^aF5c+TdUp0b%5CLmZbvyJo0St+ zzmw3s$VlN{_?jw$IElPq_4|1Lrnt--1*a=PNjEsF3qOQo-t|+Kx!hijg`KY8<$E80 zx?9q;&=y$OJN=4Lu?M8I_HioM-?>BW;0p9)2-4E?$}-5Z+}kPJ?70fcIUI$zRy&U)$7$2@lxNzZ?^Zuf4O&dc6Tlh-IZx9 zcAo1z`u4Q1F)oaCYi2F%^0OrFi^{{EOf|wXrIAP6SNoY}l$}S;J$uh$jyfOj^t$dX z#-@lzmW$cO^iy0|=gv!6yBlwk4fgn|VFv{OaDJPxFF-w!$y9WhHP~Y_z4wZ!^)}9R zl!N8sI;B-|w285q`U`G3)5KQzL<!YDq1hhh#%f82|Fq2g%ICJ^^&ML((&IN)qF(trHaA|}uRy5v z2h$iu`84XXTGHSr;}~1?v%6XYpS%z9(=M~+^E-*>9bIA486re-N;gvi=S+{WKWF=% z`af7dl<{x~_SGig*sB-v*ReKk(ui75d&_KlulEYJg{Wz8x8Wzfj@M7moco2v^K--Q zJAx{`!7m3>Y3#~YlmpPDyZ&Sf>6F!OTPh~q1?OTClVLOQ97#OqaIXk5g6+5tzQ48z zZ1Cn!vyEIA2pAE=35|DgOe#2v>$_*rqI7nRw!C(c(*!qcWiej-oC6DWhHPyWe&Kl1_l2dTy z^GLSl#~rBGl5eBa8p#OA9U7Asottp_5wSwjZ@k5FGF7RV;ptu-XfJ{-^c1{TAyDJG zwpKmbRCx)X;C>t-FX#=*a^B1~w3{5Ea)7wKL@~g)#lr!K5R~zISjx`P|SZm5h z7Tj>nDHc8E`D+U1GpbTI)J2vVy83#90@qs-~swy;=4H5NYMHh z{w?M!>pufuL0EkpqHXuoD;>vu>by^3GR{xsU=$0{gxUMXJ&KX)M7yUAgvt90R!HSS zVhG5>k<5_|vKaFFo$%meFNK)zxhV)3k`eFG`@y|d?c`@F^U1GP?}Y&Nx8)<3z`YMB z!C8xc;RjdW1TxX}@>eHt3|lZk02G|NGN_*XFOv za9|YryWW2Z{wrV6jsGI%6_?T93morprZ6~RPpEK?$__GjZ|dbuNL*yUUF)cAc=3qv zbEH-+t8?WCwKdTo&y9DwZMIw;BN-y~BOB)djUfb-z4%*3Z?_JuT)M^cj~yDK zbw0phi$qA}N)L2{+ioRQ*Vx@^G5R|fYDs6KA@Hkajs$+ppD4i9%c_@W?3~|@gr4WZ$K)Bb-pjpmh`+~mD{ z=N~Z;6nHb(GBN5zfsj-J8xgoOz{46j2&nlb>VIpU4=`t1HbwrK1HL8nfBUlj)ez^+ zbKhVbF8n_OrSmAUqn{`J1I0Bva6ZJ_e5x*R8^RohuSVw}V!*Q%Yz24AR!7<}t`PME zbzsVDbZ%$n$z%nXP3Q(L<`Gk2)yO=v`>yK0X>;o~+%EjRT(&xHqACOWl((9lu!f?h z18`zwiwo+H`u&Rg`Tp`Xzt$|Un)HtS?rrV6Z+G-IEPH=* zHG}H?fO-SOpvnq6?G=5%Da_l8Rle%{n|n%VVb!b>htIO}By=84DPi6k4ZP@1f`2LaQ>#P|w zZkt?dcxF9?A>;NE>40{#zzNlTngYOb=E4p;PpuaLObc=^M_gN=b>5J55ilP}ZI-$c zTnAABEYRzxyp%ROa>FE&f&Ie6pQmR`O$ph>Xv*o)*>l)al(Fh;{dcy+&$6G2|JwY0 zEydm9<6*wweJXIZp2x#BzuqaLXSxZTgdpn?w~MisKUjH>ft9pYr)Es#66EP==qU*N z4!$dM|$Us;r=CL0J0IupU- zNhYa6di%i#A0+kuS_TZvxcL_=7(^@hP`{1}s4WY8|-CW<^WLnlSxiEV-XZH6By`#&$PQR?yaG?Q-jk}ZE9Ce$Z~n~tGj8rQ zEIxnc%(?2abLS`j|7>kucIM3;%gv7F9?Qyh1%LYEC*<@<@HdaQjlb__YqK&o!8Egu z)aF?pyLu%jv2!LHmYsROKSHP|CzE zp5fJ^k~>G$%dO?IuYMC%(LMWj!KzE2Z)Trlm9SDh>puy2e1zgY8Ag$&AIOunAa)Vs zg*wo|Q*AXbcC4Be^*+l=++(r~!xX!=uIgW}GxkOUry*`xI#fMnSP=4Y^ChdwGj)sf z;tm_nV0k`6hB3sM*UILz(_6K=&xMzcEEX#D^kcB9D9HaiEBkN4-|Pzq|H)+qZhykE z;9tg`d7Iey?_MaF?9%!=VEPk|1z%S#_xCvh<#ZrvN*7P7rpyt){o+C=b4RGUt{tr1)f!O zdpUz-*lI@LaJlw2e@3G1Ox<>4pIV07myO9qtZbj zG?5}m?;yRC?*yOc-sgSqz5o0D{@?lKK+c}onc3Ny-JRLlgu>MClaVlx002Oy{6J9) z0Ps)%Kyd8>K3GB?z(NZagu_(fO5h^^NCAKaAoT{YsDwZu`1trlL`1~I#AIY-#M2htLf{jMe09|G*;6$!CkHuM5vJgZK-i3wjg=ml(_`kideYUV2TR7WZI6GT7i~Shu z`tc)(yP@p7C1w6)WusU$7F%fnRF1u=9K%*(D=V>Mm00Yj(UpeGpoWjJ4Ie>pkSzGO zp)JL~4aZPf+Zwh#P^~=@{5Z6OZ7XzWKd0T^9*Ls~ULe`o+4fm%I~F@IFfdx?KZ+e4 z#bU?aNP$Ggu-Mr`huOl&*+P)D$XO5^NTMdswMPv{VAf&K({ER#$2n= zuJe^#n2j3Oy#0zq-+jA+wy2}8KA6bS_pcFeX|40ijEXbGc-{r0ZH)SN9t@P3r;ew% zRXo+^98=LMZeo}1s(ThFpDT?2n4T5?vBhsZ;VNI2(2y~&$*Y$pj@K^T1T17&1DN71 zYw<3_*#Qzsfq%bApc{-7l!_#quayO^up`}rzG~P?8ry3)UdH%n8_yvY$JkM!8pSUN zp+9#BQO!$0?00%3RW3dZVFEl%A%|M~Lb#ozfb8ABUpA?p1AS74TE6`HSH*0=^|Yz0 zp>cG{mF3Z%ILn5NXOo8g(^_XTeqcFA2!iq@&uOgq?9VDw`>Vw78amWHNH7f!h~(fw z6Wr_D5tuJaEZC2mv$q{#F^o&ahWK6Ap%LEf*_~)iLP5A?#)>Q5$?mE6Nd%X7M+B?f zkB?&H1>||oQL!d3XD~|#T9BHCk~V|Bvp~F#*?Tp7!w5M`1gwcgIp0}ehoknvGy>6X zt5A+(#}ZJAioJXbB|*%A2d}-aRbWW!n-9#DGN$OW$UMI(p&#b>9tYP3(DpCAjpZG_ zW76q_F{dpdZ7N2fTbp^9E3MV%neJ?cHYn(hb>HtqV=30Cxn9!^eGF^Vgs z(9i)0caadlp-zmdXz*l-m0(8pA7v8$c?SM^4Cgw3ZKd!p(2LXOkB%`&^@~C2hWMn& znx$~Tv~vt%7xUIa${8uaK2ak5mo942kvZ?Ge-wKMB1aoo>=be21VCTUEI!j!qxZkxOw^LCRWwu(4$hXO+l!nwE(WPsF6 zdp*}%_fyFt8ZoLSidTbbZ;I6Xd6^7&PpvzyjXYmy){S6zWt0(~z{jkS^@mOg1odfu z)f{^)zEP8>8T1XOx-6(9PllgK+BnsEM?f=!(o%-BNLX>l2>JziD1+dGc`ts%6&lqo zc52|taCN>Xd-x#ej$~*GC%rPwrfR#>qN`7vUt$Mi zwhE5`e_NNumwHdcBqt=5x=eQK-J1n-?K-bliMh%1(Oqn4{7r{Mo~g{XTIAi_8C$V$&EiK* zst#yWzDpYu_D%b02j|jF>Pa}0p87dE?+D=e>T21!y#=Hv+4KiM#Z{9mCm74aKqu*t z8qe-Rw+Po5Cs@;84fNn|Q*c8x69Fv}c{iF=LpAn*xTy9qf$>d62=<#B`1 zYKRdGR}|HS^;TzClVNx_zllqdr$#!)2;`lHzYl=8SAbf=0C+Dn8%Ovqtaaj{GIw1m zV9B1Gvl6aS4f8Xo(XvZcKdNhqxz%K-otdPVP0AM(+)VCAuNIg>Ll)B3ArGm0U!Kc6 zk1$&z-;oCMBm3=hBtu*Ia+YBcMxu^By~^2x0;Zio7?IzCERdzagEH@m`U*{KYgz6U zrjn;?=Ph!Sv_j%!**4fvWuEb*$ky1p2hprL)o-jtL^DEl=JHAe(jyAzWVhLZ|%!5$0C2mhaqaJsbVU zv0uorTU*Zq>5F^lZzQw!nn5s$gTYDhbPV#u-PJT@ntc%M0Lr zi<=uNEfYI@!q)HM#$rbm1ZG@18Gv)1!%@Lz3{qnYpH&dTg2ZXIgr#w6##lTptL9(0 z1V=76WT0@PL(OD>n8;MX6J$YM#+&ppoFURq!almcon4jPPlo7t-TkuHg&*-?lZ=oG z>aVz{#_WZd5PX2Q_-j0_KX-XR%jfP3$2pBXKF49+`I2hG7IB&ffN%PnR@O0F*7Up3 zlEaPCuB0!utCPDfC!cKDG*zS=x!GquKNZas9s~!_LJeMV;lq7q^e7CIPEV%h&Rz^h z?nN1o_-7CM@7NMIIR8T9i$83l(e7X*IlRgYjwN~^E2Mgxk@7TJKIGo$PfoF?Rj#Wm zxt0lD^zxn|PoyMrN%A^D=<2_rsi5dIVzrGv(aUi_z0cXGv?U=bq2bwS3OS0`k$cwC z9cM-$;|z`p!qTo}iX6~k3VAH|aocE5VOvj+TnWBabt@e#DVa#5R(=@z!7EW&0W|C2 zHU*Aa7cxa)^fhiOyAYI4a`wB(^8-0jY-Z&#z0hQW?KM1%h0jl01sC@@hF9Pa0uE_p zT!$pFdbxG7)kJ2uP=O09htp7}BPDH6<+vdNDa<|PqI}G2tL}yMUy_Dtb4^4dEgL#; z-NJ0E@bjH*M@9Ig=d%6K5YuH?X^~(Ob1Yu5D&sT4>;ss|LVWhX@lGmAF4l(4X z#0gQ33HzqoQmD5Omu&B3$IdlW0!xzo7PD&OgI@+hsE!cEhrq6L|MtmG)wZ4RiUu@Z z0s2xzLQLAihLu#Sg0-(LVZ|b3%r!NvWPL0Kw_vg*gwJpYm1KB)w4nz_<04wjpo+1Q z+ZGNuMwF3 z;VLqS8j;8*&mQsKIc;^v6E@||jnoCcAoZ+qgm1`w{UZjCRGc{c_y)#^xGz3%J3Qin zZkx;$Q)1wK#Smhsz|mg?6d2czKLrJCXl>0OC(-Q@ZaOAI*p5B zLBv+6_?R62=-LA7n-a?H+;-vHKJCAoa`M#~diM`kxmn-{ke{39JrERb6QE|bika=UyjHdR=|{Le65nv&s5 z&hlSA)%u?-P~^uxUoM??&I-NxaN%km1e3yMgx_Nd4OGia^_j~}C$D5ZE{dx6SuTyH zi#1ALo$!4yEx0Yq-ow!4lSRT$5*6*7HSl8G`Tl)~!i;H{ya1fpMrxF0w8L&5DpHpu0=%Y39 zlCd>VyuR5X`Vp`qe6zfz6U7C}sDu_a!bl?Jzfl7%=+cu-t3bMs_}dyyrYG0zPps{a z3k#!tmnTBkFR85ed{g`O?IZROK4oO`EvD()e$Ti`JC;xCSMc%$O?Ex;Fco%yR8N;v!UbfGIX)&c2|?8)fOWg~ArqhYht3aWug!89HBu7r5r~mJK&mEOT%xE#;&*>B zec%)22`t~f9$#bcTN-Y@ct;aWj=;)vUqt%p`wzKS&~HCD{vyxno;;fkxcU1EVSe3f ze-`$K!e`z7AHRwFuh5?fj_JLTkvNsH)jqDfwGemk@q!7z)1#4RA*QBHHjKdAZWLozg-&ODu?^c#Nc|GcfAjb5y~v}wj>>8_<@m}>RDEwqOzL11!Yr%Y zDy^HXD<~5`D{M7));SlFn&f;~33^|FE|lMw7qF!v3kh|OkMGlsy%LLd+gSDYYgf2+ z)~YQoxa)lTY%Xfk1rzk8@${E-Y!I=Y$QO;BX@SL?xAK}p5}KHU|BxXvfL)K%G({sLsM^d*;w0(;lrWNCpU){japNf7+2-Qjm+C&_T*lca3IuiAU zg)hIlA74{QPa}0AXDPf@=$W=jFIUB6u8h}NxUG>Ei^s5jvt6{HvhT}`dcTD2spgm{ zH!Z;Yq=?te%AF8;J@=nC`YLd`J&867>}DgrQGF4p-#v?|Ec~cEYsL=Y7KsgiYj+Xl zq3$`MTUn28+OA7ID?0Y?s9iDp{Q^5~UF#>Hb43?2U$l=610jh)NV{FfjS;@HLp~-o zy6;oO#EtuFFq-d|F)@Po%9xS&-X4d43ZVGpz8c~C?B)mXl_Y@B0X@4br>)+*0-Eg9 ztk>+7ObJmsv`SlhX#!j!Z&iJemk6NwG)n99KR*~WUKL9YrFioK)1n=k-o5QF?)9$v z2A;^tHNp66_)4br^_E;?HCH>_0VyVnn$PM|{%*Y!zY1CW9*-4Ot_*9n+}}9slb2aC zV>iI#7AeLtn4 z2Py)Xk>4HhQM9W-g&!kO^q3g6I|dHd(m>H95tQ!-#OhQGKzt#Gu6%-ENZbKL<3Fm_ zb3y|}N2Se7r=I1{&2n05d^as4A91uDqEU1t-5>Rp$g`g zkzquey;5Mnt4BM7w{SLuLt&io^c#a=lAOwXDM<$OSew@I|By^kn4kJx2=WBxy zp!V$VA&WgHCgN%*!{m}DKn#g|$eihX9CkbmsfWL9?}Rj0<-K&Z4WCEE-zcF2n{XN+ zf`@!@P4@W+_N=>BF!E5Ab@x(yfm%nnc(`oR~`L9sn(h4 zb8wAkfhPjg)^~kM{c2Il;%R`0p5kT&xi+(y48PVF2d~7AJ=xHE1fUJwB1bfK6JVsd z+>eg3*Z>g?_#Zf2#6%^)NgC8RJZ>GCq9%Aqw_ReW_Ul}n!DSHs(WY(yWMQws zoOd#A1`xyN*8fA%hY4-~HnQwbYeu$FH6^{XYn@KOI|gEouyYJo(Wqdq-OBWF8%0h5 zZo%j+`>F>8k4v`MtkDBg+9am#wRj7OKwd!-2_~EHjJnV#v4MWWYt(WNY-_Y9xqA4a z;LRrIXR-et`Lh{Ozw!A<_K5S94m}9H3 zfsAK!bNfU>hN%hHk{72P9V9~q<$CQfEeowK){q6%l&10W;iO+6w$A%(d_U4+o5KJm z6s>!M1UeVp8nZtiDBSgE?uNueIud{1z3Qj~MxlcQ6PdK`5K~Sa;VFs5Viw)QH!a&n z%5p$~P3wSah1jJ*z5m7IiFE)s;Xr{bXOn6tu0gqYh7Uw{c_ZW4&te}A{>(L-;$xJl4pN9lP)qqig zmmJ98&AR!bUgZC9teCNi#L5&_;wx82WiBfYG~ z>#Uw@R@lY3Q$6p~`;G3u_=t_AFB=6E;(;rR77wcn@L@?S!JBk;O~aAT8SjroskAlt zdAlchYzBHr8~)ouHM43P98g z*ZN%u;)tB=MIL}AGkpuf98E2-nqwc0dF}of8CgIB0>cuJkgrUv6cF?~Tu7rlbhhVTdehu0w`ukyI8oq-c`G&=Ez6$`$Zz={toR=dH)0*nR%iT~g-J<_Prp4W@!PY$_cejVYz$eHxYm{Kc@&rK z#c6MFvekRmEX+@Asp()~=rflUa@%}sHmd$xSj1Jc%Et*;!MXXV2LY^D{)`N=DDHnF z?O+%?p_4{JlcXD8S7 zlalbXjZRcTZi91r$2vLcgY~SD%oC?ru~lc2!^NoQEE}>{uRC!|nE7L~OnZ4tH-(XH zWtFI)=RB*^9F+;=K}J3Ws>W|~XM)*L{En?+b&EdhZ-vNXEkE4=C*(GW9U01hH~kE= zFmyP-;ZuIx%XaU^~Xk%8S@|#e^CNeYVg7TiUW3b7`yXs)1ZXuO2gd8 z&9ydTx!o(1Tl4(mVXpUv97F9BH$WG;U;fyYbJ9$+!OFjuVs7>c?bO($S+J*^ZfZLD zC)Sy;n>&PP2Y7as4Zj8nJ%Rh0d+yF#?7ZDSFYA*2pKsmffRu3 zyffl-KiqDV@A2<--Sl~|m~pvPGog#hquhO+2K)92b(5;~Sl&0g26mJ+*z{Y_CZint zK1~NEzq`KfwVe>=4ky3>&+2_Vc`N-$W>(!!`^npl$L`Wb&u5S}-3|*>Ryktd9P~OS zL{Q#^K2-$C`D8PwxK%a$0`aCY6;u*9dvH7-Kw6k&beztN`fUfPs^KTUGMiRzUgV>u z2h_NCNg_<5Kt2L{@vW3;;vtRcJWk*Q%iSRn&+J3}6T&z>9V^nq)vw8LI#<L5@iHIzr$Y&dB*4-lj7 zqN!oZXGaj)HpCH&xRQ)7AO(Lyugo!j`?>xm&yf3DI9UAR&x-29!GHPsju7+o&XCh! zPMbpcbKciFhTj#0(GjAk=&cw$)OUM8MBh&f7_1>fa&~EuAQ7MRA#f>$&NrxM({UeU z-(op7sZ>TAs;;CD&Xua3we+2luJ*AD7}q(M%6oq?6b52{Sm47B33%?_3DAeJ@4?Sn zW}Er%e$Lc~YsHi|b8@^8h%Xy&9~pHIoKiK3D1|4u+^AfpfO(!4=-hA=dx7x z*d%mcVi)L;udypm>eGzZ7`j8Ffs-{%PP^Ic=96aOsY_JhQIF@TR)fCu@w7|#Z-I2%dzG*5EWrh6HVvEjSnu|ILJ0*qE^l{e zYYnK@edRKK`-A#0M4NzPHN5G>P{@3%%>d(5d-FFB!E{{sr-$+<2|mSF{ntVo&$vqS zW$xWyN>ImfRDsG^t_e%%4y5W1h%V*Y4oP+FKNly*ElCv;3|p;hbbO+0K}Nz&p;ml^ zm=39?48$t$2|HU>&N`ZA(FhO zf*HI7v2!JlWP+HHT4OeZ@Y@M6TD;CJJ^r+>ktF#w0PSeI#pEjUl!k|TL=F@Dp6(JDd_6F{Q+6eR=2um{)lUO%nq-2ur;c+f~lf1`D=4G=qQKeOp`r|ay%3zkTN`H z(USuD4S#A#O~p!!s&9bBuTYk;JukoCz5Jo@Q+ySS%6E0Kq$=o`Gj)sRiM>~h`gfpo5< zEL`ikzqJud^8AaWh2IzxpWGOaMZqI@@HR3`*vXxR^`5j_EA=Oit+Jj+J8h?nY}ew2 zpOCr)FZF-$5?hkA>T6=Gwy`Abj1=Ezvz0+$K355?>azBxy}e&a!*^VAUm!%{@b!Y~ zSSGHiwvk1I0#=Qvp!X8Q-B`TW#HeYeH>vVOCc{C72D}@;zO8(~4`Yg!)UGt`xn`86 z<6sLxp^aO%Tgy!hNIUz44!lJ_)MZgZ$OT{a`?T68)q!D{C*&O|7Y+DxAM|$cmS=k` zGoWl{P=_pYYhz7HxruQG7{9)2_5-|3#AH3fdcspoBe%T*StMK^%&Gn`lWcm+ZLOOm zP_Nwv1YNuH&)DEUzoTg|1{?C$wIpkLQm;<7)3L?bowFB_ktOU3u0q9ts}CXcqT9mD z1$$E7q#7~I~I3o2;5*x`LR4X zQLblqC{H&wRU&N|YHmP77(@8h@g%NRFzi}cZ_3EK)fGQ*pLGn4tB_S3LL7yCoF3A+ z{){8uUXw<(@)V4h6kRa8koS1sPw(h`Jm4+7+Uz)Ss!P-Uqa>bhSWM{)&D+|Jngaoa z%UO@h)_*3kuplEd%#afk<#V$)Up1|rHZG`AOwvI^j85WXngsoJ3grY~75J2A3Q!EQ-2MY2Z%^rL@`5sA$n$GMn*-TlQ8pE^dyGacU>F_(#|G2tX zvOdBH>6CK`ygQs~523Mvts6Gs!yaVDrB>SBqymDpYS4iulGSYgo+tBv^ZZ|OX}*&f zlGs_~%&XX4##~Ell3ggkJZc$L{Gx`JJmPf^fZyGe)nCY~3;U(QM=u6BtNC)D zWeSx_p^`WG3kQGTgNCiN7r~lB$2aIGQ~iw*Ir}%rZB)%Bf6?zMdhe-@N< z&@&bWIbs7f>SZUtc$Ay`4+9}qR3(^P#e19Y$Hz$z90Gm3F`l=dC=e#vbf&n`7FQdxPLk9tD~*=#diZ0mrC|8CFU-2UC(tZ*(tN*v{X$M{+>txC~#^FJj5=Jcp2(rXyp1e1EG`U3jt zR3Ukf#FT1`N#ZkZy)~BU;w6N(Z1wfKWtB303Kf~rM@%^pi#(D;)L59+9!{>^-q~mD`5+&bE1seP`;M(imx-oi$S%GXv>XMYJ;p^8H+!RL>J)R zJt6+Ls7%rhr05E$;rK9c`>L9^71XO4;=IPMweQECyBCAE=5Y-7q1`X(Vs)()o`5dl2mq=BCvrlvbRa4sERO^^pN9a&AsUP{ z{#7~{#hpgnt%>%G*ReSP=AN*rf}xn?*TG2#L`|w%H~vyTJ6uf0`ldsOpmDJc&a$rH z*+@AgfW|*_(F*NJzMoRKi4SJD#Oj_WH8^pF!mDCv1}?)kp9XTh3};VLrp7Ge>Lo%D z3RMp>X$7Or!timqEofDq48piVGntU*JwsIxhR88%Qe>yjE!-$$GY;%NDd}CjVB`}C z%IRDNo64stZ`SCGY~z(l!#w9n>6M!~rrAO(e}O4NRL&$^M^uaiBCi&n^;R>;L-jnP z-uqfag};~42|F*6hN`hTw6E6XnTh1+0tf+|tmL5)tQ2?Yg>+g#8Gyk(5`jEJPnL>y ztNxbBac3$cpR^8CcnKY7oDr<3zN*x93^RaZ7?hwNGMNO~4@OEA@wkbWm4jX!C1L^! zEbihof1RB&;gz)Ve`AsVX3|d*L#q@o3HSY{h5m;PKSdV1OZmi2WSVOOdBL%u&YA+e z6=3d-nWO_RG;T&;mj~b#kr0S~!*5taA&^%ycn;FpB9*Rgsn9^gJ4rTtrJnP-o8MWx zcux=O@%o7q_PBl;b3IK->$N2hYUSAb#K5G=r((hD#8}4vw`OcU+Gt>h;*)L7G0)HV z!Sol@1hb1rPINye^h*4T`Q3-6dG=~`hc_{88rvKBS9-TxV8z)%U+1#g^21qEjMjVo zmIqFs+@BD$hT?<6Va3;Xnc+g=xxA=Hk>HN=nG=>+{G_XEgUq-}$}ucY3pAoS9zY-* zWTkHZYKp#~p`3RL0;3{}9drDhuGgD>Jkb|0?fjkvv;{{JXxFsIST=Wrk`F%p%tCyEZ^OP^S zgzI$&3#L^*miee^$j0(0=Rc;pR!MLQlpxL)04KLwFb!VZRA$OKWOv0PQ{ak`j8o|1 zv^D8HdJRK1+~n5RLY?pmho=IZAxEthnk3h^3*&?Usc~8aOiLL7b9@g+ZxKR6YL6&0 z#Kk``5LDbuqs)t7@gcLJ!lYA``8JEHP+`7Q^UXWD>qc0PN$R}jx`^7KzFkr{oTrM< z(9q`F?Y}`3KKCD4H*aBxO@3M41-yKE4I8KcQ-1NK7220B{~y7vXIbnIf;R`{3ce}y z6BJn`SV(KW<|>}nDG$;{?}$Z~u_i?Np6;yP-iRjJ`2CT*M-vZW&LqN~!_T*=6rEc& z_DxC4T1FXQ&RS4tLK@5Yucel=ohRKeiK5ngSwz0)#hahA{bZ-#@Cs(PsaL-9taRgO z=!vCeIv0_vcAZFqj6tUb2vg%ZWLGt$-@@P=Cat_1hg(~;KHmfffO?!T1E>;S(SfUv zaon7t0UFfblZg1m#rpp;weA1BajpTu+xl|*?exStbeb^O70}A*&#R6$MbclGIBP!mnx)qYYkwiv50RQF*=$ZSHL58^d?&M- z(0cWACtaHIR}y-?K1*XYu|9ttW7)aAp2VPksJy27fyt3A9xXi=MDaU>m1|mmB+|m{ zI5jyjEp#k8&P+63;hJO$0-`3sDr;z_U-@z;V zD7hYmp8&Y6R(-oZOoN|QmgROB&2$nU8UuSBc+W|cub|Ja`@5!5nWyg0^3)H5rJ3MN zNP_%+4P34<5%iHD@2Hdh<1XYy6{e~$yqL3^p}&;)plmZewdc-<6e_Z`wbkX8{fvrF zH;0d!rW$tKbDmqlN%o}+(C*)V0~N8MAN?kNP4V;U5V~4lZ%~$((2eh=aJb7w^|0&~ zwoh(wk8XDQ6BW2?=<1Q<3)b9YEJQneUj$ssIoUp>4=BHr9k#7vr4(8?{h@JPyKzcu+i`BVAiSNx~-^U7JL!RYTjQc?~PAC@kx zEbQu0)VPv&(#lf|9jrTHnD`)cQVA%sIFsViOHNp9jnfI?)7x11>QSW`;OqUJptfxIH5kaYF$*} z&{l$akxEFEGdY~>1U}&yxyQVghrA;YifbOap0N6!zjaaS5(`7z*VE+RS9QFU8^e1Q zAIX<&e7{M3-0(6m=wv+EcPX|_!Glag!K?CShT?~3a_m=E?|L~Z6hFi0Jh!X~xtIO8 zu{6W%=Z8K}w3Mdhbq){`1@ymb$luyhD2tq^TQTEY>-8UQ1QP`4cn7>o+m*uD7A@Ef z$hax;-3=kOTVm&K>`peu zK`FTz;}~E?0&N1M-|GV2;EvxO=u+eI2Tp0#Eq>QpDR_MRQCvj}-*36$@tooqQQp3J zA-L}~o?Wf?Ei@jiueKZ(CJ??-#Qa?8H9mB|W>i{|pXFntC*J`FT{rjV%0hZ=K7e*| z$zb2a@ZxvN5gyN3)V&210M}Od>`?aDL9zpnQAwEOn6c-T!Su@fIWq63 zl&Itl@KBu7Z)Zd%M!#fTt$jf97q6J+n=y+Cek0pIc60o{jFCVin%UaUybS!}Jk}IM zvP?O155Ob~o)>MKlt54l9eC^VL4sQE|1+lbfAbxVocMI8M8ng3I~H-X9Z>T%j*(QR zE~@372lM5o!)Q3DCB$_I0FE9CsCBp%1Ev_JZXQNeREr1WQ$i7P^h{nug$hXn9N@7B zY6R#8asz1XzrjB%Q91bm4`GUzKIK4CDZ}(Q`GA7=-K172;HG%X)|Ft3j zSAIu_A+S@!uc;gvI(OCty*4R8A8I>`un3_TEef7_VMXc+;31Q%!NkB4{8`HL!qM$#d!HxH4N$)Pe$EHiB4WcSOvsh^V59O{t>? zZ`nK=oP8IFQ94>{_{m3h@1y{mWkVH8W3j79TrsT{GvNBr>Hw?*3=URDvdJjl22W^g zhbvyY3@I>*%K&!GRQeT zz==N?!wIgFdrX)ya>N5Bh+mBqX-2>GqJEe3MQk^!LS*`OkzJJI{o;1R}i6^Ekb>o`TXzzkmV5h!bIZ ztTYce#rtJJo8O)7Se(w^)&@O&0fAXCwN(}Zqsk!EfnCCsN)m%_trUz1W6$?c11%|t z5;is?Ts44L#l&~Y3Vq<#sj)X8B1v8VtFHu>J7RpLph~dpKK?6)l^UrrNM0Li2v}PE z3ZM<_>ermh{9=AQ2!UxYxI-8&399rihO4ixX%j=$QhCCE&vx92HL7z}Wf!fgQ_Jh4 z{AEE1-PL}F2bU@^6sfXn)K?_6Syp}Yl1qbp@G&XxSgQRpf^5;kD>?rb)QNmKi*QIo ztz3r{^-nrtcm=n~!(MpbwvQH+3)GbbGe88mQ^k}?m#?6}b3QrU5UY6p3Y0}LP!Z1t zaK{xsM{;t0vEo5deZ;i9tF9qlw5WREOb#<_iLn;0i!q(zcDX>>B6Grx><(LE!jlZFG~L5P}Fs=8pcTyDP%9&vM{u!NYF?J9}yMtNz$|Bl@8&PWKRl)T>FY)X1F9aQH>ArIt@GSve_%MHk^O z^U$02x0L)AT-r-Zwb75MWu%6d^z^bGwvHS=nM{>m_is$-h>JCq85KqddKisI*cRt*{k*gM6JuNSO<8R>9jO``Y)bVNl@OAVPTPP;lKd* z*FU)#7Fnt@`2TX)Ax~rlwgeJJkB$y-XfzF5F}W7LDggENCFVES8OwOC&w_7>9gp8X ziJ(8}5@JcPVCDtS2UY7D3DPnn{qbXVzj8yd zR@Qb2|9q3$$xp^XFM3%5<++xqx<)VFT%A6<07LnssxRMd6kk%N=r^P$_yaPqwOYh5ngDzQ1bf@m?Q+RR9+ z;6zLg9j&Pxxy}1$Ogn0!cW#<EEDP9arh<7Rc|9gHz-BS6n(}1*D;V9zk|d3Bl6$fjb+{4_YP%n zMo}Z!a0^-}!#_PcplW^Z9ka!4(6OxXM7tv$60Kci9rXML^3O$f8(O}g1;D$8VmOvJ zczrU@s_!>QzB1WGbmASnQRnJHRUz>S{!9|E2nMSayIRT3{pq-2KvA$8Y19h zYCr*=qD-5zZR@nrq2IWP>d5`fOgt#=|BM%Y?qcONy4&T68L=xLI^kmm?XIl%IxJ{{ zf_Qs)V^Y(1?FY<$d77E1Vo^n1Da+8TjMdbH|I2*L7#7@m^p z1Uf{WdNh~XP_dF{tU35X8nqntx(^TGNxrRJN zY3NAjui=Ec?Gr+aes9@ZEE)O z0&6{Bkh?D?$655kqQ=EIr-4gZ=rtuOjRpxvDKcb^P|UU=Bf*aEo$N&s1uNg<3zN!Q z-~p|lkL?<|&WsjRw#adbP=HA4;#`&3g~ccm9*nklrrY|pR}0H8_{!7e>&E4OkM^DQ zft!yzM^S!jpY_P=+^>22Fe0Tc+EErs_HU*s>D`D3QOJ)Bl(k)tW{Q9=@J%c>0_C56UkS1-+Vzr)D{*Ut|e?ZNMMx@ zC^-343($?Y43O|(37(maHWkIjRv@oVVb z54%4hq=-X>h^^c^19Z@C<4jg_n=81J;`79!U6Z;{j)hF~>jeuIa3|->yl9Y&>Ki)cyHD5#C40g zt>KBF)wOU7qjxrE7s9ULRrJ!!#R>8!c}xt_V!xj(zA*5&$Qesb8kM`{wqf|GHr2n% zBWK0KveKhg`=oSS)W3CdVh-z(Ymrm)O>D+vIPrxM=kH$C6JjC9Ncy()Idi#HUNZ}u zuxf{yxr1+g=~P}ndQJp|t{p_>-V#d7XQa>+Kk|=YwEP-tS~MI?oJ9}2F;^7R?9uq0 zS@$LD*Vpp5N_}hPMLVKW?^{SmeRF3~b59&kx*yPn9Pz9={L-s*zcy9LCpaXWbqjIO zHXZrRK8*GYiQ#r`Wq9KS3I-wWb}6nsmgDP!(>XJuWPYac)z^OibYMfCB`eFQ07j#` zY_yV<36=)td2D?glu~YT1qz7rTunRbwmB)e-hdH!E(h_EC|9SkW!gQy=AO7IDaZP-4ej{H*c0UMq$eYbQ?Hf4W@L}%17;^Ut7Sua(b05CG|-+dzG-~8ghfnDIg4ap03Cc$i8M*lj_Mc zBFn)BN0sH&Xm5=LchcpROTtUdT71cfVH0~R1>brn*pQ2xyK|QJuVs^|hNH8!2aZhe z(?=qi`EDc+nKj3i1I}gpw~YU%|6tIG#2-Rzoy=@#J_9gStT{hzr|&jaq>>-`p*%T zh9ac#@019uW7n0lNtenHM;?^;#!<=h6f)}-*M(*csj;R{)G)vA%^GXh8HyyvSB)o1%S@H109yHGXpRdLZfW`mKk9@qtP$KW3!QX|jO!dkw|5dWlIOQnQr{Zw(TeBAJbQvU~ zg*({c$v5f`yOuQm(!ok0V-)W9wi#XVW;nnZ0?QN2$7K?LEy_l2P5AtEiepQ*;UXb4 zWAwcUc}-yg3;0t5+=H*56>9%)ZT=&UmkSE#@&H2@|5Z8?#OH-C>HZr5PBHx{ya;9r zjWdEvQ^)HP6{qnC?2>!wS zuTTG4krt0PKI3h%e4x1%$^uT1_bAe!hFm;;CYV7Bg8Je--YDhzG!SgrHELErr#x>ztGi{^p+b2}g0>PPiQL%0=D z694%_8CN);5oNDDdm`zOcRda-%n1Vn2E(X~PDnPU!Q)x^`b0pGjrJDXQcm0ZE-oNw znYyF?;OZZpd>Nty$Ehp_Q14=v%gSi3MT-M=W{NG(*8n%{!S6QU;fZ@qs$n>d1Q)(I z#pRYH@9b-q$TI}#JZM#LA}1N*xFno^1i<*%AT$Ep@PuGw7?+aF^))cX2z*z8D)>Kb zTz@oFSsb5VWgA6xGJ{YwAwngrFOEVC

hH3oVy)$R+VR!#}_q_Mcd-vV<{c-R8d_SKf;8ua6G@y^Q zxrFM-bv>lU2HHA3_q(Oc7}31OCdH@rr)dXbM!BIZ^HBp=HMJ z83UvA-Z8=dIP5xK4!%|O!bne7;pvC__|ol9m2Mt?`Ef!R7`A*k4wekOWoLr^s-2Zg!?KFG0=djGr!vHPBPAXp_HM`5$#rKQ}M62?*4@Vs#4 zONwpnR)!n&tip45pE=p9UMReSs#_LdulWm$qO*u=6)T!jwQ1Er!$PrRh%)DQtkf>$ zt6NO!P^XCE2jc~7oz$A=5{#ke8;ED?NeS-7H3mKPR& z*!ej2f*!5c+}GY_&US39j-|5>75B>LIr#b@ji)p>$-M0YjF;y6KOGUo zJRUi7_47lwVWx?G9YZ0b z-70u$tOT4Mg{PFIaI3&>iRc_QfQU<6Af$fV2t7uN2w0MKV`UL?Y`-}OuWxQKsHnFA z->HE(_N!R%at6VaWFkBx*`bu=mMbYqw!pYrg*t~Xi|=_W+sqC0Vh_lGGf9?p&!4Ym zKge<)ZZSGx2aVXt4d4ixCY`oHNafahn6_v@(W4K9Q9ya|PZ8(9bF6!Wi7k5=;H&|3 z_Sp6J^(OD*dUeCs=+s)Ddp^aydbEW_)hf^1wBY&ZVg*^tT5b3>EK7 zFE*&SYyM45toZi(Rxg=w-C|}$2U2rviq}(QN9%ra!vVH#^99l-vAn8z~)qXL`CkJb1(NQ!H2718fVqVmL12jY#gS}LLDDq)L>}{$+ z!O6mPDOHJK`{VE0f_$jU87iUF$~K-0?093JW$-h`CzKD$&D?@!>xb&EyB(O)XfSym~PtzYW6%W zk`c~3Z)ausW{XZZUJFNREiD=EJRsftWi+YLAm{bbSMpb-qA~J^)7)O3G*aLC%X+e(75L{FN(?F>E;B}ySm`~Y;0bN9i1M8fjuX$+}X>;U=&qEdZXXB$}+n;gbT>;3Y9 z`N&1^EQu9bhkJ#83U8K664d-GXsQ&fA=uL|ZRZ#QwFN>%EGPgYw}q&-6ORgKV@8 hJYR)I*B&SSgyZ%u-ahTuv8+&5banE;*E$BJ{RI-xnr;99 diff --git a/man/model_bootstrap.Rd b/man/model_bootstrap.Rd index 7fc9668..36477ba 100644 --- a/man/model_bootstrap.Rd +++ b/man/model_bootstrap.Rd @@ -187,7 +187,8 @@ mb_gam$model_stats mb_gam$model_coefs # Plot ALE -gridExtra::grid.arrange(grobs = mb_gam$ale$plots, ncol = 2) +mb_gam$ale$plots |> + patchwork::wrap_plots() } diff --git a/vignettes/ale-intro.Rmd b/vignettes/ale-intro.Rmd index 5d4d423..32e3a4d 100644 --- a/vignettes/ale-intro.Rmd +++ b/vignettes/ale-intro.Rmd @@ -32,7 +32,7 @@ library(dplyr) ## diamonds dataset -For this introduction, we use the `diamonds` dataset, built-in with the `ggplot2` graphics system. We cleaned the original version by [removing duplicates](https://lorentzen.ch/index.php/2021/04/16/a-curious-fact-on-the-diamonds-dataset/ "errors in the diamonds dataset") and invalid entries where the length (x), width (y), or depth (z) is 0. +For this introduction, we use the `diamonds` dataset, included with the `{ggplot2}` graphics system. We cleaned the original version by [removing duplicates](https://lorentzen.ch/index.php/2021/04/16/a-curious-fact-on-the-diamonds-dataset/ "errors in the diamonds dataset") and invalid entries where the length (x), width (y), or depth (z) is 0. ```{r diamonds_print} # Clean up some invalid entries @@ -130,7 +130,6 @@ The `ale()` function returns a list with various elements. The two main ones are ale_gam_diamonds <- ale( diamonds_test, gam_diamonds, model_packages = 'mgcv', # required for parallel processing - silent = TRUE, # progress bars disabled for the vignette parallel = 2 # CRAN limit (delete this line on your own computer) ) ``` @@ -144,11 +143,12 @@ To access the plot for a specific variable, we can call it by its variable name ale_gam_diamonds$plots$carat ``` -To iterate the list and plot all the ALE plots, we provide here some demonstration code using the `gridExtra` package for arranging multiple plots in a common plot grid using `gridExtra::grid.arrange`. We need to pass the list of plots to the `grobs` argument and we can specify that we want two plots per row with the `ncol` argument. +To iterate the list and plot all the ALE plots, we provide here some demonstration code using the `patchwork` package for arranging multiple plots in a common plot grid using `patchwork::wrap_plots()`. We need to pass the list of plots to the `grobs` argument and we can specify that we want two plots per row with the `ncol` argument. ```{r print ale_simple, fig.width=7, fig.height=11} # Print all plots -gridExtra::grid.arrange(grobs = ale_gam_diamonds$plots, ncol = 2) +ale_gam_diamonds$plots |> + patchwork::wrap_plots(ncol = 2) ``` ## Bootstrapped ALE @@ -173,12 +173,12 @@ ale_gam_diamonds_boot <- ale( # Normally boot_it should be set to 100, but just 10 here for a faster demonstration boot_it = 10, model_packages = 'mgcv', # required for parallel processing - silent = TRUE, # progress bars disabled for the vignette parallel = 2 # CRAN limit (delete this line on your own computer) ) # Bootstrapping produces confidence intervals -gridExtra::grid.arrange(grobs = ale_gam_diamonds_boot$plots, ncol = 2) +ale_gam_diamonds_boot$plots |> + patchwork::wrap_plots(ncol = 2) ``` In this case, the bootstrapped results are mostly similar to single (non-bootstrapped) ALE result. In principle, we should always bootstrap the results and trust only in bootstrapped results. The most unusual result is that values of `x_length` (the length of the diamond) from 6.2 mm or so and higher are associated with lower diamond prices. When we compare this with the `y_width` value (width of the diamond), we suspect that when both the length and width (that is, the size) of a diamond become increasingly large, the price increases so much more rapidly with the width than with the length that the width has an inordinately high effect that is tempered by a decreased effect of the length at those high values. This would be worth further exploration for real analysis, but here we are just introducing the key features of the package. @@ -192,7 +192,6 @@ Another advantage of ALE is that it provides data for two-way interactions betwe ale_ixn_gam_diamonds <- ale_ixn( diamonds_test, gam_diamonds, model_packages = 'mgcv', # required for parallel processing - silent = TRUE, # progress bars disabled for the vignette parallel = 2 # CRAN limit (delete this line on your own computer) ) @@ -200,17 +199,21 @@ ale_ixn_gam_diamonds <- ale_ixn( Like the `ale()` function, the `ale_ixn()` returns a list with one element per input x1 variable, as well as a `.common_data` element with details about the outcome (y) variable. However, in this case, each variable's element consists of a list of all the x2 variables for which the x1 interaction is calculated. Each x2 element then has two elements: the ALE data for that variable and a `ggplot` plot object that plots that ALE data. In the interaction plots, the x1 variable is always shown on the x axis and the x2 variable on the y axis. -Again, we provide here some demonstration code to plot all the ALE plots. It is a little more complex this time because of the two levels of interacting variables in the output data, so we use the `purrr` package to iterate the list structure. `purrr::walk` takes a list as its first argument and then we specify an anonymous function for what we want to do with each element of the list. We specify the anonymous function as `\(.x1) {...}` where `.x1` in our case represents each individual element of `ale_ixn_gam_diamonds$plots` in turn, that is, a sublist of plots with which the x1 variable interacts. We print the plots of all the x1 interactions as a combined grid of plots with `gridExtra::grid.arrange`, as before. +Again, we provide here some demonstration code to plot all the ALE plots. It is a little more complex this time because of the two levels of interacting variables in the output data, so we use the `purrr` package to iterate the list structure. `purrr::walk()` takes a list as its first argument and then we specify an anonymous function for what we want to do with each element of the list. We specify the anonymous function as `\(.x1) {...}` where `.x1` in our case represents each individual element of `ale_ixn_gam_diamonds$plots` in turn, that is, a sublist of plots with which the x1 variable interacts. We print the plots of all the x1 interactions as a combined grid of plots with `patchwork::wrap_plots()`, as before. ```{r print all ale_ixn, fig.width=7, fig.height=7} # Print all interaction plots ale_ixn_gam_diamonds$plots |> - purrr::walk(\(.x1) { # extract list of x1 ALE outputs - gridExtra::grid.arrange(grobs = .x1, ncol = 2) # plot each x1 plot + # extract list of x1 ALE outputs + purrr::walk(\(.x1) { + # plot all x2 plots in each .x1 element + patchwork::wrap_plots(.x1, ncol = 2) |> + print() }) + ``` -Because we are printing all plots together with the same `gridExtra::grid.arrange` statement, some of them might appear vertically distorted because each plot is forced to be of the same height. For more fine-tuned presentation, we would need to refer to a specific plot. For example, we can print the interaction plot between carat and depth by referring to it thus: `ale_ixn_gam_diamonds$plots$carat$depth`. +Because we are printing all plots together with the same `patchwork::wrap_plots()` statement, some of them might appear vertically distorted because each plot is forced to be of the same height. For more fine-tuned presentation, we would need to refer to a specific plot. For example, we can print the interaction plot between carat and depth by referring to it thus: `ale_ixn_gam_diamonds$plots$carat$depth`. ```{r print specific ixn, fig.width=5, fig.height=3} ale_ixn_gam_diamonds$plots$carat$depth diff --git a/vignettes/ale-small-datasets.Rmd b/vignettes/ale-small-datasets.Rmd index e787501..a2fae7e 100644 --- a/vignettes/ale-small-datasets.Rmd +++ b/vignettes/ale-small-datasets.Rmd @@ -94,12 +94,12 @@ Note that for now, we run `ale` with no bootstrapping (the default) because smal ale_lm_attitude_simple <- ale( attitude, lm_attitude, model_packages = 'mgcv', # required for parallel processing - silent = TRUE, # progress bars disabled for the vignette parallel = 2 # CRAN limit (delete this line on your own computer) ) # Print all plots -gridExtra::grid.arrange(grobs = ale_lm_attitude_simple$plots, ncol = 2) +ale_lm_attitude_simple$plots |> + patchwork::wrap_plots(ncol = 2) ``` This visualization confirms what we see in the model coefficients above: complaints have a strong positive effect on ratings and learning has a more moderate effect. However, the ALE indicates a stronger effect of advance than the regression coefficients suggest. The other variables have relatively little effect on ratings. We will see shortly that proper bootstrapping of the model can shed some light on the discrepancies. @@ -112,14 +112,16 @@ We can use `ale_ixn()` to visualize the possible existence of any interactions: ale_lm_attitude_ixn <- ale_ixn( attitude, lm_attitude, model_packages = 'mgcv', # required for parallel processing - silent = TRUE, # progress bars disabled for the vignette parallel = 2 # CRAN limit (delete this line on your own computer) ) # Print plots ale_lm_attitude_ixn$plots |> - purrr::walk(\(.x1) { # extract list of x1 ALE outputs - gridExtra::grid.arrange(grobs = .x1, ncol = 2) # plot all x1 plots + # extract list of x1 ALE outputs + purrr::walk(\(.x1) { + # plot all x2 plots in each .x1 element + patchwork::wrap_plots(.x1, ncol = 2) |> + print() }) ``` @@ -144,7 +146,6 @@ mb_lm <- model_bootstrap( lm_attitude, boot_it = 10, # 100 by default but reduced here for a faster demonstration model_packages = 'mgcv', # required for parallel processing - silent = TRUE, # progress bars disabled for the vignette parallel = 2 # CRAN limit (delete this line on your own computer) ) ``` @@ -173,7 +174,8 @@ mb_lm$model_coefs Here we can visualize the results of the ALE plots. ```{r lm_full_ale, fig.width=7, fig.height=7} -gridExtra::grid.arrange(grobs = mb_lm$ale$plots, ncol = 2) +mb_lm$ale$plots |> + patchwork::wrap_plots(ncol = 2) ``` The key to interpreting the effects in these models is contrasting the grey bootstrapped confidence bands surrounding the average (median) ALE effect with the thin horizontal grey band labelled 'median $\pm$ 2.5%'. Anything within $\pm$ 2.5% of the median is in the 5% middle of the data. Only bootstrapped effects that are clearly beyond this middle band may be considered significant. By this criteria, considering that the median rating was 65.5%, we can conclude that: @@ -206,11 +208,11 @@ To understand which variables were responsible for this relationship, the result ale_gam_attitude_simple <- ale( attitude, gam_attitude, model_packages = 'mgcv', # required for parallel processing - silent = TRUE, # progress bars disabled for the vignette parallel = 2 # CRAN limit (delete this line on your own computer) ) -gridExtra::grid.arrange(grobs = ale_gam_attitude_simple$plots, ncol = 2) +ale_gam_attitude_simple$plots |> + patchwork::wrap_plots(ncol = 2) ``` Compared to the OLS results above, the GAM results provide quite a surprise concerning the shape of the effect of employees' perceptions that their department is too critical--it seems that both low criticism and very high criticism negatively affect ratings. However, before trying to interpret these results, we must remember that results that are not bootstrapped are simply not reliable. So, let us see what bootstrapping will give us. @@ -221,7 +223,6 @@ mb_gam <- model_bootstrap( gam_attitude, boot_it = 10, # 100 by default but reduced here for a faster demonstration model_packages = 'mgcv', # required for parallel processing - silent = TRUE, # progress bars disabled for the vignette parallel = 2 # CRAN limit (delete this line on your own computer) ) mb_gam$model_stats @@ -232,7 +233,8 @@ mb_gam$model_coefs ``` ```{r gam_full_ale, fig.width=7, fig.height=7} -gridExtra::grid.arrange(grobs = mb_gam$ale$plots, ncol = 2) +mb_gam$ale$plots |> + patchwork::wrap_plots(ncol = 2) ``` The bootstrapped GAM results tell a rather different story from the OLS results. In this case, the bootstrap confidence bands of all the variables (even of complaints) fully overlap with the entirety of the median non-significance region. Even the average slopes have vanished from all variables except for complaint, where it remains positive, yet insignificant because of the wide confidence interval. @@ -273,10 +275,9 @@ mb_gam_non_standard <- model_bootstrap( data = boot_data)', boot_it = 10, # 100 by default but reduced here for a faster demonstration model_packages = 'mgcv', # required for parallel processing - silent = TRUE, # progress bars disabled for the vignette parallel = 2 # CRAN limit (delete this line on your own computer) ) mb_gam_non_standard$model_stats ``` -Everything else works as normal. +Everything else works as usual. diff --git a/vignettes/ale-statistics.Rmd b/vignettes/ale-statistics.Rmd index 9cd455f..98508f4 100644 --- a/vignettes/ale-statistics.Rmd +++ b/vignettes/ale-statistics.Rmd @@ -140,9 +140,7 @@ mb_gam <- model_bootstrap( tidy_options = list(parametric = TRUE), # tidy_options = list(parametric = NULL), boot_it = 40, # 100 by default but reduced here for a faster demonstration - silent = TRUE, # progress bars disabled for the vignette parallel = 2 # CRAN limit (delete this line on your own computer) - ) ``` @@ -187,7 +185,8 @@ ALE was developed to graphically display the relationship between predictor vari ### ALE plots ```{r all ALE plots, fig.width=7, fig.height=10} -gridExtra::grid.arrange(grobs = mb_gam$ale$plots, ncol = 2) +mb_gam$ale$plots |> + patchwork::wrap_plots(ncol = 2) ``` We can see that most variables seem to have some sort of mean effect across various values. However, for statistical inference, our focus must be on the bootstrap intervals. Crucial to our interpretation is the middle grey band that indicates the median ± 2.5%, that is, the middle 5% of all average mathematics achievement scores (`math_avg`) values in the dataset. We call this the "median band". The idea is that if any predictor can do no better than influencing `math_avg` to fall within this middle median band, then it only has a minimal effect. For an effect to be considered statistically significant, there should be no overlap between the confidence regions of a predictor variable and the median band. (We use 5% by default, but the value can be changed with the `median_band_pct` argument.) diff --git a/vignettes/ale-x-datatypes.Rmd b/vignettes/ale-x-datatypes.Rmd index b370ad3..42dd1e0 100644 --- a/vignettes/ale-x-datatypes.Rmd +++ b/vignettes/ale-x-datatypes.Rmd @@ -81,12 +81,12 @@ Now we generate ALE data from the `var_cars` GAM model and plot it. cars_ale <- ale( var_cars, cm, model_packages = 'mgcv', #required for parallel processing - silent = TRUE, # progress bars disabled for the vignette parallel = 2 # CRAN limit (delete this line on your own computer) ) # Print all plots -gridExtra::grid.arrange(grobs = cars_ale$plots, ncol = 2) +cars_ale$plots |> + patchwork::wrap_plots(ncol = 2) ``` We can see that `ale` has no trouble modelling any of the datatypes in our sample (logical, factor, ordered, integer, or double). It plots line charts for the numeric predictors and column charts for everything else. @@ -99,14 +99,16 @@ We can also generate and plot the ALE data for all two-way interactions. cars_ale_ixn <- ale_ixn( var_cars, cm, model_packages = 'mgcv', #required for parallel processing - silent = TRUE, # progress bars disabled for the vignette parallel = 2 # CRAN limit (delete this line on your own computer) ) # Print plots cars_ale_ixn$plots |> - purrr::walk(\(.x1) { # extract list of x1 ALE outputs - gridExtra::grid.arrange(grobs = .x1, ncol = 2) # plot all x1 plots + # extract list of x1 ALE outputs + purrr::walk(\(.x1) { + # plot all x2 plots in each .x1 element + patchwork::wrap_plots(.x1, ncol = 2) |> + print() }) ``` @@ -120,12 +122,12 @@ mb <- model_bootstrap( cm, boot_it = 10, # 100 by default but reduced here for a faster demonstration model_packages = 'mgcv', #required for parallel processing - silent = TRUE, # progress bars disabled for the vignette parallel = 2, # CRAN limit (delete this line on your own computer) seed = 2 # workaround to avoid random error on such a small dataset ) -gridExtra::grid.arrange(grobs = mb$ale$plots, ncol = 2) +mb$ale$plots |> + patchwork::wrap_plots(ncol = 2) ``` (By default, `model_bootstrap()` creates 100 bootstrap samples but, so that this illustration runs faster, we demonstrate it here with only 10 iterations.) diff --git a/vignettes/articles/ale-ALEPlot.Rmd b/vignettes/articles/ale-ALEPlot.Rmd index d852858..c6fb8c0 100644 --- a/vignettes/articles/ale-ALEPlot.Rmd +++ b/vignettes/articles/ale-ALEPlot.Rmd @@ -160,10 +160,10 @@ Here are some notable differences compared to `ALEPlot`: Since the plots are saved as a list, they can easily be printed out all at once: -```{r ale nnet one-way plots, fig.width=3.5, fig.height=10} -# , fig.asp=3 is OK +```{r ale nnet one-way plots, fig.width=7, fig.height=5} # Print plots -gridExtra::grid.arrange(grobs = nn_ale$plots, ncol = 1) +nn_ale$plots |> + patchwork::wrap_plots() ``` The `{ale}` package plots have various features that enhance interpretability: @@ -182,16 +182,17 @@ nn_ale <- ale( model_packages = 'nnet' # required for parallel processing ) -gridExtra::grid.arrange(grobs = nn_ale$plots, ncol = 2) +nn_ale$plots |> + patchwork::wrap_plots() ``` With these zero-centred plots, the full range of y values and the rug plots give some context that aids interpretation. (If the rugs look slightly different, it is because they are randomly jittered to avoid overplotting.) The `{ale}` also produces interaction plots. Unlike `ALEPlot`, it does so with a separate dedicated function, `ale_ixn()`. By default, it calculates all possible two-way interactions between all variables in the data. -Because the variables interact with each other, the output data structure is a two-layer list, so the print code is slightly more complicated. However, the sample code we provide here using functions from the `purrr` package for iterating through lists and `gridExtra` package for arranging plots are reusable in any application. +Because the variables interact with each other, the output data structure is a two-layer list, so the print code is slightly more complicated. However, the sample code we provide here using functions from the `purrr` package for iterating through lists and `patchwork` package for arranging plots are reusable in any application. -```{r ale nnet ixn, fig.width=7, fig.height=10} +```{r ale nnet ixn, fig.width=7, fig.height=5} # Create and plot interactions nn_ale_ixn <- ale_ixn( DAT, nnet.DAT, pred_type = "raw", @@ -200,8 +201,11 @@ nn_ale_ixn <- ale_ixn( # Print plots nn_ale_ixn$plots |> - purrr::walk(\(.x1) { # extract list of x1 ALE outputs - gridExtra::grid.arrange(grobs = .x1, ncol = 1) # plot all x1 plots + # extract list of x1 ALE outputs + purrr::walk(\(.x1) { + # plot all x2 plots in each .x1 element + patchwork::wrap_plots(.x1, ncol = 2, nrow = 2) |> + print() }) ``` @@ -221,7 +225,7 @@ The next code example from the `{ALEPlot}` package analyzes a real dataset with ## R code for Example 3 ## Load relevant packages library(ALEPlot) -library(gbm) +library(gbm, quietly = TRUE) ## Read data and fit a boosted tree supervised learning model data(census, package = 'ale') # load ale package version of the data @@ -306,7 +310,8 @@ gbm_ale_link <- url('https://github.com/tripartio/ale/raw/main/download/gbm_ale_ readRDS() # Print plots -gridExtra::grid.arrange(grobs = gbm_ale_link$plots, ncol = 2) +gbm_ale_link$plots |> + patchwork::wrap_plots(ncol = 2) ``` Now we generate ALE data for all two-way interactions and then plot them. Again, note the interaction between `age` and `hours_per_week`. The interaction is minimal except for the extremely high cases of hours per week. @@ -329,8 +334,11 @@ gbm_ale_ixn_link <- url('https://github.com/tripartio/ale/raw/main/download/gbm_ # Print plots gbm_ale_ixn_link$plots |> - purrr::walk(\(.x1) { # extract list of x1 ALE outputs - gridExtra::grid.arrange(grobs = .x1, ncol = 2) # plot all x1 interaction plots + # extract list of x1 ALE outputs + purrr::walk(\(.x1) { + # plot all x2 plots in each .x1 element + patchwork::wrap_plots(.x1, ncol = 2) |> + print() }) ``` @@ -374,7 +382,8 @@ gbm_ale_prob <- url('https://github.com/tripartio/ale/raw/main/download/gbm_ale_ readRDS() # Print plots -gridExtra::grid.arrange(grobs = gbm_ale_prob$plots, ncol = 2) +gbm_ale_prob$plots |> + patchwork::wrap_plots(ncol = 2) ``` Finally, we again generate two-way interactions, this time based on probabilities instead of on log odds. However, probabilities might not be the best choice for indicating interactions because, as we see from the rugs in the one-way ALE plots, the GBM model heavily concentrates its probabilities in the extremes near 0 and 1. Thus, the plots' suggestions of strong interactions are likely exaggerated. In this case, the log odds ALEs shown above are probably more relevant. @@ -396,7 +405,10 @@ gbm_ale_ixn_prob <- url('https://github.com/tripartio/ale/raw/main/download/gbm_ # Print plots gbm_ale_ixn_prob$plots |> - purrr::walk(\(.x1) { # extract list of x1 ALE outputs - gridExtra::grid.arrange(grobs = .x1, ncol = 2) # plot all x1 plots + # extract list of x1 ALE outputs + purrr::walk(\(.x1) { + # plot all x2 plots in each .x1 element + patchwork::wrap_plots(.x1, ncol = 2) |> + print() }) ```