diff --git a/articles/Fixed_obj_fun.html b/articles/Fixed_obj_fun.html index ceb6da6..ab349e7 100644 --- a/articles/Fixed_obj_fun.html +++ b/articles/Fixed_obj_fun.html @@ -191,9 +191,9 @@+## 1 10 0.038 1000 0.22s 525310970 Wed Dec 4 05:41:15 2024 +## 2 20 0.05 1000 0.22s 529914981 Wed Dec 4 05:41:15 2024 +## 3 30 0.056 1000 0.24s 602604143 Wed Dec 4 05:41:15 2024Exporting objects example
## # A tibble: 3 × 6 ## N p REPLICATIONS SIM_TIME SEED COMPLETED ## <dbl> <dbl> <dbl> <chr> <int> <chr> -## 1 10 0.038 1000 0.23s 525310970 Wed Dec 4 05:32:45 2024 -## 2 20 0.05 1000 0.22s 529914981 Wed Dec 4 05:32:45 2024 -## 3 30 0.056 1000 0.25s 602604143 Wed Dec 4 05:32:45 2024
By placing objects in a list and passing this to
fixed_objects, the objects are safely exported to all
relevant functions. Furthermore, running this code in parallel will also
diff --git a/articles/Saving-results.html b/articles/Saving-results.html
index 5aa962f..75979d0 100644
--- a/articles/Saving-results.html
+++ b/articles/Saving-results.html
@@ -150,7 +150,7 @@
save = TRUE (Defa
-## [1] "SIMDESIGN-TEMPFILE_fv-az1756-196.rds"## [1] "SIMDESIGN-TEMPFILE_fv-az1759-216.rds"Notice here that the simulation stopped at 67% because the third
design condition threw too many consecutive errors (this is a built-in
fail-safe in SimDesign). To imitate a type of crash/power
@@ -215,7 +215,7 @@
@@ -265,7 +265,7 @@## Design: 2/3; Replications: 1000; RAM Used: 71.8 Mb; Total Time: 0.17s ## Conditions: N=20 ## -## Design: 3/3; Replications: 1000; RAM Used: 71.8 Mb; Total Time: 0.34s +## Design: 3/3; Replications: 1000; RAM Used: 71.8 Mb; Total Time: 0.33s ## Conditions: N=30 ##
diff --git a/articles/SimDesign-intro.html b/articles/SimDesign-intro.html index 34805f2..d098085 100644 --- a/articles/SimDesign-intro.html +++ b/articles/SimDesign-intro.html @@ -338,25 +338,25 @@Putting it all together## Design: 1/8; Replications: 1000; RAM Used: 70.5 Mb; Total Time: 0.00s ## Conditions: sample_size=30, distribution=norm ## -## Design: 2/8; Replications: 1000; RAM Used: 71.5 Mb; Total Time: 0.21s +## Design: 2/8; Replications: 1000; RAM Used: 71.5 Mb; Total Time: 0.20s ## Conditions: sample_size=60, distribution=norm ## ## Design: 3/8; Replications: 1000; RAM Used: 71.5 Mb; Total Time: 0.40s ## Conditions: sample_size=120, distribution=norm ## -## Design: 4/8; Replications: 1000; RAM Used: 71.6 Mb; Total Time: 0.61s +## Design: 4/8; Replications: 1000; RAM Used: 71.6 Mb; Total Time: 0.60s ## Conditions: sample_size=240, distribution=norm ## -## Design: 5/8; Replications: 1000; RAM Used: 71.6 Mb; Total Time: 0.83s +## Design: 5/8; Replications: 1000; RAM Used: 71.6 Mb; Total Time: 0.82s ## Conditions: sample_size=30, distribution=chi ## -## Design: 6/8; Replications: 1000; RAM Used: 71.6 Mb; Total Time: 1.02s +## Design: 6/8; Replications: 1000; RAM Used: 71.6 Mb; Total Time: 1.01s ## Conditions: sample_size=60, distribution=chi ## -## Design: 7/8; Replications: 1000; RAM Used: 71.7 Mb; Total Time: 1.22s +## Design: 7/8; Replications: 1000; RAM Used: 71.7 Mb; Total Time: 1.21s ## Conditions: sample_size=120, distribution=chi ## -## Design: 8/8; Replications: 1000; RAM Used: 71.7 Mb; Total Time: 1.43s +## Design: 8/8; Replications: 1000; RAM Used: 71.7 Mb; Total Time: 1.41s ## Conditions: sample_size=240, distribution=chi ##
diff --git a/pkgdown.yml b/pkgdown.yml index 2469887..8312ec8 100644 --- a/pkgdown.yml +++ b/pkgdown.yml @@ -9,7 +9,7 @@ articles: Parallel-computing: Parallel-computing.html Saving-results: Saving-results.html SimDesign-intro: SimDesign-intro.html -last_built: 2024-12-04T05:32Z +last_built: 2024-12-04T05:40Z urls: reference: http://philchalmers.github.io/SimDesign/reference article: http://philchalmers.github.io/SimDesign/articles diff --git a/reference/reSummarise.html b/reference/reSummarise.html index 3db3e23..8513ffa 100644 --- a/reference/reSummarise.html +++ b/reference/reSummarise.html @@ -219,9 +219,9 @@Examples#> # A tibble: 3 × 8 #> N mean median REPLICATIONS SIM_TIME RAM_USED SEED COMPLETED #> <dbl> <dbl> <dbl> <dbl> <chr> <chr> <int> <chr> -#> 1 10 10.192 10.230 50 0.01s 159.3 Mb 1771457935 Wed Dec 4 05:3… -#> 2 20 10.146 10.084 50 0.01s 159.3 Mb 590590962 Wed Dec 4 05:3… -#> 3 30 10.257 10.328 50 0.01s 159.3 Mb 273992574 Wed Dec 4 05:3… +#> 1 10 10.192 10.230 50 0.01s 159.3 Mb 1771457935 Wed Dec 4 05:4… +#> 2 20 10.146 10.084 50 0.01s 159.3 Mb 590590962 Wed Dec 4 05:4… +#> 3 30 10.257 10.328 50 0.01s 159.3 Mb 273992574 Wed Dec 4 05:4… # same summarise but with bootstrapping res2 <- reSummarise(Summarise, results = res, boot_method = 'basic') diff --git a/reference/rint.html b/reference/rint.html index b61c59d..eed900e 100644 --- a/reference/rint.html +++ b/reference/rint.html @@ -122,7 +122,7 @@
Examples# compare speed to sample() system.time(x <- rint(1000, min = 100, max = 1e8)) #> user system elapsed -#> 0.001 0.000 0.000 +#> 0 0 0 system.time(x2 <- sample(100:1e8, 1000, replace = TRUE)) #> user system elapsed #> 0 0 0 diff --git a/reference/runSimulation.html b/reference/runSimulation.html index 1f43788..e80a00f 100644 --- a/reference/runSimulation.html +++ b/reference/runSimulation.html @@ -960,14 +960,14 @@
Examples#> Conditions: N=30 #> #> -#> Simulation complete. Total execution time: 0.01s +#> Simulation complete. Total execution time: 0.00s Final #> # A tibble: 3 × 7 #> N mu SE REPLICATIONS SIM_TIME SEED COMPLETED #> <dbl> <dbl> <dbl> <dbl> <chr> <int> <chr> -#> 1 10 10.202 1.4729 2 0.00s 533810122 Wed Dec 4 05:32:34 20… -#> 2 20 10.885 0.31864 2 0.00s 1340659367 Wed Dec 4 05:32:34 20… -#> 3 30 9.6268 1.2548 2 0.00s 881068069 Wed Dec 4 05:32:34 20… +#> 1 10 10.202 1.4729 2 0.00s 533810122 Wed Dec 4 05:41:05 20… +#> 2 20 10.885 0.31864 2 0.00s 1340659367 Wed Dec 4 05:41:05 20… +#> 3 30 9.6268 1.2548 2 0.00s 881068069 Wed Dec 4 05:41:05 20… SimResults(Final) #> # A tibble: 6 × 2 #> N mean @@ -994,14 +994,14 @@
Examples#> Conditions: N=30 #> #> -#> Simulation complete. Total execution time: 0.00s +#> Simulation complete. Total execution time: 0.01s Final_rep #> # A tibble: 3 × 7 #> N mu SE REPLICATIONS SIM_TIME SEED COMPLETED #> <dbl> <dbl> <dbl> <dbl> <chr> <int> <chr> -#> 1 10 10.202 1.4729 2 0.00s 533810122 Wed Dec 4 05:32:34 20… -#> 2 20 10.885 0.31864 2 0.00s 1340659367 Wed Dec 4 05:32:34 20… -#> 3 30 9.6268 1.2548 2 0.00s 881068069 Wed Dec 4 05:32:34 20… +#> 1 10 10.202 1.4729 2 0.00s 533810122 Wed Dec 4 05:41:05 20… +#> 2 20 10.885 0.31864 2 0.00s 1340659367 Wed Dec 4 05:41:05 20… +#> 3 30 9.6268 1.2548 2 0.00s 881068069 Wed Dec 4 05:41:05 20… SimResults(Final_rep) #> # A tibble: 6 × 2 #> N mean @@ -1189,7 +1189,7 @@
Examples#> Design: 1/36; Replications: 2 Total Time: 0.00s #> Conditions: sample_size=30, group_size_ratio=1, standard_deviation_ratio=0.5 #> -#> Design: 2/36; Replications: 2 Total Time: 0.01s +#> Design: 2/36; Replications: 2 Total Time: 0.00s #> Conditions: sample_size=60, group_size_ratio=1, standard_deviation_ratio=0.5 #> #> Design: 3/36; Replications: 2 Total Time: 0.01s @@ -1210,7 +1210,7 @@
Examples#> Design: 8/36; Replications: 2 Total Time: 0.03s #> Conditions: sample_size=120, group_size_ratio=4, standard_deviation_ratio=0.5 #> -#> Design: 9/36; Replications: 2 Total Time: 0.04s +#> Design: 9/36; Replications: 2 Total Time: 0.03s #> Conditions: sample_size=30, group_size_ratio=8, standard_deviation_ratio=0.5 #> #> Design: 10/36; Replications: 2 Total Time: 0.04s @@ -1231,13 +1231,13 @@
Examples#> Design: 15/36; Replications: 2 Total Time: 0.06s #> Conditions: sample_size=90, group_size_ratio=1, standard_deviation_ratio=1 #> -#> Design: 16/36; Replications: 2 Total Time: 0.07s +#> Design: 16/36; Replications: 2 Total Time: 0.06s #> Conditions: sample_size=120, group_size_ratio=1, standard_deviation_ratio=1 #> #> Design: 17/36; Replications: 2 Total Time: 0.07s #> Conditions: sample_size=30, group_size_ratio=4, standard_deviation_ratio=1 #> -#> Design: 18/36; Replications: 2 Total Time: 0.08s +#> Design: 18/36; Replications: 2 Total Time: 0.07s #> Conditions: sample_size=60, group_size_ratio=4, standard_deviation_ratio=1 #> #> Design: 19/36; Replications: 2 Total Time: 0.08s @@ -1252,13 +1252,13 @@
Examples#> Design: 22/36; Replications: 2 Total Time: 0.09s #> Conditions: sample_size=60, group_size_ratio=8, standard_deviation_ratio=1 #> -#> Design: 23/36; Replications: 2 Total Time: 0.10s +#> Design: 23/36; Replications: 2 Total Time: 0.09s #> Conditions: sample_size=90, group_size_ratio=8, standard_deviation_ratio=1 #> #> Design: 24/36; Replications: 2 Total Time: 0.10s #> Conditions: sample_size=120, group_size_ratio=8, standard_deviation_ratio=1 #> -#> Design: 25/36; Replications: 2 Total Time: 0.11s +#> Design: 25/36; Replications: 2 Total Time: 0.10s #> Conditions: sample_size=30, group_size_ratio=1, standard_deviation_ratio=2 #> #> Design: 26/36; Replications: 2 Total Time: 0.11s @@ -1267,35 +1267,35 @@
Examples#> Design: 27/36; Replications: 2 Total Time: 0.13s #> Conditions: sample_size=90, group_size_ratio=1, standard_deviation_ratio=2 #> -#> Design: 28/36; Replications: 2 Total Time: 0.14s +#> Design: 28/36; Replications: 2 Total Time: 0.13s #> Conditions: sample_size=120, group_size_ratio=1, standard_deviation_ratio=2 #> #> Design: 29/36; Replications: 2 Total Time: 0.14s #> Conditions: sample_size=30, group_size_ratio=4, standard_deviation_ratio=2 #> -#> Design: 30/36; Replications: 2 Total Time: 0.15s +#> Design: 30/36; Replications: 2 Total Time: 0.14s #> Conditions: sample_size=60, group_size_ratio=4, standard_deviation_ratio=2 #> -#> Design: 31/36; Replications: 2 Total Time: 0.15s +#> Design: 31/36; Replications: 2 Total Time: 0.14s #> Conditions: sample_size=90, group_size_ratio=4, standard_deviation_ratio=2 #> #> Design: 32/36; Replications: 2 Total Time: 0.15s #> Conditions: sample_size=120, group_size_ratio=4, standard_deviation_ratio=2 #> -#> Design: 33/36; Replications: 2 Total Time: 0.16s +#> Design: 33/36; Replications: 2 Total Time: 0.15s #> Conditions: sample_size=30, group_size_ratio=8, standard_deviation_ratio=2 #> #> Design: 34/36; Replications: 2 Total Time: 0.16s #> Conditions: sample_size=60, group_size_ratio=8, standard_deviation_ratio=2 #> -#> Design: 35/36; Replications: 2 Total Time: 0.17s +#> Design: 35/36; Replications: 2 Total Time: 0.16s #> Conditions: sample_size=90, group_size_ratio=8, standard_deviation_ratio=2 #> -#> Design: 36/36; Replications: 2 Total Time: 0.17s +#> Design: 36/36; Replications: 2 Total Time: 0.16s #> Conditions: sample_size=120, group_size_ratio=8, standard_deviation_ratio=2 #> #> -#> Simulation complete. Total execution time: 0.18s +#> Simulation complete. Total execution time: 0.17s res #> # A tibble: 36 × 9 #> sample_size group_size_ratio standard_deviation_ratio welch independent diff --git a/search.json b/search.json index 10cc1e5..7634bc8 100644 --- a/search.json +++ b/search.json @@ -1 +1 @@ -[{"path":"http://philchalmers.github.io/SimDesign/articles/Catch_errors.html","id":"define-the-functions","dir":"Articles","previous_headings":"","what":"Define the functions","title":"Managing warning and error messages","text":"usual, define functions interest. simulation just example errors tracked SimDesign, well throw manual error case data re-drawn based user’s decision (e.g., model converges, fails number predefined iterations).","code":"library(SimDesign) # SimFunctions(comments=FALSE) Design <- createDesign(N = c(10,20,30)) Generate <- function(condition, fixed_objects) { ret <- with(condition, rnorm(N)) ret } Analyse <- function(condition, dat, fixed_objects) { whc <- sample(c(0,1,2,3), 1, prob = c(.7, .20, .05, .05)) if(whc == 0){ ret <- mean(dat) } else if(whc == 1){ ret <- t.test() # missing arguments } else if(whc == 2){ ret <- t.test('invalid') # invalid arguments } else if(whc == 3){ # throw error manually stop('Manual error thrown') } # manual warnings if(sample(c(TRUE, FALSE), 1, prob = c(.1, .9))) warning('This warning happens rarely') if(sample(c(TRUE, FALSE), 1, prob = c(.5, .5))) warning('This warning happens much more often') ret } Summarise <- function(condition, results, fixed_objects) { ret <- c(bias = bias(results, 0)) ret }"},{"path":"http://philchalmers.github.io/SimDesign/articles/Catch_errors.html","id":"run-the-simulation","dir":"Articles","previous_headings":"","what":"Run the simulation","title":"Managing warning and error messages","text":"’ll immediately notice output object counts error warning messages appended result object. useful determine just problematic errors warnings based frequency alone. Furthermore, specific frequency errors/warnings occurred also included design condition (t.test.default() error, inputs supplied, occurred often manually thrown error well invalid-input error) extracting inspecting SimExtract(results, = 'errors') SimExtract(results, = 'warnings'). Finally, SimDesign built-safety feature controlled max_errors argument avoid getting stuck infinite redrawing loops. default, 50 errors consecutively returned simulation condition halted, warning message printed console indicating last observed fatal error. safety features built-many consecutive stop() calls generally indicates major problem simulation code fixed continuing. However, encountering fatal errors given simulation condition remainder simulation experiment still executed normal, problematic conditions combinations NA placeholders assigned rows final output object. entire experiment unexpectedly terminate due one problematic row conditions Design, instead conditions can inspected debugged later time. course, inspecting code directly, simulation manually halted terminal errors can attended immediately (e.g., using Ctrl + c, clicking ‘Stop’ icon Rstudio).","code":"result <- runSimulation(Design, replications = 100, generate=Generate, analyse=Analyse, summarise=Summarise) ## ## Design: 1/3; Replications: 100; RAM Used: 67.9 Mb; Total Time: 0.00s ## Conditions: N=10 ## ## Design: 2/3; Replications: 100; RAM Used: 69.1 Mb; Total Time: 0.05s ## Conditions: N=20 ## ## Design: 3/3; Replications: 100; RAM Used: 69.2 Mb; Total Time: 0.10s ## Conditions: N=30 ## print(result) ## # A tibble: 3 × 9 ## N bias REPLICATIONS SIM_TIME RAM_USED SEED COMPLETED ERRORS ##
## 1 10 0.061138 100 0.05s 69.1 Mb 1140350788 Wed Dec 4 05… 53 ## 2 20 0.014295 100 0.05s 69.2 Mb 312928385 Wed Dec 4 05… 52 ## 3 30 0.017927 100 0.04s 69.3 Mb 866248189 Wed Dec 4 05… 42 ## # ℹ 1 more variable: WARNINGS SimExtract(result, what = 'errors') ## N ERROR: Error in t.test.default(\"invalid\") : not enough 'x' observations\\n ## 1 10 12 ## 2 20 9 ## 3 30 10 ## ERROR: Error in t.test.default() : argument \"x\" is missing, with no default\\n ## 1 31 ## 2 38 ## 3 25 ## ERROR: Manual error thrown\\n ## 1 10 ## 2 5 ## 3 7"},{"path":"http://philchalmers.github.io/SimDesign/articles/Catch_errors.html","id":"what-to-do-explicit-debugging","dir":"Articles","previous_headings":"","what":"What to do (explicit debugging)","title":"Managing warning and error messages","text":"errors occur often (fatal way) respective design conditions either extracted simulation inspected determine can fixed (e.g., providing better starting values, increasing convergence criteria/number iterations, etc). instance, say fourth row design object raised number error messages inspected . One useful approach debug 4th row instance error raised, can achieved using following: error flag used enter R’s debugger first instance error, -4 indicates 4th row design evaluated. also one instance changing warning messages error messages (.e., runSimulation(..., extra_options = list(warnings_as_errors=TRUE))) particularly useful state generated warning can inspected directly. Note similar arguments can made explicitly debugging functions generate-analyse-summarise chain (e.g., debug = 'analyse-4'), though less useful debugging (useful initial code design).","code":"runSimulation(..., debug = 'error-4')"},{"path":"http://philchalmers.github.io/SimDesign/articles/Catch_errors.html","id":"manual-debugging-via-try","dir":"Articles","previous_headings":"What to do (explicit debugging)","what":"Manual debugging via try()","title":"Managing warning and error messages","text":"Failing approach, manually wrapping problematic functions try() call. Adding line ((object, 'try-error')) browser() jump location/replication object unexpectedly witnessed, though admittedly clunky approach using debug. Nevertheless, jumping exact location error occurred, particularly case analyse() function throwing multiple error messages, greatly help determine exactly went wrong simulation state, allowing quickly locate fix issue.","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/Catch_errors.html","id":"extracting-error-seeds-for-hard-to-find-bugs","dir":"Articles","previous_headings":"What to do (explicit debugging)","what":"Extracting error seeds for hard-to-find bugs","title":"Managing warning and error messages","text":"alternative approach locating errors general use information stored within SimDesign objects time completion. default, .Random.seed states associated errors stored within final object, can extracted using SimExtract(..., ='error_seeds') option. function returns data.frame object seed stored column-wise, associated error message contained column name (allowed coerced valid column name make easier use $ operator). example, Given seeds, replicating exact error can achieved ) extracting single column integer vector, b) passing vector load_seed input. example, replicating first error message can achieved follows, makes sense immediately go debugging mode via debug inputs. Note: important manually select correct Design row using error extraction approach; otherwise, seed clearly replicate exact problem state. .Random.seed state loaded exact state, always related state well (case c typed debugger, somehow error harder find walking debug mode). Hence, users must type Q exit debugger better understood nature error message first-hand.","code":"seeds <- SimExtract(result, what = 'error_seeds') head(seeds[,1:3]) ## # A tibble: 6 × 3 ## Design_row_1.1..Error.in.t.tes…¹ Design_row_1.2..Manu…² Design_row_1.3..Erro…³ ## ## 1 10403 10403 10403 ## 2 624 21 65 ## 3 -159038368 -230613222 -230613222 ## 4 1905303777 203707493 203707493 ## 5 -371375826 1161141503 1161141503 ## 6 -1012234281 549195142 549195142 ## # ℹ abbreviated names: ## # ¹Design_row_1.1..Error.in.t.test.default.....argument..x..is.missing..with.no.default., ## # ²Design_row_1.2..Manual.error.thrown., ## # ³Design_row_1.3..Error.in.t.test.default.....argument..x..is.missing..with.no.default. picked_seed <- seeds$Design_row_1.1..Error.in.t.test.default..invalid.....not.enough..x..observations. # debug analyse() for first row of Design object via debug='analyse-1' runSimulation(Design, replications = 100, load_seed=picked_seed, debug='analyse-1', generate=Generate, analyse=Analyse, summarise=Summarise)"},{"path":"http://philchalmers.github.io/SimDesign/articles/Catch_errors.html","id":"converting-warings-to-errors-explicitly","dir":"Articles","previous_headings":"","what":"Converting warings to errors explicitly","title":"Managing warning and error messages","text":"occasion functions return warning message either border (treated ) error messages influence veracity simulation results. examples may include models appear ‘converge’ non-nonsensical parameter estimates (e.g., negative variances, non-positive definite correlation matrices, maximum iterations reached numerical searching algorithm, etc). However, issues non-fatal third-party software (.e., functions written developer simulation) may simply raise warning message inspection, whereas Monte Carlo simulation experiment issues generally treated invalid (though frequency occurrence still tracked, default SimDesign). resolve issue, avoid using nuclear option setting option(warn=2) treat warnings errors simulation, function manageWarnings() can used explicit convert known warning message strings errors disrupt code execution allowing warning messages continue raised. example, function utilized simulation whenever serious warning message raised explicitly converted error using internal grepl() test. now converts previous warning message error message, thereby correctly disrupting flow Monte Carlo simulation experiment prompting new call Generate(). course, warning error messages tallied resulting runSimulation() object, though now serious warnings tallied disruptive errors instead warnings continued normally.","code":"myfun <- function() { if(sample(c(TRUE, FALSE), 1, prob = c(.1, .9))) warning('This warning is serious') if(sample(c(TRUE, FALSE), 1, prob = c(.5, .5))) warning('This warning is no big deal') return(1) } set.seed(1) out <- myfun() set.seed(2) out <- myfun() ## Warning in myfun(): This warning is no big deal set.seed(7) out <- myfun() ## Warning in myfun(): This warning is serious set.seed(1) out1 <- manageWarnings(myfun(), warning2error='This warning is serious') out1 ## [1] 1 set.seed(2) out2 <- manageWarnings(myfun(), warning2error='This warning is serious') ## Warning in myfun(): This warning is no big deal out2 ## [1] 1 set.seed(7) out3 <- manageWarnings(myfun(), warning2error='This warning is serious') ## Error: This warning is serious"},{"path":"http://philchalmers.github.io/SimDesign/articles/Fixed_obj_fun.html","id":"including-fixed-objects","dir":"Articles","previous_headings":"","what":"Including fixed objects","title":"Exporting objects and functions from the workspace","text":"R fun language computer programming statistics, ’s without ’s quirks. instance, R generally recursive strategy attempting find objects within functions. object can’t found, R start look outside function’s environment see object can located , , look within even higher-level environments… recursive search continues searches object user workspace/Global environment, object can’t found error thrown. strange feature programmers come languages, writing simulations may cause severely unwanted issues. tutorial demonstrates make sure required user-defined objects visible SimDesign.","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/Fixed_obj_fun.html","id":"scoping","dir":"Articles","previous_headings":"","what":"Scoping","title":"Exporting objects and functions from the workspace","text":"demonstrate issue, let’s define two objects function uses objects. evaluated, objects visible user, can seen typing R console typing ls(). Functions define objects name also able locate values. behavior indeed bit strange, ’s one R’s quirks. Unfortunately, running code parallel across different cores objects visible, therefore must exported using methods (e.g., parallel package done clusterExport()). Exporting objects cluster fixes issue. reasoning applies functions defined R workspace well, including functions defined within external R packages. Hence, order use functions packages must either explicitly loaded require() library() within distributed code, referenced via Namespace :: operator (e.g., mvtnorm::rmvtnorm()).","code":"obj1 <- 10 obj2 <- 20 myfun <- function(x) obj1 + obj2 myfun(1) ## [1] 30 library(parallel) cl <- makeCluster(2) res <- try(parSapply(cl=cl, 1:4, myfun)) res ## Error in checkForRemoteErrors(val) : ## 2 nodes produced errors; first error: object 'obj1' not found clusterExport(cl=cl, c('obj1', 'obj2')) parSapply(cl=cl, 1:4, myfun) ## [1] 30 30 30 30"},{"path":"http://philchalmers.github.io/SimDesign/articles/Fixed_obj_fun.html","id":"exporting-objects-example","dir":"Articles","previous_headings":"","what":"Exporting objects example","title":"Exporting objects and functions from the workspace","text":"order make objects safely visible SimDesign strategy simple: wrap desired objects named list (object), pass fixed_objects argument. , elements can indexed using $ operator () function, whatever method may convenient. Note, however, required defined objects functions — SimDesign automatically makes user-defined functions available across nodes. aside, alternative approach simply define/source objects within respective SimDesign functions; way clearly visible runtime. following fixed_objects approach really useful defined objects contain large amount code. placing objects list passing fixed_objects, objects safely exported relevant functions. Furthermore, running code parallel also valid consequence (see ) objects properly exported core. , remember required R objects, user-defined functions!","code":"library(SimDesign) #SimFunctions(comments = FALSE) ### Define design conditions and number of replications Design <- createDesign(N = c(10, 20, 30)) replications <- 1000 # define custom functions and objects (or use source() to read these in from an external file) SD <- 2 my_gen_fun <- function(n, sd) rnorm(n, sd = sd) my_analyse_fun <- function(x) c(p = t.test(x)$p.value) fixed_objects <- list(SD=SD) #--------------------------------------------------------------------------- Generate <- function(condition, fixed_objects) { Attach(condition) # make condition names available (e.g., N) # further, can use with() to use 'SD' directly instead of 'fixed_objects$SD' ret <- with(fixed_objects, my_gen_fun(N, sd=SD)) ret } Analyse <- function(condition, dat, fixed_objects) { ret <- my_analyse_fun(dat) ret } Summarise <- function(condition, results, fixed_objects) { ret <- EDR(results, alpha = .05) ret } #--------------------------------------------------------------------------- ### Run the simulation res <- runSimulation(Design, replications, verbose=FALSE, fixed_objects=fixed_objects, generate=Generate, analyse=Analyse, summarise=Summarise, debug='none') res ## # A tibble: 3 × 6 ## N p REPLICATIONS SIM_TIME SEED COMPLETED ## ## 1 10 0.038 1000 0.23s 525310970 Wed Dec 4 05:32:45 2024 ## 2 20 0.05 1000 0.22s 529914981 Wed Dec 4 05:32:45 2024 ## 3 30 0.056 1000 0.25s 602604143 Wed Dec 4 05:32:45 2024 res <- runSimulation(Design, replications, verbose=FALSE, fixed_objects=fixed_objects, generate=Generate, analyse=Analyse, summarise=Summarise, debug='none', parallel = TRUE)"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"introduction","dir":"Articles","previous_headings":"","what":"Introduction","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"purpose vignette demonstrate utilize SimDesign context distributing many jobs across independent computing environments, high-performance computing (HPC) clusters, way allows reproducibility simulation conditions, resubmission jobs case incomplete collection results within specified HPC execution time budget, ensure random number generation across entire simulation (subsequent resubmissions, required) properly manged throughout given batch nature job. following text examples primarily managing larger simulations, often thousands replications many simulation conditions (.e., rows defined design object) generally require non-trivial amounts computing resources execute (hence, need super-computing resources job schedulers like Slurm, TORQUE, MAUI, among others). information Slurm’s Job Array support particular, vignette uses example, see https://slurm.schedmd.com/job_array.html","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"standard-setup-on-hpc-cluster","dir":"Articles","previous_headings":"","what":"Standard setup on HPC cluster","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"start, structure simulation code used later distribute jobs HPC scheduler effectively usual generate-analyse-summarise workflow described runSimulation(), organizational exceptions. , always good place start designing, testing, debugging simulation experiment submitting HPC clusters. IMPORTANT: vast majority bugs coding logic work consider moving next step involving HPC clusters. code well vetted step later jobs evaluated HPC cluster waste time resources (garbage-, garbage-).","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"example","dir":"Articles","previous_headings":"Standard setup on HPC cluster","what":"Example","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"Suppose following simulation evaluated, though time constraint reasons possible execute single computer (smaller network computers) therefore submitted HPC cluster. following script (hypothetically written file called SimDesign_simulation.R) contains simulation experiment whose instructions submitted Slurm scheduler. , sbatch utility used along set instructions specifying type hardware required file slurmInstructions.slurm. R side simulation, defined code must grab available cores (minus 1) detectable via parallelly::availableCores(), occurs automatically using runSimulation(..., parallel=TRUE). standard runSimulation(..., parallel=TRUE) setup 10,000 replications distributed available computing cores evaluated independently across three row conditions design object. However, process executed sequence: design[1, ] evaluated first , 10,000 replications collected, design[2, ] evaluated complete, design[3, ], . well, order approach optimal HPC cluster must assign job containing large number resources; specifically, higher RAM CPUs. demonstrate, following slurmInstructions.slurm file larger number CPUs requested allocating computational structure/cluster associated job, well larger amounts RAM. job request computing cluster built 192 GB RAM 96 CPUs (across whatever computing nodes available; likely 2 ), SimDesign_simulation.R evaluated , submitted scheduler via sbatch slurmInstructions.slurm.","code":"# SimDesign::SimFunctions() library(SimDesign) Design <- createDesign(N = c(10, 20, 30)) Generate <- function(condition, fixed_objects) { dat <- with(condition, rnorm(N, 10, 5)) # distributed N(10, 5) dat } Analyse <- function(condition, dat, fixed_objects) { ret <- c(mean=mean(dat), median=median(dat)) # mean/median of sample data ret } Summarise <- function(condition, results, fixed_objects){ colMeans(results) } # standard setup (not ideal for HPC clusters as parallelization # occurs within the design conditions, not across) res <- runSimulation(design=Design, replications=10000, generate=Generate, analyse=Analyse, summarise=Summarise, parallel=TRUE, filename='mysim') #!/bin/bash #SBATCH --job-name=\"My simulation (multiple CPUs)\" #SBATCH --mail-type=ALL #SBATCH --mail-user=somewhere@out.there #SBATCH --output=/dev/null ## (optional) delete .out files #SBATCH --time=12:00:00 ## HH:MM:SS #SBATCH --cpus-per-task=96 ## Build a computer with 96 cores #SBATCH --mem-per-cpu=2G ## Build a computer with 192GB of RAM module load r Rscript --vanilla SimDesign_simulation.R"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"limitations","dir":"Articles","previous_headings":"Standard setup on HPC cluster","what":"Limitations","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"generally effective distributing computational load, limitations approach: Due row-evaluation nature design conditions, computing cores point sit idle waiting remaining experiments complete job. occurs row design input (.e., per simulation condition) , simulation experiments many conditions evaluate suffer due rolling overhead, resulting wasted resource management (kind users HPC cluster) ultimately results jobs take longer complete Problematic fewer resource jobs given higher priority scheduler, may taking resources away researchers cores sit idle Using 96 CPU cluster result 96x speedup 1 CPU job. fact, larger allocated cluster, worse performance efficiency request 96 CPUs 192 GB RAM take considerably longer allocate compared requesting 96 independent computing arrays 1 CPU 2 GB RAM (latter approach described next section) concert point 3), results jobs ) take longer get started sit longer queue b) may distribute load efficiently enough, thereby resulting larger wall time necessary Setting seed condition ensure within design condition random numbers high quality, however guarantee repeated use set.seed() result high-quality random numbers (see next section example) Hence, repeated job submissions type, even unique seeds per condition, may generate high quality numbers repeated many times (alternative isolate design row submit row unique job, demonstrated near end vignette) Note avoid wasting time swapping/paging, schedulers never allocate jobs whose memory requirements exceed amount available memory address computational inefficiencies/wasted resources, one can instead switch cluster-based approach array distribution approach, discussed next section.","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"array-jobs","dir":"Articles","previous_headings":"","what":"Array jobs","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"HPC computing often optimal distribute replications conditions simultaneously unique computing nodes (termed arrays) effectively break problem several mini-batches (e.g., split simulation 1000+ independent pieces, collect results later). , design object runSimulation() structure readily lend optimal distribution array scheduler manage. Nevertheless, core components still useful initial code design, testing, debugging, therefore serve necessary first step writing simulation experiment code prior submitting HPC cluster. defining testing simulation ensure works expected, now comes time setup components required organizing HPC cluster submission using runArraySimulation() function.","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"converting-runsimulation-workflow-to-one-for-runarraysimulation","dir":"Articles","previous_headings":"Array jobs","what":"Converting runSimulation() workflow to one for runArraySimulation()","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"job runArraySimulation() utilize relevant information defined .sh .slurm script. done extracting information provided scheduler (specifically, via arrayID), used select specific subsets design rows. However, unlike runSimulation() function runArraySimulation() designed control important information pertaining .Random.seeds relevant distribution information allow rows design object contain repeated experimental condition information. allows design rows replication information optimally distributed HPC cluster. following example presents essential modifications required move single runSimulation() workflow batch workflow suitable runArraySimulation() Slurm scheduler.","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"expand-the-standard-simulation-design-object-for-each-array-id","dir":"Articles","previous_headings":"Array jobs","what":"Expand the standard simulation design object for each array ID","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"Suppose 300 computing cores independently available HPC cluster, though availability cores trickle function schedulers decided availability. case, strategy split 3 * 10000 = 30000 condition--replications experiments across gradually available resources, jobs evaluated parallel different times. Given specifications, may decide 300 computing nodes requested scheduler evaluate exactly 100 replications (nrow(design) * 10000 / 300 = 100). case, expandDesign() used repeat row condition 100 times, creating new expanded design object 300 rows instead 3. accommodate new rows, associated replications now defined according many replications used within --distributed conditions, need equal (see ). approach assumes design condition equally balanced terms computing time resources, though case (e.g., last condition contains notably higher computing times first two conditions) repeat_conditions can specified vector instead, repeat_conditions = c(100, 100, 1000), latter portion associated 10 replications per distributed node instead 100. Regardless whether expanded design balanced unbalanced row resulting expanded design object assigned unique computing array node, identified according Slurm assigned array ID.","code":"rc <- 100 # number of times the design row was repeated Design300 <- expandDesign(Design, repeat_conditions = rc) Design300 ## # A tibble: 300 × 1 ## N ## ## 1 10 ## 2 10 ## 3 10 ## 4 10 ## 5 10 ## 6 10 ## 7 10 ## 8 10 ## 9 10 ## 10 10 ## # ℹ 290 more rows # target replication number for each condition rep_target <- 10000 # replications per row in Design300 replications <- rep(rep_target / rc, nrow(Design300)) rc <- c(100, 100, 1000) DesignUnbalanced <- expandDesign(Design, repeat_conditions = rc) DesignUnbalanced ## # A tibble: 1,200 × 1 ## N ## ## 1 10 ## 2 10 ## 3 10 ## 4 10 ## 5 10 ## 6 10 ## 7 10 ## 8 10 ## 9 10 ## 10 10 ## # ℹ 1,190 more rows rep_target <- 10000 replicationsUnbalanced <- rep(rep_target / rc, times = rc) head(replicationsUnbalanced) ## [1] 100 100 100 100 100 100 table(replicationsUnbalanced) ## replicationsUnbalanced ## 10 100 ## 1000 200"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"construct-and-record-proper-random-seeds","dir":"Articles","previous_headings":"Array jobs","what":"Construct and record proper random seeds","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"principle, expanded Design300 object passed runSimulation(Design300, replications=100, ...) evaluate repeated conditions, row now replicated 100 times. However, now issue respect random seed management use functions set.seed() friends longer viable. repeated use set.seed() guarantee independent high-quality random numbers different instances. example, issue generally problem standard runSimulation() approach within design condition high quality random numbers used default, potentially repeated number sequences across conditions highly unlikely affect quality overall simulation experiment (conditions typically generate manage random numbers different ways due varying simulation factors, sample sizes, variance conditions, fitted models, number variables, type probability distributions use, ). However, expandDesign() setup likelihood witnessing correlated/redundant random samples increases quickly, particularly problematic within distributed replication set; hence, special care must taken ensure proper seeds distributed job array. Fortunately, seeds easy manage genSeeds() function using two-step approach, internally managed runArraySimulation() supplying initial seed (iseed) value associated array ID (arrayID). utilize L’Ecuyer’s (1999) method, constructs sequentially computed .Random.seed states ensure independence across replications conditions. Note iseed must constant across job arrays, make sure define iseed ! discussed FAQ section bottom, associated value also allow generation new .Random.seed elements () second third set simulation jobs submitted HPC cluster later time must also generate simulated data independent initial submission(s).","code":"set.seed(0) x <- runif(100) set.seed(1) y <- runif(100) plot(x, y) ## seemingly independent plot(x[-1], y[-100]) ## subsets perfectly correlated # genSeeds() # do this once on the main node/home computer and store the number! iseed <- 1276149341"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"extract-array-id-information-from-the--slurm-script","dir":"Articles","previous_headings":"Array jobs","what":"Extract array ID information from the .slurm script","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"submitting HPC cluster ’ll need include information scheduler distribute simulation experiment workers. Slurm systems, may script following, stored suitable .slurm file: reference later, label file simulation.slurm file must submitted scheduler ’s time. top part .slurm file provides BASH instructions Slurm scheduler via #SBATCH statements. case, many array jobs queue (1 300), much memory use per job (2GB), time limits (12 hours), ; see SBATCH details. important input focus context #SBATCH –array=1-300 used Slurm scheduler assign unique ID array job. scheduler take defined mySimDesignScript.R script send 300 independent resources (1 CPU 4GB RAM, case), independent jobs assigned unique array ID number within --array=1-300 range (e.g., distribution first computing resource assigned arrayID=1, second resource arrayID=2, ). runArraySimulation() function used subset Design300 object row; hence, array range must correspond row identifiers design object proper subsetting! Collecting single number assigned Slurm scheduler also easy. Just include obtain associated array ID, example single integer value 1 300. value used final execution step via runArraySimulation(..., arrayID=arrayID), finally turn .","code":"#!/bin/bash #SBATCH --job-name=\"My simulation (array jobs, distributing conditions + replications)\" #SBATCH --mail-type=ALL #SBATCH --mail-user=somewhere@out.there #SBATCH --output=/dev/null ## (optional) delete .out files #SBATCH --time=12:00:00 ## HH:MM:SS #SBATCH --mem-per-cpu=4G ## 4GB of RAM per cpu #SBATCH --cpus-per-task=1 #SBATCH --array=1-300 ## Slurm schedulers often allow up to 10,000 arrays module load r Rscript --vanilla mySimDesignScript.R # get assigned array ID (default uses type = 'slurm') arrayID <- getArrayID()"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"organize-information-for-runarraysimulation","dir":"Articles","previous_headings":"Array jobs","what":"Organize information for runArraySimulation()","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"prerequisite steps place ’re finally ready pass information runArraySimulation(), effectively wrapper runSimulation() suppresses verbose outputs, takes subsets Design300 object given supplied arrayID (objects, replications, seeds, etc), forces evaluation single CPU (hence, #SBATCH --cpus-per-task=1 used default, unless parallelization occur within replications, via OpenMP), manages random number generation seeds tractable way, saves SimDesign results file names based filename argument suffixes associated arrayID (e.g., filename='mysim' save files mysim-1.rds array 1, mysim-2.rds array 2, …, mysim-300.rds array 300). ’s ! store mysim-#.rds files directory job submitted, somewhat messy side, may also want specify directory name store simulation files . Hence, main (.e., landing) location associated ssh account create directory, using something like mkdir mysimfiles (R, dir.create('mysimfiles')) location .R .slurm files stored. following can used store 300 collected .rds files, making use dirname argument. Regardless, hard part done , though information included way control list input necessary, including explicit time/resource limits R executions within array jobs (see FAQ section information).","code":"# run the simulation on subset based on arrayID subset information runArraySimulation(design=Design300, replications=replications, generate=Generate, analyse=Analyse, summarise=Summarise, iseed=iseed, arrayID=arrayID, filename='mysim') # run the simulation on subset based on arrayID subset information runArraySimulation(design=Design300, replications=replications, generate=Generate, analyse=Analyse, summarise=Summarise, iseed=iseed, arrayID=arrayID, dirname='mysimfiles', filename='mysim')"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"putting-it-all-together","dir":"Articles","previous_headings":"Array jobs","what":"Putting it all together","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"complete submission script collecting everything presented . assumes .R file simulation code stored file mySimDesignScript.R, suitable Slurm instruction file created file simulation.slurm, points mySimDesignScript.R includes relevant R modules, directory called mysimfiles/ created storing files computer used submit array job file submitted job scheduler via sbatch, pointing .slurm instructions. complete can now go get beer, coffee, whatever else tickles fancy celebrate hard part .","code":"library(SimDesign) Design <- createDesign(N = c(10, 20, 30)) Generate <- function(condition, fixed_objects) { dat <- with(condition, rnorm(N, 10, 5)) # distributed N(10, 5) dat } Analyse <- function(condition, dat, fixed_objects) { ret <- c(mean=mean(dat), median=median(dat)) # mean/median of sample data ret } Summarise <- function(condition, results, fixed_objects){ colMeans(results) } # expand the design to create 300 rows with associated replications rc <- 100 Design300 <- expandDesign(Design, repeat_conditions = rc) rep_target <- 10000 replications <- rep(rep_target / rc, nrow(Design300)) # genSeeds() # do this once on the main node/home computer, and store the number! iseed <- 1276149341 # get assigned array ID (default uses type = 'slurm') arrayID <- getArrayID() # run the simulation on subset based on arrayID subset information runArraySimulation(design=Design300, replications=replications, generate=Generate, analyse=Analyse, summarise=Summarise, iseed=iseed, arrayID=arrayID, dirname='mysimfiles', filename='mysim') sbatch simulation.slurm"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"post-evaluation-combine-the-files","dir":"Articles","previous_headings":"","what":"Post-evaluation: Combine the files","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"time elapsed, job evaluation now complete, ’ll access complete set simulation files store file names mysim-#.rds. final step process collect independent results simulation object resembles returned canonical runSimulation() function. Fortunately, easy SimCollect(). must point point working directory containing simulation files use SimCollect(): function detects Design300 rows belong original Design object, collapse meta-statistic results, stored results information accordingly. fuss, mess. course, ’ll want store object later use complete collection results 300 array jobs, organized one neat little (object) package. now consider moving \"final_sim.rds\" Slurm landing node onto home computer via scp favourite method (e.g., using WinSCP Windows). also move saved *.rds files ssh landing case need inspect files (e.g., debugging purposes).","code":"library(SimDesign) # automatically checks whether all saved files are present via SimCheck() Final <- SimCollect('mysimfiles/') Final # A tibble: 3 × 8 N mean median REPLICATIONS SIM_TIME COMPLETED 1 10 9.9973 9.9934 10000 23.42s Thu Apr 4 11:50:11 2024 2 20 10.007 10.015 10000 24.24s Thu Apr 4 11:50:35 2024 3 30 10.003 10.007 10000 24.39s Thu Apr 4 11:51:00 2024 # save the aggregated simulation object for subsequent analyses saveRDS(Final, \"../final_sim.rds\")"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"array-jobs-and-multicore-computing-simultaneously","dir":"Articles","previous_headings":"","what":"Array jobs and multicore computing simultaneously","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"course, nothing really stops mixing matching ideas related multicore computing array jobs Slurm HPC clusters. example, wanted take original design object submit batches instead (e.g., submit one rows design object array job), within array multicore processing requested, something like following work: associated .R file containing, case, nine simulation conditions across rows Design. Depending intensity conditions, may choose distribute one row Design object array (multirow=TRUE following), otherwise natural choice distribute row Design object assigned array. complete, function SimCollect() can used put simulation results together given nine saved files (nine files also saved multirow set TRUE #SBATCH --array=1-3 used instead stored per-row basis). type hybrid approach middle ground submitting complete job (top vignette) condition + replication distributed load previous section, though similar overhead + inefficiency issues (though less , array jobs evaluated independently). Note row’s take different amounts time evaluate strategy can prove less efficient (e.g., first two rows may take 2 hours complete, third row may take 12 hours complete), case nuanced array2row() function defined help explicit balance load computing cluster.","code":"#!/bin/bash #SBATCH --job-name=\"My simulation (arrays + multiple CPUs)\" #SBATCH --mail-type=ALL #SBATCH --mail-user=somewhere@out.there #SBATCH --output=/dev/null ## (optional) delete .out files #SBATCH --time=04:00:00 ## HH:MM:SS #SBATCH --mem-per-cpu=4G ## Build a computing cluster with 64GB of RAM #SBATCH --cpus-per-task=16 ## 16 CPUs per array, likely built from 1 node #SBATCH --array=1-9 ## 9 array jobs module load r Rscript --vanilla mySimDesignScript.R library(SimDesign) Design <- createDesign(N = c(10, 20, 30), SD = c(1,2,3)) Generate <- function(condition, fixed_objects) { dat <- with(condition, rnorm(N, 10, sd=SD)) # distributed N(10, 5) dat } Analyse <- function(condition, dat, fixed_objects) { ret <- c(mean=mean(dat), median=median(dat)) # mean/median of sample data ret } Summarise <- function(condition, results, fixed_objects){ colMeans(results) } Design ## # A tibble: 9 × 2 ## N SD ## ## 1 10 1 ## 2 20 1 ## 3 30 1 ## 4 10 2 ## 5 20 2 ## 6 30 2 ## 7 10 3 ## 8 20 3 ## 9 30 3 # get array ID arrayID <- getArrayID() multirow <- FALSE # submit multiple rows of Design object to array? if(multirow){ # If selecting multiple design rows per array, such as the first 3 rows, # then next 3 rows, and so on, something like the following would work ## For arrayID=1, rows 1 through 3 are evaluated ## For arrayID=2, rows 4 through 6 are evaluated ## For arrayID=3, rows 7 through 9 are evaluated array2row <- function(arrayID) 1:3 + 3 * (arrayID-1) } else { # otherwise, use one row per respective arrayID array2row <- function(arrayID) arrayID } # Make sure parallel=TRUE flag is on to use all available cores! runArraySimulation(design=Design, replications=10000, generate=Generate, analyse=Analyse, summarise=Summarise, iseed=iseed, dirname='mysimfiles', filename='mysim', parallel=TRUE, arrayID=arrayID, array2row=array2row)"},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"helpful-slurm-commands","dir":"Articles","previous_headings":"Extra information (FAQs)","what":"Helpful Slurm commands","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"addition using sbatch submit jobs, following contains useful Slurm commands.","code":"sbatch # submit job file to Slurm scheduler squeue -u # what jobs are currently queued/running for a specific user sshare -U # check the share usage for a specific user scancel # cancel a specific job scancel -u # cancel all queued and running jobs for a specific user"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"my-hpc-cluster-excution-timeram-is-limited-and-terminates-before-the-simulation-is-complete","dir":"Articles","previous_headings":"Extra information (FAQs)","what":"My HPC cluster excution time/RAM is limited and terminates before the simulation is complete","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"issue important whenever HPC cluster mandatory time/RAM limits job submissions, array job may complete within assigned resources — hence, properly managed, discard valid replication information abruptly terminated. Unfortunately, likely occurrence, largely function unsure long simulation condition/replication take complete distributed across arrays (conditions/replications take longer others, difficult perfectly knowledgeable information beforehand) large final objects grow simulation progresses. avoid time/resource waste strongly recommended add max_time /max_RAM argument control list (see help(runArraySimulation) supported specifications), less Slurm specifications. control flags halt runArraySimulation() executions early return complete simulation results point. However, work arguments non-trivially less allocated Slurm resources; otherwise, ’ll run risk job terminates SimDesign functions chance store successfully completed replications. Setting around 90-95% respective #SBATCH --time= #SBATCH --mem-per-cpu= inputs , however, sufficient cases. course, session time early implies target replications missed first job submission batch. Therefore, covered next section, new job must submitted scheduler mindful initial simulation history (particularly, .Random.seed states).","code":"# Return successful results up to the 11 hour mark, and terminate early # if more than 3.5 GB of RAM are required to store the internal results runArraySimulation(design=Design300, replications=replications, generate=Generate, analyse=Analyse, summarise=Summarise, iseed=iseed, arrayID=arrayID, dirname='mysimfiles', filename='mysim', control=list(max_time=\"11:00:00\", max_RAM=\"3.5GB\"))"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"uploading-array-jobs-related-to-previous-array-submissions","dir":"Articles","previous_headings":"Extra information (FAQs)","what":"Uploading array jobs related to previous array submissions","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"Related early termination issue missing replication information event complete set replication information collected. Obtaining missing information clearly requires second (third) submission simulation job, though obviously conditions collected replication results problematic. Moreover, performed care avoid redundant random data generation strings, ultimately resulting sub-optimal results. start, locate simulation conditions aggregated result meet target replication criteria. obtained via inspection aggregated results via informative (less memory intensive) SimCollect(..., check.=TRUE) flag. Next, build new simulation structure containing missing information components. point, can return logic organizing simulation script job, distributing information across many array jobs necessary fill missing information. However, must careful random number generators per row subDesign original submission job. fix case straightforward well: simply create continuation previous logic, new elements treated additional rows resulting object though part initial job submission. , now just glue new subDesign information original expanded version, though telling scheduler evaluate new rows #SBATCH --array specification (technically unnecessary, conceptually clear keeps simulation files array IDs consistent). , approach simply expands original simulation 300 array jobs one 400 array jobs though added structure intended part initial design (obviously wasn’t, organized ). Finally, .slurm submission file longer want evaluate first 1-300 cases, .rds files already evaluated, instead want change --array line Submit job compute missing replication information, stores files working directory new information stored mysim-301.rds mysim-400.rds. example, now total 400 files saved. complete, run one last time, now reads complete set 400 stored files instead previous 300, thereby obtaining complete set high-quality simulation results. Rinse repeat issue appears yet second submission.","code":"Final <- SimCollect('mysimfiles/') Final # A tibble: 3 × 8 N mean median REPLICATIONS SIM_TIME COMPLETED 1 10 9.9973 9.9934 9000 23.42s Thu Apr 4 11:50:11 2024 2 20 10.007 10.015 10000 24.24s Thu Apr 4 11:50:35 2024 3 30 10.003 10.007 8000 24.39s Thu Apr 4 11:51:00 2024 Missed <- aggregate_simulations(files=dir(), check.only=TRUE) Missed # A tibble: 4 × 3 N MISSED_REPLICATIONS TARGET_REPLICATIONS 1 10 1000 10000 2 30 2000 10000 subDesign <- subset(Missed, select=N) replications_missed <- subset(Missed, select=MISSED_REPLICATIONS) subDesign ## # A tibble: 2 × 1 ## N ## ## 1 10 ## 2 30 replications_missed ## [1] 1000 2000 rc <- 50 Design_left <- expandDesign(subDesign, rc) # smaller number of reps per array Design_left ## # A tibble: 100 × 1 ## N ## ## 1 10 ## 2 10 ## 3 10 ## 4 10 ## 5 10 ## 6 10 ## 7 10 ## 8 10 ## 9 10 ## 10 10 ## # ℹ 90 more rows replications_left <- rep(replications_missed/rc, each=rc) table(replications_left) ## replications_left ## 20 40 ## 50 50 # new total design and replication objects Design_total <- rbind(Design300, Design_left) nrow(Design_total) ## [1] 400 replications_total <- c(replications, replications_left) table(replications_total) ## replications_total ## 20 40 100 ## 50 50 300 # this *must* be the same as the original submission! iseed <- 1276149341 #SBATCH --array=1-300 #SBATCH --array=301-400 # See if any missing still SimCollect('mysimfiles', check.only=TRUE) # Obtain complete simulation results Final <- SimCollect('mysimfiles')"},{"path":"http://philchalmers.github.io/SimDesign/articles/MultipleAnalyses.html","id":"description-of-structure","dir":"Articles","previous_headings":"","what":"Description of structure","title":"Multiple analysis functions","text":"usual work-flow SimDesign requires first calling SimFunctions() generate working template, following. uses default nAnalyses=1 generate single Analyse() function. context multiple analysis functions, however, users may interested passing number analysis functions believe need simulation (e.g., analyzing tt-test setup compare Welch versus independent samples t-test, two analysis functions used). Passing nAnalyses=2 SimFunctions() creates following template: Notice case two Analyse.#() definitions constructed, passed runSimulation() organized named list. names list ultimately attached names analysis objects ambiguity outputted information. However, inputs Analyse() functions always , dat object formed Generate() call passed Analyse definitions (hence, generate data held constant across respective analyses). template course modified replace less useful names Analyse.#() components. default users want change something like Analyse.some_statistic, Analyse.some_other_statistic, …, Analyse.some_other_other_statistic, , number Analyse.# function definitions ultimately end runSimulation(..., Analyse=list()) input. Supplying better names named list component also recommended used name associated results Summarise() step. Finally, note rules objects object naming typical single Analyse function still apply properly checked internally suitable names consistency. independently defined Analyse functions also interchangable removable/replaceable, makes structure Generate-Analyse-Summarise setup modular respect analysis components.","code":"SimDesign::SimFunctions() ## #------------------------------------------------------------------- ## ## library(SimDesign) ## ## Design <- createDesign(factor1 = NA, ## factor2 = NA) ## ## #------------------------------------------------------------------- ## ## Generate <- function(condition, fixed_objects) { ## dat <- data.frame() ## dat ## } ## ## Analyse <- function(condition, dat, fixed_objects) { ## ret <- nc(stat1 = NaN, stat2 = NaN) ## ret ## } ## ## Summarise <- function(condition, results, fixed_objects) { ## ret <- c(bias = NaN, RMSE = NaN) ## ret ## } ## ## #------------------------------------------------------------------- ## ## res <- runSimulation(design=Design, replications=2, generate=Generate, ## analyse=Analyse, summarise=Summarise) ## res SimDesign::SimFunctions(nAnalyses = 2) ## #------------------------------------------------------------------- ## ## library(SimDesign) ## ## Design <- createDesign(factor1 = NA, ## factor2 = NA) ## ## #------------------------------------------------------------------- ## ## Generate <- function(condition, fixed_objects) { ## dat <- data.frame() ## dat ## } ## ## Analyse.A1 <- function(condition, dat, fixed_objects) { ## ret <- nc(stat1 = NaN, stat2 = NaN) ## ret ## } ## ## Analyse.A2 <- function(condition, dat, fixed_objects) { ## ret <- nc(stat1 = NaN, stat2 = NaN) ## ret ## } ## ## #------------------------------------------------------------------- ## ## Summarise <- function(condition, results, fixed_objects) { ## ret <- c(bias = NaN, RMSE = NaN) ## ret ## } ## ## #------------------------------------------------------------------- ## ## res <- runSimulation(design=Design, replications=2,generate=Generate, ## analyse=list(A1=Analyse.A1, A2=Analyse.A2), ## summarise=Summarise) ## res"},{"path":"http://philchalmers.github.io/SimDesign/articles/MultipleAnalyses.html","id":"an-example","dir":"Articles","previous_headings":"Description of structure","what":"An example","title":"Multiple analysis functions","text":"following code adopted Wiki, details simulation obtained source. particular formulation mirt lavaan package analyses completely isolated respective functions, principle therefore analyzed independently future simulation studies. adds nicer layer potential modularity Analyse portion SimDesign framework, re-using modifying previous SimDesign code less painful.","code":"library(SimDesign) # SimFunctions(nAnalyses = 2) sample_sizes <- c(250, 500, 1000) nitems <- c(10, 20) Design <- createDesign(sample_size = sample_sizes, nitems = nitems) # create list of additional parameters which are fixed across conditions set.seed(1) pars_10 <- rbind(a = round(rlnorm(10, .3, .5)/1.702, 2), d = round(rnorm(10, 0, .5)/1.702, 2)) pars_20 <- rbind(a = round(rlnorm(20, .3, .5)/1.702, 2), d = round(rnorm(20, 0, .5)/1.702, 2)) pars <- list(ten=pars_10, twenty=pars_20) P_logit <- function(a, d, Theta) exp(a * Theta + d) / (1 + exp(a * Theta + d)) P_ogive <- function(a, d, Theta) pnorm(a * Theta + d) Generate <- function(condition, fixed_objects) { N <- condition$sample_size nitems <- condition$nitems nitems_name <- ifelse(nitems == 10, 'ten', 'twenty') #extract objects from fixed_objects a <- fixed_objects[[nitems_name]]['a', ] d <- fixed_objects[[nitems_name]]['d', ] dat <- matrix(NA, N, nitems) colnames(dat) <- paste0('item_', 1:nitems) Theta <- rnorm(N) for(j in 1:nitems){ p <- P_ogive(a[j], d[j], Theta) for(i in 1:N) dat[i,j] <- sample(c(1,0), 1, prob = c(p[i], 1 - p[i])) } as.data.frame(dat) #data.frame works nicer with lavaan } Analyse.FIML <- function(condition, dat, fixed_objects) { mod <- mirt(dat, 1L, verbose=FALSE) if(!extract.mirt(mod, 'converged')) stop('mirt did not converge') cfs <- mirt::coef(mod, simplify = TRUE, digits = Inf) FIML_as <- cfs$items[,1L] / 1.702 ret <- c(as=unname(FIML_as)) ret } Analyse.DWLS <- function(condition, dat, fixed_objects) { nitems <- condition$nitems lavmod <- paste0('F =~ ', paste0('NA*', colnames(dat)[1L], ' + '), paste0(colnames(dat)[-1L], collapse = ' + '), '\\nF ~~ 1*F') lmod <- sem(lavmod, dat, ordered = colnames(dat)) if(!lavInspect(lmod, 'converged')) stop('lavaan did not converge') cfs2 <- lavaan::coef(lmod) DWLS_alpha <- cfs2[1L:nitems] const <- sqrt(1 - DWLS_alpha^2) DWLS_as <- DWLS_alpha / const ret <- c(as=unname(DWLS_as)) ret } Summarise <- function(condition, results, fixed_objects) { nitems <- condition$nitems nitems_name <- ifelse(nitems == 10, 'ten', 'twenty') #extract objects from fixed_objects a <- fixed_objects[[nitems_name]]['a', ] pop <- c(a, a) obt_bias <- bias(results, pop) obt_RMSE <- RMSE(results, pop) ret <- c(bias=obt_bias, RMSE=obt_RMSE) ret } res <- runSimulation(Design, replications=100, verbose=FALSE, parallel=TRUE, generate=Generate, analyse=list(FIML=Analyse.FIML, DWLS=Analyse.DWLS), summarise=Summarise, filename = 'mirt_lavaan', packages=c('mirt', 'lavaan'), fixed_objects=pars) res ## # A tibble: 6 × 86 ## sample_size nitems bias.FIML.as1 bias.FIML.as2 bias.FIML.as3 bias.FIML.as4 ## ## 1 250 10 0.0099887 0.013457 -0.020161 0.18616 ## 2 500 10 -0.021136 0.018720 -0.018173 0.10424 ## 3 1000 10 -0.0039473 -0.0014778 -0.013393 0.077890 ## 4 250 20 0.060647 0.011787 0.064688 -0.026314 ## 5 500 20 0.019202 0.045376 0.021837 -0.0063587 ## 6 1000 20 0.019976 0.018608 0.0079817 -0.011498 ## # ℹ 80 more variables: bias.FIML.as5 , bias.FIML.as6 , ## # bias.FIML.as7 , bias.FIML.as8 , bias.FIML.as9 , ## # bias.FIML.as10 , bias.DWLS.as1 , bias.DWLS.as2 , ## # bias.DWLS.as3 , bias.DWLS.as4 , bias.DWLS.as5 , ## # bias.DWLS.as6 , bias.DWLS.as7 , bias.DWLS.as8 , ## # bias.DWLS.as9 , bias.DWLS.as10 , RMSE.FIML.as1 , ## # RMSE.FIML.as2 , RMSE.FIML.as3 , RMSE.FIML.as4 , …"},{"path":"http://philchalmers.github.io/SimDesign/articles/MultipleAnalyses.html","id":"analyseif","dir":"Articles","previous_headings":"","what":"AnalyseIf()","title":"Multiple analysis functions","text":"situations analysis functions defined analyse list applied certain design conditions, users can include AnalyseIf() definition beginning respective functions ensure analyses executed provided logical TRUE. logical ensures data generation conditions suitable analysis function investigated; otherwise, skipped generate-analyse-summarise work-flow. continuation , say investigator also interested recovering slope parameters factor analysis model observed indicator variable continuous well discrete. Design definition may therefore look like following. Provided Generate() step utilized indicators character vector, imply dat object returned Generate() consist discrete continuous data. case continuous indicator variables, lavaan used supports indicator types; however, mirt . , ensure analysis function pertaining lavaan used one include following replacement definition used mirt, now includes AnalyseIf() logical given indicators variable’s state. Using definition final object returned runSimulation() provide suitable NA placeholders (appropriate). continuous indicators results presented though mirt never used continuous indicator conditions controlled Design object. Note similar logic can made multiple Generate() functions, templated via SimDesign::SimFunctions(nGenerate = 2), requiring specific GenerateIf() tests indicate generation function used. interested type structure, following used separate discrete/continuous data generation per Design row:","code":"Design <- createDesign(sample_size = sample_sizes, nitems = nitems, indicators = c('discrete', 'continuous')) Design ## # A tibble: 12 × 3 ## sample_size nitems indicators ## ## 1 250 10 discrete ## 2 500 10 discrete ## 3 1000 10 discrete ## 4 250 20 discrete ## 5 500 20 discrete ## 6 1000 20 discrete ## 7 250 10 continuous ## 8 500 10 continuous ## 9 1000 10 continuous ## 10 250 20 continuous ## 11 500 20 continuous ## 12 1000 20 continuous Analyse.FIML <- function(condition, dat, fixed_objects) { AnalyseIf(condition$indicators == 'discrete') # equivalently: # AnalyseIf(indicators == 'discrete', condition) # with(condition, AnalyseIf(indicators == 'discrete')) mod <- mirt(dat, 1L, verbose=FALSE) if(!extract.mirt(mod, 'converged')) stop('mirt did not converge') cfs <- coef(mod, simplify = TRUE, digits = Inf) FIML_as <- cfs$items[,1L] / 1.702 ret <- c(as=unname(FIML_as)) ret } Generate.G1 <- function(condition, fixed_objects) { GenerateIf(condition$indicators == 'discrete') ... dat } Generate.G2 <- function(condition, fixed_objects) { GenerateIf(condition$indicators == 'continuous') ... dat } res <- runSimulation(design=Design, replications=1000, generate=list(G1=Generate.G1, G2=Generate.G2), analyse=list(DWLS=Analyse.DWLS, FIML=Analyse.FIML), summarise=Summarise)"},{"path":"http://philchalmers.github.io/SimDesign/articles/MultipleAnalyses.html","id":"applying-one-analyse-function-per-condition","dir":"Articles","previous_headings":"AnalyseIf()","what":"Applying one analyse function per-condition","title":"Multiple analysis functions","text":"Interestingly, AnalyseIf() also used select one analysis function time given components Design object. instance, Design definition constructed using analysis functions supplied defined one analysis function applied time simulation experiment. Note case need append ‘MML’ ‘DWLS’ results objects becomes redundant method column Design object, analyse list input specified unnamed list (cf. earlier input named, appended MML. DWLS. results output Summarise()). Users may find natural setup merge analysis information single Analyse() definition. downside, however, analysis function applied different generated datasets, theoretically unbiased ramifications analysis functions throw errors different rates (even explicitly supplying seed vector input runSimulation()).","code":"Design <- createDesign(sample_size = sample_sizes, nitems = nitems, method = c('FIML', 'DWLS')) Design ## # A tibble: 12 × 3 ## sample_size nitems method ## ## 1 250 10 FIML ## 2 500 10 FIML ## 3 1000 10 FIML ## 4 250 20 FIML ## 5 500 20 FIML ## 6 1000 20 FIML ## 7 250 10 DWLS ## 8 500 10 DWLS ## 9 1000 10 DWLS ## 10 250 20 DWLS ## 11 500 20 DWLS ## 12 1000 20 DWLS Analyse.FIML <- function(condition, dat, fixed_objects) { AnalyseIf(method == 'FIML', condition) #... } Analyse.DWLS <- function(condition, dat, fixed_objects) { AnalyseIf(method == 'DWLS', condition) # ... } # ... res <- runSimulation(Design, replications=100, verbose=FALSE, parallel=TRUE, generate=Generate, analyse=list(Analyse.FIML, Analyse.DWLS), summarise=Summarise, filename = 'mirt_lavaan', packages=c('mirt', 'lavaan'), fixed_objects=pars)"},{"path":"http://philchalmers.github.io/SimDesign/articles/Parallel-computing.html","id":"introduction","dir":"Articles","previous_headings":"","what":"Introduction","title":"Parallel computing information","text":"purpose vignette highlight internally defined structures SimDesign supports distributing simulation experiment across computing cores, networks, clusters. particular, document includes runSimulation() function distributes workload across replications per simulation condition basis. Hence, replcations row experiment defined within design object distributed parallel, ensures independent replications within condition follow proper random number generation control. logic presented within document based notion front-end user , principle, access computing cores (e.g., can connected via ssh, available locally), simulation condition defined design object reflect independent experiments (default using createDesign(); see expandDesign() non-independent structure supported approaches described document). situations computing architecture directly available, high performance computing (HPC) super computers, simply picking independent computers running batches simulation code computer (purpose collapsing later via aggregate_simulations()), see vignette “Distributing jobs high-performance computing (HPC)” managing random number generation require additional care.","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/Parallel-computing.html","id":"local-parallel-computing","dir":"Articles","previous_headings":"Introduction","what":"Local parallel computing","title":"Parallel computing information","text":"default, SimDesign’s runSimulation() function executed single core. However, setting argument runSimulation(..., parallel = TRUE) automatically define cluster object using one core less system available (detect via parallelly::availableCores()). allows straightforward way construct suitable, locally supported cluster object parallel processing just active computer. Depending verbose progress flags, progress distributed replication also printed console indicate amount estimated time remaining selection simulation condition complete. process repeated condition supplied design object rows evaluated. setup painless way construct distribute independent replications per condition, within evaluated condition (.e., row design object) high-quality random numbers automatically used via Pierre L’Ecuyer’s (1999) multiple streams method, limited number cores available. Alternatively, though bit extra effort, users may also define cluster computing object way runSimulation(..., cl) object, can used link computing resources able communicate via ssh, thereby expanding number available computing cores detected parallel::detectCores() friends.","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/Parallel-computing.html","id":"network-computing","dir":"Articles","previous_headings":"Introduction","what":"Network computing","title":"Parallel computing information","text":"access set computers can linked via secure-shell (ssh) LAN network Network computing (.k.., Beowulf cluster) may viable useful option. setup generally requires master node SimDesign installed, slave/master nodes required R packages pre-installed (Unix utilities dsh useful purpose). Finally, master node must ssh access slave nodes, slave node must ssh access master node, cluster object (cl) parallel package must manually defined master node. Setup network computing generally straightforward requires specification ) respective IP addresses within defined R script, b) user name (different master node’s user name; otherwise, ) required). Linux, may also important include relevant information host names IP addresses /etc/hosts file master slave nodes, ensure selected port (passed parallel::makeCluster()) master node hindered firewall. example, using following code master (primary) node spawn 7 slave (secondary) 1 master node, separate computer network associated IP address spawn additional 6 slave nodes. Information collected master node, also files objects saved using associated save/filename inputs runSimulation(). object cl passed runSimulation() master node computations distributed across respective IP addresses. Finally, ’s usually good practice use stopCluster(cl) simulations said done release communication computers, code shows. provided suitable names respective slave node, well master, can define cl object using instead (rather supplying IP addresses R script). requires master node slave nodes defined /etc/hosts ~/.ssh/config files, slave nodes require master node files (2 IP addresses required slave). Following setup, assuming user name across nodes, cl object instead defined case local cluster definition first section, random numbers automatically organized via Pierre L’Ecuyer’s (1999) method ensure quality number generation. similar setup can also used via recently supported future interface (see ).","code":"library(parallel) primary <- '192.168.2.1' IPs <- list(list(host=primary, user='myname', ncore=8), list(host='192.168.2.2', user='myname', ncore=6)) spec <- lapply(IPs, function(IP) rep(list(list(host=IP$host, user=IP$user)), IP$ncore)) spec <- unlist(spec, recursive=FALSE) cl <- makeCluster(master=primary, spec=spec, type = 'PSOCK') Final <- runSimulation(..., cl=cl) stopCluster(cl) library(parallel) primary <- 'master' IPs <- list(list(host=primary, ncore=8), list(host='slave', ncore=6)) spec <- lapply(IPs, function(IP) rep(list(list(host=IP$host)), IP$ncore)) spec <- unlist(spec, recursive=FALSE) cl <- makeCluster(master=primary, spec=spec, type = 'PSOCK') Final <- runSimulation(..., cl=cl) stopCluster(cl)"},{"path":"http://philchalmers.github.io/SimDesign/articles/Parallel-computing.html","id":"using-the-future-framework","dir":"Articles","previous_headings":"Introduction","what":"Using the future framework","title":"Parallel computing information","text":"future framework (see help(future, package = 'future')) can also used distributing asynchronous function evaluations simulation replication changing logical input runSimulation(..., parallel = TRUE/FALSE) character vector runSimulation(..., parallel = 'future'). work, computation plan must pre-specified via future::plan(). example, initialize local two-worker parallel processing computational plan one can use follow: benefit using future framework automatic support many distinct back-ends, , instance, HPC clusters control distribution jobs via Slurm TORQUE (e.g., see future.batchtools package). progress reporting progressr package required intended wrapper around runSimulation(). Specifically, wrap function with_progress() around runSimulation() specified type handler() use, via following. Finally, parallel computations complete sure manually reset computation plan free workers via benefit future framework provides unified distribution framework parallel (multisession, multicore, cluster) non-parallel (sequential) processing, manages random number generation correctly Monte Carlo simulations, demonstrated wide variety tools can applied interactive sessions.","code":"library(future) plan(multisession, workers = 2) res <- runSimulation(design=Design, replications=1000, generate=Generate, analyse=Analyse, summarise=Summarise, parallel = 'future') library(progressr) # Rstudio style handler (if using RStudio) handlers(\"rstudio\") # or using the cli package for terminal-based progress handlers('cli') # See help(progressr) for additional options and details # to use progressr, wrap/pipe inside with_progress() res <- with_progress(runSimulation(design=Design, replications=1000, generate=Generate, analyse=Analyse, summarise=Summarise, parallel = 'future')) plan(sequential) # release workers"},{"path":"http://philchalmers.github.io/SimDesign/articles/Saving-results.html","id":"option-save-true-default-is-true","dir":"Articles","previous_headings":"","what":"Option: save = TRUE (Default is TRUE)","title":"Saving simulation results and state","text":"save flag triggers whether temporary results saved hard-disk case power outages crashes. flag used results can easily restored automatically simulation can continue left hardware problems dealt . fact, modifications code required runSimulation() automatically detect temporary files resume (long resumed computer node; otherwise, see save_details list). simple example, say N=30N=30 condition something went terribly wrong simulation crashed. However, first two design conditions perfectly fine. save flag helpful state lost results still useful. Finally, supplying filename argument safely save aggregate simulation results hard-drive future reference; however, won’t called simulation complete. Check temporary file still exists. Notice simulation stopped 67% third design condition threw many consecutive errors (built-fail-safe SimDesign). imitate type crash/power outage, control = list(stop_on_fatal = TRUE) input; otherwise, simulation continue normally terminal conditions though place NA placeholders terminal condition. fix portion code simulation can restarted previous state continue normal. Therefore, event unforeseen program execution crashes time lost. Check files exist. Notice complete, temporary file removed hard-drive. Relatedly, .Random.seed states successful replication can saved passing control = list(store_Random.seeds = TRUE)) , though generally useful exceptional circumstances (e.g., generate-analyse results unusual throw warning error message, yet inspected interactively).","code":"Analyse <- function(condition, dat, fixed_objects) { if(condition$N == 30) stop('Danger Will Robinson!') ret <- c(p = t.test(dat)$p.value) ret } res <- runSimulation(Design, replications = 1000, save=TRUE, filename='my-simple-sim', generate=Generate, analyse=Analyse, summarise=Summarise, control = list(stop_on_fatal = TRUE)) ## ## Design: 1/3; Replications: 1000; RAM Used: 67.9 Mb; Total Time: 0.00s ## Conditions: N=10 ## ## Design: 2/3; Replications: 1000; RAM Used: 68.9 Mb; Total Time: 0.18s ## Conditions: N=20 ## ## Design: 3/3; Replications: 1000; RAM Used: 68.9 Mb; Total Time: 0.35s ## Conditions: N=30 ## files <- dir() files[grepl('SIMDESIGN', files)] ## [1] \"SIMDESIGN-TEMPFILE_fv-az1756-196.rds\" Analyse <- function(condition, dat, fixed_objects) { ret <- c(p = t.test(dat)$p.value) ret } res <- runSimulation(Design, replications = 1000, save=TRUE, filename='my-simple-sim', generate=Generate, analyse=Analyse, summarise=Summarise) ## ## Design: 3/3; Replications: 1000 Total Time: 0.35s ## Conditions: N=30 ## files <- dir() files[grepl('SIMDESIGN', files)] ## character(0) files[grepl('my-simp', files)] ## [1] \"my-simple-sim.rds\""},{"path":"http://philchalmers.github.io/SimDesign/articles/Saving-results.html","id":"store_results-true-by-default-though-not-recommended-if-ram-is-suspected-to-be-an-issue","dir":"Articles","previous_headings":"","what":"store_results (TRUE by default, though not recommended if RAM is suspected to be an issue)","title":"Saving simulation results and state","text":"Passing store_results = TRUE stores results object information passed Summarise() returned object. allows inspection simulation results, potential use functions reSummarise() provide meta-summaries simulation later time. simulation complete, results can extracted using SimResults(res) (generally SimExtract(res, = 'results')). example, Note used number replications/design conditions small enough warrant storage; otherwise, R session may run memory (RAM) simulation progresses. Otherwise, save_results = TRUE described recommended approach resolve potential memory issues.","code":"# store_results=TRUE by default res <- runSimulation(Design, replications = 3, generate=Generate, analyse=Analyse, summarise=Summarise) ## ## Design: 1/3; Replications: 3; RAM Used: 69.6 Mb; Total Time: 0.00s ## Conditions: N=10 ## ## Design: 2/3; Replications: 3; RAM Used: 69.5 Mb; Total Time: 0.00s ## Conditions: N=20 ## ## Design: 3/3; Replications: 3; RAM Used: 69.5 Mb; Total Time: 0.00s ## Conditions: N=30 ## results <- SimResults(res) results ## # A tibble: 9 × 2 ## N p ## ## 1 10 0.363 ## 2 10 0.816 ## 3 10 0.555 ## 4 20 0.674 ## 5 20 0.481 ## 6 20 0.120 ## 7 30 0.664 ## 8 30 0.527 ## 9 30 0.848"},{"path":"http://philchalmers.github.io/SimDesign/articles/Saving-results.html","id":"option-save_results-true-false-by-default-recommended-during-official-simulation-run-if-ram-is-an-issue","dir":"Articles","previous_headings":"","what":"Option: save_results = TRUE (FALSE by default; recommended during official simulation run if RAM is an issue)","title":"Saving simulation results and state","text":"Finally, save_results argument output results elements passed Summarise() separate .rds files containing analysis results condition information. option supported primarily simulations anticipated memory storage issues, results written hard-drive released memory. Note using save_results save flag automatically set TRUE ensure simulation state correctly tracked. can see three .rds files saved folder computer node name prefixed 'SimDesign-results' character string. .rds file contains respective simulation results (including errors warnings), can read directly readRDS(): , equivalently, SimResults() function SimResults() function added benefit can read-simulation results (recommended RAM can hold information), simply hand pick ones inspected. example, saved results can inspected: need arise remove results directory SimClean() function easiest way remove unwanted files directories.","code":"res <- runSimulation(Design, replications = 1000, save_results=TRUE, generate=Generate, analyse=Analyse, summarise=Summarise) ## ## Design: 1/3; Replications: 1000; RAM Used: 71.8 Mb; Total Time: 0.00s ## Conditions: N=10 ## ## Design: 2/3; Replications: 1000; RAM Used: 71.8 Mb; Total Time: 0.17s ## Conditions: N=20 ## ## Design: 3/3; Replications: 1000; RAM Used: 71.8 Mb; Total Time: 0.34s ## Conditions: N=30 ## dir <- dir() directory <- dir[grepl('SimDesign-results', dir)] dir(directory) ## [1] \"results-row-1.rds\" \"results-row-2.rds\" \"results-row-3.rds\" row1 <- readRDS(paste0(directory, '/results-row-1.rds')) str(row1) ## List of 6 ## $ condition : tibble [1 × 1] (S3: tbl_df/tbl/data.frame) ## ..$ N: num 10 ## $ results :'data.frame': 1000 obs. of 1 variable: ## ..$ p: num [1:1000] 0.976 0.397 0.143 0.595 0.453 ... ## $ errors : 'table' int[0 (1d)] ## ..- attr(*, \"dimnames\")=List of 1 ## .. ..$ : NULL ## $ error_seeds : NULL ## $ warnings : 'table' int[0 (1d)] ## ..- attr(*, \"dimnames\")=List of 1 ## .. ..$ warnings: NULL ## $ warning_seeds: NULL row1$condition ## # A tibble: 1 × 1 ## N ## ## 1 10 head(row1$results) ## p ## 1 0.9759 ## 2 0.3974 ## 3 0.1430 ## 4 0.5947 ## 5 0.4534 ## 6 0.2007 # first row row1 <- SimResults(res, which = 1) str(row1) ## tibble [1,000 × 2] (S3: tbl_df/tbl/data.frame) ## $ N: num [1:1000] 10 10 10 10 10 10 10 10 10 10 ... ## $ p: num [1:1000] 0.976 0.397 0.143 0.595 0.453 ... input <- SimResults(res) str(input) ## tibble [3,000 × 2] (S3: tbl_df/tbl/data.frame) ## $ N: num [1:3000] 10 10 10 10 10 10 10 10 10 10 ... ## $ p: num [1:3000] 0.976 0.397 0.143 0.595 0.453 ... SimClean(results = TRUE)"},{"path":"http://philchalmers.github.io/SimDesign/articles/Saving-results.html","id":"recommendations","dir":"Articles","previous_headings":"","what":"Recommendations","title":"Saving simulation results and state","text":"general recommendation running simulations supply filename = 'some_simulation_name' simulation finally ready run time (particularly simulations take long time finish), leave default save = TRUE store_results = TRUE track temporary files event unexpected crashes store results objects future inspection (need arise). aggregation simulation results often interested approach ensure results stored succinct manner later analyses. However, RAM suspected issue simulation progresses using save_results = TRUE strongly recommended avoid memory-based storage issues.","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/SimDesign-intro.html","id":"a-general-overview","dir":"Articles","previous_headings":"","what":"A general overview","title":"Introduction to the SimDesign package","text":"loading SimDesign package, begin defining required user-constructed functions. expedite process, call SimFunctions() create template filled , necessary functional arguments pre-assigned, body functions need modified. documentation argument can found respective R help files, however organization simple conceptually. begin, following code copied saved external source (.e., text) file. Alternatively, lazy (read: efficient) just don’t like copy--pasting, SimFunctions() can write output file providing filename argument. following creates file (mysim.R) containing simulation design/execution required user-defined functions. larger simulations, may want use two files, ’d prefer helpful comments included can achieved singlefile comments arguments, respectively. choice using single file entirely matter preference, influence overall simulation implementation. However, wish include separate analysis generate functions arguments nGenerate nAnalyse can useful compartmentalize generally distinct portions code (e.g., one analyse function fitting extracting components structural equation model, one analyse function fitting extracting information form item response theory model).","code":"library(SimDesign) SimFunctions() #------------------------------------------------------------------- library(SimDesign) Design <- createDesign(factor1 = NA, factor2 = NA) #------------------------------------------------------------------- Generate <- function(condition, fixed_objects) { dat <- data.frame() dat } Analyse <- function(condition, dat, fixed_objects) { ret <- nc(stat1 = NaN, stat2 = NaN) ret } Summarise <- function(condition, results, fixed_objects) { ret <- c(bias = NaN, RMSE = NaN) ret } #------------------------------------------------------------------- res <- runSimulation(design=Design, replications=2, generate=Generate, analyse=Analyse, summarise=Summarise) res SimDesign::SimFunctions('mysim') SimFunctions('mysim', singlefile = FALSE, comments = TRUE)"},{"path":"http://philchalmers.github.io/SimDesign/articles/SimDesign-intro.html","id":"simulation-determine-estimator-efficiency","dir":"Articles","previous_headings":"","what":"Simulation: Determine estimator efficiency","title":"Introduction to the SimDesign package","text":"toy example, let’s consider following question can investigated SimDesign: Question: trimming affect recovering mean distribution? Investigate using different sample sizes Gaussian χ2\\chi^2 distributions. Also, demonstrate effect using median recover mean.","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/SimDesign-intro.html","id":"define-the-conditions","dir":"Articles","previous_headings":"Simulation: Determine estimator efficiency","what":"Define the conditions","title":"Introduction to the SimDesign package","text":"First, define condition combinations investigated. case wish study 4 different sample sizes, use symmetric skewed distribution. use createDesign() extremely helpful create completely crossed-design combination (8 total). row Design represents unique condition studied simulation. case, first condition studied comes row 1, N=30N=30 distribution normal.","code":"Design <- createDesign(sample_size = c(30, 60, 120, 240), distribution = c('norm', 'chi')) Design ## # A tibble: 8 × 2 ## sample_size distribution ## ## 1 30 norm ## 2 60 norm ## 3 120 norm ## 4 240 norm ## 5 30 chi ## 6 60 chi ## 7 120 chi ## 8 240 chi"},{"path":"http://philchalmers.github.io/SimDesign/articles/SimDesign-intro.html","id":"define-the-functions","dir":"Articles","previous_headings":"Simulation: Determine estimator efficiency","what":"Define the functions","title":"Introduction to the SimDesign package","text":"first start defining data generation functional component. argument accepted function condition, always single row Design data.frame object class data.frame. Conditions run sequentially row 1 last row Design. also possible pass fixed_objects object function including fixed sets population parameters conditions, however simple simulation input required. can see , Generate() return numeric vector length NN containing data analysed population mean 3 (χ2\\chi^2 distribution mean equal df). Next, define analyse component analyse said data: function accepts data previously returned Generate() (dat), condition vector previously mentioned. point, may conceptually think first two functions evaluated independently RR times obtain RR sets results. words, wanted number replications 100, first two functions independently run (least) 100 times, results Analyse() stored, need summarise 100 elements meaningful meta statistics describe empirical properties. computing meta-statistics bias, root mean-square error, detection rates, primary importance. Unsurprisingly, , purpose summarise component: , condition defined , results matrix containing results Analyse(), row represents result returned respective replication, number columns equal length single vector returned Analyse(). sounds much complicated — really need know simulation RR x 4 matrix called results available build suitable summary . results matrix, apply() useful apply function respective row. bias RMSE obtained respective statistic, overall result returned vector. Stopping moment thinking carefully, know condition paired unique vector returned Summarise(). Therefore, might thinking result returned simulation rectangular form, matrix, data.frame, tibble. Well, ’d right!","code":"Generate <- function(condition, fixed_objects) { N <- condition$sample_size dist <- condition$distribution if(dist == 'norm'){ dat <- rnorm(N, mean = 3) } else if(dist == 'chi'){ dat <- rchisq(N, df = 3) } dat } Analyse <- function(condition, dat, fixed_objects) { M0 <- mean(dat) M1 <- mean(dat, trim = .1) M2 <- mean(dat, trim = .2) med <- median(dat) ret <- c(mean_no_trim=M0, mean_trim.1=M1, mean_trim.2=M2, median=med) ret } Summarise <- function(condition, results, fixed_objects) { obs_bias <- bias(results, parameter = 3) obs_RMSE <- RMSE(results, parameter = 3) ret <- c(bias=obs_bias, RMSE=obs_RMSE, RE=RE(obs_RMSE)) ret }"},{"path":"http://philchalmers.github.io/SimDesign/articles/SimDesign-intro.html","id":"putting-it-all-together","dir":"Articles","previous_headings":"Simulation: Determine estimator efficiency","what":"Putting it all together","title":"Introduction to the SimDesign package","text":"last stage SimDesign work-flow pass four defined elements runSimulation() function , unsurprisingly given ’s name, runs simulation. numerous options available function, investigated reading help(runSimulation) HTML file. Options performing simulations parallel, storing/resuming temporary results, debugging functions, available. simply request condition run 1000 times single processor, finally store results object called results. runSimulation(..., store_results = TRUE) used, current default package RAM issue, complete stored results can viewed using can seen printed results res object, result Summarise() function paired respective condition, meta-statistics properly named, three additional columns appended results: REPLICATIONS, indicates many time conditions performed, SIM_TIME, indicating time (seconds) took completely finish respective conditions, SEED, indicates random seeds used SimDesign condition (reproducibility). call View() R console may also nice way sift res object.","code":"res <- runSimulation(Design, replications = 1000, generate=Generate, analyse=Analyse, summarise=Summarise) ## ## Design: 1/8; Replications: 1000; RAM Used: 70.5 Mb; Total Time: 0.00s ## Conditions: sample_size=30, distribution=norm ## ## Design: 2/8; Replications: 1000; RAM Used: 71.5 Mb; Total Time: 0.21s ## Conditions: sample_size=60, distribution=norm ## ## Design: 3/8; Replications: 1000; RAM Used: 71.5 Mb; Total Time: 0.40s ## Conditions: sample_size=120, distribution=norm ## ## Design: 4/8; Replications: 1000; RAM Used: 71.6 Mb; Total Time: 0.61s ## Conditions: sample_size=240, distribution=norm ## ## Design: 5/8; Replications: 1000; RAM Used: 71.6 Mb; Total Time: 0.83s ## Conditions: sample_size=30, distribution=chi ## ## Design: 6/8; Replications: 1000; RAM Used: 71.6 Mb; Total Time: 1.02s ## Conditions: sample_size=60, distribution=chi ## ## Design: 7/8; Replications: 1000; RAM Used: 71.7 Mb; Total Time: 1.22s ## Conditions: sample_size=120, distribution=chi ## ## Design: 8/8; Replications: 1000; RAM Used: 71.7 Mb; Total Time: 1.43s ## Conditions: sample_size=240, distribution=chi ## # Final simulation object res ## # A tibble: 8 × 19 ## sample_size distribution bias.mean_no_trim bias.mean_trim.1 bias.mean_trim.2 ## ## 1 30 norm -0.0048080 -0.0046834 -0.0029265 ## 2 60 norm 0.0030546 0.0039219 0.0031080 ## 3 120 norm 0.0011275 0.00078027 0.00057227 ## 4 240 norm 0.00095261 0.0011309 0.00070515 ## 5 30 chi 0.022222 -0.29863 -0.44355 ## 6 60 chi -0.0042835 -0.34631 -0.49384 ## 7 120 chi -0.0050244 -0.34566 -0.48818 ## 8 240 chi -0.000058300 -0.34650 -0.48943 ## # ℹ 14 more variables: bias.median , RMSE.mean_no_trim , ## # RMSE.mean_trim.1 , RMSE.mean_trim.2 , RMSE.median , ## # RE.mean_no_trim , RE.mean_trim.1 , RE.mean_trim.2 , ## # RE.median , REPLICATIONS , SIM_TIME , RAM_USED , ## # SEED , COMPLETED # Extract complete set of stored results results <- SimResults(res) results ## # A tibble: 8,000 × 6 ## sample_size distribution mean_no_trim mean_trim.1 mean_trim.2 median ## ## 1 30 norm 3.18 3.14 3.15 3.17 ## 2 30 norm 2.84 2.80 2.78 2.86 ## 3 30 norm 3.29 3.21 3.14 2.95 ## 4 30 norm 3.34 3.37 3.34 3.28 ## 5 30 norm 2.78 2.77 2.77 2.76 ## 6 30 norm 3.15 3.08 3.06 3.03 ## 7 30 norm 3.03 3.00 3.00 2.99 ## 8 30 norm 3.50 3.51 3.55 3.55 ## 9 30 norm 2.95 2.91 2.90 2.94 ## 10 30 norm 2.98 2.99 3.01 2.95 ## # ℹ 7,990 more rows"},{"path":"http://philchalmers.github.io/SimDesign/articles/SimDesign-intro.html","id":"interpreting-the-results","dir":"Articles","previous_headings":"Simulation: Determine estimator efficiency","what":"Interpreting the results","title":"Introduction to the SimDesign package","text":"case, visually inspecting simulation table enough understand occurring, though Monte Carlo simulations use ANOVAs, marginalized tables, graphics used capture essentially phenomenon results. Monte Carlo simulations just like collecting analysing data experiments, advice put analysis hats present data though data collected real world. particular simulation, readily apparent using un-adjusted mean adequately recover population mean little bias. precision also seems increase sample sizes increase, indicated decreasing RMSE statistics. Generally, trimming causes less efficiency estimates, greater amounts trimming results even less efficiency, using median proxy estimate mean least effective method. witnessed rather clearly following table, prints relative efficiency estimators: Finally, χ2\\chi^2 distribution investigated un-adjusted mean accurately portrayed population mean. isn’t surprising, trimmed mean , , making inferences population trimmed mean, median making inferences , well, median. distributions investigation symmetric statistics able draw inferences inferences mean population.","code":"REs <- res[,grepl('RE\\\\.', colnames(res))] data.frame(Design, REs) ## sample_size distribution RE.mean_no_trim RE.mean_trim.1 RE.mean_trim.2 ## 1 30 norm 1 1.0 1.1 ## 2 60 norm 1 1.1 1.1 ## 3 120 norm 1 1.1 1.2 ## 4 240 norm 1 1.1 1.1 ## 5 30 chi 1 1.3 1.9 ## 6 60 chi 1 2.0 3.2 ## 7 120 chi 1 3.3 5.7 ## 8 240 chi 1 5.7 10.5 ## RE.median ## 1 1.4 ## 2 1.6 ## 3 1.5 ## 4 1.6 ## 5 2.8 ## 6 5.1 ## 7 9.2 ## 8 17.1"},{"path":"http://philchalmers.github.io/SimDesign/articles/SimDesign-intro.html","id":"conceptual-walk-through-of-what-runsimulation-is-doing","dir":"Articles","previous_headings":"","what":"Conceptual walk-through of what runSimulation() is doing","title":"Introduction to the SimDesign package","text":"following conceptual breakdown runSimulation() actually behind scenes. demonstrate results first condition (row 1 Design) show function returns. single replication Monte Carlo simulation results following objects: can see Generate() returns numeric vector accepted Analyse(). Analyse() function completes analysis portion using generated data, returns named vector observed parameter estimates. course, single replication, therefore really meaningful grand scheme things; , must repeated number times. matrix stored results contains 1000 parameter estimates returned statistic. obtained, can move summarising output Summarise() function obtain average estimates, associated sampling error, efficiency, . process repeated row Design object entire simulation study complete. course, runSimulation() much conceptual outline, exists. Namely, errors warnings controlled tracked, data re-drawn needed, parallel processing supported, debugging easier debug input (inserting browser() directly), temporary full results can saved external files, simulation state can saved/restored, build-safety features included, . point, however, user bogged nitty-gritty details setting simulation work-flow/features; instead, focusing time important generate-analyse-summarise steps, organized body functions, required obtain interesting simulation results. , point designing computer simulation experiment understand resulting output, become master aspects select computing language pertaining object storage, parallel processing, RAM storage, defensive coding, progress reporting, reproducibility, post-processing, …, ad nauseam. access examples instructions feel free visit package wiki Github","code":"(condition <- Design[1, ]) ## # A tibble: 1 × 2 ## sample_size distribution ## ## 1 30 norm dat <- Generate(condition) dat ## [1] 2.37 3.18 2.16 4.60 3.33 2.18 3.49 3.74 3.58 2.69 4.51 3.39 2.38 0.79 4.12 ## [16] 2.96 2.98 3.94 3.82 3.59 3.92 3.78 3.07 1.01 3.62 2.94 2.84 1.53 2.52 3.42 res <- Analyse(condition, dat) res ## mean_no_trim mean_trim.1 mean_trim.2 median ## 3.1 3.2 3.2 3.3 # repeat 1000x results <- matrix(0, 1000, 4) colnames(results) <- names(res) for(i in 1:1000){ dat <- Generate(condition) res <- Analyse(condition, dat) results[i, ] <- res } head(results) ## mean_no_trim mean_trim.1 mean_trim.2 median ## [1,] 3.1 3.1 3.1 2.9 ## [2,] 3.1 3.1 3.1 3.1 ## [3,] 3.1 3.1 3.0 2.8 ## [4,] 2.7 2.6 2.6 2.7 ## [5,] 3.2 3.2 3.2 3.0 ## [6,] 3.1 3.1 3.0 3.1 Summarise(condition, results) ## bias.mean_no_trim bias.mean_trim.1 bias.mean_trim.2 bias.median ## -0.0011 -0.0031 -0.0035 -0.0037 ## RMSE.mean_no_trim RMSE.mean_trim.1 RMSE.mean_trim.2 RMSE.median ## 0.1739 0.1777 0.1859 0.2146 ## RE.mean_no_trim RE.mean_trim.1 RE.mean_trim.2 RE.median ## 1.0000 1.0442 1.1425 1.5225"},{"path":"http://philchalmers.github.io/SimDesign/authors.html","id":null,"dir":"","previous_headings":"","what":"Authors","title":"Authors and Citation","text":"Phil Chalmers. Author, maintainer. Matthew Sigal. Contributor. Ogreden Oguzhan. Contributor. Mikko Ronkko. Contributor.","code":""},{"path":"http://philchalmers.github.io/SimDesign/authors.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Authors and Citation","text":"R. Philip Chalmers & Adkins, M. C. (2020). Writing effective reliable Monte Carlo simulations SimDesign package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248","code":"@Article{, title = {Writing effective and reliable {Monte Carlo} simulations with the {SimDesign} package}, author = {R. Philip Chalmers and Mark C. Adkins}, journal = {The Quantitative Methods for Psychology}, year = {2020}, volume = {16}, number = {4}, pages = {248--280}, doi = {10.20982/tqmp.16.4.p248}, }"},{"path":"http://philchalmers.github.io/SimDesign/index.html","id":"simdesign-","dir":"","previous_headings":"","what":"Structure for Organizing Monte Carlo Simulation Designs","title":"Structure for Organizing Monte Carlo Simulation Designs","text":"Structure Organizing Monte Carlo Simulation Designs","code":""},{"path":"http://philchalmers.github.io/SimDesign/index.html","id":"installation","dir":"","previous_headings":"","what":"Installation","title":"Structure for Organizing Monte Carlo Simulation Designs","text":"install latest stable version package CRAN, please use following R console: install Github version package devtools, type following (assuming already installed devtools package CRAN).","code":"install.packages('SimDesign') library('devtools') install_github('philchalmers/SimDesign')"},{"path":"http://philchalmers.github.io/SimDesign/index.html","id":"getting-started","dir":"","previous_headings":"","what":"Getting started","title":"Structure for Organizing Monte Carlo Simulation Designs","text":"discription pertaining philosophy general workflow package helpful first read following: Chalmers, R. Philip, Adkins, Mark C. (2020) Writing Effective Reliable Monte Carlo Simulations SimDesign Package, Quantitative Methods Psychology, 16(4), 248-280. doi: 10.20982/tqmp.16.4.p248 Coding examples found within article range relatively simple (e.g., re-implementation one Hallgren’s (2013) simulation study examples, well possible extensions simulation design) advanced real-world simulation experiments (e.g., Flora Curran’s (2004) simulation study). additional information instructions use package please refer examples associated Github wiki.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Analyse.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute estimates and statistics — Analyse","title":"Compute estimates and statistics — Analyse","text":"Compute relevant test statistics, parameter estimates, detection rates, . computational heavy lifting portion Monte Carlo simulation. Users may define single Analysis function perform analyses function environment, may define list named functions runSimulation allow modularized approach performing analyses independent blocks (share generated data). Note suitable Generate function supplied function can used generate analyse Monte Carlo data (though general setup recommended larger simulations).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Analyse.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute estimates and statistics — Analyse","text":"","code":"Analyse(condition, dat, fixed_objects)"},{"path":"http://philchalmers.github.io/SimDesign/reference/Analyse.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute estimates and statistics — Analyse","text":"condition single row design input (data.frame), indicating simulation conditions dat dat object returned Generate function (usually data.frame, matrix, vector, list) fixed_objects object passed runSimulation","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Analyse.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute estimates and statistics — Analyse","text":"returns named numeric vector data.frame values interest (e.g., p-values, effects sizes, etc), list containing values interest (e.g., separate matrix vector parameter estimates corresponding elements parameters). data.frame returned 1 row objects wrapped suitable list objects","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Analyse.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Compute estimates and statistics — Analyse","text":"cases, may easier change output named list containing different parameter configurations (e.g., determining RMSE values large set population parameters). use try functions generally required function Analyse internally wrapped try call. Therefore, function stops early cause function halt internally, message triggered stop recorded, Generate called obtain different dataset. said, may useful users throw stop commands data re-drawn reasons (e.g., estimated model terminated correctly maximum number iterations reached).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Analyse.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Compute estimates and statistics — Analyse","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/Analyse.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Compute estimates and statistics — Analyse","text":"","code":"if (FALSE) { # \\dontrun{ analyse <- function(condition, dat, fixed_objects) { # require packages/define functions if needed, or better yet index with the :: operator require(stats) mygreatfunction <- function(x) print('Do some stuff') #wrap computational statistics in try() statements to control estimation problems welch <- t.test(DV ~ group, dat) ind <- stats::t.test(DV ~ group, dat, var.equal=TRUE) # In this function the p values for the t-tests are returned, # and make sure to name each element, for future reference ret <- c(welch = welch$p.value, independent = ind$p.value) return(ret) } # A more modularized example approach analysis_welch <- function(condition, dat, fixed_objects) { welch <- t.test(DV ~ group, dat) ret <- c(p=welch$p.value) ret } analysis_ind <- function(condition, dat, fixed_objects) { ind <- t.test(DV ~ group, dat, var.equal=TRUE) ret <- c(p=ind$p.value) ret } # pass functions as a named list # runSimulation(..., analyse=list(welch=analyse_welch, independent=analysis_ind)) } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/AnalyseIf.html","id":null,"dir":"Reference","previous_headings":"","what":"Perform a test that indicates whether a given Analyse() function should be executed — AnalyseIf","title":"Perform a test that indicates whether a given Analyse() function should be executed — AnalyseIf","text":"function designed prevent specific analysis function executions design conditions met. Primarily useful analyse argument runSimulation input named list object, however analysis functions interesting/compatible generated data therefore skipped.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/AnalyseIf.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Perform a test that indicates whether a given Analyse() function should be executed — AnalyseIf","text":"","code":"AnalyseIf(x, condition = NULL)"},{"path":"http://philchalmers.github.io/SimDesign/reference/AnalyseIf.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Perform a test that indicates whether a given Analyse() function should be executed — AnalyseIf","text":"x logical statement evaluate. statement evaluates TRUE remainder defined function evaluated condition (optional) current design condition. need supplied expression x evaluates valid logical (e.g., use Attach(condition) prior using AnalyseIf, use (condition, AnalyseIf(someLogicalTest)))","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/AnalyseIf.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Perform a test that indicates whether a given Analyse() function should be executed — AnalyseIf","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/AnalyseIf.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Perform a test that indicates whether a given Analyse() function should be executed — AnalyseIf","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/AnalyseIf.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Perform a test that indicates whether a given Analyse() function should be executed — AnalyseIf","text":"","code":"if (FALSE) { # \\dontrun{ Design <- createDesign(N=c(10,20,30), var.equal = c(TRUE, FALSE)) Generate <- function(condition, fixed_objects) { Attach(condition) dat <- data.frame(DV = rnorm(N*2), IV = gl(2, N, labels=c('G1', 'G2'))) dat } # always run this analysis for each row in Design Analyse1 <- function(condition, dat, fixed_objects) { mod <- t.test(DV ~ IV, data=dat) mod$p.value } # Only perform analysis when variances are equal and N = 20 or 30 Analyse2 <- function(condition, dat, fixed_objects) { AnalyseIf(var.equal && N %in% c(20, 30), condition) mod <- t.test(DV ~ IV, data=dat, var.equal=TRUE) mod$p.value } Summarise <- function(condition, results, fixed_objects) { ret <- EDR(results, alpha=.05) ret } #------------------------------------------------------------------- # append names 'Welch' and 'independent' to associated output res <- runSimulation(design=Design, replications=100, generate=Generate, analyse=list(Welch=Analyse1, independent=Analyse2), summarise=Summarise) res # leave results unnamed res <- runSimulation(design=Design, replications=100, generate=Generate, analyse=list(Analyse1, Analyse2), summarise=Summarise) } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/Attach.html","id":null,"dir":"Reference","previous_headings":"","what":"Attach objects for easier reference — Attach","title":"Attach objects for easier reference — Attach","text":"behaviour function similar attach, however environment specific, therefore remains defined given function rather Global Environment. Hence, function much safer use attach, incidentally never used code. useful primarily convenience function prefer call variable names condition directly rather indexing condition$sample_size (condition, sample_size), example.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Attach.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Attach objects for easier reference — Attach","text":"","code":"Attach( ..., omit = NULL, check = TRUE, attach_listone = TRUE, RStudio_flags = FALSE )"},{"path":"http://philchalmers.github.io/SimDesign/reference/Attach.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Attach objects for easier reference — Attach","text":"... comma separated list data.frame, tibble, list, matrix objects containing (column) elements placed current working environment omit optional character vector containing names objects attached current environment. instance, objects named '' 'b' attached use omit = c('', 'b'). NULL (default) objects attached check logical; check see function accidentally replace previously defined variables names condition? Default TRUE, avoid error attach_listone logical; element assign list length one assign first element list associated name. generally avoids adding often unnecessary list 1 index, name <- list[[1L]] RStudio_flags logical; print R script output comments disable flagged missing variables RStudio? Requires form Attach(Design, RStudio_flags=TRUE) interactive debugging session Attach(condition, RStudio_flags=TRUE)","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Attach.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Attach objects for easier reference — Attach","text":"Note using RStudio \"Warn variable used definition scope\" diagnostic flag using Attach() raise suspensions. suppress issues, can either disable flags (atomic solution) evaluate following output R console place output working simulation file. Attach(Design, RStudio_flags = TRUE)","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Attach.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Attach objects for easier reference — Attach","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/Attach.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Attach objects for easier reference — Attach","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Attach.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Attach objects for easier reference — Attach","text":"","code":"Design <- createDesign(N1=c(10,20), N2=c(10,20), sd=c(1,2)) Design #> # A tibble: 8 × 3 #> N1 N2 sd #> #> 1 10 10 1 #> 2 20 10 1 #> 3 10 20 1 #> 4 20 20 1 #> 5 10 10 2 #> 6 20 10 2 #> 7 10 20 2 #> 8 20 20 2 # does not use Attach() Generate <- function(condition, fixed_objects ) { # condition = single row of Design input (e.g., condition <- Design[1,]) N1 <- condition$N1 N2 <- condition$N2 sd <- condition$sd group1 <- rnorm(N1) group2 <- rnorm(N2, sd=sd) dat <- data.frame(group = c(rep('g1', N1), rep('g2', N2)), DV = c(group1, group2)) dat } # similar to above, but using the Attach() function instead of indexing Generate <- function(condition, fixed_objects ) { Attach(condition) # N1, N2, and sd are now 'attached' and visible group1 <- rnorm(N1) group2 <- rnorm(N2, sd=sd) dat <- data.frame(group = c(rep('g1', N1), rep('g2', N2)), DV = c(group1, group2)) dat } ##################### # NOTE: if you're using RStudio with code diagnostics on then evaluate + add the # following output to your source file to manually support the flagged variables Attach(Design, RStudio_flags=TRUE) #> # !diagnostics suppress=N1,N2,sd # Below is the same example, however with false positive missing variables suppressed # when # !diagnostics ... is added added to the source file(s) # !diagnostics suppress=N1,N2,sd Generate <- function(condition, fixed_objects ) { Attach(condition) # N1, N2, and sd are now 'attached' and visible group1 <- rnorm(N1) group2 <- rnorm(N2, sd=sd) dat <- data.frame(group = c(rep('g1', N1), rep('g2', N2)), DV = c(group1, group2)) dat }"},{"path":"http://philchalmers.github.io/SimDesign/reference/BF_sim.html","id":null,"dir":"Reference","previous_headings":"","what":"Example simulation from Brown and Forsythe (1974) — BF_sim","title":"Example simulation from Brown and Forsythe (1974) — BF_sim","text":"Example results Brown Forsythe (1974) article robust estimators variance ratio tests. Statistical tests organized columns unique design conditions organized rows. See BF_sim_alternative alternative form simulation. Code simulation available wiki (https://github.com/philchalmers/SimDesign/wiki).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/BF_sim.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Example simulation from Brown and Forsythe (1974) — BF_sim","text":"Brown, M. B. Forsythe, . B. (1974). Robust tests equality variances. Journal American Statistical Association, 69(346), 364–367. Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/BF_sim.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Example simulation from Brown and Forsythe (1974) — BF_sim","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/BF_sim.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Example simulation from Brown and Forsythe (1974) — BF_sim","text":"","code":"if (FALSE) { # \\dontrun{ data(BF_sim) head(BF_sim) #Type I errors subset(BF_sim, var_ratio == 1) } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/BF_sim_alternative.html","id":null,"dir":"Reference","previous_headings":"","what":"(Alternative) Example simulation from Brown and Forsythe (1974) — BF_sim_alternative","title":"(Alternative) Example simulation from Brown and Forsythe (1974) — BF_sim_alternative","text":"Example results Brown Forsythe (1974) article robust estimators variance ratio tests. Statistical tests distributions organized columns unique design conditions organized rows. See BF_sim alternative form simulation distributions also included rows. Code simulation available wiki (https://github.com/philchalmers/SimDesign/wiki).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/BF_sim_alternative.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"(Alternative) Example simulation from Brown and Forsythe (1974) — BF_sim_alternative","text":"Brown, M. B. Forsythe, . B. (1974). Robust tests equality variances. Journal American Statistical Association, 69(346), 364–367. Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/BF_sim_alternative.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"(Alternative) Example simulation from Brown and Forsythe (1974) — BF_sim_alternative","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/BF_sim_alternative.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"(Alternative) Example simulation from Brown and Forsythe (1974) — BF_sim_alternative","text":"","code":"if (FALSE) { # \\dontrun{ data(BF_sim_alternative) head(BF_sim_alternative) #' #Type I errors subset(BF_sim_alternative, var_ratio == 1) } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/Bradley1978.html","id":null,"dir":"Reference","previous_headings":"","what":"Bradley's (1978) empirical robustness interval — Bradley1978","title":"Bradley's (1978) empirical robustness interval — Bradley1978","text":"Robustness interval criteria empirical detection rate estimates empirical coverage estimates defined Bradley (1978). See EDR ECR obtain estimates.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Bradley1978.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Bradley's (1978) empirical robustness interval — Bradley1978","text":"","code":"Bradley1978( rate, alpha = 0.05, type = \"liberal\", CI = FALSE, out.logical = FALSE, out.labels = c(\"conservative\", \"robust\", \"liberal\"), unname = FALSE )"},{"path":"http://philchalmers.github.io/SimDesign/reference/Bradley1978.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Bradley's (1978) empirical robustness interval — Bradley1978","text":"rate (optional) numeric vector containing empirical detection rate(s) empirical confidence interval estimates. supplied character vector elements defined .labels logical vector returned indicating whether detection rate estimate considered 'robust'. input empirical coverage rate argument CI must set TRUE. input missing, interval criteria printed console alpha Type error rate evaluated (default .05) type character vector indicating type interval classification use. Default 'liberal', however can 'stringent' use Bradley's stringent robustness criteria CI logical; robust interval constructed empirical detection rates (FALSE) empirical coverage rates (TRUE)? .logical logical; output vector TRUE/FALSE indicating whether supplied empirical detection rate/CI considered \"robust\"? Default FALSE, case .labels elements used instead .labels character vector length three indicating classification labels according desired robustness interval unname logical; apply unname results remove variable names?","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Bradley1978.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Bradley's (1978) empirical robustness interval — Bradley1978","text":"Bradley, J. V. (1978). Robustness? British Journal Mathematical Statistical Psychology, 31, 144-152. Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/Bradley1978.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Bradley's (1978) empirical robustness interval — Bradley1978","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Bradley1978.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Bradley's (1978) empirical robustness interval — Bradley1978","text":"","code":"# interval criteria used for empirical detection rates Bradley1978() #> liberal.lower liberal.upper #> 0.025 0.075 Bradley1978(type = 'stringent') #> stringent.lower stringent.upper #> 0.045 0.055 Bradley1978(alpha = .01, type = 'stringent') #> stringent.lower stringent.upper #> 0.009 0.011 # intervals applied to empirical detection rate estimates edr <- c(test1 = .05, test2 = .027, test3 = .051, test4 = .076, test5 = .024) Bradley1978(edr) #> test1 test2 test3 test4 test5 #> \"robust\" \"robust\" \"robust\" \"liberal\" \"conservative\" Bradley1978(edr, out.logical=TRUE) # is robust? #> test1 test2 test3 test4 test5 #> TRUE TRUE TRUE FALSE FALSE ##### # interval criteria used for coverage estimates Bradley1978(CI = TRUE) #> liberal.lower liberal.upper #> 0.925 0.975 Bradley1978(CI = TRUE, type = 'stringent') #> stringent.lower stringent.upper #> 0.945 0.955 Bradley1978(CI = TRUE, alpha = .01, type = 'stringent') #> stringent.lower stringent.upper #> 0.989 0.991 # intervals applied to empirical coverage rate estimates ecr <- c(test1 = .950, test2 = .973, test3 = .949, test4 = .924, test5 = .976) Bradley1978(ecr, CI=TRUE) #> test1 test2 test3 test4 test5 #> \"robust\" \"robust\" \"robust\" \"liberal\" \"conservative\" Bradley1978(ecr, CI=TRUE, out.logical=TRUE) # is robust? #> test1 test2 test3 test4 test5 #> TRUE TRUE TRUE FALSE FALSE"},{"path":"http://philchalmers.github.io/SimDesign/reference/CC.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute congruence coefficient — CC","title":"Compute congruence coefficient — CC","text":"Computes congruence coefficient, also known \"unadjusted\" correlation Tucker's congruence coefficient.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/CC.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute congruence coefficient — CC","text":"","code":"CC(x, y = NULL, unname = FALSE)"},{"path":"http://philchalmers.github.io/SimDesign/reference/CC.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute congruence coefficient — CC","text":"x vector data.frame/matrix containing variables use. vector input y required, otherwise congruence coefficient computed bivariate combinations y (optional) second vector input use x vector unname logical; apply unname results remove variable names?","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/CC.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Compute congruence coefficient — CC","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/CC.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compute congruence coefficient — CC","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/CC.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Compute congruence coefficient — CC","text":"","code":"vec1 <- runif(1000) vec2 <- runif(1000) CC(vec1, vec2) #> [1] 0.7539623 # compare to cor() cor(vec1, vec2) #> [1] 0.009919872 # column input df <- data.frame(vec1, vec2, vec3 = runif(1000)) CC(df) #> vec1 vec2 vec3 #> vec1 1.0000000 0.7539623 0.7528033 #> vec2 0.7539623 1.0000000 0.7574734 #> vec3 0.7528033 0.7574734 1.0000000 cor(df) #> vec1 vec2 vec3 #> vec1 1.000000000 0.009919872 0.02368507 #> vec2 0.009919872 1.000000000 0.02560829 #> vec3 0.023685075 0.025608293 1.00000000"},{"path":"http://philchalmers.github.io/SimDesign/reference/ECR.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute empirical coverage rates — ECR","title":"Compute empirical coverage rates — ECR","text":"Computes detection rate determining empirical coverage rates given set estimated confidence intervals. Note using 1 - ECR(CIs, parameter) provide empirical detection rate. Also supports computing average width CIs, may useful comparing efficiency CI estimators.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/ECR.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute empirical coverage rates — ECR","text":"","code":"ECR( CIs, parameter, tails = FALSE, CI_width = FALSE, complement = FALSE, names = NULL, unname = FALSE )"},{"path":"http://philchalmers.github.io/SimDesign/reference/ECR.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute empirical coverage rates — ECR","text":"CIs numeric vector matrix confidence interval values given parameter value, first element/column indicates lower confidence interval second element/column upper confidence interval. vector length 2 passed instead returned value either 1 0 indicate whether parameter value within interval, respectively. Otherwise, input must matrix even number columns parameter numeric scalar indicating fixed parameter value. Alternative, numeric vector object length equal number rows CIs (use compare sets parameters ) tails logical; TRUE returns vector length 2 indicate proportion times parameter lower higher supplied interval, respectively. mainly useful coverage region expected symmetric, therefore generally required. Note 1 - sum(ECR(CIs, parameter, tails=TRUE)) == ECR(CIs, parameter) CI_width logical; rather returning overall coverage rate, return average width CIs instead? Useful comparing efficiency different CI estimators complement logical; rather computing proportion population parameters within CI, return proportion outside advertised CI (1 - ECR = alpha). case one value provided, normally return 0 outside CI 1 inside, values switched (useful using, example, CI tests significance parameters) names optional character vector used name returned object. Generally useful one CI estimate investigated unname logical; apply unname results remove variable names?","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/ECR.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Compute empirical coverage rates — ECR","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/ECR.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compute empirical coverage rates — ECR","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/ECR.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Compute empirical coverage rates — ECR","text":"","code":"CIs <- matrix(NA, 100, 2) for(i in 1:100){ dat <- rnorm(100) CIs[i,] <- t.test(dat)$conf.int } ECR(CIs, 0) #> [1] 0.92 ECR(CIs, 0, tails = TRUE) #> [1] 0.03 0.05 ECR(CIs, 0, complement = TRUE) # proportion outside interval #> [1] 0.08 # single vector input CI <- c(-1, 1) ECR(CI, 0) #> [1] 1 ECR(CI, 0, complement = TRUE) #> [1] 0 ECR(CI, 2) #> [1] 0 ECR(CI, 2, complement = TRUE) #> [1] 1 ECR(CI, 2, tails = TRUE) #> [1] 0 1 # parameters of the same size as CI parameters <- 1:10 CIs <- cbind(parameters - runif(10), parameters + runif(10)) parameters <- parameters + rnorm(10) ECR(CIs, parameters) #> [1] 0.6 # average width of CIs ECR(CIs, parameters, CI_width=TRUE) #> [1] 1.107228 # ECR() for multiple CI estimates in the same object parameter <- 10 CIs <- data.frame(lowerCI_1=parameter - runif(10), upperCI_1=parameter + runif(10), lowerCI_2=parameter - 2*runif(10), upperCI_2=parameter + 2*runif(10)) head(CIs) #> lowerCI_1 upperCI_1 lowerCI_2 upperCI_2 #> 1 9.868010 10.95595 8.056701 11.70004 #> 2 9.540787 10.46340 9.198803 10.49114 #> 3 9.990340 10.39441 8.744864 11.23700 #> 4 9.095354 10.05217 9.778473 10.56988 #> 5 9.670788 10.71086 9.079359 11.79476 #> 6 9.392610 10.17076 8.889488 11.87073 ECR(CIs, parameter) #> [1] 1 1 ECR(CIs, parameter, tails=TRUE) #> [1] 0 0 0 0 ECR(CIs, parameter, CI_width=TRUE) #> [1] 0.8840042 1.9404326 # often a good idea to provide names for the output ECR(CIs, parameter, names = c('this', 'that')) #> this that #> 1 1 ECR(CIs, parameter, CI_width=TRUE, names = c('this', 'that')) #> this that #> 0.8840042 1.9404326 ECR(CIs, parameter, tails=TRUE, names = c('this', 'that')) #> this_lower this_upper that_lower that_upper #> 0 0 0 0"},{"path":"http://philchalmers.github.io/SimDesign/reference/EDR.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute the empirical detection/rejection rate for Type I errors and Power — EDR","title":"Compute the empirical detection/rejection rate for Type I errors and Power — EDR","text":"Computes detection/rejection rate determining empirical Type error power rates using information p-values.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/EDR.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute the empirical detection/rejection rate for Type I errors and Power — EDR","text":"","code":"EDR(p, alpha = 0.05, unname = FALSE)"},{"path":"http://philchalmers.github.io/SimDesign/reference/EDR.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute the empirical detection/rejection rate for Type I errors and Power — EDR","text":"p numeric vector matrix/data.frame p-values desired statistical estimator. matrix, statistic must organized column, number rows equal number replications alpha detection threshold (typical values .10, .05, .01). Default .05 unname logical; apply unname results remove variable names?","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/EDR.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Compute the empirical detection/rejection rate for Type I errors and Power — EDR","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/EDR.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compute the empirical detection/rejection rate for Type I errors and Power — EDR","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/EDR.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Compute the empirical detection/rejection rate for Type I errors and Power — EDR","text":"","code":"rates <- numeric(100) for(i in 1:100){ dat <- rnorm(100) rates[i] <- t.test(dat)$p.value } EDR(rates) #> [1] 0.07 EDR(rates, alpha = .01) #> [1] 0 # multiple rates at once rates <- cbind(runif(1000), runif(1000)) EDR(rates) #> [1] 0.034 0.043"},{"path":"http://philchalmers.github.io/SimDesign/reference/Generate.html","id":null,"dir":"Reference","previous_headings":"","what":"Generate data — Generate","title":"Generate data — Generate","text":"Generate data single row design input (see runSimulation). R contains numerous approaches generate data, contained base package, well SimDesign (e.g., rmgh, rValeMaurelli, rHeadrick). However majority can found external packages. See CRAN's list possible distributions : https://CRAN.R-project.org/view=Distributions. Note function technically can omitted data generation provided Analyse step, though general recommended.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Generate.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Generate data — Generate","text":"","code":"Generate(condition, fixed_objects)"},{"path":"http://philchalmers.github.io/SimDesign/reference/Generate.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Generate data — Generate","text":"condition single row design input (data.frame), indicating simulation conditions fixed_objects object passed runSimulation","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Generate.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Generate data — Generate","text":"returns single object containing data analyzed (usually vector, matrix, data.frame), list","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Generate.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Generate data — Generate","text":"use try functions generally required function Generate internally wrapped try call. Therefore, function stops early cause function halt internally, message triggered stop recorded, Generate called obtain different dataset. said, may useful users throw stop commands data re-drawn reasons (e.g., estimated model terminated correctly maximum number iterations reached).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Generate.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Generate data — Generate","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/Generate.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Generate data — Generate","text":"","code":"if (FALSE) { # \\dontrun{ generate <- function(condition, fixed_objects) { N1 <- condition$sample_sizes_group1 N2 <- condition$sample_sizes_group2 sd <- condition$standard_deviations group1 <- rnorm(N1) group2 <- rnorm(N2, sd=sd) dat <- data.frame(group = c(rep('g1', N1), rep('g2', N2)), DV = c(group1, group2)) # just a silly example of a simulated parameter pars <- list(random_number = rnorm(1)) list(dat=dat, parameters=pars) } # similar to above, but using the Attach() function instead of indexing generate <- function(condition, fixed_objects) { Attach(condition) N1 <- sample_sizes_group1 N2 <- sample_sizes_group2 sd <- standard_deviations group1 <- rnorm(N1) group2 <- rnorm(N2, sd=sd) dat <- data.frame(group = c(rep('g1', N1), rep('g2', N2)), DV = c(group1, group2)) dat } generate2 <- function(condition, fixed_objects) { mu <- sample(c(-1,0,1), 1) dat <- rnorm(100, mu) dat #return simple vector (discard mu information) } generate3 <- function(condition, fixed_objects) { mu <- sample(c(-1,0,1), 1) dat <- data.frame(DV = rnorm(100, mu)) dat } } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/GenerateIf.html","id":null,"dir":"Reference","previous_headings":"","what":"Perform a test that indicates whether a given Generate() function should be executed — GenerateIf","title":"Perform a test that indicates whether a given Generate() function should be executed — GenerateIf","text":"function designed prevent specific generate function executions design conditions met. Primarily useful generate argument runSimulation input named list object, however applied specific design condition (otherwise, data generation moves next function list).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/GenerateIf.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Perform a test that indicates whether a given Generate() function should be executed — GenerateIf","text":"","code":"GenerateIf(x, condition = NULL)"},{"path":"http://philchalmers.github.io/SimDesign/reference/GenerateIf.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Perform a test that indicates whether a given Generate() function should be executed — GenerateIf","text":"x logical statement evaluate. statement evaluates TRUE remainder defined function evaluated condition (optional) current design condition. need supplied expression x evaluates valid logical (e.g., use Attach(condition) prior using AnalyseIf, use (condition, AnalyseIf(someLogicalTest)))","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/GenerateIf.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Perform a test that indicates whether a given Generate() function should be executed — GenerateIf","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/GenerateIf.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Perform a test that indicates whether a given Generate() function should be executed — GenerateIf","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/GenerateIf.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Perform a test that indicates whether a given Generate() function should be executed — GenerateIf","text":"","code":"if (FALSE) { # \\dontrun{ # SimFunctions(nGenerate = 2) Design <- createDesign(N=c(10,20,30), var.equal = c(TRUE, FALSE)) Generate.G1 <- function(condition, fixed_objects) { GenerateIf(condition$var.equal == FALSE) # only run when unequal vars Attach(condition) dat <- data.frame(DV = c(rnorm(N), rnorm(N, sd=2)), IV = gl(2, N, labels=c('G1', 'G2'))) dat } Generate.G2 <- function(condition, fixed_objects) { Attach(condition) dat <- data.frame(DV = rnorm(N*2), IV = gl(2, N, labels=c('G1', 'G2'))) dat } # always run this analysis for each row in Design Analyse <- function(condition, dat, fixed_objects) { mod <- t.test(DV ~ IV, data=dat) mod$p.value } Summarise <- function(condition, results, fixed_objects) { ret <- EDR(results, alpha=.05) ret } #------------------------------------------------------------------- # append names 'Welch' and 'independent' to associated output res <- runSimulation(design=Design, replications=1000, generate=list(G1=Generate.G1, G2=Generate.G2), analyse=Analyse, summarise=Summarise) res } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/IRMSE.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute the integrated root mean-square error — IRMSE","title":"Compute the integrated root mean-square error — IRMSE","text":"Computes average/cumulative deviation given two continuous functions optional function representing probability density function. one-dimensional integration supported.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/IRMSE.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute the integrated root mean-square error — IRMSE","text":"","code":"IRMSE( estimate, parameter, fn, density = function(theta, ...) 1, lower = -Inf, upper = Inf, ... )"},{"path":"http://philchalmers.github.io/SimDesign/reference/IRMSE.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute the integrated root mean-square error — IRMSE","text":"estimate vector parameter estimates parameter vector population parameters fn continuous function first argument integrated second argument vector parameters parameter estimates. function represents implied continuous function uses sample estimates population parameters density (optional) density function used marginalize (.e., average), first argument integrated, must form density(theta, ...) density(theta, param1, param2), param1 placeholder name hyper-parameters associated probability density function. omitted cumulative different respective functions computed instead lower lower bound begin numerical integration upper upper bound finish numerical integration ... additional parameters pass fnest, fnparam, density, integrate,","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/IRMSE.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute the integrated root mean-square error — IRMSE","text":"returns single numeric term indicating average/cumulative deviation given supplied continuous functions","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/IRMSE.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Compute the integrated root mean-square error — IRMSE","text":"integrated root mean-square error (IRMSE) form $$IRMSE(\\theta) = \\sqrt{\\int [f(\\theta, \\hat{\\psi}) - f(\\theta, \\psi)]^2 g(\\theta, ...)}$$ \\(g(\\theta, ...)\\) density function used marginalize continuous sample (\\(f(\\theta, \\hat{\\psi})\\)) population (\\(f(\\theta, \\psi)\\)) functions.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/IRMSE.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Compute the integrated root mean-square error — IRMSE","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/IRMSE.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compute the integrated root mean-square error — IRMSE","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/IRMSE.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Compute the integrated root mean-square error — IRMSE","text":"","code":"# logistic regression function with one slope and intercept fn <- function(theta, param) 1 / (1 + exp(-(param[1] + param[2] * theta))) # sample and population sets est <- c(-0.4951, 1.1253) pop <- c(-0.5, 1) theta <- seq(-10,10,length.out=1000) plot(theta, fn(theta, pop), type = 'l', col='red', ylim = c(0,1)) lines(theta, fn(theta, est), col='blue', lty=2) # cumulative result (i.e., standard integral) IRMSE(est, pop, fn) #> [1] 0.05879362 # integrated RMSE result by marginalizing over a N(0,1) distribution den <- function(theta, mean, sd) dnorm(theta, mean=mean, sd=sd) IRMSE(est, pop, fn, den, mean=0, sd=1) #> [1] 0.01933435 # this specification is equivalent to the above den2 <- function(theta, ...) dnorm(theta, ...) IRMSE(est, pop, fn, den2, mean=0, sd=1) #> [1] 0.01933435"},{"path":"http://philchalmers.github.io/SimDesign/reference/MAE.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute the mean absolute error — MAE","title":"Compute the mean absolute error — MAE","text":"Computes average absolute deviation sample estimate parameter value. Accepts estimate parameter values, well estimate values deviation form.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/MAE.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute the mean absolute error — MAE","text":"","code":"MAE(estimate, parameter = NULL, type = \"MAE\", percent = FALSE, unname = FALSE)"},{"path":"http://philchalmers.github.io/SimDesign/reference/MAE.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute the mean absolute error — MAE","text":"estimate numeric vector, matrix/data.frame, list parameter estimates. vector, length equal number replications. matrix/data.frame number rows must equal number replications. list objects looped using rules first translating information one-dimensional vectors re-creating structure upon return parameter numeric scalar/vector matrix indicating fixed parameter values. single value supplied estimate matrix/data.frame value recycled column; otherwise, element associated respective column estimate input. NULL, assumed estimate input deviation form (therefore mean(abs(estimate)) returned) type type deviation compute. Can 'MAE' (default) mean absolute error, 'NMSE' normalized MAE (MAE / (max(estimate) - min(estimate))), 'SMSE' standardized MAE (MAE / sd(estimate)) percent logical; change returned result percentage multiplying 100? Default FALSE unname logical; apply unname results remove variable names?","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/MAE.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute the mean absolute error — MAE","text":"returns numeric vector indicating overall mean absolute error estimates","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/MAE.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Compute the mean absolute error — MAE","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/MAE.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compute the mean absolute error — MAE","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/MAE.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Compute the mean absolute error — MAE","text":"","code":"pop <- 1 samp <- rnorm(100, 1, sd = 0.5) MAE(samp, pop) #> [1] 0.3545376 dev <- samp - pop MAE(dev) #> [1] 0.3545376 MAE(samp, pop, type = 'NMAE') #> [1] 0.1193529 MAE(samp, pop, type = 'SMAE') #> [1] 0.7859817 # matrix input mat <- cbind(M1=rnorm(100, 2, sd = 0.5), M2 = rnorm(100, 2, sd = 1)) MAE(mat, parameter = 2) #> M1 M2 #> 0.4552233 0.8681354 # same, but with data.frame df <- data.frame(M1=rnorm(100, 2, sd = 0.5), M2 = rnorm(100, 2, sd = 1)) MAE(df, parameter = c(2,2)) #> M1 M2 #> 0.4040587 0.7386515 # parameters of the same size parameters <- 1:10 estimates <- parameters + rnorm(10) MAE(estimates, parameters) #> [1] 0.8352983"},{"path":"http://philchalmers.github.io/SimDesign/reference/MSRSE.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute the relative performance behavior of collections of standard errors — MSRSE","title":"Compute the relative performance behavior of collections of standard errors — MSRSE","text":"mean-square relative standard error (MSRSE) compares standard error estimates standard deviation respective parameter estimates. Values close 1 indicate behavior standard errors closely matched sampling variability parameter estimates.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/MSRSE.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute the relative performance behavior of collections of standard errors — MSRSE","text":"","code":"MSRSE(SE, SD, percent = FALSE, unname = FALSE)"},{"path":"http://philchalmers.github.io/SimDesign/reference/MSRSE.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute the relative performance behavior of collections of standard errors — MSRSE","text":"SE numeric scalar/vector indicating average standard errors across replications, matrix collected standard error estimates used compute average standard errors. column/element input corresponds column/element SD SD numeric scalar/vector indicating standard deviation across replications, matrix collected parameter estimates used compute standard deviations. column/element input corresponds column/element SE percent logical; change returned result percentage multiplying 100? Default FALSE unname logical; apply unname results remove variable names?","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/MSRSE.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute the relative performance behavior of collections of standard errors — MSRSE","text":"returns vector ratios indicating relative performance standard error estimates observed parameter standard deviation. Values less 1 indicate standard errors larger standard deviation parameters (hence, SEs interpreted conservative), values greater 1 smaller standard deviation parameters (.e., liberal SEs)","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/MSRSE.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Compute the relative performance behavior of collections of standard errors — MSRSE","text":"Mean-square relative standard error (MSRSE) expressed $$MSRSE = \\frac{E(SE(\\psi)^2)}{SD(\\psi)^2} = \\frac{1/R * \\sum_{r=1}^R SE(\\psi_r)^2}{SD(\\psi)^2}$$ \\(SE(\\psi_r)\\) represents estimate standard error \\(r\\)th simulation replication, \\(SD(\\psi)\\) represents standard deviation estimate parameters across \\(R\\) replications. Note \\(SD(\\psi)^2\\) used, corresponds variance \\(\\psi\\).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/MSRSE.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Compute the relative performance behavior of collections of standard errors — MSRSE","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/MSRSE.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compute the relative performance behavior of collections of standard errors — MSRSE","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/MSRSE.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Compute the relative performance behavior of collections of standard errors — MSRSE","text":"","code":"Generate <- function(condition, fixed_objects) { X <- rep(0:1, each = 50) y <- 10 + 5 * X + rnorm(100, 0, .2) data.frame(y, X) } Analyse <- function(condition, dat, fixed_objects) { mod <- lm(y ~ X, dat) so <- summary(mod) ret <- c(SE = so$coefficients[,\"Std. Error\"], est = so$coefficients[,\"Estimate\"]) ret } Summarise <- function(condition, results, fixed_objects) { MSRSE(SE = results[,1:2], SD = results[,3:4]) } results <- runSimulation(replications=500, generate=Generate, analyse=Analyse, summarise=Summarise) #> #> Replications: 500; RAM Used: 131 Mb; #> Conditions: dummy_run=NA #> results #> # A tibble: 1 × 7 #> `SE.(Intercept)` SE.X REPLICATIONS SIM_TIME RAM_USED SEED COMPLETED #> #> 1 0.98822 1.1781 500 0.48s 132.1 Mb 402147477 Wed Dec 4 0…"},{"path":"http://philchalmers.github.io/SimDesign/reference/PBA.html","id":null,"dir":"Reference","previous_headings":"","what":"Probabilistic Bisection Algorithm — PBA","title":"Probabilistic Bisection Algorithm — PBA","text":"function PBA searches specified interval root (.e., zero) function f(x) respect first argument. However, function differs deterministic cousins uniroot f may contain stochastic error components, instead provides Bayesian interval root likely lie. Note assumed E[f(x)] non-decreasing x root search interval (evaluated approximately check.interval=TRUE). See Waeber, Frazier, Henderson (2013) details.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/PBA.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probabilistic Bisection Algorithm — PBA","text":"","code":"PBA( f, interval, ..., p = 0.6, integer = FALSE, tol = if (integer) 0.01 else 1e-04, maxiter = 300L, miniter = 100L, wait.time = NULL, f.prior = NULL, resolution = 10000L, check.interval = TRUE, check.interval.only = FALSE, verbose = TRUE ) # S3 method for class 'PBA' print(x, ...) # S3 method for class 'PBA' plot(x, type = \"posterior\", main = \"Probabilistic Bisection Posterior\", ...)"},{"path":"http://philchalmers.github.io/SimDesign/reference/PBA.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probabilistic Bisection Algorithm — PBA","text":"f noisy function root sought interval vector containing end-points interval searched root ... additional named arguments passed f p assumed constant probability correct responses (must > 0.5) integer logical; values root considered integer numeric? former uses discreet grid track updates, latter currently creates grid resolution points tol tolerance criteria convergence based average f(x) evaluations maxiter maximum number iterations (default 300) miniter minimum number iterations (default 100) wait.time (optional) instead terminating specific estimate criteria satisfied (e.g., tol), terminate specific wait time. Input specified either numeric vector seconds character vector formatted timeFormater. Note users increase number maxiter well termination can occur either maximum iterations satisfied specified wait time elapsed (whichever occurs first) f.prior density function indicating likely location prior (e.g., root within [0,1] dunif works, otherwise custom functions required) resolution constant indicating number equally spaced grid points track integer = FALSE. check.interval logical; initial check made determine whether f(interval[1L]) f(interval[2L]) opposite signs? Default TRUE check.interval.logical; return TRUE FALSE test whether likely root given interval? Setting TRUE can useful unsure root location interval may want use higher replication input SimSolve verbose logical; iterations estimate printed console? x object class PBA type type plot draw PBA object. Can either 'posterior' 'history' plot PBA posterior distribution mediation iteration history main plot title","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/PBA.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probabilistic Bisection Algorithm — PBA","text":"Horstein, M. (1963). Sequential transmission using noiseless feedback. IEEE Trans. Inform. Theory, 9(3):136-143. Waeber, R., Frazier, P. . & Henderson, S. G. (2013). Bisection Search Noisy Responses. SIAM Journal Control Optimization, Society Industrial & Applied Mathematics (SIAM), 51, 2261-2279.","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/PBA.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probabilistic Bisection Algorithm — PBA","text":"","code":"# find x that solves f(x) - b = 0 for the following f.root <- function(x, b = .6) 1 / (1 + exp(-x)) - b f.root(.3) #> [1] -0.02555748 xs <- seq(-3,3, length.out=1000) plot(xs, f.root(xs), type = 'l', ylab = \"f(x)\", xlab='x', las=1) abline(h=0, col='red') retuni <- uniroot(f.root, c(0,1)) retuni #> $root #> [1] 0.4054644 #> #> $f.root #> [1] -1.772764e-07 #> #> $iter #> [1] 4 #> #> $init.it #> [1] NA #> #> $estim.prec #> [1] 6.103516e-05 #> abline(v=retuni$root, col='blue', lty=2) # PBA without noisy root retpba <- PBA(f.root, c(0,1)) #> Iter: 1; Median = 0.500; E(f(x)) = 0.02 Iter: 2; Median = 0.417; E(f(x)) = 0.01 Iter: 3; Median = 0.347; E(f(x)) = 0.00 Iter: 4; Median = 0.405; E(f(x)) = 0.00 Iter: 5; Median = 0.472; E(f(x)) = 0.01 Iter: 6; Median = 0.413; E(f(x)) = 0.00 Iter: 7; Median = 0.367; E(f(x)) = 0.00 Iter: 8; Median = 0.407; E(f(x)) = 0.00 Iter: 9; Median = 0.374; E(f(x)) = 0.00 Iter: 10; Median = 0.403; E(f(x)) = 0.00 Iter: 11; Median = 0.440; E(f(x)) = 0.00 Iter: 12; Median = 0.406; E(f(x)) = 0.00 Iter: 13; Median = 0.384; E(f(x)) = 0.00 Iter: 14; Median = 0.404; E(f(x)) = 0.00 Iter: 15; Median = 0.425; E(f(x)) = 0.00 Iter: 16; Median = 0.406; E(f(x)) = 0.00 Iter: 17; Median = 0.391; E(f(x)) = 0.00 Iter: 18; Median = 0.405; E(f(x)) = 0.00 Iter: 19; Median = 0.415; E(f(x)) = 0.00 Iter: 20; Median = 0.406; E(f(x)) = 0.00 Iter: 21; Median = 0.396; E(f(x)) = 0.00 Iter: 22; Median = 0.405; E(f(x)) = 0.00 Iter: 23; Median = 0.410; E(f(x)) = 0.00 Iter: 24; Median = 0.405; E(f(x)) = 0.00 Iter: 25; Median = 0.409; E(f(x)) = 0.00 Iter: 26; Median = 0.405; E(f(x)) = 0.00 Iter: 27; Median = 0.408; E(f(x)) = 0.00 Iter: 28; Median = 0.406; E(f(x)) = 0.00 Iter: 29; Median = 0.404; E(f(x)) = 0.00 Iter: 30; Median = 0.406; E(f(x)) = 0.00 Iter: 31; Median = 0.404; E(f(x)) = 0.00 Iter: 32; Median = 0.405; E(f(x)) = 0.00 Iter: 33; Median = 0.406; E(f(x)) = 0.00 Iter: 34; Median = 0.406; E(f(x)) = 0.00 Iter: 35; Median = 0.405; E(f(x)) = 0.00 Iter: 36; Median = 0.406; E(f(x)) = 0.00 Iter: 37; Median = 0.405; E(f(x)) = 0.00 Iter: 38; Median = 0.405; E(f(x)) = 0.00 Iter: 39; Median = 0.406; E(f(x)) = 0.00 Iter: 40; Median = 0.406; E(f(x)) = 0.00 Iter: 41; Median = 0.405; E(f(x)) = 0.00 Iter: 42; Median = 0.406; E(f(x)) = 0.00 Iter: 43; Median = 0.405; E(f(x)) = 0.00 Iter: 44; Median = 0.405; E(f(x)) = 0.00 Iter: 45; Median = 0.406; E(f(x)) = 0.00 Iter: 46; Median = 0.405; E(f(x)) = 0.00 Iter: 47; Median = 0.406; E(f(x)) = 0.00 Iter: 48; Median = 0.406; E(f(x)) = 0.00 Iter: 49; Median = 0.406; E(f(x)) = 0.00 Iter: 50; Median = 0.406; E(f(x)) = 0.00 Iter: 51; Median = 0.405; E(f(x)) = 0.00 Iter: 52; Median = 0.405; E(f(x)) = 0.00 Iter: 53; Median = 0.406; E(f(x)) = 0.00 Iter: 54; Median = 0.406; E(f(x)) = 0.00 Iter: 55; Median = 0.405; E(f(x)) = 0.00 Iter: 56; Median = 0.406; E(f(x)) = 0.00 Iter: 57; Median = 0.405; E(f(x)) = 0.00 Iter: 58; Median = 0.405; E(f(x)) = 0.00 Iter: 59; Median = 0.405; E(f(x)) = 0.00 Iter: 60; Median = 0.406; E(f(x)) = 0.00 Iter: 61; Median = 0.405; E(f(x)) = 0.00 Iter: 62; Median = 0.406; E(f(x)) = 0.00 Iter: 63; Median = 0.406; E(f(x)) = 0.00 Iter: 64; Median = 0.405; E(f(x)) = 0.00 Iter: 65; Median = 0.406; E(f(x)) = 0.00 Iter: 66; Median = 0.406; E(f(x)) = 0.00 Iter: 67; Median = 0.405; E(f(x)) = 0.00 Iter: 68; Median = 0.405; E(f(x)) = 0.00 Iter: 69; Median = 0.406; E(f(x)) = 0.00 Iter: 70; Median = 0.406; E(f(x)) = 0.00 Iter: 71; Median = 0.406; E(f(x)) = 0.00 Iter: 72; Median = 0.406; E(f(x)) = 0.00 Iter: 73; Median = 0.406; E(f(x)) = 0.00 Iter: 74; Median = 0.405; E(f(x)) = 0.00 Iter: 75; Median = 0.406; E(f(x)) = 0.00 Iter: 76; Median = 0.405; E(f(x)) = 0.00 Iter: 77; Median = 0.406; E(f(x)) = 0.00 Iter: 78; Median = 0.405; E(f(x)) = 0.00 Iter: 79; Median = 0.405; E(f(x)) = 0.00 Iter: 80; Median = 0.405; E(f(x)) = 0.00 Iter: 81; Median = 0.405; E(f(x)) = 0.00 Iter: 82; Median = 0.406; E(f(x)) = 0.00 Iter: 83; Median = 0.406; E(f(x)) = 0.00 Iter: 84; Median = 0.406; E(f(x)) = 0.00 Iter: 85; Median = 0.406; E(f(x)) = 0.00 Iter: 86; Median = 0.406; E(f(x)) = 0.00 Iter: 87; Median = 0.406; E(f(x)) = 0.00 Iter: 88; Median = 0.406; E(f(x)) = 0.00 Iter: 89; Median = 0.406; E(f(x)) = 0.00 Iter: 90; Median = 0.406; E(f(x)) = 0.00 Iter: 91; Median = 0.406; E(f(x)) = 0.00 Iter: 92; Median = 0.406; E(f(x)) = 0.00 Iter: 93; Median = 0.406; E(f(x)) = 0.00 Iter: 94; Median = 0.405; E(f(x)) = 0.00 Iter: 95; Median = 0.406; E(f(x)) = 0.00 Iter: 96; Median = 0.406; E(f(x)) = 0.00 Iter: 97; Median = 0.406; E(f(x)) = 0.00 Iter: 98; Median = 0.406; E(f(x)) = 0.00 Iter: 99; Median = 0.405; E(f(x)) = 0.00 Iter: 100; Median = 0.406; E(f(x)) = 0.00 Iter: 101; Median = 0.405; E(f(x)) = 0.00 Iter: 102; Median = 0.405; E(f(x)) = 0.00 Iter: 103; Median = 0.405; E(f(x)) = 0.00 Iter: 104; Median = 0.405; E(f(x)) = 0.00 Iter: 105; Median = 0.406; E(f(x)) = 0.00 Iter: 106; Median = 0.406; E(f(x)) = 0.00 Iter: 107; Median = 0.406; E(f(x)) = 0.00 Iter: 108; Median = 0.405; E(f(x)) = 0.00 Iter: 109; Median = 0.405; E(f(x)) = 0.00 Iter: 110; Median = 0.405; E(f(x)) = 0.00 Iter: 111; Median = 0.406; E(f(x)) = 0.00 Iter: 112; Median = 0.405; E(f(x)) = 0.00 Iter: 113; Median = 0.405; E(f(x)) = 0.00 Iter: 114; Median = 0.406; E(f(x)) = 0.00 Iter: 115; Median = 0.406; E(f(x)) = 0.00 Iter: 116; Median = 0.406; E(f(x)) = 0.00 Iter: 117; Median = 0.406; E(f(x)) = 0.00 Iter: 118; Median = 0.406; E(f(x)) = 0.00 Iter: 119; Median = 0.406; E(f(x)) = 0.00 Iter: 120; Median = 0.405; E(f(x)) = 0.00 Iter: 121; Median = 0.406; E(f(x)) = 0.00 Iter: 122; Median = 0.405; E(f(x)) = 0.00 Iter: 123; Median = 0.406; E(f(x)) = 0.00 Iter: 124; Median = 0.405; E(f(x)) = 0.00 Iter: 125; Median = 0.406; E(f(x)) = 0.00 Iter: 126; Median = 0.405; E(f(x)) = 0.00 Iter: 127; Median = 0.405; E(f(x)) = 0.00 Iter: 128; Median = 0.405; E(f(x)) = 0.00 Iter: 129; Median = 0.405; E(f(x)) = 0.00 Iter: 130; Median = 0.405; E(f(x)) = 0.00 Iter: 131; Median = 0.406; E(f(x)) = 0.00 Iter: 132; Median = 0.406; E(f(x)) = 0.00 Iter: 133; Median = 0.406; E(f(x)) = 0.00 Iter: 134; Median = 0.406; E(f(x)) = 0.00 Iter: 135; Median = 0.405; E(f(x)) = 0.00 Iter: 136; Median = 0.405; E(f(x)) = 0.00 Iter: 137; Median = 0.406; E(f(x)) = 0.00 Iter: 138; Median = 0.406; E(f(x)) = 0.00 Iter: 139; Median = 0.405; E(f(x)) = 0.00 Iter: 140; Median = 0.405; E(f(x)) = 0.00 Iter: 141; Median = 0.405; E(f(x)) = 0.00 Iter: 142; Median = 0.405; E(f(x)) = 0.00 Iter: 143; Median = 0.405; E(f(x)) = 0.00 Iter: 144; Median = 0.406; E(f(x)) = 0.00 Iter: 145; Median = 0.405; E(f(x)) = 0.00 Iter: 146; Median = 0.405; E(f(x)) = 0.00 Iter: 147; Median = 0.406; E(f(x)) = 0.00 Iter: 148; Median = 0.405; E(f(x)) = 0.00 Iter: 149; Median = 0.406; E(f(x)) = 0.00 Iter: 150; Median = 0.406; E(f(x)) = 0.00 Iter: 151; Median = 0.405; E(f(x)) = 0.00 Iter: 152; Median = 0.405; E(f(x)) = 0.00 Iter: 153; Median = 0.406; E(f(x)) = 0.00 Iter: 154; Median = 0.405; E(f(x)) = 0.00 Iter: 155; Median = 0.406; E(f(x)) = 0.00 Iter: 156; Median = 0.405; E(f(x)) = 0.00 Iter: 157; Median = 0.405; E(f(x)) = 0.00 Iter: 158; Median = 0.405; E(f(x)) = 0.00 Iter: 159; Median = 0.405; E(f(x)) = 0.00 Iter: 160; Median = 0.405; E(f(x)) = 0.00 Iter: 161; Median = 0.406; E(f(x)) = 0.00 Iter: 162; Median = 0.405; E(f(x)) = 0.00 Iter: 163; Median = 0.406; E(f(x)) = 0.00 Iter: 164; Median = 0.405; E(f(x)) = 0.00 Iter: 165; Median = 0.405; E(f(x)) = 0.00 Iter: 166; Median = 0.406; E(f(x)) = 0.00 Iter: 167; Median = 0.406; E(f(x)) = 0.00 Iter: 168; Median = 0.405; E(f(x)) = 0.00 Iter: 169; Median = 0.405; E(f(x)) = 0.00 Iter: 170; Median = 0.406; E(f(x)) = 0.00 Iter: 171; Median = 0.406; E(f(x)) = 0.00 retpba #> $root #> [1] 0.4055406 #> #> $terminated_early #> [1] TRUE #> #> $time #> [1] 0.12s #> #> $iterations #> [1] 172 #> retpba$root #> [1] 0.4055406 plot(retpba) plot(retpba, type = 'history') # Same problem, however root function is now noisy. Hence, need to solve # fhat(x) - b + e = 0, where E(e) = 0 f.root_noisy <- function(x) 1 / (1 + exp(-x)) - .6 + rnorm(1, sd=.02) sapply(rep(.3, 10), f.root_noisy) #> [1] -0.039123616 -0.001791425 -0.023427438 -0.028433850 -0.066480674 #> [6] -0.006044210 -0.018417790 -0.017867592 -0.033764287 -0.023880012 # uniroot \"converges\" unreliably set.seed(123) uniroot(f.root_noisy, c(0,1))$root #> [1] 0.3748233 uniroot(f.root_noisy, c(0,1))$root #> [1] 0.3785736 uniroot(f.root_noisy, c(0,1))$root #> [1] 0.4954932 # probabilistic bisection provides better convergence retpba.noise <- PBA(f.root_noisy, c(0,1)) #> Iter: 1; Median = 0.500; E(f(x)) = 0.02 Iter: 2; Median = 0.417; E(f(x)) = 0.02 Iter: 3; Median = 0.347; E(f(x)) = 0.00 Iter: 4; Median = 0.405; E(f(x)) = 0.00 Iter: 5; Median = 0.472; E(f(x)) = 0.00 Iter: 6; Median = 0.413; E(f(x)) = 0.00 Iter: 7; Median = 0.462; E(f(x)) = 0.01 Iter: 8; Median = 0.420; E(f(x)) = 0.00 Iter: 9; Median = 0.455; E(f(x)) = 0.00 Iter: 10; Median = 0.426; E(f(x)) = 0.01 Iter: 11; Median = 0.388; E(f(x)) = 0.00 Iter: 12; Median = 0.421; E(f(x)) = 0.00 Iter: 13; Median = 0.396; E(f(x)) = 0.01 Iter: 14; Median = 0.352; E(f(x)) = 0.00 Iter: 15; Median = 0.390; E(f(x)) = 0.00 Iter: 16; Median = 0.415; E(f(x)) = 0.00 Iter: 17; Median = 0.394; E(f(x)) = 0.01 Iter: 18; Median = 0.366; E(f(x)) = 0.01 Iter: 19; Median = 0.392; E(f(x)) = 0.00 Iter: 20; Median = 0.409; E(f(x)) = 0.00 Iter: 21; Median = 0.424; E(f(x)) = 0.00 Iter: 22; Median = 0.412; E(f(x)) = 0.00 Iter: 23; Median = 0.396; E(f(x)) = 0.00 Iter: 24; Median = 0.380; E(f(x)) = 0.00 Iter: 25; Median = 0.356; E(f(x)) = 0.00 Iter: 26; Median = 0.377; E(f(x)) = 0.00 Iter: 27; Median = 0.360; E(f(x)) = 0.00 Iter: 28; Median = 0.374; E(f(x)) = 0.00 Iter: 29; Median = 0.363; E(f(x)) = 0.00 Iter: 30; Median = 0.373; E(f(x)) = 0.00 Iter: 31; Median = 0.388; E(f(x)) = 0.00 Iter: 32; Median = 0.398; E(f(x)) = 0.00 Iter: 33; Median = 0.391; E(f(x)) = 0.00 Iter: 34; Median = 0.396; E(f(x)) = 0.00 Iter: 35; Median = 0.409; E(f(x)) = 0.00 Iter: 36; Median = 0.417; E(f(x)) = 0.00 Iter: 37; Median = 0.410; E(f(x)) = 0.00 Iter: 38; Median = 0.400; E(f(x)) = 0.00 Iter: 39; Median = 0.393; E(f(x)) = 0.00 Iter: 40; Median = 0.398; E(f(x)) = 0.00 Iter: 41; Median = 0.408; E(f(x)) = 0.00 Iter: 42; Median = 0.399; E(f(x)) = 0.00 Iter: 43; Median = 0.394; E(f(x)) = 0.00 Iter: 44; Median = 0.398; E(f(x)) = 0.00 Iter: 45; Median = 0.406; E(f(x)) = 0.00 Iter: 46; Median = 0.399; E(f(x)) = 0.00 Iter: 47; Median = 0.405; E(f(x)) = 0.00 Iter: 48; Median = 0.410; E(f(x)) = 0.00 Iter: 49; Median = 0.406; E(f(x)) = 0.00 Iter: 50; Median = 0.400; E(f(x)) = 0.00 Iter: 51; Median = 0.405; E(f(x)) = 0.00 Iter: 52; Median = 0.401; E(f(x)) = 0.00 Iter: 53; Median = 0.398; E(f(x)) = 0.00 Iter: 54; Median = 0.401; E(f(x)) = 0.00 Iter: 55; Median = 0.398; E(f(x)) = 0.00 Iter: 56; Median = 0.400; E(f(x)) = 0.00 Iter: 57; Median = 0.403; E(f(x)) = 0.00 Iter: 58; Median = 0.400; E(f(x)) = 0.00 Iter: 59; Median = 0.399; E(f(x)) = 0.00 Iter: 60; Median = 0.396; E(f(x)) = 0.00 Iter: 61; Median = 0.399; E(f(x)) = 0.00 Iter: 62; Median = 0.397; E(f(x)) = 0.00 Iter: 63; Median = 0.394; E(f(x)) = 0.00 Iter: 64; Median = 0.391; E(f(x)) = 0.00 Iter: 65; Median = 0.394; E(f(x)) = 0.00 Iter: 66; Median = 0.392; E(f(x)) = 0.00 Iter: 67; Median = 0.393; E(f(x)) = 0.00 Iter: 68; Median = 0.395; E(f(x)) = 0.00 Iter: 69; Median = 0.394; E(f(x)) = 0.00 Iter: 70; Median = 0.395; E(f(x)) = 0.00 Iter: 71; Median = 0.394; E(f(x)) = 0.00 Iter: 72; Median = 0.393; E(f(x)) = 0.00 Iter: 73; Median = 0.394; E(f(x)) = 0.00 Iter: 74; Median = 0.394; E(f(x)) = 0.00 Iter: 75; Median = 0.396; E(f(x)) = 0.00 Iter: 76; Median = 0.398; E(f(x)) = 0.00 Iter: 77; Median = 0.396; E(f(x)) = 0.00 Iter: 78; Median = 0.398; E(f(x)) = 0.00 Iter: 79; Median = 0.397; E(f(x)) = 0.00 Iter: 80; Median = 0.395; E(f(x)) = 0.00 Iter: 81; Median = 0.397; E(f(x)) = 0.00 Iter: 82; Median = 0.398; E(f(x)) = 0.00 Iter: 83; Median = 0.397; E(f(x)) = 0.00 Iter: 84; Median = 0.395; E(f(x)) = 0.00 Iter: 85; Median = 0.397; E(f(x)) = 0.00 Iter: 86; Median = 0.398; E(f(x)) = 0.00 Iter: 87; Median = 0.398; E(f(x)) = 0.00 Iter: 88; Median = 0.399; E(f(x)) = 0.00 Iter: 89; Median = 0.398; E(f(x)) = 0.00 Iter: 90; Median = 0.398; E(f(x)) = 0.00 Iter: 91; Median = 0.397; E(f(x)) = 0.00 Iter: 92; Median = 0.396; E(f(x)) = 0.00 Iter: 93; Median = 0.397; E(f(x)) = 0.00 Iter: 94; Median = 0.398; E(f(x)) = 0.00 Iter: 95; Median = 0.398; E(f(x)) = 0.00 Iter: 96; Median = 0.398; E(f(x)) = 0.00 Iter: 97; Median = 0.397; E(f(x)) = 0.00 Iter: 98; Median = 0.397; E(f(x)) = 0.00 Iter: 99; Median = 0.398; E(f(x)) = 0.00 Iter: 100; Median = 0.398; E(f(x)) = 0.00 Iter: 101; Median = 0.398; E(f(x)) = 0.00 Iter: 102; Median = 0.400; E(f(x)) = 0.00 Iter: 103; Median = 0.399; E(f(x)) = 0.00 Iter: 104; Median = 0.398; E(f(x)) = 0.00 Iter: 105; Median = 0.399; E(f(x)) = 0.00 Iter: 106; Median = 0.398; E(f(x)) = 0.00 Iter: 107; Median = 0.398; E(f(x)) = 0.00 Iter: 108; Median = 0.398; E(f(x)) = 0.00 Iter: 109; Median = 0.398; E(f(x)) = 0.00 Iter: 110; Median = 0.398; E(f(x)) = 0.00 Iter: 111; Median = 0.398; E(f(x)) = 0.00 Iter: 112; Median = 0.398; E(f(x)) = 0.00 Iter: 113; Median = 0.398; E(f(x)) = 0.00 Iter: 114; Median = 0.398; E(f(x)) = 0.00 Iter: 115; Median = 0.398; E(f(x)) = 0.00 Iter: 116; Median = 0.398; E(f(x)) = 0.00 Iter: 117; Median = 0.398; E(f(x)) = 0.00 Iter: 118; Median = 0.398; E(f(x)) = 0.00 Iter: 119; Median = 0.398; E(f(x)) = 0.00 Iter: 120; Median = 0.398; E(f(x)) = 0.00 Iter: 121; Median = 0.398; E(f(x)) = 0.00 Iter: 122; Median = 0.398; E(f(x)) = 0.00 Iter: 123; Median = 0.398; E(f(x)) = 0.00 Iter: 124; Median = 0.398; E(f(x)) = 0.00 Iter: 125; Median = 0.398; E(f(x)) = 0.00 Iter: 126; Median = 0.398; E(f(x)) = 0.00 Iter: 127; Median = 0.398; E(f(x)) = 0.00 Iter: 128; Median = 0.398; E(f(x)) = 0.00 Iter: 129; Median = 0.398; E(f(x)) = 0.00 Iter: 130; Median = 0.398; E(f(x)) = 0.00 Iter: 131; Median = 0.398; E(f(x)) = 0.00 Iter: 132; Median = 0.398; E(f(x)) = 0.00 Iter: 133; Median = 0.398; E(f(x)) = 0.00 Iter: 134; Median = 0.399; E(f(x)) = 0.00 Iter: 135; Median = 0.399; E(f(x)) = 0.00 Iter: 136; Median = 0.398; E(f(x)) = 0.00 Iter: 137; Median = 0.398; E(f(x)) = 0.00 retpba.noise #> $root #> [1] 0.3984398 #> #> $terminated_early #> [1] TRUE #> #> $time #> [1] 0.09s #> #> $iterations #> [1] 138 #> plot(retpba.noise) plot(retpba.noise, type = 'history') if (FALSE) { # \\dontrun{ # ignore termination criteria and instead run for 30 seconds or 30000 iterations retpba.noise_30sec <- PBA(f.root_noisy, c(0,1), wait.time = \"0:30\", maxiter=30000) retpba.noise_30sec } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/RAB.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute the relative absolute bias of multiple estimators — RAB","title":"Compute the relative absolute bias of multiple estimators — RAB","text":"Computes relative absolute bias given bias estimates multiple estimators.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RAB.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute the relative absolute bias of multiple estimators — RAB","text":"","code":"RAB(x, percent = FALSE, unname = FALSE)"},{"path":"http://philchalmers.github.io/SimDesign/reference/RAB.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute the relative absolute bias of multiple estimators — RAB","text":"x numeric vector bias estimates (see bias), first element used reference percent logical; change returned result percentage multiplying 100? Default FALSE unname logical; apply unname results remove variable names?","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RAB.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute the relative absolute bias of multiple estimators — RAB","text":"returns vector absolute bias ratios indicating relative bias effects compared first estimator. Values less 1 indicate better bias estimates first estimator, values greater 1 indicate worse bias first estimator","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RAB.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Compute the relative absolute bias of multiple estimators — RAB","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RAB.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compute the relative absolute bias of multiple estimators — RAB","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RAB.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Compute the relative absolute bias of multiple estimators — RAB","text":"","code":"pop <- 1 samp1 <- rnorm(5000, 1) bias1 <- bias(samp1, pop) samp2 <- rnorm(5000, 1) bias2 <- bias(samp2, pop) RAB(c(bias1, bias2)) #> [1] 1.00000 31.51306 RAB(c(bias1, bias2), percent = TRUE) # as a percentage #> [1] 100.000 3151.306"},{"path":"http://philchalmers.github.io/SimDesign/reference/RD.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute the relative difference — RD","title":"Compute the relative difference — RD","text":"Computes relative difference statistic form (est - pop)/ pop, equivalent form est/pop - 1. matrices supplied equivalent matrix variant used form (est - pop) * solve(pop). Values closer 0 indicate better relative parameter recovery. Note single variable inputs equivalent bias(..., type = 'relative').","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RD.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute the relative difference — RD","text":"","code":"RD(est, pop, as.vector = TRUE, unname = FALSE)"},{"path":"http://philchalmers.github.io/SimDesign/reference/RD.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute the relative difference — RD","text":"est numeric vector, matrix/data.frame, list containing parameter estimates pop numeric vector matrix containing true parameter values. Must comparable dimension est .vector logical; always wrap result .vector function returning? unname logical; apply unname results remove variable names?","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RD.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute the relative difference — RD","text":"returns vector matrix depending inputs whether .vector used","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RD.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Compute the relative difference — RD","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RD.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compute the relative difference — RD","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RD.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Compute the relative difference — RD","text":"","code":"# vector pop <- seq(1, 100, length.out=9) est1 <- pop + rnorm(9, 0, .2) (rds <- RD(est1, pop)) #> [1] 9.161195e-02 -2.856299e-03 -4.743365e-03 -5.027428e-05 7.186801e-04 #> [6] 8.588291e-05 3.142036e-03 2.113068e-03 1.682020e-03 summary(rds) #> Min. 1st Qu. Median Mean 3rd Qu. Max. #> -4.743e-03 -5.027e-05 7.187e-04 1.019e-02 2.113e-03 9.161e-02 # matrix pop <- matrix(c(1:8, 10), 3, 3) est2 <- pop + rnorm(9, 0, .2) RD(est2, pop, as.vector = FALSE) #> [,1] [,2] [,3] #> [1,] -0.09783211 0.6298721 -0.43147808 #> [2,] -0.32422225 0.3607876 -0.07267172 #> [3,] 0.04717509 -0.2042294 0.12395488 (rds <- RD(est2, pop)) #> [1] -0.09783211 -0.32422225 0.04717509 0.62987208 0.36078763 -0.20422939 #> [7] -0.43147808 -0.07267172 0.12395488 summary(rds) #> Min. 1st Qu. Median Mean 3rd Qu. Max. #> -0.431478 -0.204229 -0.072672 0.003484 0.123955 0.629872"},{"path":"http://philchalmers.github.io/SimDesign/reference/RE.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute the relative efficiency of multiple estimators — RE","title":"Compute the relative efficiency of multiple estimators — RE","text":"Computes relative efficiency given RMSE (default) MSE values multiple estimators.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RE.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute the relative efficiency of multiple estimators — RE","text":"","code":"RE(x, MSE = FALSE, percent = FALSE, unname = FALSE)"},{"path":"http://philchalmers.github.io/SimDesign/reference/RE.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute the relative efficiency of multiple estimators — RE","text":"x numeric vector root mean square error values (see RMSE), first element used reference. Otherwise, object contain MSE values flag MSE = TRUE also included MSE logical; input value mean squared errors instead root mean square errors? percent logical; change returned result percentage multiplying 100? Default FALSE unname logical; apply unname results remove variable names?","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RE.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute the relative efficiency of multiple estimators — RE","text":"returns vector variance ratios indicating relative efficiency compared first estimator. Values less 1 indicate better efficiency first estimator, values greater 1 indicate worse efficiency first estimator","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RE.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Compute the relative efficiency of multiple estimators — RE","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RE.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compute the relative efficiency of multiple estimators — RE","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RE.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Compute the relative efficiency of multiple estimators — RE","text":"","code":"pop <- 1 samp1 <- rnorm(100, 1, sd = 0.5) RMSE1 <- RMSE(samp1, pop) samp2 <- rnorm(100, 1, sd = 1) RMSE2 <- RMSE(samp2, pop) RE(c(RMSE1, RMSE2)) #> [1] 1.000000 4.072193 RE(c(RMSE1, RMSE2), percent = TRUE) # as a percentage #> [1] 100.0000 407.2193 # using MSE instead mse <- c(RMSE1, RMSE2)^2 RE(mse, MSE = TRUE) #> [1] 1.000000 4.072193"},{"path":"http://philchalmers.github.io/SimDesign/reference/RMSE.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute the (normalized) root mean square error — RMSE","title":"Compute the (normalized) root mean square error — RMSE","text":"Computes average deviation (root mean square error; also known root mean square deviation) sample estimate parameter value. Accepts estimate parameter values, well estimate values deviation form.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RMSE.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute the (normalized) root mean square error — RMSE","text":"","code":"RMSE( estimate, parameter = NULL, type = \"RMSE\", MSE = FALSE, percent = FALSE, unname = FALSE ) RMSD( estimate, parameter = NULL, type = \"RMSE\", MSE = FALSE, percent = FALSE, unname = FALSE )"},{"path":"http://philchalmers.github.io/SimDesign/reference/RMSE.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute the (normalized) root mean square error — RMSE","text":"estimate numeric vector, matrix/data.frame, list parameter estimates. vector, length equal number replications. matrix/data.frame, number rows must equal number replications. list objects looped using rules first translating information one-dimensional vectors re-creating structure upon return parameter numeric scalar/vector indicating fixed parameter values. single value supplied estimate matrix/data.frame value recycled column; otherwise, element associated respective column estimate input. NULL assumed estimate input deviation form (therefore sqrt(mean(estimate^2)) returned) type type deviation compute. Can 'RMSE' (default) root mean square-error, 'NRMSE' normalized RMSE (RMSE / (max(estimate) - min(estimate))), 'SRMSE' standardized RMSE (RMSE / sd(estimate)), 'CV' coefficient variation, 'RMSLE' root mean-square log-error MSE logical; return mean square error equivalent results instead root mean-square error (words, result squared)? Default FALSE percent logical; change returned result percentage multiplying 100? Default FALSE unname logical; apply unname results remove variable names?","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RMSE.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute the (normalized) root mean square error — RMSE","text":"returns numeric vector indicating overall average deviation estimates","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RMSE.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Compute the (normalized) root mean square error — RMSE","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/RMSE.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compute the (normalized) root mean square error — RMSE","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RMSE.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Compute the (normalized) root mean square error — RMSE","text":"","code":"pop <- 1 samp <- rnorm(100, 1, sd = 0.5) RMSE(samp, pop) #> [1] 0.4888468 dev <- samp - pop RMSE(dev) #> [1] 0.4888468 RMSE(samp, pop, type = 'NRMSE') #> [1] 0.2246974 RMSE(dev, type = 'NRMSE') #> [1] 0.2246974 RMSE(dev, pop, type = 'SRMSE') #> [1] 2.255327 RMSE(samp, pop, type = 'CV') #> [1] 0.486093 RMSE(samp, pop, type = 'RMSLE') #> [1] 0.2590176 # percentage reported RMSE(samp, pop, type = 'NRMSE') #> [1] 0.2246974 RMSE(samp, pop, type = 'NRMSE', percent = TRUE) #> [1] 22.46974 # matrix input mat <- cbind(M1=rnorm(100, 2, sd = 0.5), M2 = rnorm(100, 2, sd = 1)) RMSE(mat, parameter = 2) #> M1 M2 #> 0.4933475 1.0596005 RMSE(mat, parameter = c(2, 3)) #> M1 M2 #> 0.4933475 1.5255310 # different parameter associated with each column mat <- cbind(M1=rnorm(1000, 2, sd = 0.25), M2 = rnorm(1000, 3, sd = .25)) RMSE(mat, parameter = c(2,3)) #> M1 M2 #> 0.2513247 0.2531064 # same, but with data.frame df <- data.frame(M1=rnorm(100, 2, sd = 0.5), M2 = rnorm(100, 2, sd = 1)) RMSE(df, parameter = c(2,2)) #> M1 M2 #> 0.4434231 1.0532604 # parameters of the same size parameters <- 1:10 estimates <- parameters + rnorm(10) RMSE(estimates, parameters) #> [1] 0.5072955"},{"path":"http://philchalmers.github.io/SimDesign/reference/RSE.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute the relative standard error ratio — RSE","title":"Compute the relative standard error ratio — RSE","text":"Computes relative standard error ratio given set estimated standard errors (SE) deviation across R simulation replications (SD). ratio formed finding expectation SE terms, compares expectation general variability respective parameter estimates across R replications (ratio equal 1). used roughly evaluate whether SEs advertised given estimation method matches sampling variability respective estimates across samples.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RSE.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute the relative standard error ratio — RSE","text":"","code":"RSE(SE, ests, unname = FALSE)"},{"path":"http://philchalmers.github.io/SimDesign/reference/RSE.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute the relative standard error ratio — RSE","text":"SE numeric matrix SE estimates across replications (extracted results object Summarise step). Alternatively, can vector containing mean SE estimates across R simulation replications ests numeric matrix object containing parameter estimates investigation found within Summarise function. input used compute standard deviation/variance estimates column evaluate well expected SE matches standard deviation unname logical; apply unname results remove variable names?","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RSE.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute the relative standard error ratio — RSE","text":"returns vector variance ratios, (RSV = SE^2/SD^2)","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RSE.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Compute the relative standard error ratio — RSE","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RSE.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compute the relative standard error ratio — RSE","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RSE.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Compute the relative standard error ratio — RSE","text":"","code":"R <- 10000 par_ests <- cbind(rnorm(R), rnorm(R, sd=1/10), rnorm(R, sd=1/15)) colnames(par_ests) <- paste0(\"par\", 1:3) (SDs <- colSDs(par_ests)) #> par1 par2 par3 #> 1.00319636 0.09950705 0.06681960 SEs <- cbind(1 + rnorm(R, sd=.01), 1/10 + + rnorm(R, sd=.01), 1/15 + rnorm(R, sd=.01)) (E_SEs <- colMeans(SEs)) #> [1] 1.00018127 0.10001882 0.06673368 RSE(SEs, par_ests) #> par1 par2 par3 #> 0.9969945 1.0051431 0.9987141 # equivalent to the form colMeans(SEs) / SDs #> par1 par2 par3 #> 0.9969945 1.0051431 0.9987141"},{"path":"http://philchalmers.github.io/SimDesign/reference/RobbinsMonro.html","id":null,"dir":"Reference","previous_headings":"","what":"Robbins-Monro (1951) stochastic root-finding algorithm — RobbinsMonro","title":"Robbins-Monro (1951) stochastic root-finding algorithm — RobbinsMonro","text":"Function performs stochastic root solving provided f(x) using Robbins-Monro (1951) algorithm. Differs deterministic cousins uniroot f may contain stochastic error components, root obtained running average method provided noise filter (see also PBA). Assumes E[f(x)] non-decreasing x.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RobbinsMonro.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Robbins-Monro (1951) stochastic root-finding algorithm — RobbinsMonro","text":"","code":"RobbinsMonro( f, p, ..., Polyak_Juditsky = FALSE, maxiter = 500L, miniter = 100L, k = 3L, tol = 1e-05, verbose = TRUE, fn.a = function(iter, a = 1, b = 1/2, c = 0, ...) a/(iter + c)^b ) # S3 method for class 'RM' print(x, ...) # S3 method for class 'RM' plot(x, par = 1, main = NULL, Polyak_Juditsky = FALSE, ...)"},{"path":"http://philchalmers.github.io/SimDesign/reference/RobbinsMonro.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Robbins-Monro (1951) stochastic root-finding algorithm — RobbinsMonro","text":"f noisy function root sought p vector starting values passed f(p, ...) ... additional named arguments passed f Polyak_Juditsky logical; apply Polyak Juditsky (1992) running-average method? Returns final running average estimate using Robbins-Monro updates (also applies plot). Note used step-sizes sufficiently large Robbins-Monro ability stochastically explore around root (just approach one side, occurs using small steps) maxiter maximum number iterations (default 500) miniter minimum number iterations (default 100) k number consecutive tol criteria required terminating tol tolerance criteria convergence changes updated p elements. Must achieved k (default 3) successive occasions verbose logical; iterations estimate printed console? fn.function create coefficient Robbins-Monro noise filter. Requires first argument current iteration (iter), provide one arguments, (optionally) .... Sequence function form recommended Spall (2000). Note different function provided must satisfy property \\(\\sum^\\infty_{=1} a_i = \\infty\\) \\(\\sum^\\infty_{=1} a_i^2 < \\infty\\) x object class RM par parameter original vector p include plot main plot title","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/RobbinsMonro.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Robbins-Monro (1951) stochastic root-finding algorithm — RobbinsMonro","text":"Polyak, B. T. Juditsky, . B. (1992). Acceleration Stochastic Approximation Averaging. SIAM Journal Control Optimization, 30(4):838. Robbins, H. Monro, S. (1951). stochastic approximation method. Ann.Math.Statistics, 22:400-407. Spall, J.C. (2000). Adaptive stochastic approximation simultaneous perturbation method. IEEE Trans. Autom. Control 45, 1839-1853.","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/RobbinsMonro.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Robbins-Monro (1951) stochastic root-finding algorithm — RobbinsMonro","text":"","code":"# find x that solves f(x) - b = 0 for the following f.root <- function(x, b = .6) 1 / (1 + exp(-x)) - b f.root(.3) #> [1] -0.02555748 xs <- seq(-3,3, length.out=1000) plot(xs, f.root(xs), type = 'l', ylab = \"f(x)\", xlab='x') abline(h=0, col='red') retuni <- uniroot(f.root, c(0,1)) retuni #> $root #> [1] 0.4054644 #> #> $f.root #> [1] -1.772764e-07 #> #> $iter #> [1] 4 #> #> $init.it #> [1] NA #> #> $estim.prec #> [1] 6.103516e-05 #> abline(v=retuni$root, col='blue', lty=2) # Robbins-Monro without noisy root, start with p=.9 retrm <- RobbinsMonro(f.root, .9) #> Iter: 1; Max change in p = 0.111 Iter: 2; Max change in p = 0.062 Iter: 3; Max change in p = 0.043 Iter: 4; Max change in p = 0.032 Iter: 5; Max change in p = 0.026 Iter: 6; Max change in p = 0.021 Iter: 7; Max change in p = 0.018 Iter: 8; Max change in p = 0.015 Iter: 9; Max change in p = 0.013 Iter: 10; Max change in p = 0.011 Iter: 11; Max change in p = 0.010 Iter: 12; Max change in p = 0.009 Iter: 13; Max change in p = 0.008 Iter: 14; Max change in p = 0.007 Iter: 15; Max change in p = 0.007 Iter: 16; Max change in p = 0.006 Iter: 17; Max change in p = 0.005 Iter: 18; Max change in p = 0.005 Iter: 19; Max change in p = 0.005 Iter: 20; Max change in p = 0.004 Iter: 21; Max change in p = 0.004 Iter: 22; Max change in p = 0.004 Iter: 23; Max change in p = 0.003 Iter: 24; Max change in p = 0.003 Iter: 25; Max change in p = 0.003 Iter: 26; Max change in p = 0.003 Iter: 27; Max change in p = 0.003 Iter: 28; Max change in p = 0.002 Iter: 29; Max change in p = 0.002 Iter: 30; Max change in p = 0.002 Iter: 31; Max change in p = 0.002 Iter: 32; Max change in p = 0.002 Iter: 33; Max change in p = 0.002 Iter: 34; Max change in p = 0.002 Iter: 35; Max change in p = 0.002 Iter: 36; Max change in p = 0.002 Iter: 37; Max change in p = 0.001 Iter: 38; Max change in p = 0.001 Iter: 39; Max change in p = 0.001 Iter: 40; Max change in p = 0.001 Iter: 41; Max change in p = 0.001 Iter: 42; Max change in p = 0.001 Iter: 43; Max change in p = 0.001 Iter: 44; Max change in p = 0.001 Iter: 45; Max change in p = 0.001 Iter: 46; Max change in p = 0.001 Iter: 47; Max change in p = 0.001 Iter: 48; Max change in p = 0.001 Iter: 49; Max change in p = 0.001 Iter: 50; Max change in p = 0.001 Iter: 51; Max change in p = 0.001 Iter: 52; Max change in p = 0.001 Iter: 53; Max change in p = 0.001 Iter: 54; Max change in p = 0.001 Iter: 55; Max change in p = 0.001 Iter: 56; Max change in p = 0.001 Iter: 57; Max change in p = 0.001 Iter: 58; Max change in p = 0.001 Iter: 59; Max change in p = 0.001 Iter: 60; Max change in p = 0.000 Iter: 61; Max change in p = 0.000 Iter: 62; Max change in p = 0.000 Iter: 63; Max change in p = 0.000 Iter: 64; Max change in p = 0.000 Iter: 65; Max change in p = 0.000 Iter: 66; Max change in p = 0.000 Iter: 67; Max change in p = 0.000 Iter: 68; Max change in p = 0.000 Iter: 69; Max change in p = 0.000 Iter: 70; Max change in p = 0.000 Iter: 71; Max change in p = 0.000 Iter: 72; Max change in p = 0.000 Iter: 73; Max change in p = 0.000 Iter: 74; Max change in p = 0.000 Iter: 75; Max change in p = 0.000 Iter: 76; Max change in p = 0.000 Iter: 77; Max change in p = 0.000 Iter: 78; Max change in p = 0.000 Iter: 79; Max change in p = 0.000 Iter: 80; Max change in p = 0.000 Iter: 81; Max change in p = 0.000 Iter: 82; Max change in p = 0.000 Iter: 83; Max change in p = 0.000 Iter: 84; Max change in p = 0.000 Iter: 85; Max change in p = 0.000 Iter: 86; Max change in p = 0.000 Iter: 87; Max change in p = 0.000 Iter: 88; Max change in p = 0.000 Iter: 89; Max change in p = 0.000 Iter: 90; Max change in p = 0.000 Iter: 91; Max change in p = 0.000 Iter: 92; Max change in p = 0.000 Iter: 93; Max change in p = 0.000 Iter: 94; Max change in p = 0.000 Iter: 95; Max change in p = 0.000 Iter: 96; Max change in p = 0.000 Iter: 97; Max change in p = 0.000 Iter: 98; Max change in p = 0.000 Iter: 99; Max change in p = 0.000 Iter: 100; Max change in p = 0.000 Iter: 101; Max change in p = 0.000 Iter: 102; Max change in p = 0.000 Iter: 103; Max change in p = 0.000 Iter: 104; Max change in p = 0.000 Iter: 105; Max change in p = 0.000 Iter: 106; Max change in p = 0.000 Iter: 107; Max change in p = 0.000 Iter: 108; Max change in p = 0.000 Iter: 109; Max change in p = 0.000 Iter: 110; Max change in p = 0.000 Iter: 111; Max change in p = 0.000 Iter: 112; Max change in p = 0.000 Iter: 113; Max change in p = 0.000 Iter: 114; Max change in p = 0.000 Iter: 115; Max change in p = 0.000 Iter: 116; Max change in p = 0.000 Iter: 117; Max change in p = 0.000 Iter: 118; Max change in p = 0.000 Iter: 119; Max change in p = 0.000 Iter: 120; Max change in p = 0.000 Iter: 121; Max change in p = 0.000 Iter: 122; Max change in p = 0.000 Iter: 123; Max change in p = 0.000 Iter: 124; Max change in p = 0.000 Iter: 125; Max change in p = 0.000 Iter: 126; Max change in p = 0.000 Iter: 127; Max change in p = 0.000 Iter: 128; Max change in p = 0.000 Iter: 129; Max change in p = 0.000 Iter: 130; Max change in p = 0.000 Iter: 131; Max change in p = 0.000 Iter: 132; Max change in p = 0.000 Iter: 133; Max change in p = 0.000 Iter: 134; Max change in p = 0.000 Iter: 135; Max change in p = 0.000 Iter: 136; Max change in p = 0.000 Iter: 137; Max change in p = 0.000 Iter: 138; Max change in p = 0.000 Iter: 139; Max change in p = 0.000 Iter: 140; Max change in p = 0.000 Iter: 141; Max change in p = 0.000 Iter: 142; Max change in p = 0.000 Iter: 143; Max change in p = 0.000 Iter: 144; Max change in p = 0.000 Iter: 145; Max change in p = 0.000 Iter: 146; Max change in p = 0.000 Iter: 147; Max change in p = 0.000 Iter: 148; Max change in p = 0.000 Iter: 149; Max change in p = 0.000 Iter: 150; Max change in p = 0.000 Iter: 151; Max change in p = 0.000 Iter: 152; Max change in p = 0.000 Iter: 153; Max change in p = 0.000 Iter: 154; Max change in p = 0.000 Iter: 155; Max change in p = 0.000 Iter: 156; Max change in p = 0.000 Iter: 157; Max change in p = 0.000 Iter: 158; Max change in p = 0.000 Iter: 159; Max change in p = 0.000 Iter: 160; Max change in p = 0.000 Iter: 161; Max change in p = 0.000 Iter: 162; Max change in p = 0.000 Iter: 163; Max change in p = 0.000 Iter: 164; Max change in p = 0.000 Iter: 165; Max change in p = 0.000 Iter: 166; Max change in p = 0.000 Iter: 167; Max change in p = 0.000 Iter: 168; Max change in p = 0.000 Iter: 169; Max change in p = 0.000 Iter: 170; Max change in p = 0.000 Iter: 171; Max change in p = 0.000 Iter: 172; Max change in p = 0.000 Iter: 173; Max change in p = 0.000 Iter: 174; Max change in p = 0.000 Iter: 175; Max change in p = 0.000 Iter: 176; Max change in p = 0.000 Iter: 177; Max change in p = 0.000 Iter: 178; Max change in p = 0.000 Iter: 179; Max change in p = 0.000 Iter: 180; Max change in p = 0.000 Iter: 181; Max change in p = 0.000 Iter: 182; Max change in p = 0.000 Iter: 183; Max change in p = 0.000 Iter: 184; Max change in p = 0.000 Iter: 185; Max change in p = 0.000 Iter: 186; Max change in p = 0.000 Iter: 187; Max change in p = 0.000 Iter: 188; Max change in p = 0.000 Iter: 189; Max change in p = 0.000 Iter: 190; Max change in p = 0.000 Iter: 191; Max change in p = 0.000 Iter: 192; Max change in p = 0.000 Iter: 193; Max change in p = 0.000 Iter: 194; Max change in p = 0.000 Iter: 195; Max change in p = 0.000 Iter: 196; Max change in p = 0.000 Iter: 197; Max change in p = 0.000 Iter: 198; Max change in p = 0.000 Iter: 199; Max change in p = 0.000 Iter: 200; Max change in p = 0.000 Iter: 201; Max change in p = 0.000 Iter: 202; Max change in p = 0.000 Iter: 203; Max change in p = 0.000 Iter: 204; Max change in p = 0.000 Iter: 205; Max change in p = 0.000 Iter: 206; Max change in p = 0.000 Iter: 207; Max change in p = 0.000 Iter: 208; Max change in p = 0.000 Iter: 209; Max change in p = 0.000 Iter: 210; Max change in p = 0.000 Iter: 211; Max change in p = 0.000 Iter: 212; Max change in p = 0.000 retrm #> [1] 0.4060382 plot(retrm) # Same problem, however root function is now noisy. Hence, need to solve # fhat(x) - b + e = 0, where E(e) = 0 f.root_noisy <- function(x) 1 / (1 + exp(-x)) - .6 + rnorm(1, sd=.02) sapply(rep(.3, 10), f.root_noisy) #> [1] -0.054412751 -0.036751696 -0.025966561 -0.014003517 0.004254432 #> [6] -0.035781165 -0.045079269 -0.059757656 -0.035333325 -0.043135155 # uniroot \"converges\" unreliably set.seed(123) uniroot(f.root_noisy, c(0,1))$root #> [1] 0.3748233 uniroot(f.root_noisy, c(0,1))$root #> [1] 0.3785736 uniroot(f.root_noisy, c(0,1))$root #> [1] 0.4954932 # Robbins-Monro provides better convergence retrm.noise <- RobbinsMonro(f.root_noisy, .9) #> Iter: 1; Max change in p = 0.129 Iter: 2; Max change in p = 0.072 Iter: 3; Max change in p = 0.049 Iter: 4; Max change in p = 0.035 Iter: 5; Max change in p = 0.027 Iter: 6; Max change in p = 0.017 Iter: 7; Max change in p = 0.012 Iter: 8; Max change in p = 0.010 Iter: 9; Max change in p = 0.007 Iter: 10; Max change in p = 0.009 Iter: 11; Max change in p = 0.001 Iter: 12; Max change in p = 0.021 Iter: 13; Max change in p = 0.014 Iter: 14; Max change in p = 0.000 Iter: 15; Max change in p = 0.004 Iter: 16; Max change in p = 0.003 Iter: 17; Max change in p = 0.009 Iter: 18; Max change in p = 0.004 Iter: 19; Max change in p = 0.005 Iter: 20; Max change in p = 0.003 Iter: 21; Max change in p = 0.003 Iter: 22; Max change in p = 0.009 Iter: 23; Max change in p = 0.002 Iter: 24; Max change in p = 0.009 Iter: 25; Max change in p = 0.004 Iter: 26; Max change in p = 0.004 Iter: 27; Max change in p = 0.002 Iter: 28; Max change in p = 0.003 Iter: 29; Max change in p = 0.003 Iter: 30; Max change in p = 0.000 Iter: 31; Max change in p = 0.000 Iter: 32; Max change in p = 0.002 Iter: 33; Max change in p = 0.002 Iter: 34; Max change in p = 0.003 Iter: 35; Max change in p = 0.003 Iter: 36; Max change in p = 0.001 Iter: 37; Max change in p = 0.004 Iter: 38; Max change in p = 0.008 Iter: 39; Max change in p = 0.001 Iter: 40; Max change in p = 0.007 Iter: 41; Max change in p = 0.004 Iter: 42; Max change in p = 0.001 Iter: 43; Max change in p = 0.001 Iter: 44; Max change in p = 0.004 Iter: 45; Max change in p = 0.000 Iter: 46; Max change in p = 0.003 Iter: 47; Max change in p = 0.001 Iter: 48; Max change in p = 0.000 Iter: 49; Max change in p = 0.001 Iter: 50; Max change in p = 0.002 Iter: 51; Max change in p = 0.000 Iter: 52; Max change in p = 0.002 Iter: 53; Max change in p = 0.000 Iter: 54; Max change in p = 0.001 Iter: 55; Max change in p = 0.003 Iter: 56; Max change in p = 0.002 Iter: 57; Max change in p = 0.001 Iter: 58; Max change in p = 0.003 Iter: 59; Max change in p = 0.003 Iter: 60; Max change in p = 0.002 Iter: 61; Max change in p = 0.001 Iter: 62; Max change in p = 0.002 Iter: 63; Max change in p = 0.004 Iter: 64; Max change in p = 0.001 Iter: 65; Max change in p = 0.005 Iter: 66; Max change in p = 0.004 Iter: 67; Max change in p = 0.001 Iter: 68; Max change in p = 0.003 Iter: 69; Max change in p = 0.002 Iter: 70; Max change in p = 0.001 Iter: 71; Max change in p = 0.001 Iter: 72; Max change in p = 0.001 Iter: 73; Max change in p = 0.002 Iter: 74; Max change in p = 0.000 Iter: 75; Max change in p = 0.002 Iter: 76; Max change in p = 0.004 Iter: 77; Max change in p = 0.001 Iter: 78; Max change in p = 0.002 Iter: 79; Max change in p = 0.001 Iter: 80; Max change in p = 0.002 Iter: 81; Max change in p = 0.003 Iter: 82; Max change in p = 0.000 Iter: 83; Max change in p = 0.001 Iter: 84; Max change in p = 0.001 Iter: 85; Max change in p = 0.000 Iter: 86; Max change in p = 0.001 Iter: 87; Max change in p = 0.002 Iter: 88; Max change in p = 0.002 Iter: 89; Max change in p = 0.001 Iter: 90; Max change in p = 0.002 Iter: 91; Max change in p = 0.001 Iter: 92; Max change in p = 0.000 Iter: 93; Max change in p = 0.004 Iter: 94; Max change in p = 0.001 Iter: 95; Max change in p = 0.001 Iter: 96; Max change in p = 0.000 Iter: 97; Max change in p = 0.002 Iter: 98; Max change in p = 0.000 Iter: 99; Max change in p = 0.003 Iter: 100; Max change in p = 0.001 Iter: 101; Max change in p = 0.000 Iter: 102; Max change in p = 0.001 Iter: 103; Max change in p = 0.004 Iter: 104; Max change in p = 0.002 Iter: 105; Max change in p = 0.003 Iter: 106; Max change in p = 0.002 Iter: 107; Max change in p = 0.004 Iter: 108; Max change in p = 0.003 Iter: 109; Max change in p = 0.002 Iter: 110; Max change in p = 0.000 Iter: 111; Max change in p = 0.003 Iter: 112; Max change in p = 0.003 Iter: 113; Max change in p = 0.003 Iter: 114; Max change in p = 0.001 Iter: 115; Max change in p = 0.002 Iter: 116; Max change in p = 0.002 Iter: 117; Max change in p = 0.004 Iter: 118; Max change in p = 0.002 Iter: 119; Max change in p = 0.002 Iter: 120; Max change in p = 0.002 Iter: 121; Max change in p = 0.001 Iter: 122; Max change in p = 0.002 Iter: 123; Max change in p = 0.000 Iter: 124; Max change in p = 0.000 Iter: 125; Max change in p = 0.001 Iter: 126; Max change in p = 0.000 Iter: 127; Max change in p = 0.002 Iter: 128; Max change in p = 0.000 Iter: 129; Max change in p = 0.002 Iter: 130; Max change in p = 0.002 Iter: 131; Max change in p = 0.002 Iter: 132; Max change in p = 0.006 Iter: 133; Max change in p = 0.001 Iter: 134; Max change in p = 0.001 Iter: 135; Max change in p = 0.001 Iter: 136; Max change in p = 0.001 Iter: 137; Max change in p = 0.001 Iter: 138; Max change in p = 0.001 Iter: 139; Max change in p = 0.000 Iter: 140; Max change in p = 0.000 Iter: 141; Max change in p = 0.000 Iter: 142; Max change in p = 0.004 Iter: 143; Max change in p = 0.001 Iter: 144; Max change in p = 0.002 Iter: 145; Max change in p = 0.000 Iter: 146; Max change in p = 0.001 Iter: 147; Max change in p = 0.001 Iter: 148; Max change in p = 0.001 Iter: 149; Max change in p = 0.002 Iter: 150; Max change in p = 0.002 Iter: 151; Max change in p = 0.001 Iter: 152; Max change in p = 0.001 Iter: 153; Max change in p = 0.000 Iter: 154; Max change in p = 0.000 Iter: 155; Max change in p = 0.002 Iter: 156; Max change in p = 0.000 Iter: 157; Max change in p = 0.001 Iter: 158; Max change in p = 0.001 Iter: 159; Max change in p = 0.000 Iter: 160; Max change in p = 0.000 Iter: 161; Max change in p = 0.000 Iter: 162; Max change in p = 0.001 Iter: 163; Max change in p = 0.002 Iter: 164; Max change in p = 0.003 Iter: 165; Max change in p = 0.001 Iter: 166; Max change in p = 0.002 Iter: 167; Max change in p = 0.001 Iter: 168; Max change in p = 0.002 Iter: 169; Max change in p = 0.003 Iter: 170; Max change in p = 0.002 Iter: 171; Max change in p = 0.000 Iter: 172; Max change in p = 0.001 Iter: 173; Max change in p = 0.001 Iter: 174; Max change in p = 0.001 Iter: 175; Max change in p = 0.001 Iter: 176; Max change in p = 0.001 Iter: 177; Max change in p = 0.003 Iter: 178; Max change in p = 0.000 Iter: 179; Max change in p = 0.000 Iter: 180; Max change in p = 0.000 Iter: 181; Max change in p = 0.002 Iter: 182; Max change in p = 0.001 Iter: 183; Max change in p = 0.001 Iter: 184; Max change in p = 0.002 Iter: 185; Max change in p = 0.001 Iter: 186; Max change in p = 0.001 Iter: 187; Max change in p = 0.002 Iter: 188; Max change in p = 0.002 Iter: 189; Max change in p = 0.001 Iter: 190; Max change in p = 0.001 Iter: 191; Max change in p = 0.002 Iter: 192; Max change in p = 0.001 Iter: 193; Max change in p = 0.001 Iter: 194; Max change in p = 0.000 Iter: 195; Max change in p = 0.001 Iter: 196; Max change in p = 0.001 Iter: 197; Max change in p = 0.001 Iter: 198; Max change in p = 0.001 Iter: 199; Max change in p = 0.003 Iter: 200; Max change in p = 0.000 Iter: 201; Max change in p = 0.000 Iter: 202; Max change in p = 0.001 Iter: 203; Max change in p = 0.001 Iter: 204; Max change in p = 0.002 Iter: 205; Max change in p = 0.000 Iter: 206; Max change in p = 0.001 Iter: 207; Max change in p = 0.000 Iter: 208; Max change in p = 0.001 Iter: 209; Max change in p = 0.001 Iter: 210; Max change in p = 0.001 Iter: 211; Max change in p = 0.002 Iter: 212; Max change in p = 0.002 Iter: 213; Max change in p = 0.000 Iter: 214; Max change in p = 0.003 Iter: 215; Max change in p = 0.000 Iter: 216; Max change in p = 0.002 Iter: 217; Max change in p = 0.001 Iter: 218; Max change in p = 0.001 Iter: 219; Max change in p = 0.000 Iter: 220; Max change in p = 0.001 Iter: 221; Max change in p = 0.000 Iter: 222; Max change in p = 0.000 Iter: 223; Max change in p = 0.002 Iter: 224; Max change in p = 0.000 Iter: 225; Max change in p = 0.001 Iter: 226; Max change in p = 0.001 Iter: 227; Max change in p = 0.000 Iter: 228; Max change in p = 0.002 Iter: 229; Max change in p = 0.001 Iter: 230; Max change in p = 0.001 Iter: 231; Max change in p = 0.000 Iter: 232; Max change in p = 0.002 Iter: 233; Max change in p = 0.003 Iter: 234; Max change in p = 0.002 Iter: 235; Max change in p = 0.000 Iter: 236; Max change in p = 0.002 Iter: 237; Max change in p = 0.001 Iter: 238; Max change in p = 0.000 Iter: 239; Max change in p = 0.001 Iter: 240; Max change in p = 0.001 Iter: 241; Max change in p = 0.001 Iter: 242; Max change in p = 0.002 Iter: 243; Max change in p = 0.001 Iter: 244; Max change in p = 0.001 Iter: 245; Max change in p = 0.000 Iter: 246; Max change in p = 0.000 Iter: 247; Max change in p = 0.001 Iter: 248; Max change in p = 0.000 Iter: 249; Max change in p = 0.002 Iter: 250; Max change in p = 0.001 Iter: 251; Max change in p = 0.000 Iter: 252; Max change in p = 0.000 Iter: 253; Max change in p = 0.000 Iter: 254; Max change in p = 0.000 Iter: 255; Max change in p = 0.000 Iter: 256; Max change in p = 0.002 Iter: 257; Max change in p = 0.000 Iter: 258; Max change in p = 0.000 Iter: 259; Max change in p = 0.001 Iter: 260; Max change in p = 0.001 Iter: 261; Max change in p = 0.001 Iter: 262; Max change in p = 0.001 Iter: 263; Max change in p = 0.002 Iter: 264; Max change in p = 0.000 Iter: 265; Max change in p = 0.002 Iter: 266; Max change in p = 0.002 Iter: 267; Max change in p = 0.000 Iter: 268; Max change in p = 0.001 Iter: 269; Max change in p = 0.001 Iter: 270; Max change in p = 0.001 Iter: 271; Max change in p = 0.001 Iter: 272; Max change in p = 0.001 Iter: 273; Max change in p = 0.001 Iter: 274; Max change in p = 0.000 Iter: 275; Max change in p = 0.003 Iter: 276; Max change in p = 0.000 Iter: 277; Max change in p = 0.001 Iter: 278; Max change in p = 0.002 Iter: 279; Max change in p = 0.001 Iter: 280; Max change in p = 0.001 Iter: 281; Max change in p = 0.002 Iter: 282; Max change in p = 0.001 Iter: 283; Max change in p = 0.001 Iter: 284; Max change in p = 0.001 Iter: 285; Max change in p = 0.000 Iter: 286; Max change in p = 0.002 Iter: 287; Max change in p = 0.002 Iter: 288; Max change in p = 0.001 Iter: 289; Max change in p = 0.000 Iter: 290; Max change in p = 0.001 Iter: 291; Max change in p = 0.002 Iter: 292; Max change in p = 0.001 Iter: 293; Max change in p = 0.001 Iter: 294; Max change in p = 0.001 Iter: 295; Max change in p = 0.000 Iter: 296; Max change in p = 0.001 Iter: 297; Max change in p = 0.001 Iter: 298; Max change in p = 0.000 Iter: 299; Max change in p = 0.001 Iter: 300; Max change in p = 0.002 Iter: 301; Max change in p = 0.001 Iter: 302; Max change in p = 0.002 Iter: 303; Max change in p = 0.000 Iter: 304; Max change in p = 0.002 Iter: 305; Max change in p = 0.002 Iter: 306; Max change in p = 0.000 Iter: 307; Max change in p = 0.001 Iter: 308; Max change in p = 0.000 Iter: 309; Max change in p = 0.001 Iter: 310; Max change in p = 0.001 Iter: 311; Max change in p = 0.002 Iter: 312; Max change in p = 0.000 Iter: 313; Max change in p = 0.000 Iter: 314; Max change in p = 0.002 Iter: 315; Max change in p = 0.000 Iter: 316; Max change in p = 0.001 Iter: 317; Max change in p = 0.001 Iter: 318; Max change in p = 0.000 Iter: 319; Max change in p = 0.001 Iter: 320; Max change in p = 0.002 Iter: 321; Max change in p = 0.001 Iter: 322; Max change in p = 0.000 Iter: 323; Max change in p = 0.001 Iter: 324; Max change in p = 0.000 Iter: 325; Max change in p = 0.000 Iter: 326; Max change in p = 0.000 Iter: 327; Max change in p = 0.003 Iter: 328; Max change in p = 0.003 Iter: 329; Max change in p = 0.000 Iter: 330; Max change in p = 0.001 Iter: 331; Max change in p = 0.000 Iter: 332; Max change in p = 0.001 Iter: 333; Max change in p = 0.001 Iter: 334; Max change in p = 0.000 Iter: 335; Max change in p = 0.000 Iter: 336; Max change in p = 0.001 Iter: 337; Max change in p = 0.001 Iter: 338; Max change in p = 0.001 Iter: 339; Max change in p = 0.003 Iter: 340; Max change in p = 0.002 Iter: 341; Max change in p = 0.001 Iter: 342; Max change in p = 0.001 Iter: 343; Max change in p = 0.000 Iter: 344; Max change in p = 0.001 Iter: 345; Max change in p = 0.001 Iter: 346; Max change in p = 0.000 Iter: 347; Max change in p = 0.000 Iter: 348; Max change in p = 0.002 Iter: 349; Max change in p = 0.000 Iter: 350; Max change in p = 0.000 Iter: 351; Max change in p = 0.000 Iter: 352; Max change in p = 0.001 Iter: 353; Max change in p = 0.000 Iter: 354; Max change in p = 0.001 Iter: 355; Max change in p = 0.000 Iter: 356; Max change in p = 0.001 Iter: 357; Max change in p = 0.000 Iter: 358; Max change in p = 0.000 Iter: 359; Max change in p = 0.001 Iter: 360; Max change in p = 0.002 Iter: 361; Max change in p = 0.001 Iter: 362; Max change in p = 0.001 Iter: 363; Max change in p = 0.001 Iter: 364; Max change in p = 0.002 Iter: 365; Max change in p = 0.001 Iter: 366; Max change in p = 0.001 Iter: 367; Max change in p = 0.001 Iter: 368; Max change in p = 0.000 Iter: 369; Max change in p = 0.000 Iter: 370; Max change in p = 0.001 Iter: 371; Max change in p = 0.001 Iter: 372; Max change in p = 0.000 Iter: 373; Max change in p = 0.001 Iter: 374; Max change in p = 0.002 Iter: 375; Max change in p = 0.000 Iter: 376; Max change in p = 0.001 Iter: 377; Max change in p = 0.001 Iter: 378; Max change in p = 0.000 Iter: 379; Max change in p = 0.001 Iter: 380; Max change in p = 0.000 Iter: 381; Max change in p = 0.001 Iter: 382; Max change in p = 0.000 Iter: 383; Max change in p = 0.000 Iter: 384; Max change in p = 0.003 Iter: 385; Max change in p = 0.000 Iter: 386; Max change in p = 0.000 Iter: 387; Max change in p = 0.001 Iter: 388; Max change in p = 0.001 Iter: 389; Max change in p = 0.002 Iter: 390; Max change in p = 0.000 Iter: 391; Max change in p = 0.000 Iter: 392; Max change in p = 0.001 Iter: 393; Max change in p = 0.001 Iter: 394; Max change in p = 0.000 Iter: 395; Max change in p = 0.002 Iter: 396; Max change in p = 0.000 Iter: 397; Max change in p = 0.002 Iter: 398; Max change in p = 0.001 Iter: 399; Max change in p = 0.000 Iter: 400; Max change in p = 0.000 Iter: 401; Max change in p = 0.000 Iter: 402; Max change in p = 0.001 Iter: 403; Max change in p = 0.000 Iter: 404; Max change in p = 0.001 Iter: 405; Max change in p = 0.001 Iter: 406; Max change in p = 0.001 Iter: 407; Max change in p = 0.002 Iter: 408; Max change in p = 0.001 Iter: 409; Max change in p = 0.001 Iter: 410; Max change in p = 0.000 Iter: 411; Max change in p = 0.001 Iter: 412; Max change in p = 0.000 Iter: 413; Max change in p = 0.001 Iter: 414; Max change in p = 0.000 Iter: 415; Max change in p = 0.000 Iter: 416; Max change in p = 0.001 Iter: 417; Max change in p = 0.002 Iter: 418; Max change in p = 0.002 Iter: 419; Max change in p = 0.001 Iter: 420; Max change in p = 0.001 Iter: 421; Max change in p = 0.000 Iter: 422; Max change in p = 0.001 Iter: 423; Max change in p = 0.001 Iter: 424; Max change in p = 0.003 Iter: 425; Max change in p = 0.001 Iter: 426; Max change in p = 0.000 Iter: 427; Max change in p = 0.000 Iter: 428; Max change in p = 0.000 Iter: 429; Max change in p = 0.001 Iter: 430; Max change in p = 0.000 Iter: 431; Max change in p = 0.001 Iter: 432; Max change in p = 0.000 Iter: 433; Max change in p = 0.001 Iter: 434; Max change in p = 0.001 Iter: 435; Max change in p = 0.001 Iter: 436; Max change in p = 0.002 Iter: 437; Max change in p = 0.000 Iter: 438; Max change in p = 0.001 Iter: 439; Max change in p = 0.002 Iter: 440; Max change in p = 0.000 Iter: 441; Max change in p = 0.001 Iter: 442; Max change in p = 0.000 Iter: 443; Max change in p = 0.000 Iter: 444; Max change in p = 0.000 Iter: 445; Max change in p = 0.002 Iter: 446; Max change in p = 0.000 Iter: 447; Max change in p = 0.000 Iter: 448; Max change in p = 0.000 Iter: 449; Max change in p = 0.000 Iter: 450; Max change in p = 0.000 Iter: 451; Max change in p = 0.001 Iter: 452; Max change in p = 0.000 Iter: 453; Max change in p = 0.001 Iter: 454; Max change in p = 0.001 Iter: 455; Max change in p = 0.001 Iter: 456; Max change in p = 0.001 Iter: 457; Max change in p = 0.001 Iter: 458; Max change in p = 0.000 Iter: 459; Max change in p = 0.000 Iter: 460; Max change in p = 0.001 Iter: 461; Max change in p = 0.001 Iter: 462; Max change in p = 0.001 Iter: 463; Max change in p = 0.001 Iter: 464; Max change in p = 0.001 Iter: 465; Max change in p = 0.000 Iter: 466; Max change in p = 0.000 Iter: 467; Max change in p = 0.000 Iter: 468; Max change in p = 0.000 Iter: 469; Max change in p = 0.001 Iter: 470; Max change in p = 0.001 Iter: 471; Max change in p = 0.001 Iter: 472; Max change in p = 0.001 Iter: 473; Max change in p = 0.001 Iter: 474; Max change in p = 0.000 Iter: 475; Max change in p = 0.001 Iter: 476; Max change in p = 0.002 Iter: 477; Max change in p = 0.000 Iter: 478; Max change in p = 0.000 Iter: 479; Max change in p = 0.000 Iter: 480; Max change in p = 0.000 Iter: 481; Max change in p = 0.001 Iter: 482; Max change in p = 0.000 Iter: 483; Max change in p = 0.001 Iter: 484; Max change in p = 0.001 Iter: 485; Max change in p = 0.000 Iter: 486; Max change in p = 0.000 Iter: 487; Max change in p = 0.001 Iter: 488; Max change in p = 0.000 Iter: 489; Max change in p = 0.001 Iter: 490; Max change in p = 0.000 Iter: 491; Max change in p = 0.001 Iter: 492; Max change in p = 0.000 Iter: 493; Max change in p = 0.001 Iter: 494; Max change in p = 0.000 Iter: 495; Max change in p = 0.001 Iter: 496; Max change in p = 0.000 Iter: 497; Max change in p = 0.000 Iter: 498; Max change in p = 0.001 Iter: 499; Max change in p = 0.000 Iter: 500; Max change in p = 0.001 retrm.noise #> [1] 0.401403 plot(retrm.noise) # different power (b) for fn.a() retrm.b2 <- RobbinsMonro(f.root_noisy, .9, b = .01) #> Iter: 1; Max change in p = 0.083 Iter: 2; Max change in p = 0.096 Iter: 3; Max change in p = 0.111 Iter: 4; Max change in p = 0.063 Iter: 5; Max change in p = 0.056 Iter: 6; Max change in p = 0.027 Iter: 7; Max change in p = 0.002 Iter: 8; Max change in p = 0.009 Iter: 9; Max change in p = 0.006 Iter: 10; Max change in p = 0.001 Iter: 11; Max change in p = 0.023 Iter: 12; Max change in p = 0.019 Iter: 13; Max change in p = 0.026 Iter: 14; Max change in p = 0.020 Iter: 15; Max change in p = 0.025 Iter: 16; Max change in p = 0.016 Iter: 17; Max change in p = 0.047 Iter: 18; Max change in p = 0.022 Iter: 19; Max change in p = 0.011 Iter: 20; Max change in p = 0.023 Iter: 21; Max change in p = 0.019 Iter: 22; Max change in p = 0.002 Iter: 23; Max change in p = 0.025 Iter: 24; Max change in p = 0.041 Iter: 25; Max change in p = 0.004 Iter: 26; Max change in p = 0.012 Iter: 27; Max change in p = 0.003 Iter: 28; Max change in p = 0.011 Iter: 29; Max change in p = 0.012 Iter: 30; Max change in p = 0.018 Iter: 31; Max change in p = 0.018 Iter: 32; Max change in p = 0.036 Iter: 33; Max change in p = 0.024 Iter: 34; Max change in p = 0.006 Iter: 35; Max change in p = 0.013 Iter: 36; Max change in p = 0.011 Iter: 37; Max change in p = 0.009 Iter: 38; Max change in p = 0.001 Iter: 39; Max change in p = 0.018 Iter: 40; Max change in p = 0.033 Iter: 41; Max change in p = 0.012 Iter: 42; Max change in p = 0.012 Iter: 43; Max change in p = 0.006 Iter: 44; Max change in p = 0.031 Iter: 45; Max change in p = 0.010 Iter: 46; Max change in p = 0.004 Iter: 47; Max change in p = 0.033 Iter: 48; Max change in p = 0.004 Iter: 49; Max change in p = 0.004 Iter: 50; Max change in p = 0.001 Iter: 51; Max change in p = 0.020 Iter: 52; Max change in p = 0.017 Iter: 53; Max change in p = 0.016 Iter: 54; Max change in p = 0.003 Iter: 55; Max change in p = 0.014 Iter: 56; Max change in p = 0.012 Iter: 57; Max change in p = 0.017 Iter: 58; Max change in p = 0.023 Iter: 59; Max change in p = 0.047 Iter: 60; Max change in p = 0.027 Iter: 61; Max change in p = 0.028 Iter: 62; Max change in p = 0.000 Iter: 63; Max change in p = 0.015 Iter: 64; Max change in p = 0.020 Iter: 65; Max change in p = 0.004 Iter: 66; Max change in p = 0.032 Iter: 67; Max change in p = 0.036 Iter: 68; Max change in p = 0.040 Iter: 69; Max change in p = 0.016 Iter: 70; Max change in p = 0.009 Iter: 71; Max change in p = 0.007 Iter: 72; Max change in p = 0.034 Iter: 73; Max change in p = 0.018 Iter: 74; Max change in p = 0.005 Iter: 75; Max change in p = 0.012 Iter: 76; Max change in p = 0.024 Iter: 77; Max change in p = 0.003 Iter: 78; Max change in p = 0.008 Iter: 79; Max change in p = 0.002 Iter: 80; Max change in p = 0.014 Iter: 81; Max change in p = 0.045 Iter: 82; Max change in p = 0.002 Iter: 83; Max change in p = 0.022 Iter: 84; Max change in p = 0.053 Iter: 85; Max change in p = 0.000 Iter: 86; Max change in p = 0.004 Iter: 87; Max change in p = 0.014 Iter: 88; Max change in p = 0.042 Iter: 89; Max change in p = 0.024 Iter: 90; Max change in p = 0.015 Iter: 91; Max change in p = 0.021 Iter: 92; Max change in p = 0.004 Iter: 93; Max change in p = 0.017 Iter: 94; Max change in p = 0.021 Iter: 95; Max change in p = 0.002 Iter: 96; Max change in p = 0.022 Iter: 97; Max change in p = 0.023 Iter: 98; Max change in p = 0.048 Iter: 99; Max change in p = 0.015 Iter: 100; Max change in p = 0.034 Iter: 101; Max change in p = 0.019 Iter: 102; Max change in p = 0.010 Iter: 103; Max change in p = 0.024 Iter: 104; Max change in p = 0.030 Iter: 105; Max change in p = 0.012 Iter: 106; Max change in p = 0.033 Iter: 107; Max change in p = 0.016 Iter: 108; Max change in p = 0.009 Iter: 109; Max change in p = 0.014 Iter: 110; Max change in p = 0.011 Iter: 111; Max change in p = 0.006 Iter: 112; Max change in p = 0.014 Iter: 113; Max change in p = 0.009 Iter: 114; Max change in p = 0.018 Iter: 115; Max change in p = 0.019 Iter: 116; Max change in p = 0.005 Iter: 117; Max change in p = 0.016 Iter: 118; Max change in p = 0.017 Iter: 119; Max change in p = 0.042 Iter: 120; Max change in p = 0.006 Iter: 121; Max change in p = 0.014 Iter: 122; Max change in p = 0.028 Iter: 123; Max change in p = 0.015 Iter: 124; Max change in p = 0.013 Iter: 125; Max change in p = 0.025 Iter: 126; Max change in p = 0.010 Iter: 127; Max change in p = 0.012 Iter: 128; Max change in p = 0.003 Iter: 129; Max change in p = 0.012 Iter: 130; Max change in p = 0.019 Iter: 131; Max change in p = 0.026 Iter: 132; Max change in p = 0.018 Iter: 133; Max change in p = 0.056 Iter: 134; Max change in p = 0.000 Iter: 135; Max change in p = 0.003 Iter: 136; Max change in p = 0.004 Iter: 137; Max change in p = 0.010 Iter: 138; Max change in p = 0.017 Iter: 139; Max change in p = 0.016 Iter: 140; Max change in p = 0.008 Iter: 141; Max change in p = 0.007 Iter: 142; Max change in p = 0.013 Iter: 143; Max change in p = 0.002 Iter: 144; Max change in p = 0.037 Iter: 145; Max change in p = 0.013 Iter: 146; Max change in p = 0.014 Iter: 147; Max change in p = 0.002 Iter: 148; Max change in p = 0.002 Iter: 149; Max change in p = 0.022 Iter: 150; Max change in p = 0.028 Iter: 151; Max change in p = 0.011 Iter: 152; Max change in p = 0.013 Iter: 153; Max change in p = 0.011 Iter: 154; Max change in p = 0.003 Iter: 155; Max change in p = 0.006 Iter: 156; Max change in p = 0.018 Iter: 157; Max change in p = 0.001 Iter: 158; Max change in p = 0.034 Iter: 159; Max change in p = 0.011 Iter: 160; Max change in p = 0.033 Iter: 161; Max change in p = 0.026 Iter: 162; Max change in p = 0.018 Iter: 163; Max change in p = 0.028 Iter: 164; Max change in p = 0.054 Iter: 165; Max change in p = 0.008 Iter: 166; Max change in p = 0.005 Iter: 167; Max change in p = 0.006 Iter: 168; Max change in p = 0.034 Iter: 169; Max change in p = 0.009 Iter: 170; Max change in p = 0.023 Iter: 171; Max change in p = 0.001 Iter: 172; Max change in p = 0.014 Iter: 173; Max change in p = 0.017 Iter: 174; Max change in p = 0.024 Iter: 175; Max change in p = 0.013 Iter: 176; Max change in p = 0.013 Iter: 177; Max change in p = 0.012 Iter: 178; Max change in p = 0.019 Iter: 179; Max change in p = 0.021 Iter: 180; Max change in p = 0.008 Iter: 181; Max change in p = 0.004 Iter: 182; Max change in p = 0.003 Iter: 183; Max change in p = 0.027 Iter: 184; Max change in p = 0.015 Iter: 185; Max change in p = 0.006 Iter: 186; Max change in p = 0.017 Iter: 187; Max change in p = 0.000 Iter: 188; Max change in p = 0.000 Iter: 189; Max change in p = 0.027 Iter: 190; Max change in p = 0.034 Iter: 191; Max change in p = 0.019 Iter: 192; Max change in p = 0.027 Iter: 193; Max change in p = 0.008 Iter: 194; Max change in p = 0.011 Iter: 195; Max change in p = 0.015 Iter: 196; Max change in p = 0.004 Iter: 197; Max change in p = 0.014 Iter: 198; Max change in p = 0.022 Iter: 199; Max change in p = 0.031 Iter: 200; Max change in p = 0.008 Iter: 201; Max change in p = 0.048 Iter: 202; Max change in p = 0.027 Iter: 203; Max change in p = 0.007 Iter: 204; Max change in p = 0.006 Iter: 205; Max change in p = 0.006 Iter: 206; Max change in p = 0.009 Iter: 207; Max change in p = 0.006 Iter: 208; Max change in p = 0.016 Iter: 209; Max change in p = 0.006 Iter: 210; Max change in p = 0.010 Iter: 211; Max change in p = 0.034 Iter: 212; Max change in p = 0.006 Iter: 213; Max change in p = 0.013 Iter: 214; Max change in p = 0.056 Iter: 215; Max change in p = 0.050 Iter: 216; Max change in p = 0.022 Iter: 217; Max change in p = 0.037 Iter: 218; Max change in p = 0.009 Iter: 219; Max change in p = 0.015 Iter: 220; Max change in p = 0.020 Iter: 221; Max change in p = 0.003 Iter: 222; Max change in p = 0.008 Iter: 223; Max change in p = 0.014 Iter: 224; Max change in p = 0.010 Iter: 225; Max change in p = 0.014 Iter: 226; Max change in p = 0.012 Iter: 227; Max change in p = 0.044 Iter: 228; Max change in p = 0.001 Iter: 229; Max change in p = 0.015 Iter: 230; Max change in p = 0.013 Iter: 231; Max change in p = 0.010 Iter: 232; Max change in p = 0.036 Iter: 233; Max change in p = 0.011 Iter: 234; Max change in p = 0.020 Iter: 235; Max change in p = 0.021 Iter: 236; Max change in p = 0.012 Iter: 237; Max change in p = 0.011 Iter: 238; Max change in p = 0.026 Iter: 239; Max change in p = 0.040 Iter: 240; Max change in p = 0.008 Iter: 241; Max change in p = 0.011 Iter: 242; Max change in p = 0.031 Iter: 243; Max change in p = 0.012 Iter: 244; Max change in p = 0.040 Iter: 245; Max change in p = 0.005 Iter: 246; Max change in p = 0.009 Iter: 247; Max change in p = 0.007 Iter: 248; Max change in p = 0.009 Iter: 249; Max change in p = 0.014 Iter: 250; Max change in p = 0.013 Iter: 251; Max change in p = 0.007 Iter: 252; Max change in p = 0.028 Iter: 253; Max change in p = 0.024 Iter: 254; Max change in p = 0.005 Iter: 255; Max change in p = 0.022 Iter: 256; Max change in p = 0.035 Iter: 257; Max change in p = 0.026 Iter: 258; Max change in p = 0.003 Iter: 259; Max change in p = 0.023 Iter: 260; Max change in p = 0.007 Iter: 261; Max change in p = 0.028 Iter: 262; Max change in p = 0.012 Iter: 263; Max change in p = 0.001 Iter: 264; Max change in p = 0.018 Iter: 265; Max change in p = 0.006 Iter: 266; Max change in p = 0.011 Iter: 267; Max change in p = 0.004 Iter: 268; Max change in p = 0.011 Iter: 269; Max change in p = 0.003 Iter: 270; Max change in p = 0.017 Iter: 271; Max change in p = 0.016 Iter: 272; Max change in p = 0.018 Iter: 273; Max change in p = 0.003 Iter: 274; Max change in p = 0.005 Iter: 275; Max change in p = 0.008 Iter: 276; Max change in p = 0.036 Iter: 277; Max change in p = 0.008 Iter: 278; Max change in p = 0.014 Iter: 279; Max change in p = 0.055 Iter: 280; Max change in p = 0.025 Iter: 281; Max change in p = 0.009 Iter: 282; Max change in p = 0.012 Iter: 283; Max change in p = 0.011 Iter: 284; Max change in p = 0.019 Iter: 285; Max change in p = 0.005 Iter: 286; Max change in p = 0.023 Iter: 287; Max change in p = 0.023 Iter: 288; Max change in p = 0.023 Iter: 289; Max change in p = 0.018 Iter: 290; Max change in p = 0.023 Iter: 291; Max change in p = 0.002 Iter: 292; Max change in p = 0.018 Iter: 293; Max change in p = 0.006 Iter: 294; Max change in p = 0.011 Iter: 295; Max change in p = 0.004 Iter: 296; Max change in p = 0.031 Iter: 297; Max change in p = 0.011 Iter: 298; Max change in p = 0.014 Iter: 299; Max change in p = 0.036 Iter: 300; Max change in p = 0.003 Iter: 301; Max change in p = 0.024 Iter: 302; Max change in p = 0.032 Iter: 303; Max change in p = 0.007 Iter: 304; Max change in p = 0.003 Iter: 305; Max change in p = 0.036 Iter: 306; Max change in p = 0.010 Iter: 307; Max change in p = 0.009 Iter: 308; Max change in p = 0.011 Iter: 309; Max change in p = 0.011 Iter: 310; Max change in p = 0.056 Iter: 311; Max change in p = 0.016 Iter: 312; Max change in p = 0.002 Iter: 313; Max change in p = 0.009 Iter: 314; Max change in p = 0.006 Iter: 315; Max change in p = 0.016 Iter: 316; Max change in p = 0.020 Iter: 317; Max change in p = 0.026 Iter: 318; Max change in p = 0.011 Iter: 319; Max change in p = 0.003 Iter: 320; Max change in p = 0.010 Iter: 321; Max change in p = 0.013 Iter: 322; Max change in p = 0.031 Iter: 323; Max change in p = 0.042 Iter: 324; Max change in p = 0.023 Iter: 325; Max change in p = 0.034 Iter: 326; Max change in p = 0.005 Iter: 327; Max change in p = 0.005 Iter: 328; Max change in p = 0.032 Iter: 329; Max change in p = 0.011 Iter: 330; Max change in p = 0.006 Iter: 331; Max change in p = 0.033 Iter: 332; Max change in p = 0.016 Iter: 333; Max change in p = 0.001 Iter: 334; Max change in p = 0.052 Iter: 335; Max change in p = 0.043 Iter: 336; Max change in p = 0.023 Iter: 337; Max change in p = 0.013 Iter: 338; Max change in p = 0.031 Iter: 339; Max change in p = 0.029 Iter: 340; Max change in p = 0.002 Iter: 341; Max change in p = 0.022 Iter: 342; Max change in p = 0.021 Iter: 343; Max change in p = 0.027 Iter: 344; Max change in p = 0.030 Iter: 345; Max change in p = 0.001 Iter: 346; Max change in p = 0.021 Iter: 347; Max change in p = 0.005 Iter: 348; Max change in p = 0.022 Iter: 349; Max change in p = 0.006 Iter: 350; Max change in p = 0.032 Iter: 351; Max change in p = 0.010 Iter: 352; Max change in p = 0.019 Iter: 353; Max change in p = 0.002 Iter: 354; Max change in p = 0.000 Iter: 355; Max change in p = 0.028 Iter: 356; Max change in p = 0.014 Iter: 357; Max change in p = 0.014 Iter: 358; Max change in p = 0.011 Iter: 359; Max change in p = 0.017 Iter: 360; Max change in p = 0.016 Iter: 361; Max change in p = 0.019 Iter: 362; Max change in p = 0.041 Iter: 363; Max change in p = 0.006 Iter: 364; Max change in p = 0.018 Iter: 365; Max change in p = 0.004 Iter: 366; Max change in p = 0.008 Iter: 367; Max change in p = 0.007 Iter: 368; Max change in p = 0.003 Iter: 369; Max change in p = 0.021 Iter: 370; Max change in p = 0.012 Iter: 371; Max change in p = 0.011 Iter: 372; Max change in p = 0.034 Iter: 373; Max change in p = 0.016 Iter: 374; Max change in p = 0.020 Iter: 375; Max change in p = 0.012 Iter: 376; Max change in p = 0.007 Iter: 377; Max change in p = 0.001 Iter: 378; Max change in p = 0.000 Iter: 379; Max change in p = 0.044 Iter: 380; Max change in p = 0.037 Iter: 381; Max change in p = 0.004 Iter: 382; Max change in p = 0.039 Iter: 383; Max change in p = 0.015 Iter: 384; Max change in p = 0.021 Iter: 385; Max change in p = 0.023 Iter: 386; Max change in p = 0.046 Iter: 387; Max change in p = 0.008 Iter: 388; Max change in p = 0.029 Iter: 389; Max change in p = 0.016 Iter: 390; Max change in p = 0.008 Iter: 391; Max change in p = 0.003 Iter: 392; Max change in p = 0.005 Iter: 393; Max change in p = 0.010 Iter: 394; Max change in p = 0.020 Iter: 395; Max change in p = 0.011 Iter: 396; Max change in p = 0.003 Iter: 397; Max change in p = 0.041 Iter: 398; Max change in p = 0.007 Iter: 399; Max change in p = 0.010 Iter: 400; Max change in p = 0.037 Iter: 401; Max change in p = 0.003 Iter: 402; Max change in p = 0.007 Iter: 403; Max change in p = 0.001 Iter: 404; Max change in p = 0.012 Iter: 405; Max change in p = 0.011 Iter: 406; Max change in p = 0.009 Iter: 407; Max change in p = 0.002 Iter: 408; Max change in p = 0.032 Iter: 409; Max change in p = 0.019 Iter: 410; Max change in p = 0.012 Iter: 411; Max change in p = 0.034 Iter: 412; Max change in p = 0.011 Iter: 413; Max change in p = 0.001 Iter: 414; Max change in p = 0.010 Iter: 415; Max change in p = 0.007 Iter: 416; Max change in p = 0.045 Iter: 417; Max change in p = 0.008 Iter: 418; Max change in p = 0.024 Iter: 419; Max change in p = 0.002 Iter: 420; Max change in p = 0.015 Iter: 421; Max change in p = 0.010 Iter: 422; Max change in p = 0.009 Iter: 423; Max change in p = 0.010 Iter: 424; Max change in p = 0.030 Iter: 425; Max change in p = 0.012 Iter: 426; Max change in p = 0.011 Iter: 427; Max change in p = 0.024 Iter: 428; Max change in p = 0.026 Iter: 429; Max change in p = 0.017 Iter: 430; Max change in p = 0.042 Iter: 431; Max change in p = 0.008 Iter: 432; Max change in p = 0.008 Iter: 433; Max change in p = 0.006 Iter: 434; Max change in p = 0.005 Iter: 435; Max change in p = 0.040 Iter: 436; Max change in p = 0.027 Iter: 437; Max change in p = 0.009 Iter: 438; Max change in p = 0.003 Iter: 439; Max change in p = 0.010 Iter: 440; Max change in p = 0.033 Iter: 441; Max change in p = 0.014 Iter: 442; Max change in p = 0.004 Iter: 443; Max change in p = 0.037 Iter: 444; Max change in p = 0.011 Iter: 445; Max change in p = 0.003 Iter: 446; Max change in p = 0.014 Iter: 447; Max change in p = 0.004 Iter: 448; Max change in p = 0.012 Iter: 449; Max change in p = 0.050 Iter: 450; Max change in p = 0.026 Iter: 451; Max change in p = 0.005 Iter: 452; Max change in p = 0.008 Iter: 453; Max change in p = 0.032 Iter: 454; Max change in p = 0.018 Iter: 455; Max change in p = 0.001 Iter: 456; Max change in p = 0.016 Iter: 457; Max change in p = 0.023 Iter: 458; Max change in p = 0.034 Iter: 459; Max change in p = 0.008 Iter: 460; Max change in p = 0.027 Iter: 461; Max change in p = 0.008 Iter: 462; Max change in p = 0.007 Iter: 463; Max change in p = 0.005 Iter: 464; Max change in p = 0.004 Iter: 465; Max change in p = 0.019 Iter: 466; Max change in p = 0.031 Iter: 467; Max change in p = 0.008 Iter: 468; Max change in p = 0.001 Iter: 469; Max change in p = 0.015 Iter: 470; Max change in p = 0.013 Iter: 471; Max change in p = 0.009 Iter: 472; Max change in p = 0.005 Iter: 473; Max change in p = 0.041 Iter: 474; Max change in p = 0.035 Iter: 475; Max change in p = 0.013 Iter: 476; Max change in p = 0.051 Iter: 477; Max change in p = 0.007 Iter: 478; Max change in p = 0.016 Iter: 479; Max change in p = 0.048 Iter: 480; Max change in p = 0.038 Iter: 481; Max change in p = 0.047 Iter: 482; Max change in p = 0.004 Iter: 483; Max change in p = 0.016 Iter: 484; Max change in p = 0.005 Iter: 485; Max change in p = 0.003 Iter: 486; Max change in p = 0.030 Iter: 487; Max change in p = 0.006 Iter: 488; Max change in p = 0.032 Iter: 489; Max change in p = 0.011 Iter: 490; Max change in p = 0.009 Iter: 491; Max change in p = 0.023 Iter: 492; Max change in p = 0.008 Iter: 493; Max change in p = 0.042 Iter: 494; Max change in p = 0.003 Iter: 495; Max change in p = 0.029 Iter: 496; Max change in p = 0.018 Iter: 497; Max change in p = 0.005 Iter: 498; Max change in p = 0.002 Iter: 499; Max change in p = 0.035 Iter: 500; Max change in p = 0.001 retrm.b2 #> [1] 0.385524 plot(retrm.b2) # use Polyak-Juditsky averaging (b should be closer to 0 to work well) retrm.PJ <- RobbinsMonro(f.root_noisy, .9, b = .01, Polyak_Juditsky = TRUE) #> Iter: 1; Max change in E(p) = 0.450 Iter: 2; Max change in E(p) = 0.119 Iter: 3; Max change in E(p) = 0.039 Iter: 4; Max change in E(p) = 0.005 Iter: 5; Max change in E(p) = 0.002 Iter: 6; Max change in E(p) = 0.008 Iter: 7; Max change in E(p) = 0.012 Iter: 8; Max change in E(p) = 0.012 Iter: 9; Max change in E(p) = 0.015 Iter: 10; Max change in E(p) = 0.011 Iter: 11; Max change in E(p) = 0.011 Iter: 12; Max change in E(p) = 0.009 Iter: 13; Max change in E(p) = 0.008 Iter: 14; Max change in E(p) = 0.008 Iter: 15; Max change in E(p) = 0.006 Iter: 16; Max change in E(p) = 0.004 Iter: 17; Max change in E(p) = 0.003 Iter: 18; Max change in E(p) = 0.005 Iter: 19; Max change in E(p) = 0.005 Iter: 20; Max change in E(p) = 0.005 Iter: 21; Max change in E(p) = 0.004 Iter: 22; Max change in E(p) = 0.004 Iter: 23; Max change in E(p) = 0.003 Iter: 24; Max change in E(p) = 0.004 Iter: 25; Max change in E(p) = 0.004 Iter: 26; Max change in E(p) = 0.003 Iter: 27; Max change in E(p) = 0.002 Iter: 28; Max change in E(p) = 0.003 Iter: 29; Max change in E(p) = 0.003 Iter: 30; Max change in E(p) = 0.003 Iter: 31; Max change in E(p) = 0.002 Iter: 32; Max change in E(p) = 0.003 Iter: 33; Max change in E(p) = 0.003 Iter: 34; Max change in E(p) = 0.002 Iter: 35; Max change in E(p) = 0.001 Iter: 36; Max change in E(p) = 0.001 Iter: 37; Max change in E(p) = 0.001 Iter: 38; Max change in E(p) = 0.001 Iter: 39; Max change in E(p) = 0.002 Iter: 40; Max change in E(p) = 0.002 Iter: 41; Max change in E(p) = 0.001 Iter: 42; Max change in E(p) = 0.001 Iter: 43; Max change in E(p) = 0.000 Iter: 44; Max change in E(p) = 0.001 Iter: 45; Max change in E(p) = 0.000 Iter: 46; Max change in E(p) = 0.001 Iter: 47; Max change in E(p) = 0.001 Iter: 48; Max change in E(p) = 0.000 Iter: 49; Max change in E(p) = 0.000 Iter: 50; Max change in E(p) = 0.001 Iter: 51; Max change in E(p) = 0.001 Iter: 52; Max change in E(p) = 0.001 Iter: 53; Max change in E(p) = 0.000 Iter: 54; Max change in E(p) = 0.001 Iter: 55; Max change in E(p) = 0.000 Iter: 56; Max change in E(p) = 0.000 Iter: 57; Max change in E(p) = 0.000 Iter: 58; Max change in E(p) = 0.000 Iter: 59; Max change in E(p) = 0.000 Iter: 60; Max change in E(p) = 0.000 Iter: 61; Max change in E(p) = 0.000 Iter: 62; Max change in E(p) = 0.000 Iter: 63; Max change in E(p) = 0.000 Iter: 64; Max change in E(p) = 0.000 Iter: 65; Max change in E(p) = 0.000 Iter: 66; Max change in E(p) = 0.000 Iter: 67; Max change in E(p) = 0.001 Iter: 68; Max change in E(p) = 0.001 Iter: 69; Max change in E(p) = 0.001 Iter: 70; Max change in E(p) = 0.001 Iter: 71; Max change in E(p) = 0.001 Iter: 72; Max change in E(p) = 0.000 Iter: 73; Max change in E(p) = 0.000 Iter: 74; Max change in E(p) = 0.000 Iter: 75; Max change in E(p) = 0.000 Iter: 76; Max change in E(p) = 0.000 Iter: 77; Max change in E(p) = 0.000 Iter: 78; Max change in E(p) = 0.000 Iter: 79; Max change in E(p) = 0.000 Iter: 80; Max change in E(p) = 0.000 Iter: 81; Max change in E(p) = 0.000 Iter: 82; Max change in E(p) = 0.001 Iter: 83; Max change in E(p) = 0.000 Iter: 84; Max change in E(p) = 0.001 Iter: 85; Max change in E(p) = 0.001 Iter: 86; Max change in E(p) = 0.001 Iter: 87; Max change in E(p) = 0.000 Iter: 88; Max change in E(p) = 0.001 Iter: 89; Max change in E(p) = 0.000 Iter: 90; Max change in E(p) = 0.000 Iter: 91; Max change in E(p) = 0.000 Iter: 92; Max change in E(p) = 0.000 Iter: 93; Max change in E(p) = 0.000 Iter: 94; Max change in E(p) = 0.000 Iter: 95; Max change in E(p) = 0.001 Iter: 96; Max change in E(p) = 0.001 Iter: 97; Max change in E(p) = 0.000 Iter: 98; Max change in E(p) = 0.000 Iter: 99; Max change in E(p) = 0.000 Iter: 100; Max change in E(p) = 0.000 Iter: 101; Max change in E(p) = 0.000 Iter: 102; Max change in E(p) = 0.000 Iter: 103; Max change in E(p) = 0.000 Iter: 104; Max change in E(p) = 0.000 Iter: 105; Max change in E(p) = 0.000 Iter: 106; Max change in E(p) = 0.000 Iter: 107; Max change in E(p) = 0.000 Iter: 108; Max change in E(p) = 0.000 Iter: 109; Max change in E(p) = 0.000 Iter: 110; Max change in E(p) = 0.000 Iter: 111; Max change in E(p) = 0.000 Iter: 112; Max change in E(p) = 0.000 Iter: 113; Max change in E(p) = 0.000 Iter: 114; Max change in E(p) = 0.000 Iter: 115; Max change in E(p) = 0.000 Iter: 116; Max change in E(p) = 0.000 Iter: 117; Max change in E(p) = 0.000 Iter: 118; Max change in E(p) = 0.000 Iter: 119; Max change in E(p) = 0.000 Iter: 120; Max change in E(p) = 0.000 Iter: 121; Max change in E(p) = 0.000 Iter: 122; Max change in E(p) = 0.000 Iter: 123; Max change in E(p) = 0.000 Iter: 124; Max change in E(p) = 0.000 Iter: 125; Max change in E(p) = 0.000 Iter: 126; Max change in E(p) = 0.000 Iter: 127; Max change in E(p) = 0.000 Iter: 128; Max change in E(p) = 0.000 Iter: 129; Max change in E(p) = 0.000 Iter: 130; Max change in E(p) = 0.000 Iter: 131; Max change in E(p) = 0.000 Iter: 132; Max change in E(p) = 0.000 Iter: 133; Max change in E(p) = 0.000 Iter: 134; Max change in E(p) = 0.000 Iter: 135; Max change in E(p) = 0.000 Iter: 136; Max change in E(p) = 0.000 Iter: 137; Max change in E(p) = 0.000 Iter: 138; Max change in E(p) = 0.000 Iter: 139; Max change in E(p) = 0.000 Iter: 140; Max change in E(p) = 0.000 Iter: 141; Max change in E(p) = 0.000 Iter: 142; Max change in E(p) = 0.000 Iter: 143; Max change in E(p) = 0.000 Iter: 144; Max change in E(p) = 0.000 Iter: 145; Max change in E(p) = 0.000 Iter: 146; Max change in E(p) = 0.000 Iter: 147; Max change in E(p) = 0.000 Iter: 148; Max change in E(p) = 0.000 Iter: 149; Max change in E(p) = 0.000 Iter: 150; Max change in E(p) = 0.000 Iter: 151; Max change in E(p) = 0.000 Iter: 152; Max change in E(p) = 0.000 Iter: 153; Max change in E(p) = 0.000 Iter: 154; Max change in E(p) = 0.000 Iter: 155; Max change in E(p) = 0.000 Iter: 156; Max change in E(p) = 0.000 Iter: 157; Max change in E(p) = 0.000 Iter: 158; Max change in E(p) = 0.000 Iter: 159; Max change in E(p) = 0.000 Iter: 160; Max change in E(p) = 0.000 Iter: 161; Max change in E(p) = 0.000 Iter: 162; Max change in E(p) = 0.000 Iter: 163; Max change in E(p) = 0.000 Iter: 164; Max change in E(p) = 0.000 Iter: 165; Max change in E(p) = 0.000 Iter: 166; Max change in E(p) = 0.000 Iter: 167; Max change in E(p) = 0.000 Iter: 168; Max change in E(p) = 0.000 Iter: 169; Max change in E(p) = 0.000 Iter: 170; Max change in E(p) = 0.000 Iter: 171; Max change in E(p) = 0.000 Iter: 172; Max change in E(p) = 0.000 Iter: 173; Max change in E(p) = 0.000 Iter: 174; Max change in E(p) = 0.000 Iter: 175; Max change in E(p) = 0.000 Iter: 176; Max change in E(p) = 0.000 Iter: 177; Max change in E(p) = 0.000 Iter: 178; Max change in E(p) = 0.000 Iter: 179; Max change in E(p) = 0.000 Iter: 180; Max change in E(p) = 0.000 Iter: 181; Max change in E(p) = 0.000 Iter: 182; Max change in E(p) = 0.000 Iter: 183; Max change in E(p) = 0.000 Iter: 184; Max change in E(p) = 0.000 Iter: 185; Max change in E(p) = 0.000 Iter: 186; Max change in E(p) = 0.000 Iter: 187; Max change in E(p) = 0.000 Iter: 188; Max change in E(p) = 0.000 Iter: 189; Max change in E(p) = 0.000 Iter: 190; Max change in E(p) = 0.000 Iter: 191; Max change in E(p) = 0.000 Iter: 192; Max change in E(p) = 0.000 Iter: 193; Max change in E(p) = 0.000 Iter: 194; Max change in E(p) = 0.000 Iter: 195; Max change in E(p) = 0.000 Iter: 196; Max change in E(p) = 0.000 Iter: 197; Max change in E(p) = 0.000 Iter: 198; Max change in E(p) = 0.000 Iter: 199; Max change in E(p) = 0.000 Iter: 200; Max change in E(p) = 0.000 Iter: 201; Max change in E(p) = 0.000 Iter: 202; Max change in E(p) = 0.000 Iter: 203; Max change in E(p) = 0.000 Iter: 204; Max change in E(p) = 0.000 Iter: 205; Max change in E(p) = 0.000 Iter: 206; Max change in E(p) = 0.000 Iter: 207; Max change in E(p) = 0.000 Iter: 208; Max change in E(p) = 0.000 Iter: 209; Max change in E(p) = 0.000 Iter: 210; Max change in E(p) = 0.000 Iter: 211; Max change in E(p) = 0.000 Iter: 212; Max change in E(p) = 0.000 Iter: 213; Max change in E(p) = 0.000 Iter: 214; Max change in E(p) = 0.000 Iter: 215; Max change in E(p) = 0.000 Iter: 216; Max change in E(p) = 0.000 Iter: 217; Max change in E(p) = 0.000 Iter: 218; Max change in E(p) = 0.000 Iter: 219; Max change in E(p) = 0.000 Iter: 220; Max change in E(p) = 0.000 Iter: 221; Max change in E(p) = 0.000 Iter: 222; Max change in E(p) = 0.000 Iter: 223; Max change in E(p) = 0.000 Iter: 224; Max change in E(p) = 0.000 Iter: 225; Max change in E(p) = 0.000 Iter: 226; Max change in E(p) = 0.000 Iter: 227; Max change in E(p) = 0.000 Iter: 228; Max change in E(p) = 0.000 Iter: 229; Max change in E(p) = 0.000 Iter: 230; Max change in E(p) = 0.000 Iter: 231; Max change in E(p) = 0.000 Iter: 232; Max change in E(p) = 0.000 Iter: 233; Max change in E(p) = 0.000 Iter: 234; Max change in E(p) = 0.000 Iter: 235; Max change in E(p) = 0.000 Iter: 236; Max change in E(p) = 0.000 Iter: 237; Max change in E(p) = 0.000 Iter: 238; Max change in E(p) = 0.000 Iter: 239; Max change in E(p) = 0.000 Iter: 240; Max change in E(p) = 0.000 Iter: 241; Max change in E(p) = 0.000 Iter: 242; Max change in E(p) = 0.000 Iter: 243; Max change in E(p) = 0.000 Iter: 244; Max change in E(p) = 0.000 Iter: 245; Max change in E(p) = 0.000 Iter: 246; Max change in E(p) = 0.000 Iter: 247; Max change in E(p) = 0.000 Iter: 248; Max change in E(p) = 0.000 Iter: 249; Max change in E(p) = 0.000 Iter: 250; Max change in E(p) = 0.000 Iter: 251; Max change in E(p) = 0.000 Iter: 252; Max change in E(p) = 0.000 Iter: 253; Max change in E(p) = 0.000 Iter: 254; Max change in E(p) = 0.000 Iter: 255; Max change in E(p) = 0.000 Iter: 256; Max change in E(p) = 0.000 Iter: 257; Max change in E(p) = 0.000 Iter: 258; Max change in E(p) = 0.000 Iter: 259; Max change in E(p) = 0.000 Iter: 260; Max change in E(p) = 0.000 Iter: 261; Max change in E(p) = 0.000 Iter: 262; Max change in E(p) = 0.000 Iter: 263; Max change in E(p) = 0.000 Iter: 264; Max change in E(p) = 0.000 Iter: 265; Max change in E(p) = 0.000 Iter: 266; Max change in E(p) = 0.000 Iter: 267; Max change in E(p) = 0.000 Iter: 268; Max change in E(p) = 0.000 Iter: 269; Max change in E(p) = 0.000 Iter: 270; Max change in E(p) = 0.000 Iter: 271; Max change in E(p) = 0.000 Iter: 272; Max change in E(p) = 0.000 Iter: 273; Max change in E(p) = 0.000 Iter: 274; Max change in E(p) = 0.000 Iter: 275; Max change in E(p) = 0.000 Iter: 276; Max change in E(p) = 0.000 Iter: 277; Max change in E(p) = 0.000 Iter: 278; Max change in E(p) = 0.000 Iter: 279; Max change in E(p) = 0.000 Iter: 280; Max change in E(p) = 0.000 Iter: 281; Max change in E(p) = 0.000 Iter: 282; Max change in E(p) = 0.000 Iter: 283; Max change in E(p) = 0.000 Iter: 284; Max change in E(p) = 0.000 Iter: 285; Max change in E(p) = 0.000 Iter: 286; Max change in E(p) = 0.000 Iter: 287; Max change in E(p) = 0.000 Iter: 288; Max change in E(p) = 0.000 Iter: 289; Max change in E(p) = 0.000 Iter: 290; Max change in E(p) = 0.000 Iter: 291; Max change in E(p) = 0.000 Iter: 292; Max change in E(p) = 0.000 Iter: 293; Max change in E(p) = 0.000 Iter: 294; Max change in E(p) = 0.000 Iter: 295; Max change in E(p) = 0.000 Iter: 296; Max change in E(p) = 0.000 Iter: 297; Max change in E(p) = 0.000 Iter: 298; Max change in E(p) = 0.000 Iter: 299; Max change in E(p) = 0.000 Iter: 300; Max change in E(p) = 0.000 Iter: 301; Max change in E(p) = 0.000 Iter: 302; Max change in E(p) = 0.000 Iter: 303; Max change in E(p) = 0.000 Iter: 304; Max change in E(p) = 0.000 Iter: 305; Max change in E(p) = 0.000 Iter: 306; Max change in E(p) = 0.000 Iter: 307; Max change in E(p) = 0.000 Iter: 308; Max change in E(p) = 0.000 Iter: 309; Max change in E(p) = 0.000 Iter: 310; Max change in E(p) = 0.000 Iter: 311; Max change in E(p) = 0.000 Iter: 312; Max change in E(p) = 0.000 Iter: 313; Max change in E(p) = 0.000 Iter: 314; Max change in E(p) = 0.000 Iter: 315; Max change in E(p) = 0.000 Iter: 316; Max change in E(p) = 0.000 Iter: 317; Max change in E(p) = 0.000 Iter: 318; Max change in E(p) = 0.000 Iter: 319; Max change in E(p) = 0.000 Iter: 320; Max change in E(p) = 0.000 Iter: 321; Max change in E(p) = 0.000 Iter: 322; Max change in E(p) = 0.000 Iter: 323; Max change in E(p) = 0.000 Iter: 324; Max change in E(p) = 0.000 Iter: 325; Max change in E(p) = 0.000 Iter: 326; Max change in E(p) = 0.000 Iter: 327; Max change in E(p) = 0.000 Iter: 328; Max change in E(p) = 0.000 Iter: 329; Max change in E(p) = 0.000 Iter: 330; Max change in E(p) = 0.000 Iter: 331; Max change in E(p) = 0.000 Iter: 332; Max change in E(p) = 0.000 Iter: 333; Max change in E(p) = 0.000 Iter: 334; Max change in E(p) = 0.000 Iter: 335; Max change in E(p) = 0.000 Iter: 336; Max change in E(p) = 0.000 Iter: 337; Max change in E(p) = 0.000 Iter: 338; Max change in E(p) = 0.000 Iter: 339; Max change in E(p) = 0.000 Iter: 340; Max change in E(p) = 0.000 Iter: 341; Max change in E(p) = 0.000 Iter: 342; Max change in E(p) = 0.000 Iter: 343; Max change in E(p) = 0.000 Iter: 344; Max change in E(p) = 0.000 Iter: 345; Max change in E(p) = 0.000 Iter: 346; Max change in E(p) = 0.000 Iter: 347; Max change in E(p) = 0.000 Iter: 348; Max change in E(p) = 0.000 Iter: 349; Max change in E(p) = 0.000 Iter: 350; Max change in E(p) = 0.000 Iter: 351; Max change in E(p) = 0.000 Iter: 352; Max change in E(p) = 0.000 Iter: 353; Max change in E(p) = 0.000 Iter: 354; Max change in E(p) = 0.000 Iter: 355; Max change in E(p) = 0.000 Iter: 356; Max change in E(p) = 0.000 Iter: 357; Max change in E(p) = 0.000 Iter: 358; Max change in E(p) = 0.000 Iter: 359; Max change in E(p) = 0.000 Iter: 360; Max change in E(p) = 0.000 Iter: 361; Max change in E(p) = 0.000 Iter: 362; Max change in E(p) = 0.000 Iter: 363; Max change in E(p) = 0.000 Iter: 364; Max change in E(p) = 0.000 Iter: 365; Max change in E(p) = 0.000 Iter: 366; Max change in E(p) = 0.000 Iter: 367; Max change in E(p) = 0.000 Iter: 368; Max change in E(p) = 0.000 Iter: 369; Max change in E(p) = 0.000 Iter: 370; Max change in E(p) = 0.000 Iter: 371; Max change in E(p) = 0.000 Iter: 372; Max change in E(p) = 0.000 Iter: 373; Max change in E(p) = 0.000 Iter: 374; Max change in E(p) = 0.000 Iter: 375; Max change in E(p) = 0.000 Iter: 376; Max change in E(p) = 0.000 Iter: 377; Max change in E(p) = 0.000 Iter: 378; Max change in E(p) = 0.000 Iter: 379; Max change in E(p) = 0.000 Iter: 380; Max change in E(p) = 0.000 Iter: 381; Max change in E(p) = 0.000 Iter: 382; Max change in E(p) = 0.000 Iter: 383; Max change in E(p) = 0.000 Iter: 384; Max change in E(p) = 0.000 Iter: 385; Max change in E(p) = 0.000 Iter: 386; Max change in E(p) = 0.000 Iter: 387; Max change in E(p) = 0.000 Iter: 388; Max change in E(p) = 0.000 Iter: 389; Max change in E(p) = 0.000 Iter: 390; Max change in E(p) = 0.000 Iter: 391; Max change in E(p) = 0.000 Iter: 392; Max change in E(p) = 0.000 Iter: 393; Max change in E(p) = 0.000 Iter: 394; Max change in E(p) = 0.000 Iter: 395; Max change in E(p) = 0.000 Iter: 396; Max change in E(p) = 0.000 Iter: 397; Max change in E(p) = 0.000 Iter: 398; Max change in E(p) = 0.000 Iter: 399; Max change in E(p) = 0.000 Iter: 400; Max change in E(p) = 0.000 Iter: 401; Max change in E(p) = 0.000 Iter: 402; Max change in E(p) = 0.000 Iter: 403; Max change in E(p) = 0.000 Iter: 404; Max change in E(p) = 0.000 Iter: 405; Max change in E(p) = 0.000 Iter: 406; Max change in E(p) = 0.000 Iter: 407; Max change in E(p) = 0.000 Iter: 408; Max change in E(p) = 0.000 Iter: 409; Max change in E(p) = 0.000 Iter: 410; Max change in E(p) = 0.000 Iter: 411; Max change in E(p) = 0.000 Iter: 412; Max change in E(p) = 0.000 Iter: 413; Max change in E(p) = 0.000 Iter: 414; Max change in E(p) = 0.000 Iter: 415; Max change in E(p) = 0.000 Iter: 416; Max change in E(p) = 0.000 Iter: 417; Max change in E(p) = 0.000 Iter: 418; Max change in E(p) = 0.000 Iter: 419; Max change in E(p) = 0.000 Iter: 420; Max change in E(p) = 0.000 Iter: 421; Max change in E(p) = 0.000 Iter: 422; Max change in E(p) = 0.000 Iter: 423; Max change in E(p) = 0.000 Iter: 424; Max change in E(p) = 0.000 Iter: 425; Max change in E(p) = 0.000 Iter: 426; Max change in E(p) = 0.000 Iter: 427; Max change in E(p) = 0.000 Iter: 428; Max change in E(p) = 0.000 Iter: 429; Max change in E(p) = 0.000 Iter: 430; Max change in E(p) = 0.000 Iter: 431; Max change in E(p) = 0.000 Iter: 432; Max change in E(p) = 0.000 Iter: 433; Max change in E(p) = 0.000 Iter: 434; Max change in E(p) = 0.000 Iter: 435; Max change in E(p) = 0.000 Iter: 436; Max change in E(p) = 0.000 Iter: 437; Max change in E(p) = 0.000 Iter: 438; Max change in E(p) = 0.000 Iter: 439; Max change in E(p) = 0.000 Iter: 440; Max change in E(p) = 0.000 Iter: 441; Max change in E(p) = 0.000 Iter: 442; Max change in E(p) = 0.000 Iter: 443; Max change in E(p) = 0.000 Iter: 444; Max change in E(p) = 0.000 Iter: 445; Max change in E(p) = 0.000 Iter: 446; Max change in E(p) = 0.000 Iter: 447; Max change in E(p) = 0.000 Iter: 448; Max change in E(p) = 0.000 Iter: 449; Max change in E(p) = 0.000 Iter: 450; Max change in E(p) = 0.000 Iter: 451; Max change in E(p) = 0.000 Iter: 452; Max change in E(p) = 0.000 Iter: 453; Max change in E(p) = 0.000 Iter: 454; Max change in E(p) = 0.000 Iter: 455; Max change in E(p) = 0.000 Iter: 456; Max change in E(p) = 0.000 Iter: 457; Max change in E(p) = 0.000 Iter: 458; Max change in E(p) = 0.000 Iter: 459; Max change in E(p) = 0.000 Iter: 460; Max change in E(p) = 0.000 Iter: 461; Max change in E(p) = 0.000 Iter: 462; Max change in E(p) = 0.000 Iter: 463; Max change in E(p) = 0.000 Iter: 464; Max change in E(p) = 0.000 Iter: 465; Max change in E(p) = 0.000 Iter: 466; Max change in E(p) = 0.000 Iter: 467; Max change in E(p) = 0.000 Iter: 468; Max change in E(p) = 0.000 Iter: 469; Max change in E(p) = 0.000 Iter: 470; Max change in E(p) = 0.000 Iter: 471; Max change in E(p) = 0.000 Iter: 472; Max change in E(p) = 0.000 Iter: 473; Max change in E(p) = 0.000 Iter: 474; Max change in E(p) = 0.000 Iter: 475; Max change in E(p) = 0.000 Iter: 476; Max change in E(p) = 0.000 Iter: 477; Max change in E(p) = 0.000 Iter: 478; Max change in E(p) = 0.000 Iter: 479; Max change in E(p) = 0.000 Iter: 480; Max change in E(p) = 0.000 Iter: 481; Max change in E(p) = 0.000 Iter: 482; Max change in E(p) = 0.000 Iter: 483; Max change in E(p) = 0.000 Iter: 484; Max change in E(p) = 0.000 Iter: 485; Max change in E(p) = 0.000 Iter: 486; Max change in E(p) = 0.000 Iter: 487; Max change in E(p) = 0.000 Iter: 488; Max change in E(p) = 0.000 Iter: 489; Max change in E(p) = 0.000 Iter: 490; Max change in E(p) = 0.000 Iter: 491; Max change in E(p) = 0.000 Iter: 492; Max change in E(p) = 0.000 Iter: 493; Max change in E(p) = 0.000 Iter: 494; Max change in E(p) = 0.000 Iter: 495; Max change in E(p) = 0.000 Iter: 496; Max change in E(p) = 0.000 Iter: 497; Max change in E(p) = 0.000 Iter: 498; Max change in E(p) = 0.000 Iter: 499; Max change in E(p) = 0.000 Iter: 500; Max change in E(p) = 0.000 retrm.PJ # final Polyak_Juditsky estimate #> [,1] #> [1,] 0.4077852 plot(retrm.PJ) # Robbins-Monro history plot(retrm.PJ, Polyak_Juditsky = TRUE) # Polyak_Juditsky history"},{"path":"http://philchalmers.github.io/SimDesign/reference/SFA.html","id":null,"dir":"Reference","previous_headings":"","what":"Surrogate Function Approximation via the Generalized Linear Model — SFA","title":"Surrogate Function Approximation via the Generalized Linear Model — SFA","text":"Given simulation executed runSimulation, potentially argument store_results = TRUE store unsummarised analysis results, fit surrogate function approximation (SFA) model results (optionally) perform root-solving step solve target quantity. See Schoemann et al. (2014) details.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SFA.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Surrogate Function Approximation via the Generalized Linear Model — SFA","text":"","code":"SFA( results, formula, family = \"binomial\", b = NULL, design = NULL, CI = 0.95, interval = NULL, ... ) # S3 method for class 'SFA' print(x, ...)"},{"path":"http://philchalmers.github.io/SimDesign/reference/SFA.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Surrogate Function Approximation via the Generalized Linear Model — SFA","text":"results data returned runSimulation. can original results object extracted results stored using store_results = TRUE included store analysis results. formula formula specify regression model family character vector indicating family GLMs use (see family) b (optional) Target quantity use root solving given fitted surrogate function (e.g., find sample size associated SFA implied power .80) design (optional) data.frame object containing information relevant surrogate model (passed newdata predict) NA value variable solved CI advertised confidence interval SFA prediction around solved target interval interval passed uniroot specified lowest highest values results respective variable used ... additional arguments pass glm x object class SFA","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SFA.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Surrogate Function Approximation via the Generalized Linear Model — SFA","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Schoemann, . M., Miller, P., Pornprasertmanit, S., Wu, W. (2014). Using Monte Carlo simulations determine power sample size planned missing designs. International Journal Behavioral Development, SAGE Publications, 38, 471-479.","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/SFA.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Surrogate Function Approximation via the Generalized Linear Model — SFA","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SFA.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Surrogate Function Approximation via the Generalized Linear Model — SFA","text":"","code":"if (FALSE) { # \\dontrun{ # create long Design object to fit surrogate over Design <- createDesign(N = 100:500, d = .2) Design #~~~~~~~~~~~~~~~~~~~~~~~~ #### Step 2 --- Define generate, analyse, and summarise functions Generate <- function(condition, fixed_objects) { Attach(condition) group1 <- rnorm(N) group2 <- rnorm(N, mean=d) dat <- data.frame(group = gl(2, N, labels=c('G1', 'G2')), DV = c(group1, group2)) dat } Analyse <- function(condition, dat, fixed_objects) { p <- c(p = t.test(DV ~ group, dat, var.equal=TRUE)$p.value) p } Summarise <- function(condition, results, fixed_objects) { ret <- EDR(results, alpha = .05) ret } #~~~~~~~~~~~~~~~~~~~~~~~~ #### Step 3 --- Estimate power over N # Use small number of replications given range of sample sizes ## note that due to the lower replications disabling the ## RAM printing will help reduce overhead sim <- runSimulation(design=Design, replications=10, generate=Generate, analyse=Analyse, summarise=Summarise, store_results=TRUE, save=FALSE, progress=FALSE, control=list(print_RAM=FALSE)) sim # total of 4010 replication sum(sim$REPLICATIONS) # use the unsummarised results for the SFA, and include p.values < alpha sim_results <- SimResults(sim) sim_results <- within(sim_results, sig <- p < .05) sim_results # fitted model sfa <- SFA(sim_results, formula = sig ~ N) sfa summary(sfa) # plot the observed and SFA expected values plot(p ~ N, sim, las=1, pch=16, main='Rejection rates with R=10') pred <- predict(sfa, type = 'response') lines(sim_results$N, pred, col='red', lty=2) # fitted model + root-solved solution given f(.) = b, # where b = target power of .8 design <- data.frame(N=NA, d=.2) sfa.root <- SFA(sim_results, formula = sig ~ N, b=.8, design=design) sfa.root # true root pwr::pwr.t.test(power=.8, d=.2) ################ # example with smaller range but higher precision Design <- createDesign(N = 375:425, d = .2) Design sim2 <- runSimulation(design=Design, replications=100, generate=Generate, analyse=Analyse, summarise=Summarise, store_results=TRUE, save=FALSE, progress=FALSE, control=list(print_RAM=FALSE)) sim2 sum(sim2$REPLICATIONS) # more replications in total # use the unsummarised results for the SFA, and include p.values < alpha sim_results <- SimResults(sim2) sim_results <- within(sim_results, sig <- p < .05) sim_results # fitted model sfa <- SFA(sim_results, formula = sig ~ N) sfa summary(sfa) # plot the observed and SFA expected values plot(p ~ N, sim2, las=1, pch=16, main='Rejection rates with R=100') pred <- predict(sfa, type = 'response') lines(sim_results$N, pred, col='red', lty=2) # fitted model + root-solved solution given f(.) = b, # where b = target power of .8 design <- data.frame(N=NA, d=.2) sfa.root <- SFA(sim_results, formula = sig ~ N, b=.8, design=design, interval=c(100, 500)) sfa.root # true root pwr::pwr.t.test(power=.8, d=.2) ################### # vary multiple parameters (e.g., sample size + effect size) to fit # multi-parameter surrogate Design <- createDesign(N = seq(from=10, to=500, by=10), d = seq(from=.1, to=.5, by=.1)) Design sim3 <- runSimulation(design=Design, replications=50, generate=Generate, analyse=Analyse, summarise=Summarise, store_results=TRUE, save=FALSE, progress=FALSE, control=list(print_RAM=FALSE)) sim3 sum(sim3$REPLICATIONS) # use the unsummarised results for the SFA, and include p.values < alpha sim_results <- SimResults(sim3) sim_results <- within(sim_results, sig <- p < .05) sim_results # additive effects (logit(sig) ~ N + d) sfa0 <- SFA(sim_results, formula = sig ~ N+d) sfa0 # multiplicative effects (logit(sig) ~ N + d + N:d) sfa <- SFA(sim_results, formula = sig ~ N*d) sfa # multiplicative better fit (sample size interacts with effect size) anova(sfa0, sfa, test = \"LRT\") summary(sfa) # plot the observed and SFA expected values library(ggplot2) sim3$pred <- predict(sfa, type = 'response', newdata=sim3) ggplot(sim3, aes(N, p, color = factor(d))) + geom_point() + geom_line(aes(y=pred)) + facet_wrap(~factor(d)) # fitted model + root-solved solution given f(.) = b, # where b = target power of .8 design <- data.frame(N=NA, d=.2) sfa.root <- SFA(sim_results, formula = sig ~ N * d, b=.8, design=design, interval=c(100, 500)) sfa.root # true root pwr::pwr.t.test(power=.8, d=.2) # root prediction where d *not* used in original data design <- data.frame(N=NA, d=.25) sfa.root <- SFA(sim_results, formula = sig ~ N * d, b=.8, design=design, interval=c(100, 500)) sfa.root # true root pwr::pwr.t.test(power=.8, d=.25) } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/Serlin2000.html","id":null,"dir":"Reference","previous_headings":"","what":"Empirical detection robustness method suggested by Serlin (2000) — Serlin2000","title":"Empirical detection robustness method suggested by Serlin (2000) — Serlin2000","text":"Hypothesis test determine whether observed empirical detection rate, coupled given robustness interval, statistically differs population value. Uses methods described Serlin (2000) well generate critical values (similar confidence intervals, define fixed window robustness). Critical values may computed without performing simulation experiment (hence, can obtained priori).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Serlin2000.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Empirical detection robustness method suggested by Serlin (2000) — Serlin2000","text":"","code":"Serlin2000(p, alpha, delta, R, CI = 0.95)"},{"path":"http://philchalmers.github.io/SimDesign/reference/Serlin2000.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Empirical detection robustness method suggested by Serlin (2000) — Serlin2000","text":"p (optional) vector containing empirical detection rate(s) tested. Omitting input compute CV1 CV2 values, including input perform one-sided hypothesis test robustness alpha Type error rate (e.g., often set .05) delta (optional) symmetric robustness interval around alpha (e.g., value .01 alpha = .05 test robustness window .04-.06) R number replications used simulation CI confidence interval alpha proportion. Default 0.95 indicates 95% interval","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Serlin2000.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Empirical detection robustness method suggested by Serlin (2000) — Serlin2000","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Serlin, R. C. (2000). Testing Robustness Monte Carlo Studies. Psychological Methods, 5, 230-240. Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Serlin2000.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Empirical detection robustness method suggested by Serlin (2000) — Serlin2000","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Serlin2000.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Empirical detection robustness method suggested by Serlin (2000) — Serlin2000","text":"","code":"# Cochran's criteria at alpha = .05 (i.e., 0.5 +- .01), assuming N = 2000 Serlin2000(p = .051, alpha = .05, delta = .01, R = 2000) #> p z(|p-a| - d)) Pr(>|z|) robust CV1 CV2 #> 1 0.051 -1.846761 0.03239089 yes 0.04125991 0.05864068 # Bradley's liberal criteria given p = .06 and .076, assuming N = 1000 Serlin2000(p = .060, alpha = .05, delta = .025, R = 1000) #> p z(|p-a| - d)) Pr(>|z|) robust CV1 CV2 #> 1 0.06 -2.176429 0.01476161 yes 0.03815878 0.06259781 Serlin2000(p = .076, alpha = .05, delta = .025, R = 1000) #> p z(|p-a| - d)) Pr(>|z|) robust CV1 CV2 #> 1 0.076 0.1450953 0.4423178 no 0.03815878 0.06259781 # multiple p-values Serlin2000(p = c(.05, .06, .07), alpha = .05, delta = .025, R = 1000) #> p z(|p-a| - d)) Pr(>|z|) robust CV1 CV2 #> 1 0.05 -3.6273813 0.0001431552 yes 0.03815878 0.06259781 #> 2 0.06 -2.1764288 0.0147616080 yes 0.03815878 0.06259781 #> 3 0.07 -0.7254763 0.2340799549 no 0.03815878 0.06259781 # CV values computed before simulation performed Serlin2000(alpha = .05, R = 2500) #> CV1 CV2 #> 1 0.04210768 0.05763959"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimAnova.html","id":null,"dir":"Reference","previous_headings":"","what":"Function for decomposing the simulation into ANOVA-based effect sizes — SimAnova","title":"Function for decomposing the simulation into ANOVA-based effect sizes — SimAnova","text":"Given results simulation runSimulation form ANOVA table (without p-values) effect sizes based eta-squared statistic. results provide approximate indications observable simulation effects, therefore ANOVA-based results generally useful exploratory rather inferential tools.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimAnova.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Function for decomposing the simulation into ANOVA-based effect sizes — SimAnova","text":"","code":"SimAnova(formula, dat, subset = NULL, rates = TRUE)"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimAnova.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Function for decomposing the simulation into ANOVA-based effect sizes — SimAnova","text":"formula R formula generally form suitable lm aov. However, dependent variable (left size equation) omitted dependent variables simulation used result return list analyses dat object returned runSimulation class 'SimDesign' subset optional argument passed subset name. Used subset results object preserving associated attributes rates logical; dependent variable consist rates (e.g., returned ECR EDR)? Default TRUE, use logit DV help stabilize proportion-based summary statistics computing parameters effect sizes","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimAnova.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Function for decomposing the simulation into ANOVA-based effect sizes — SimAnova","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimAnova.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Function for decomposing the simulation into ANOVA-based effect sizes — SimAnova","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimAnova.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Function for decomposing the simulation into ANOVA-based effect sizes — SimAnova","text":"","code":"data(BF_sim) # all results (not usually good to mix Power and Type I results together) SimAnova(alpha.05.F ~ (groups_equal + distribution)^2, BF_sim) #> SS df MS F p sig eta.sq eta.sq.part #> groups_equal 0.080 1 0.080 0.022 0.885 . 0.001 0.001 #> distribution 17.790 3 5.930 1.590 0.223 . 0.192 0.193 #> groups_equal:distribution 0.006 3 0.002 0.001 1.000 . 0.000 0.000 #> Residuals 74.598 20 3.730 NA NA 0.806 NA # only use anova for Type I error conditions SimAnova(alpha.05.F ~ (groups_equal + distribution)^2, BF_sim, subset = var_ratio == 1) #> SS df MS F p sig eta.sq eta.sq.part #> groups_equal 0.027 1 0.027 0.281 0.610 . 0.001 0.034 #> distribution 30.755 3 10.252 108.101 0.000 *** 0.975 0.976 #> groups_equal:distribution 0.002 3 0.001 0.006 0.999 . 0.000 0.002 #> Residuals 0.759 8 0.095 NA NA 0.024 NA # run all DVs at once using the same formula SimAnova(~ groups_equal * distribution, BF_sim, subset = var_ratio == 1) #> $alpha.05.F #> SS df MS F p sig eta.sq eta.sq.part #> groups_equal 0.027 1 0.027 0.281 0.610 . 0.001 0.034 #> distribution 30.755 3 10.252 108.101 0.000 *** 0.975 0.976 #> groups_equal:distribution 0.002 3 0.001 0.006 0.999 . 0.000 0.002 #> Residuals 0.759 8 0.095 NA NA 0.024 NA #> #> $alpha.05.Jacknife #> SS df MS F p sig eta.sq eta.sq.part #> groups_equal 0.077 1 0.077 3.644 0.093 . 0.016 0.313 #> distribution 4.462 3 1.487 70.265 0.000 *** 0.933 0.963 #> groups_equal:distribution 0.072 3 0.024 1.140 0.390 . 0.015 0.300 #> Residuals 0.169 8 0.021 NA NA 0.035 NA #> #> $alpha.05.Layard #> SS df MS F p sig eta.sq eta.sq.part #> groups_equal 0.004 1 0.004 0.056 0.818 . 0.000 0.007 #> distribution 10.111 3 3.370 46.579 0.000 *** 0.943 0.946 #> groups_equal:distribution 0.023 3 0.008 0.104 0.955 . 0.002 0.038 #> Residuals 0.579 8 0.072 NA NA 0.054 NA #> #> $alpha.05.Levene #> SS df MS F p sig eta.sq eta.sq.part #> groups_equal 0.024 1 0.024 1.615 0.239 . 0.006 0.168 #> distribution 4.263 3 1.421 93.943 0.000 *** 0.960 0.972 #> groups_equal:distribution 0.030 3 0.010 0.661 0.599 . 0.007 0.199 #> Residuals 0.121 8 0.015 NA NA 0.027 NA #> #> $alpha.05.W10 #> SS df MS F p sig eta.sq eta.sq.part #> groups_equal 0.040 1 0.040 0.573 0.471 . 0.021 0.067 #> distribution 1.263 3 0.421 5.956 0.020 . 0.669 0.691 #> groups_equal:distribution 0.019 3 0.006 0.092 0.963 . 0.010 0.033 #> Residuals 0.566 8 0.071 NA NA 0.299 NA #> #> $alpha.05.W50 #> SS df MS F p sig eta.sq eta.sq.part #> groups_equal 0.073 1 0.073 3.005 0.121 . 0.048 0.273 #> distribution 1.164 3 0.388 16.002 0.001 ** 0.763 0.857 #> groups_equal:distribution 0.095 3 0.032 1.304 0.338 . 0.062 0.328 #> Residuals 0.194 8 0.024 NA NA 0.127 NA #>"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimCheck.html","id":null,"dir":"Reference","previous_headings":"","what":"Check for missing files in array simulations — SimCheck","title":"Check for missing files in array simulations — SimCheck","text":"Given saved files runArraySimulation remote evaluation check whether .rds files saved. missing missing row condition numbers returned.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimCheck.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check for missing files in array simulations — SimCheck","text":"","code":"SimCheck(dir = NULL, files = NULL, min = 1L, max = NULL)"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimCheck.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check for missing files in array simulations — SimCheck","text":"dir character vector input indicating directory containing .rds files (see files) files vector file names referring saved simulation files. E.g. c('mysim-1.rds', 'mysim-2.rds', ...) min minimum number '-' deliminator. Default 1 max maximum number '-' deliminator. specified extracted attributes first file","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimCheck.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check for missing files in array simulations — SimCheck","text":"returns invisible TRUE files present FALSE otherwise","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimCheck.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Check for missing files in array simulations — SimCheck","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/SimCheck.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Check for missing files in array simulations — SimCheck","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimCheck.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Check for missing files in array simulations — SimCheck","text":"","code":"if (FALSE) { # \\dontrun{ # if files are in mysimfiles/ directory SimCheck('mysimfiles') # specifying files explicility setwd('mysimfiles/') SimCheck(files=dir()) } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimClean.html","id":null,"dir":"Reference","previous_headings":"","what":"Removes/cleans files and folders that have been saved — SimClean","title":"Removes/cleans files and folders that have been saved — SimClean","text":"function mainly used pilot studies results datasets temporarily saved runSimulation removed beginning full Monte Carlo simulation (e.g., remove files folders contained bugs/biased results).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimClean.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Removes/cleans files and folders that have been saved — SimClean","text":"","code":"SimClean( ..., dirs = NULL, temp = TRUE, results = FALSE, seeds = FALSE, save_details = list() )"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimClean.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Removes/cleans files and folders that have been saved — SimClean","text":"... one character objects indicating files remove. Used remove .rds files saved saveRDS using save filename inputs runSimulation dirs character vector indicating directories remove temp logical; remove temporary file saved passing save = TRUE? results logical; remove .rds results files saved passing save_results = TRUE? seeds logical; remove seed files saved passing save_seeds = TRUE? save_details list pertaining information files saved (see corresponding list runSimulation)","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimClean.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Removes/cleans files and folders that have been saved — SimClean","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/SimClean.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Removes/cleans files and folders that have been saved — SimClean","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimClean.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Removes/cleans files and folders that have been saved — SimClean","text":"","code":"if (FALSE) { # \\dontrun{ # remove file called 'results.rds' SimClean('results.rds') # remove default temp file SimClean() # remove customized saved-results directory called 'mydir' SimClean(results = TRUE, save_details = list(save_results_dirname = 'mydir')) } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimCollect.html","id":null,"dir":"Reference","previous_headings":"","what":"Collapse separate simulation files into a single result — SimCollect","title":"Collapse separate simulation files into a single result — SimCollect","text":"function collects aggregates results SimDesign's runSimulation single objects suitable post-analyses, combines saved results directories combines one. useful results run piece-wise one node (e.g., 500 replications one batch, 500 later date, though careful set.seed use random numbers tend correlate used) run independently across different nodes/computing cores (e.g., see runArraySimulation.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimCollect.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Collapse separate simulation files into a single result — SimCollect","text":"","code":"SimCollect( dir = NULL, files = NULL, filename = NULL, select = NULL, check.only = FALSE, target.reps = NULL, warning_details = FALSE, error_details = TRUE ) aggregate_simulations(...)"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimCollect.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Collapse separate simulation files into a single result — SimCollect","text":"dir character vector pointing directory name containing .rds files. .rds files directory used first checking status SimCheck. greater specificity use files argument files character vector containing names simulation's final .rds files. filename (optional) name .rds file save aggregate simulation file . specified results returned R console. select character vector indicating columns variables select SimExtract(='results') information. mainly useful RAM issue given simulations many stored estimates. Default includes results objects entirety, though omit internally stored simulation results pass character 'NONE' check.logical; larger simulations file sets, generated runArraySimulation, return design conditions satisfy target.reps throw warning files unexpectedly missing target.reps (optional) number replications check evaluate whether simulation files returned desired number replications. missing, highest detected value collected set replication information used warning_details logical; include aggregate warnings extracted via SimExtract? error_details logical; include aggregate errors extracted via SimExtract? ... used","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimCollect.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Collapse separate simulation files into a single result — SimCollect","text":"returns data.frame/tibble (weighted) average/aggregate simulation results","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimCollect.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Collapse separate simulation files into a single result — SimCollect","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/SimCollect.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Collapse separate simulation files into a single result — SimCollect","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimCollect.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Collapse separate simulation files into a single result — SimCollect","text":"","code":"if (FALSE) { # \\dontrun{ setwd('my_working_directory') ## run simulations to save the .rds files (or move them to the working directory) # seeds1 <- genSeeds(design) # seeds2 <- genSeeds(design, old.seeds=seeds1) # ret1 <- runSimulation(design, ..., seed=seeds1, filename='file1') # ret2 <- runSimulation(design, ..., seed=seeds2, filename='file2') # saves to the hard-drive and stores in workspace final <- SimCollect(files = c('file1.rds', 'file2.rds')) final # If filename not included, can be extracted from results # files <- c(SimExtract(ret1, 'filename'), SimExtract(ret2, 'filename')) # final <- SimCollect(files = files) ################################################# # Example where each row condition is repeated, evaluated independently, # and later collapsed into a single analysis object # Each condition repeated four times (hence, replications # should be set to desired.reps/4) Design <- createDesign(mu = c(0,5), N = c(30, 60)) Design # assume the N=60 takes longer, and should be spread out across more arrays Design_long <- expandDesign(Design, c(2,2,4,4)) Design_long replications <- c(rep(50, 4), rep(25,8)) data.frame(Design_long, replications) #------------------------------------------------------------------- Generate <- function(condition, fixed_objects) { dat <- with(condition, rnorm(N, mean=mu)) dat } Analyse <- function(condition, dat, fixed_objects) { ret <- c(mean=mean(dat), SD=sd(dat)) ret } Summarise <- function(condition, results, fixed_objects) { ret <- colMeans(results) ret } #------------------------------------------------------------------- # create directory to store all final simulation files dir.create('sim_files/') iseed <- genSeeds() # distribute jobs independently sapply(1:nrow(Design_long), \\(i) { runArraySimulation(design=Design_long, replications=replications, generate=Generate, analyse=Analyse, summarise=Summarise, arrayID=i, dirname='sim_files/', filename='job', iseed=iseed) }) |> invisible() # check that all replications satisfy target SimCollect('sim_files/', check.only = TRUE) # this would have been returned were the target.rep supposed to be 1000 SimCollect('sim_files/', check.only = TRUE, target.reps=1000) # aggregate into single object sim <- SimCollect('sim_files/') sim SimClean(dir='sim_files/') } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimDesign.html","id":null,"dir":"Reference","previous_headings":"","what":"Structure for Organizing Monte Carlo Simulation Designs — SimDesign","title":"Structure for Organizing Monte Carlo Simulation Designs — SimDesign","text":"Structure Organizing Monte Carlo Simulation Designs","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimDesign.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Structure for Organizing Monte Carlo Simulation Designs — SimDesign","text":"Provides tools help organize Monte Carlo simulations R. package controls structure back-end Monte Carlo simulations utilizing general generate-analyse-summarise strategy. functions provided control common simulation issues re-simulating non-convergent results, support parallel back-end computations proper random number generation within simulation condition, save restore temporary files, aggregate results across independent nodes, provide native support debugging. primary function organizing simulations runSimulation, array jobs submitting HPC clusters (e.g., SLURM) see runArraySimulation associated package vignettes. -depth tutorial package please refer Chalmers Adkins (2020; doi:10.20982/tqmp.16.4.p248 ). earlier didactic presentation package users can refer Sigal Chalmers (2016; doi:10.1080/10691898.2016.1246953 ). Finally, see associated wiki Github (https://github.com/philchalmers/SimDesign/wiki) tutorial material, examples, applications SimDesign real-world simulations.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimDesign.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Structure for Organizing Monte Carlo Simulation Designs — SimDesign","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimDesign.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Structure for Organizing Monte Carlo Simulation Designs — SimDesign","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimExtract.html","id":null,"dir":"Reference","previous_headings":"","what":"Function to extract extra information from SimDesign objects — SimExtract","title":"Function to extract extra information from SimDesign objects — SimExtract","text":"Function used extract error warnings messages, seeds associated error warning messages, analysis results stored final simulation object.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimExtract.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Function to extract extra information from SimDesign objects — SimExtract","text":"","code":"SimExtract(object, what, fuzzy = TRUE, append = TRUE)"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimExtract.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Function to extract extra information from SimDesign objects — SimExtract","text":"object object returned runSimulation character indicating information extract. Possible inputs include 'errors' return tibble object containing counts error messages, 'warnings' return data.frame object containing counts warning messages, 'seeds' specified random number generation seeds, 'Random.seeds' complete list .Random.seed states across replications (stored runSimulation(..., control = list(store_Random.seeds=TRUE))), 'error_seeds' 'warning_seeds' extract associated .Random.seed values associated ERROR/WARNING messages, 'results' extract simulation results option store_results passed runSimulation, 'filename' 'save_results_dirname' extracting saved file/directory name information (used), 'summarise' Summarise definition returned named list rather named numeric vector. Note 'warning_seeds' stored automatically simulations require passing store_warning_seeds = TRUE runSimulation. fuzzy logical; use fuzzy string matching reduce effectively identical messages? example, attempting invert matrix error message \"System computationally singular: reciprocal condition number = 1.92747e-17\" \"System computationally singular: reciprocal condition number = 2.15321e-16\" effectively , likely reported columns extracted output append logical; append design conditions extracting error/warning messages?","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimExtract.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Function to extract extra information from SimDesign objects — SimExtract","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimExtract.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Function to extract extra information from SimDesign objects — SimExtract","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimExtract.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Function to extract extra information from SimDesign objects — SimExtract","text":"","code":"if (FALSE) { # \\dontrun{ Generate <- function(condition, fixed_objects) { int <- sample(1:10, 1) if(int > 5) warning('GENERATE WARNING: int greater than 5') if(int == 1) stop('GENERATE ERROR: integer is 1') rnorm(5) } Analyse <- function(condition, dat, fixed_objects) { int <- sample(1:10, 1) if(int > 5) warning('ANALYSE WARNING: int greater than 5') if(int == 1) stop('ANALYSE ERROR: int is 1') c(ret = 1) } Summarise <- function(condition, results, fixed_objects) { mean(results) } res <- runSimulation(replications = 100, seed=1234, verbose=FALSE, generate=Generate, analyse=Analyse, summarise=Summarise) res SimExtract(res, what = 'errors') SimExtract(res, what = 'warnings') seeds <- SimExtract(res, what = 'error_seeds') seeds[,1:3] # replicate a specific error for debugging (type Q to exit debugger) res <- runSimulation(replications = 100, load_seed=seeds[,1], debug='analyse', generate=Generate, analyse=Analyse, summarise=Summarise) } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimFunctions.html","id":null,"dir":"Reference","previous_headings":"","what":"Template-based generation of the Generate-Analyse-Summarise functions — SimFunctions","title":"Template-based generation of the Generate-Analyse-Summarise functions — SimFunctions","text":"function prints template versions required Design Generate-Analyse-Summarise functions SimDesign run simulations. Templated output comes complete correct inputs, class outputs, optional comments help initial definitions. Use start Monte Carlo simulation study. Following definition SimDesign template file please refer detailed information runSimulation edit template make working simulation study.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimFunctions.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Template-based generation of the Generate-Analyse-Summarise functions — SimFunctions","text":"","code":"SimFunctions( filename = NULL, dir = getwd(), save_structure = \"single\", extra_file = FALSE, nAnalyses = 1, nGenerate = 1, summarise = TRUE, comments = FALSE, openFiles = TRUE, spin_header = TRUE, SimSolve = FALSE )"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimFunctions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Template-based generation of the Generate-Analyse-Summarise functions — SimFunctions","text":"filename character vector indicating whether output saved two respective files containing simulation design functional components, respectively. Using option generally recommended approach beginning write Monte Carlo simulation dir directory write files . Default working directory save_structure character indicating number files break simulation code filename included (default 'single' one file). save_structure = 'double' output saved two separate files containing functions design definitions, save_structure = '' generate, analyse, summarise, execution code area saved separate files. purpose structure multiple structured files often makes organization debugging slightly easier larger Monte Carlo simulations, though principle files stored single R script extra_file logical; extra file saved containing user-defined functions objects? Default FALSE nAnalyses number analysis functions create (default 1). Increasing value argument independent analysis performed allows function definitions better partitioned potentially modular nGenerate number generate functions create (default 1). Increase value argument data generation functions different isolated (otherwise, much common generate steps, default 1 preferred). Otherwise, nGenerate == 0 generate function provided instead data-generation step can defined analysis function(s) (recommended smaller simulations) summarise include summarise function? Default TRUE comments logical; include helpful comments? Default FALSE openFiles logical; files generated, open text editor (e.g., Rstudio running scripts open new tab)? spin_header logical; include basic knitr::spin header allow simulation knitted? Default TRUE. less familiar spin documents see https://bookdown.org/yihui/rmarkdown-cookbook/spin.html details SimSolve logical; template generated intended SimSolve implementation? Default FALSE","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimFunctions.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Template-based generation of the Generate-Analyse-Summarise functions — SimFunctions","text":"recommended approach organizing Monte Carlo simulation files first save template generated function hard-drive passing suitable filename argument (, users interacting R via RStudio IDE, also open template file saved). larger simulations, two separate files also used (achieved changing .files), may easier debugging/sourcing simulation code; however, matter preference change functionality package.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimFunctions.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Template-based generation of the Generate-Analyse-Summarise functions — SimFunctions","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/SimFunctions.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Template-based generation of the Generate-Analyse-Summarise functions — SimFunctions","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimFunctions.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Template-based generation of the Generate-Analyse-Summarise functions — SimFunctions","text":"","code":"SimFunctions() #> #------------------------------------------------------------------- #> #> library(SimDesign) #> #> Design <- createDesign(factor1 = NA, #> factor2 = NA) #> #> #------------------------------------------------------------------- #> #> Generate <- function(condition, fixed_objects) { #> dat <- data.frame() #> dat #> } #> #> Analyse <- function(condition, dat, fixed_objects) { #> ret <- nc(stat1 = NaN, stat2 = NaN) #> ret #> } #> #> Summarise <- function(condition, results, fixed_objects) { #> ret <- c(bias = NaN, RMSE = NaN) #> ret #> } #> #> #------------------------------------------------------------------- #> #> res <- runSimulation(design=Design, replications=2, generate=Generate, #> analyse=Analyse, summarise=Summarise) #> res #> SimFunctions(comments = TRUE) #with helpful comments #> #------------------------------------------------------------------- #> #> library(SimDesign) #> #> ### Define design conditions #> Design <- createDesign(factor1 = NA, #> factor2 = NA) #> #> #------------------------------------------------------------------- #> #> ### Define essential simulation functions #> #> Generate <- function(condition, fixed_objects) { #> # Define data generation code ... #> #> # Return a vector, matrix, data.frame, or list #> dat <- data.frame() #> dat #> } #> #> Analyse <- function(condition, dat, fixed_objects) { #> # Run statistical analyses of interest ... #> #> # Return a named vector or list #> ret <- nc(stat1 = NaN, stat2 = NaN) #> ret #> } #> #> Summarise <- function(condition, results, fixed_objects) { #> # Summarise the simulation results ... #> #> # Return a named vector of results #> ret <- c(bias = NaN, RMSE = NaN) #> ret #> } #> #> #------------------------------------------------------------------- #> #> ### Run the simulation #> res <- runSimulation(design=Design, replications=2, generate=Generate, #> analyse=Analyse, summarise=Summarise) #> res #> if (FALSE) { # \\dontrun{ # write output files to a single file with comments SimFunctions('mysim', comments = TRUE) # Multiple analysis functions for optional partitioning SimFunctions(nAnalyses = 2) SimFunctions(nAnalyses = 3) # Multiple analysis + generate functions SimFunctions(nAnalyses = 2, nGenerate=2) # save multiple files for the purpose of designing larger simulations # (also include extra_file for user-defined objects/functions) SimFunctions('myBigSim', save_structure = 'all', nAnalyses = 3, nGenerate=2, extra_file = TRUE) } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimResults.html","id":null,"dir":"Reference","previous_headings":"","what":"Function to read in saved simulation results — SimResults","title":"Function to read in saved simulation results — SimResults","text":"runSimulation passed flag save_results = TRUE row results corresponding design object stored suitable sub-directory individual .rds files. users use readRDS directly read files , convenience function read desired rows automatically given returned object simulation. Can used read 1 .rds files (1 file read result stored list).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimResults.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Function to read in saved simulation results — SimResults","text":"","code":"SimResults(obj, which, prefix = \"results-row\", wd = getwd())"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimResults.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Function to read in saved simulation results — SimResults","text":"obj object returned runSimulation save_results = TRUE store_results used. former remaining function arguments can useful reading specific files numeric vector indicating rows read . missing, rows read prefix character indicating prefix used stored files wd working directory; default found getwd.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimResults.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Function to read in saved simulation results — SimResults","text":"returned result either nested list (length() > 1) single list (length() == 1) containing simulation results. read-result refers list 4 elements: condition associate row (ID) conditions respective design object results object returned analyse function, potentially simplified matrix data.frame errors table containing message number errors caused generate-analyse steps rerun. inspected carefully indicate validity issues simulation noted warnings table containing message number non-fatal warnings arose analyse step. inspected carefully indicate validity issues simulation noted","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimResults.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Function to read in saved simulation results — SimResults","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimResults.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Function to read in saved simulation results — SimResults","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimResults.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Function to read in saved simulation results — SimResults","text":"","code":"if (FALSE) { # \\dontrun{ # store results (default behaviour) sim <- runSimulation(..., store_results = TRUE) SimResults(sim) # store results to drive if RAM issues are present obj <- runSimulation(..., save_results = TRUE) # row 1 results row1 <- SimResults(obj, 1) # rows 1:5, stored in a named list rows_1to5 <- SimResults(obj, 1:5) # all results rows_all <- SimResults(obj) } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimShiny.html","id":null,"dir":"Reference","previous_headings":"","what":"Generate a basic Monte Carlo simulation GUI template — SimShiny","title":"Generate a basic Monte Carlo simulation GUI template — SimShiny","text":"function generates suitable stand-alone code shiny package create simple web-interfaces performing single condition Monte Carlo simulations. template generated relatively minimalistic, allows user quickly easily edit saved files customize associated shiny elements see fit.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimShiny.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Generate a basic Monte Carlo simulation GUI template — SimShiny","text":"","code":"SimShiny(filename = NULL, dir = getwd(), design, ...)"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimShiny.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Generate a basic Monte Carlo simulation GUI template — SimShiny","text":"filename optional name text file save server UI components (e.g., 'mysimGUI.R'). omitted, code printed R console instead dir directory write files . Default working directory design design object runSimulation ... arguments passed runSimulation. Note design object used directly, instead provides options selected GUI","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimShiny.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Generate a basic Monte Carlo simulation GUI template — SimShiny","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/SimShiny.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Generate a basic Monte Carlo simulation GUI template — SimShiny","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimShiny.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Generate a basic Monte Carlo simulation GUI template — SimShiny","text":"","code":"if (FALSE) { # \\dontrun{ Design <- createDesign(sample_size = c(30, 60, 90, 120), group_size_ratio = c(1, 4, 8), standard_deviation_ratio = c(.5, 1, 2)) Generate <- function(condition, fixed_objects) { N <- condition$sample_size grs <- condition$group_size_ratio sd <- condition$standard_deviation_ratio if(grs < 1){ N2 <- N / (1/grs + 1) N1 <- N - N2 } else { N1 <- N / (grs + 1) N2 <- N - N1 } group1 <- rnorm(N1) group2 <- rnorm(N2, sd=sd) dat <- data.frame(group = c(rep('g1', N1), rep('g2', N2)), DV = c(group1, group2)) dat } Analyse <- function(condition, dat, fixed_objects) { welch <- t.test(DV ~ group, dat) ind <- t.test(DV ~ group, dat, var.equal=TRUE) # In this function the p values for the t-tests are returned, # and make sure to name each element, for future reference ret <- c(welch = welch$p.value, independent = ind$p.value) ret } Summarise <- function(condition, results, fixed_objects) { #find results of interest here (e.g., alpha < .1, .05, .01) ret <- EDR(results, alpha = .05) ret } # test that it works # Final <- runSimulation(design=Design, replications=5, # generate=Generate, analyse=Analyse, summarise=Summarise) # print code to console SimShiny(design=Design, generate=Generate, analyse=Analyse, summarise=Summarise, verbose=FALSE) # save shiny code to file SimShiny('app.R', design=Design, generate=Generate, analyse=Analyse, summarise=Summarise, verbose=FALSE) # run the application shiny::runApp() shiny::runApp(launch.browser = TRUE) # in web-browser } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimSolve.html","id":null,"dir":"Reference","previous_headings":"","what":"One Dimensional Root (Zero) Finding in Simulation Experiments — SimSolve","title":"One Dimensional Root (Zero) Finding in Simulation Experiments — SimSolve","text":"Function provides stochastic root-finding approach solve specific quantities simulation experiments (e.g., solving specific sample size meet target power rate) using Probablistic Bisection Algorithm Bolstering Interpolations (ProBABLI; Chalmers, 2024). structure follows three functional steps outlined runSimulation, however portions design input taken variables estimated rather fixed, additional constant b required order solve root equation f(x) - b = 0.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimSolve.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"One Dimensional Root (Zero) Finding in Simulation Experiments — SimSolve","text":"","code":"SimSolve( design, interval, b, generate, analyse, summarise, replications = list(burnin.iter = 15L, burnin.reps = 100L, max.reps = 500L, min.total.reps = 9000L, increase.by = 10L), integer = TRUE, formula = y ~ poly(x, 2), family = \"binomial\", parallel = FALSE, cl = NULL, save = TRUE, resume = TRUE, method = \"ProBABLI\", wait.time = NULL, ncores = parallelly::availableCores(omit = 1L), type = ifelse(.Platform$OS.type == \"windows\", \"PSOCK\", \"FORK\"), maxiter = 100L, check.interval = TRUE, verbose = TRUE, control = list(), predCI = 0.95, predCI.tol = NULL, ... ) # S3 method for class 'SimSolve' summary(object, tab.only = FALSE, reps.cutoff = 300, ...) # S3 method for class 'SimSolve' plot(x, y, ...)"},{"path":"http://philchalmers.github.io/SimDesign/reference/SimSolve.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"One Dimensional Root (Zero) Finding in Simulation Experiments — SimSolve","text":"design tibble data.frame object containing Monte Carlo simulation conditions studied, row represents unique condition column factor varied (see createDesign). However, exactly one column object row must specified NA placeholders indicate missing value estimated via select stochastic optimizer interval vector length two, matrix nrow(design) rows two columns, containing end-points interval searched per row condition. vector interval used rows supplied design object b single constant used solve root equation f(x) - b = 0 generate generate function. See runSimulation analyse analysis function. See runSimulation summarise summary function returns single number corresponding function evaluation f(x) equation f(x) = b solved root f(x) - b = 0. Unlike standard runSimulation() definitions input required. information function specification, see runSimulation replications named list vector indicating number replication use design condition per PBA iteration. default input list arguments burnin.iter = 15L, specifying number burn-iterations used, burnin.reps = 100L indicate many replications use burn-iteration, max.reps = 500L prevent replications increasing higher number, min.total.reps = 9000L avoid termination replications explored (lower bound replication budget), increase.= 10L indicate many replications increase burn-stage. Default can overwritten explicit definition (e.g., replications = list(increase.= 25L)). Vector inputs can specify exact replications respective iteration. general rule, early iterations relatively low initial searches avoid unnecessary computations locating approximate location root, number replications gradually increase burn- reduce sampling variability. integer logical; values root considered integer numeric? TRUE bolstered directional decisions made PBA function based collected sampling history formula regression formula use interpolate = TRUE. Default fits orthogonal polynomial degree 2 family family argument passed glm. default 'binomial' family used, function defaults power analysis setups isolated results passed summarise return 0/1s, however families used summarise returns something else (e.g., solving particular standard error 'gaussian' family appropriate). Note individual results analyse steps used (.e., aggregate summarise meaningful) set control = list(summarise.reg_data = TRUE) override default behaviour, thereby using aggregate information weights parallel parallel computing slower simulation experiments (see runSimulation details) cl see runSimulation save logical; store temporary file case crashes. detected working directory automatically loaded resume (see runSimulation similar behaviour) resume logical; temporary SimDesign file detected simulation resume location? Keeping TRUE generally recommended, however disabled using SimSolve within runSimulation avoid reading improper save states method optimizer method use. Default stochastic root-finder 'ProBABLI', can also deterministic options 'Brent' (uses function uniroot) 'bisection' classical bisection method. using deterministic root-finders replications must either equal single constant reflect number replication use per deterministic iteration vector length maxiter indicate replications use per iteration wait.time (optional) argument passed PBA indicate time wait (specified minutes) per row Design object rather using pre-determined termination criteria based estimates. example, three three conditions defined Design, wait.time=\"5\", total search time till terminate 15 minutes regardless independently specified termination criteria control. See timeFormater alternative specifications ncores see runSimulation type type cluster (see makeCluster) plotting type use. type used plot can 'density' plot density iteration history burn-stage, 'iterations' bubble plot inverse replication weights. specified default PBA plots provided (see PBA) maxiter maximum number iterations (default 100) except wait.time specified (automatically increased 3000 avoid early termination) check.interval logical; initial check made determine whether f(interval[1L]) f(interval[2L]) opposite signs? FALSE, specified interval assumed contain root, f(interval[1]) < 0 f(interval[2] > 0. Default TRUE verbose logical; print information console? control list algorithm control parameters. specified, defaults described used. tol tolerance criteria early termination (.1 integer = TRUE searches; .00025 non-integer searches rel.tol relative tolerance criteria early termination (default .0001) k.success number consecutive tolerance successes given rel.tol tol criteria (default 3) bolster logical; PBA evaluations use bolstering based previous evaluations? Default TRUE, though applicable integer = TRUE interpolate.R number replications collect prior performing interpolation step (default 3000 accounting data exclusion burnin.iter). Setting 0 disable interpolation computations include_reps logical; include column condition elements indicate many replications currently evaluated? Mainly useful tuning within ProBABLI iteration desirable (e.g., increasing/decreasing bootstrap draws search progresses). Default FALSE summarise.reg_data logical; aggregate results Summarise (along associated weights) used interpolation steps, raw data Analyse step? Set TRUE individual results Analyse give less meaningful information predCI advertised confidence interval probability final model-based prediction target b given root input estimate. Returned element summary() list output predCI.tol (optional) rather relying changes successive estimates (default), predicting CI consistently within supplied tolerance range search terminated. provides termination behaviour based predicted precision root solutions rather stability history, therefore can used obtain estimates particular level advertised accuracy. example, solving sample size value (N) solution associated b = .80 requires advertised 95 consistently [.795, .805] predCI.tol = .01 used reflect tolerance range ... additional arguments pasted PBA object object class 'SimSolve' tab.logical; print (reduced) table estimates? reps.cutoff integer indicating rows omit output number replications less value x object class 'SimSolve' y design row plot. omitted defaults 1","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimSolve.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"One Dimensional Root (Zero) Finding in Simulation Experiments — SimSolve","text":"filled-design object containing associated lower upper interval estimates stochastic optimization","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimSolve.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"One Dimensional Root (Zero) Finding in Simulation Experiments — SimSolve","text":"Root finding performed using progressively bolstered version probabilistic bisection algorithm (PBA) find associated root given noisy simulation objective function evaluations. Information collected throughout search make accurate predictions associated root via interpolation. interpolations fail, last iteration PBA search returned best guess. greater advertised accuracy ProBABLI, termination criteria can based width advertised predicting interval (via predCI.tol) specifying long investigator willing wait final estimates (via wait.time, longer wait times lead progressively better accuracy final estimates).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimSolve.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"One Dimensional Root (Zero) Finding in Simulation Experiments — SimSolve","text":"Chalmers, R. P. (2024). Solving Variables Monte Carlo Simulation Experiments: Stochastic Root-Solving Approach. Psychological Methods. DOI: 10.1037/met0000689 Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/SimSolve.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"One Dimensional Root (Zero) Finding in Simulation Experiments — SimSolve","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/SimSolve.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"One Dimensional Root (Zero) Finding in Simulation Experiments — SimSolve","text":"","code":"if (FALSE) { # \\dontrun{ ########################## ## A Priori Power Analysis ########################## # GOAL: Find specific sample size in each group for independent t-test # corresponding to a power rate of .8 # # For ease of the setup, assume the groups are the same size, and the mean # difference corresponds to Cohen's d values of .2, .5, and .8 # This example can be solved numerically using the pwr package (see below), # though the following simulation setup is far more general and can be # used for any generate-analyse combination of interest # SimFunctions(SimSolve=TRUE) #### Step 1 --- Define your conditions under study and create design data.frame. #### However, use NA placeholder for sample size as it must be solved, #### and add desired power rate to object Design <- createDesign(N = NA, d = c(.2, .5, .8), sig.level = .05) Design # solve for NA's #~~~~~~~~~~~~~~~~~~~~~~~~ #### Step 2 --- Define generate, analyse, and summarise functions Generate <- function(condition, fixed_objects) { Attach(condition) group1 <- rnorm(N) group2 <- rnorm(N, mean=d) dat <- data.frame(group = gl(2, N, labels=c('G1', 'G2')), DV = c(group1, group2)) dat } Analyse <- function(condition, dat, fixed_objects) { p <- t.test(DV ~ group, dat, var.equal=TRUE)$p.value p } Summarise <- function(condition, results, fixed_objects) { # Must return a single number corresponding to f(x) in the # root equation f(x) = b ret <- c(power = EDR(results, alpha = condition$sig.level)) ret } #~~~~~~~~~~~~~~~~~~~~~~~~ #### Step 3 --- Optimize N over the rows in design ### (For debugging) may want to see if simulation code works as intended first ### for some given set of inputs # runSimulation(design=createDesign(N=100, d=.8, sig.level=.05), # replications=10, generate=Generate, analyse=Analyse, # summarise=Summarise) # Initial search between N = [10,500] for each row using the default # integer solver (integer = TRUE). In this example, b = target power solved <- SimSolve(design=Design, b=.8, interval=c(10, 500), generate=Generate, analyse=Analyse, summarise=Summarise) solved summary(solved) plot(solved, 1) plot(solved, 2) plot(solved, 3) # also can plot median history and estimate precision plot(solved, 1, type = 'history') plot(solved, 1, type = 'density') plot(solved, 1, type = 'iterations') # verify with true power from pwr package library(pwr) pwr.t.test(d=.2, power = .8) # sig.level/alpha = .05 by default pwr.t.test(d=.5, power = .8) pwr.t.test(d=.8, power = .8) # use estimated N results to see how close power was N <- solved$N pwr.t.test(d=.2, n=N[1]) pwr.t.test(d=.5, n=N[2]) pwr.t.test(d=.8, n=N[3]) # with rounding N <- ceiling(solved$N) pwr.t.test(d=.2, n=N[1]) pwr.t.test(d=.5, n=N[2]) pwr.t.test(d=.8, n=N[3]) ### failing analytic formula, confirm results with more precise ### simulation via runSimulation() ### (not required, if accuracy is important then ProBABLI should be run longer) # csolved <- solved # csolved$N <- ceiling(solved$N) # confirm <- runSimulation(design=csolved, replications=10000, parallel=TRUE, # generate=Generate, analyse=Analyse, # summarise=Summarise) # confirm # Similarly, terminate if the prediction interval is consistently predicted # to be between [.795, .805]. Note that maxiter increased as well solved_predCI <- SimSolve(design=Design, b=.8, interval=c(10, 500), generate=Generate, analyse=Analyse, summarise=Summarise, maxiter=200, predCI.tol=.01) solved_predCI summary(solved_predCI) # note that predCI.b are all within [.795, .805] N <- solved_predCI$N pwr.t.test(d=.2, n=N[1]) pwr.t.test(d=.5, n=N[2]) pwr.t.test(d=.8, n=N[3]) # Alternatively, and often more realistically, wait.time can be used # to specify how long the user is willing to wait for a final estimate. # Solutions involving more iterations will be more accurate, # and therefore it is recommended to run the ProBABLI root-solver as long # the analyst can tolerate if the most accurate estimates are desired. # Below executes the simulation for 2 minutes per condition solved_2min <- SimSolve(design=Design[1, ], b=.8, interval=c(10, 500), generate=Generate, analyse=Analyse, summarise=Summarise, wait.time=\"2\") solved_2min summary(solved_2min) # use estimated N results to see how close power was N <- solved_2min$N pwr.t.test(d=.2, n=N[1]) #------------------------------------------------ ####################### ## Sensitivity Analysis ####################### # GOAL: solve effect size d given sample size and power inputs (inputs # for root no longer required to be an integer) # Generate-Analyse-Summarise functions identical to above, however # Design input includes NA for d element Design <- createDesign(N = c(100, 50, 25), d = NA, sig.level = .05) Design # solve for NA's #~~~~~~~~~~~~~~~~~~~~~~~~ #### Step 2 --- Define generate, analyse, and summarise functions (same as above) #~~~~~~~~~~~~~~~~~~~~~~~~ #### Step 3 --- Optimize d over the rows in design # search between d = [.1, 2] for each row # In this example, b = target power # note that integer = FALSE to allow smooth updates of d solved <- SimSolve(design=Design, b = .8, interval=c(.1, 2), generate=Generate, analyse=Analyse, summarise=Summarise, integer=FALSE) solved summary(solved) plot(solved, 1) plot(solved, 2) plot(solved, 3) # plot median history and estimate precision plot(solved, 1, type = 'history') plot(solved, 1, type = 'density') plot(solved, 1, type = 'iterations') # verify with true power from pwr package library(pwr) pwr.t.test(n=100, power = .8) pwr.t.test(n=50, power = .8) pwr.t.test(n=25, power = .8) # use estimated d results to see how close power was pwr.t.test(n=100, d = solved$d[1]) pwr.t.test(n=50, d = solved$d[2]) pwr.t.test(n=25, d = solved$d[3]) ### failing analytic formula, confirm results with more precise ### simulation via runSimulation() (not required; if accuracy is important ### PROBABLI should just be run longer) # confirm <- runSimulation(design=solved, replications=10000, parallel=TRUE, # generate=Generate, analyse=Analyse, # summarise=Summarise) # confirm #------------------------------------------------ ##################### ## Criterion Analysis ##################### # GOAL: solve Type I error rate (alpha) given sample size, effect size, and # power inputs (inputs for root no longer required to be an integer). Only useful # when Type I error is less important than achieving the desired 1-beta (power) Design <- createDesign(N = 50, d = c(.2, .5, .8), sig.level = NA) Design # solve for NA's # all other function definitions same as above # search for alpha within [.0001, .8] solved <- SimSolve(design=Design, b = .8, interval=c(.0001, .8), generate=Generate, analyse=Analyse, summarise=Summarise, integer=FALSE) solved summary(solved) plot(solved, 1) plot(solved, 2) plot(solved, 3) # plot median history and estimate precision plot(solved, 1, type = 'history') plot(solved, 1, type = 'density') plot(solved, 1, type = 'iterations') # verify with true power from pwr package library(pwr) pwr.t.test(n=50, power = .8, d = .2, sig.level=NULL) pwr.t.test(n=50, power = .8, d = .5, sig.level=NULL) pwr.t.test(n=50, power = .8, d = .8, sig.level=NULL) # use estimated alpha results to see how close power was pwr.t.test(n=50, d = .2, sig.level=solved$sig.level[1]) pwr.t.test(n=50, d = .5, sig.level=solved$sig.level[2]) pwr.t.test(n=50, d = .8, sig.level=solved$sig.level[3]) ### failing analytic formula, confirm results with more precise ### simulation via runSimulation() (not required; if accuracy is important ### PROBABLI should just be run longer) # confirm <- runSimulation(design=solved, replications=10000, parallel=TRUE, # generate=Generate, analyse=Analyse, # summarise=Summarise) # confirm } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/Summarise.html","id":null,"dir":"Reference","previous_headings":"","what":"Summarise simulated data using various population comparison statistics — Summarise","title":"Summarise simulated data using various population comparison statistics — Summarise","text":"collapses simulation results within condition composite estimates RMSE, bias, Type error rates, coverage rates, etc. See See Also section useful functions used within Summarise.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Summarise.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Summarise simulated data using various population comparison statistics — Summarise","text":"","code":"Summarise(condition, results, fixed_objects)"},{"path":"http://philchalmers.github.io/SimDesign/reference/Summarise.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Summarise simulated data using various population comparison statistics — Summarise","text":"condition single row design input runSimulation (data.frame), indicating simulation conditions results tibble data frame (Analyse returned named numeric vector length) list (Analyse returned list multi-rowed data.frame) containing analysis results Analyse, cell stored unique row/list element fixed_objects object passed runSimulation","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Summarise.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Summarise simulated data using various population comparison statistics — Summarise","text":"best results return named numeric vector data.frame desired meta-simulation results. Named list objects can also returned, however subsequent results must extracted via SimExtract","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/Summarise.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Summarise simulated data using various population comparison statistics — Summarise","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/Summarise.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Summarise simulated data using various population comparison statistics — Summarise","text":"","code":"if (FALSE) { # \\dontrun{ summarise <- function(condition, results, fixed_objects) { #find results of interest here (alpha < .1, .05, .01) lessthan.05 <- EDR(results, alpha = .05) # return the results that will be appended to the design input ret <- c(lessthan.05=lessthan.05) ret } } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/addMissing.html","id":null,"dir":"Reference","previous_headings":"","what":"Add missing values to a vector given a MCAR, MAR, or MNAR scheme — addMissing","title":"Add missing values to a vector given a MCAR, MAR, or MNAR scheme — addMissing","text":"Given input vector, replace elements vector missing values according scheme. Default method replaces input values MCAR scheme (average 10% values replaced NAs). MAR MNAR supported replacing default FUN argument.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/addMissing.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Add missing values to a vector given a MCAR, MAR, or MNAR scheme — addMissing","text":"","code":"addMissing(y, fun = function(y, rate = 0.1, ...) rep(rate, length(y)), ...)"},{"path":"http://philchalmers.github.io/SimDesign/reference/addMissing.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Add missing values to a vector given a MCAR, MAR, or MNAR scheme — addMissing","text":"y input vector contain missing data form NA's fun user defined function indicating missing data mechanism element y. Function must return vector probability values length equal length y. value returned vector indicates probability respective element y replaced NA. Function must contain argument y, representing input vector, however number additional arguments can included ... additional arguments passed FUN","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/addMissing.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Add missing values to a vector given a MCAR, MAR, or MNAR scheme — addMissing","text":"input vector y sampled NA values (according FUN scheme)","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/addMissing.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Add missing values to a vector given a MCAR, MAR, or MNAR scheme — addMissing","text":"Given input vector y, relevant variables inside (X) outside (Z) data-set, three types missingness : MCAR Missing completely random (MCAR). realized randomly sampling values input vector (y) irrespective possible values X Z. Therefore missing values randomly sampled depend data characteristics truly random MAR Missing random (MAR). realized values dataset (X) predict missing data mechanism y; conceptually equivalent \\(P(y = NA | X)\\). requires user define custom missing data function MNAR Missing random (MNAR). similar MAR except missing mechanism comes value y variables outside working dataset; conceptually equivalent \\(P(y = NA | X, Z, y)\\). requires user define custom missing data function","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/addMissing.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Add missing values to a vector given a MCAR, MAR, or MNAR scheme — addMissing","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/addMissing.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Add missing values to a vector given a MCAR, MAR, or MNAR scheme — addMissing","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/addMissing.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Add missing values to a vector given a MCAR, MAR, or MNAR scheme — addMissing","text":"","code":"if (FALSE) { # \\dontrun{ set.seed(1) y <- rnorm(1000) ## 10% missing rate with default FUN head(ymiss <- addMissing(y), 10) ## 50% missing with default FUN head(ymiss <- addMissing(y, rate = .5), 10) ## missing values only when female and low X <- data.frame(group = sample(c('male', 'female'), 1000, replace=TRUE), level = sample(c('high', 'low'), 1000, replace=TRUE)) head(X) fun <- function(y, X, ...){ p <- rep(0, length(y)) p[X$group == 'female' & X$level == 'low'] <- .2 p } ymiss <- addMissing(y, X, fun=fun) tail(cbind(ymiss, X), 10) ## missingness as a function of elements in X (i.e., a type of MAR) fun <- function(y, X){ # missingness with a logistic regression approach df <- data.frame(y, X) mm <- model.matrix(y ~ group + level, df) cfs <- c(-5, 2, 3) #intercept, group, and level coefs z <- cfs %*% t(mm) plogis(z) } ymiss <- addMissing(y, X, fun=fun) tail(cbind(ymiss, X), 10) ## missing values when y elements are large (i.e., a type of MNAR) fun <- function(y) ifelse(abs(y) > 1, .4, 0) ymiss <- addMissing(y, fun=fun) tail(cbind(y, ymiss), 10) } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/bias.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute (relative/standardized) bias summary statistic — bias","title":"Compute (relative/standardized) bias summary statistic — bias","text":"Computes (relative) bias sample estimate parameter value. Accepts estimate parameter values, well estimate values deviation form. relative bias requested estimate parameter inputs required.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/bias.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute (relative/standardized) bias summary statistic — bias","text":"","code":"bias( estimate, parameter = NULL, type = \"bias\", abs = FALSE, percent = FALSE, unname = FALSE )"},{"path":"http://philchalmers.github.io/SimDesign/reference/bias.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute (relative/standardized) bias summary statistic — bias","text":"estimate numeric vector, matrix/data.frame, list parameter estimates. vector, length equal number replications. matrix/data.frame, number rows must equal number replications. list objects looped using rules first translating information one-dimensional vectors re-creating structure upon return parameter numeric scalar/vector indicating fixed parameters. single value supplied estimate matrix/data.frame value recycled column; otherwise, element associated respective column estimate input. NULL assumed estimate input deviation form (therefore mean(estimate)) returned) type type bias statistic return. Default ('bias') computes standard bias (average difference sample population), 'relative' computes relative bias statistic (.e., divide bias value parameter; note multiplying 100 gives \"percent bias\" measure, Type error rates (\\(\\alpha\\)) supplied result \"percentage error\"), 'abs_relative' computes relative bias absolute values parameters used denominator rather (potentially) signed input values, 'standardized' computes standardized bias estimate (standard bias divided standard deviation sample estimates) abs logical; find absolute bias parameters estimates? effectively just applies abs transformation returned result. Default FALSE percent logical; change returned result percentage multiplying 100? Default FALSE unname logical; apply unname results remove variable names?","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/bias.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute (relative/standardized) bias summary statistic — bias","text":"returns numeric vector indicating overall (relative/standardized) bias estimates","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/bias.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Compute (relative/standardized) bias summary statistic — bias","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/bias.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compute (relative/standardized) bias summary statistic — bias","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/bias.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Compute (relative/standardized) bias summary statistic — bias","text":"","code":"pop <- 2 samp <- rnorm(100, 2, sd = 0.5) bias(samp, pop) #> [1] 0.0398212 bias(samp, pop, type = 'relative') #> [1] 0.0199106 bias(samp, pop, type = 'standardized') #> [1] 0.0800926 dev <- samp - pop bias(dev) #> [1] 0.0398212 # equivalent here bias(mean(samp), pop) #> [1] 0.0398212 # matrix input mat <- cbind(M1=rnorm(100, 2, sd = 0.5), M2 = rnorm(100, 2, sd = 1)) bias(mat, parameter = 2) #> M1 M2 #> -0.01850358 0.07757825 bias(mat, parameter = 2, type = 'relative') #> M1 M2 #> -0.00925179 0.03878912 bias(mat, parameter = 2, type = 'standardized') #> M1 M2 #> -0.03281049 0.08895343 # different parameter associated with each column mat <- cbind(M1=rnorm(1000, 2, sd = 0.25), M2 = rnorm(1000, 3, sd = .25)) bias(mat, parameter = c(2,3)) #> M1 M2 #> -0.005719856 -0.003655121 # same, but with data.frame df <- data.frame(M1=rnorm(100, 2, sd = 0.5), M2 = rnorm(100, 2, sd = 1)) bias(df, parameter = c(2,2)) #> M1 M2 #> 0.04919156 0.07691533 # parameters of the same size parameters <- 1:10 estimates <- parameters + rnorm(10) bias(estimates, parameters) #> [1] -0.03489829 # relative difference dividing by the magnitude of parameters bias(estimates, parameters, type = 'abs_relative') #> [1] -0.1708617 # relative bias as a percentage bias(estimates, parameters, type = 'abs_relative', percent = TRUE) #> [1] -17.08617 # percentage error (PE) statistic given alpha (Type I error) and EDR() result # edr <- EDR(results, alpha = .05) edr <- c(.04, .05, .06, .08) bias(matrix(edr, 1L), .05, type = 'relative', percent = TRUE) #> [1] -20 0 20 60"},{"path":"http://philchalmers.github.io/SimDesign/reference/bootPredict.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute prediction estimates for the replication size using bootstrap MSE estimates — bootPredict","title":"Compute prediction estimates for the replication size using bootstrap MSE estimates — bootPredict","text":"function computes bootstrap mean-square error estimates approximate sampling behavior meta-statistics SimDesign's summarise functions. single design condition supplied, simulation max(Rstar) replications performed whereby generate-analyse results collected. obtaining replication values, replications drawn (replacement) using differing sizes Rstar approximate bootstrap MSE behavior given different replication sizes. Finally, given bootstrap estimates linear regression models fitted using predictor term one_sqrtR = 1 / sqrt(Rstar) allow extrapolation replication sizes observed Rstar. information method subsequent bootstrap MSE plots, refer Koehler, Brown, Haneuse (2009).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/bootPredict.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute prediction estimates for the replication size using bootstrap MSE estimates — bootPredict","text":"","code":"bootPredict( condition, generate, analyse, summarise, fixed_objects = NULL, ..., Rstar = seq(100, 500, by = 100), boot_draws = 1000 ) boot_predict(...)"},{"path":"http://philchalmers.github.io/SimDesign/reference/bootPredict.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute prediction estimates for the replication size using bootstrap MSE estimates — bootPredict","text":"condition data.frame consisting one row original design input object used within runSimulation generate see runSimulation analyse see runSimulation summarise see runSimulation fixed_objects see runSimulation ... additional arguments passed runSimulation Rstar vector containing size bootstrap subsets obtain. Default investigates vector [100, 200, 300, 400, 500] compute respective MSE terms boot_draws number bootstrap replications draw. Default 1000","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/bootPredict.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute prediction estimates for the replication size using bootstrap MSE estimates — bootPredict","text":"returns list linear model objects (via lm) meta-statistics returned summarise() function","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/bootPredict.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Compute prediction estimates for the replication size using bootstrap MSE estimates — bootPredict","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Koehler, E., Brown, E., & Haneuse, S. J.-P. . (2009). Assessment Monte Carlo Error Simulation-Based Statistical Analyses. American Statistician, 63, 155-162. Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/bootPredict.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compute prediction estimates for the replication size using bootstrap MSE estimates — bootPredict","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/bootPredict.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Compute prediction estimates for the replication size using bootstrap MSE estimates — bootPredict","text":"","code":"set.seed(4321) Design <- createDesign(sigma = c(1, 2)) #------------------------------------------------------------------- Generate <- function(condition, fixed_objects) { dat <- rnorm(100, 0, condition$sigma) dat } Analyse <- function(condition, dat, fixed_objects) { CIs <- t.test(dat)$conf.int names(CIs) <- c('lower', 'upper') ret <- c(mean = mean(dat), CIs) ret } Summarise <- function(condition, results, fixed_objects) { ret <- c(mu_bias = bias(results[,\"mean\"], 0), mu_coverage = ECR(results[,c(\"lower\", \"upper\")], parameter = 0)) ret } if (FALSE) { # \\dontrun{ # boot_predict supports only one condition at a time out <- bootPredict(condition=Design[1L, , drop=FALSE], generate=Generate, analyse=Analyse, summarise=Summarise) out # list of fitted linear model(s) # extract first meta-statistic mu_bias <- out$mu_bias dat <- model.frame(mu_bias) print(dat) # original R metric plot R <- 1 / dat$one_sqrtR^2 plot(R, dat$MSE, type = 'b', ylab = 'MSE', main = \"Replications by MSE\") plot(MSE ~ one_sqrtR, dat, main = \"Bootstrap prediction plot\", xlim = c(0, max(one_sqrtR)), ylim = c(0, max(MSE)), ylab = 'MSE', xlab = expression(1/sqrt(R))) beta <- coef(mu_bias) abline(a = 0, b = beta, lty = 2, col='red') # what is the replication value when x-axis = .02? What's its associated expected MSE? 1 / .02^2 # number of replications predict(mu_bias, data.frame(one_sqrtR = .02)) # y-axis value # approximately how many replications to obtain MSE = .001? (beta / .001)^2 } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/clusterSetRNGSubStream.html","id":null,"dir":"Reference","previous_headings":"","what":"Set RNG sub-stream for Pierre L'Ecuyer's RngStreams — clusterSetRNGSubStream","title":"Set RNG sub-stream for Pierre L'Ecuyer's RngStreams — clusterSetRNGSubStream","text":"Sets sub-stream RNG state within Pierre L'Ecuyer's (1999) algorithm. used within distributed array jobs suitable L'Ecuyer's (1999) distributed array, array defined use multi-core processing. See clusterSetRNGStream information.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/clusterSetRNGSubStream.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Set RNG sub-stream for Pierre L'Ecuyer's RngStreams — clusterSetRNGSubStream","text":"","code":"clusterSetRNGSubStream(cl, seed)"},{"path":"http://philchalmers.github.io/SimDesign/reference/clusterSetRNGSubStream.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Set RNG sub-stream for Pierre L'Ecuyer's RngStreams — clusterSetRNGSubStream","text":"cl cluster parallel snow package, (NULL) registered cluster seed integer vector length 7 given .Random.seed L'Ecuyer-CMR RNG use. SeeRNG valid values","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/clusterSetRNGSubStream.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Set RNG sub-stream for Pierre L'Ecuyer's RngStreams — clusterSetRNGSubStream","text":"invisible NULL","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/colVars.html","id":null,"dir":"Reference","previous_headings":"","what":"Form Column Standard Deviation and Variances — colVars","title":"Form Column Standard Deviation and Variances — colVars","text":"Form column standard deviation variances numeric arrays (data frames).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/colVars.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Form Column Standard Deviation and Variances — colVars","text":"","code":"colVars(x, na.rm = FALSE, unname = FALSE) colSDs(x, na.rm = FALSE, unname = FALSE)"},{"path":"http://philchalmers.github.io/SimDesign/reference/colVars.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Form Column Standard Deviation and Variances — colVars","text":"x array two dimensions containing numeric, complex, integer logical values, numeric data frame na.rm logical; remove missing values respective column? unname logical; apply unname results remove variable names?","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/colVars.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Form Column Standard Deviation and Variances — colVars","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/colVars.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Form Column Standard Deviation and Variances — colVars","text":"","code":"results <- matrix(rnorm(100), ncol=4) colnames(results) <- paste0('stat', 1:4) colVars(results) #> stat1 stat2 stat3 stat4 #> 0.7607727 0.4891335 0.7170886 1.3992792 colSDs(results) #> stat1 stat2 stat3 stat4 #> 0.8722229 0.6993808 0.8468108 1.1829113 results[1,1] <- NA colSDs(results) #> stat1 stat2 stat3 stat4 #> NA 0.6993808 0.8468108 1.1829113 colSDs(results, na.rm=TRUE) #> stat1 stat2 stat3 stat4 #> 0.8855740 0.6993808 0.8468108 1.1829113 colSDs(results, na.rm=TRUE, unname=TRUE) #> [1] 0.8855740 0.6993808 0.8468108 1.1829113"},{"path":"http://philchalmers.github.io/SimDesign/reference/createDesign.html","id":null,"dir":"Reference","previous_headings":"","what":"Create the simulation design object — createDesign","title":"Create the simulation design object — createDesign","text":"Create partially fully-crossed data object reflecting unique simulation design conditions. row returned object represents unique simulation condition, column represents named factor variables study.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/createDesign.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create the simulation design object — createDesign","text":"","code":"createDesign( ..., subset, fractional = NULL, tibble = TRUE, stringsAsFactors = FALSE ) # S3 method for class 'Design' print(x, list2char = TRUE, pillar.sigfig = 5, ...)"},{"path":"http://philchalmers.github.io/SimDesign/reference/createDesign.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create the simulation design object — createDesign","text":"... comma separated list named input objects representing simulation factors completely cross. Note arguments passed expand.grid perform complete crossings subset (optional) logical vector indicating elements rows keep create partially crossed simulation design fractional fractional design matrix returned FrF2 package. Note order factor names/labels associated respective ... inputs tibble logical; return tibble object instead data.frame? Default TRUE stringsAsFactors logical; character variable inputs coerced factors building data.frame? Default FALSE x object returned createDesign list2char logical; tibble object re-evaluate list elements character vectors better printing levels? Note change original classes object, just printed. Default TRUE pillar.sigfig number significant digits print. Default 5","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/createDesign.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create the simulation design object — createDesign","text":"tibble data.frame containing simulation experiment conditions evaluated runSimulation","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/createDesign.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Create the simulation design object — createDesign","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/createDesign.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Create the simulation design object — createDesign","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/createDesign.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create the simulation design object — createDesign","text":"","code":"if (FALSE) { # \\dontrun{ # modified example from runSimulation() Design <- createDesign(N = c(10, 20), SD = c(1, 2)) Design # remove N=10, SD=2 row from initial definition Design <- createDesign(N = c(10, 20), SD = c(1, 2), subset = !(N == 10 & SD == 2)) Design # example with list inputs Design <- createDesign(N = c(10, 20), SD = c(1, 2), combo = list(c(0,0), c(0,0,1))) Design # notice levels printed (not typical for tibble) print(Design, list2char = FALSE) # standard tibble output Design <- createDesign(N = c(10, 20), SD = c(1, 2), combo = list(c(0,0), c(0,0,1)), combo2 = list(c(5,10,5), c(6,7))) Design print(Design, list2char = FALSE) # standard tibble output ########## ## fractional factorial example library(FrF2) # help(FrF2) # 7 factors in 32 runs fr <- FrF2(32,7) dim(fr) fr[1:6,] # Create working simulation design given -1/1 combinations fDesign <- createDesign(sample_size=c(100,200), mean_diff=c(.25, 1, 2), variance.ratio=c(1,4, 8), equal_size=c(TRUE, FALSE), dists=c('norm', 'skew'), same_dists=c(TRUE, FALSE), symmetric=c(TRUE, FALSE), # remove same-normal combo subset = !(symmetric & dists == 'norm'), fractional=fr) fDesign } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/expandDesign.html","id":null,"dir":"Reference","previous_headings":"","what":"Create the simulation design object — expandDesign","title":"Create the simulation design object — expandDesign","text":"Repeat design row specified number times. primarily used cluster computing jobs distributed batches replications later aggregated complete simulation object (see runArraySimulation SimCollect).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/expandDesign.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create the simulation design object — expandDesign","text":"","code":"expandDesign(Design, repeat_conditions)"},{"path":"http://philchalmers.github.io/SimDesign/reference/expandDesign.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create the simulation design object — expandDesign","text":"Design object created createDesign rows repeated optimal HPC schedulers repeat_conditions integer vector used repeat design row specified number times. Can either single integer, repeats row many times, integer vector equal number total rows created object. argument useful distributing independent row conditions cluster computing environments, particularly different replication information. example, 1000 replications total target condition repeated 4 rows 250 replications per row required across repeated conditions. See SimCollect combining simulation objects complete","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/expandDesign.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create the simulation design object — expandDesign","text":"tibble data.frame containing simulation experiment conditions evaluated runSimulation","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/expandDesign.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Create the simulation design object — expandDesign","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/expandDesign.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Create the simulation design object — expandDesign","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/expandDesign.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create the simulation design object — expandDesign","text":"","code":"if (FALSE) { # \\dontrun{ # repeat each row 4 times (for cluster computing) Design <- createDesign(N = c(10, 20), SD.equal = c(TRUE, FALSE)) Design4 <- expandDesign(Design, 4) Design4 # repeat first two rows 2x and the rest 4 times (for cluster computing # where first two conditions are faster to execute) Design <- createDesign(SD.equal = c(TRUE, FALSE), N = c(10, 100, 1000)) Design24 <- expandDesign(Design, c(2,2,rep(4, 4))) Design24 } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/genSeeds.html","id":null,"dir":"Reference","previous_headings":"","what":"Generate random seeds — genSeeds","title":"Generate random seeds — genSeeds","text":"Generate seeds passed runSimulation's seed input. Values sampled 1 2147483647, generated using L'Ecuyer-CMRG's (2002) method (returning either list arrayID omitted, specific row value list arrayID included).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/genSeeds.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Generate random seeds — genSeeds","text":"","code":"genSeeds(design = 1L, iseed = NULL, arrayID = NULL, old.seeds = NULL) gen_seeds(...)"},{"path":"http://philchalmers.github.io/SimDesign/reference/genSeeds.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Generate random seeds — genSeeds","text":"design design matrix requires unique seed per condition, number indicating number seeds generate. Default generates one number iseed initial set.seed number used generate sequence independent seeds according L'Ecuyer-CMRG (2002) method. recommended whenever quality random number generation required across similar (identical) simulation jobs (e.g., see runArraySimulation). arrayID specified return list associated seed full design arrayID (optional) single integer input corresponding specific row design object using iseed input. used functions runArraySimulation pull specific seed rather manage complete list, therefore memory efficient old.seeds (optional) vector matrix last seeds used previous simulations avoid repeating seed subsequent run. Note approach used sparingly seeds set frequently likely correlate, therefore provide less optimal random number behaviour (e.g., performing simulation two runs achieve 5000 * 2 = 10,000 replications likely reasonable, simulations 100 * 2 = 200 replications likely sub-optimal). Length must equal number rows design ... nothing","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/genSeeds.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Generate random seeds — genSeeds","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/genSeeds.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Generate random seeds — genSeeds","text":"","code":"# generate 1 seed (default) genSeeds() #> [1] 1243352586 # generate 5 unique seeds genSeeds(5) #> [1] 886678571 865383938 1718793689 1802720933 413067025 # generate from nrow(design) design <- createDesign(factorA=c(1,2,3), factorB=letters[1:3]) seeds <- genSeeds(design) seeds #> [1] 595688410 1809449992 1887503261 1000739687 1129367556 717434384 1123050453 #> [8] 474408704 1310164663 # construct new seeds that are independent from original (use this sparingly) newseeds <- genSeeds(design, old.seeds=seeds) newseeds #> [1] 1607262655 390300427 731677967 298354824 27984649 225437218 746949893 #> [8] 1602927401 937942413 # can be done in batches too newseeds2 <- genSeeds(design, old.seeds=cbind(seeds, newseeds)) cbind(seeds, newseeds, newseeds2) # all unique #> seeds newseeds newseeds2 #> [1,] 595688410 1607262655 992053320 #> [2,] 1809449992 390300427 1416086442 #> [3,] 1887503261 731677967 823141978 #> [4,] 1000739687 298354824 1246508874 #> [5,] 1129367556 27984649 1619914930 #> [6,] 717434384 225437218 1610796999 #> [7,] 1123050453 746949893 187037883 #> [8,] 474408704 1602927401 1156242821 #> [9,] 1310164663 937942413 2079951069 ############ # generate seeds for runArraySimulation() (iseed <- genSeeds()) # initial seed #> [1] 65720644 seed_list <- genSeeds(design, iseed=iseed) seed_list #> [[1]] #> [1] 10407 379889883 659117664 -2055402783 1234182958 1566802391 #> [7] 1478353964 #> #> [[2]] #> [1] 10407 -1996363988 993937887 1454248844 -1855571605 1976941944 #> [7] -1338551852 #> #> [[3]] #> [1] 10407 -711630543 1740484843 1339480259 -153618978 -1565624872 #> [7] 1457706395 #> #> [[4]] #> [1] 10407 -103215903 -1613204698 379013845 -1130009889 244099792 #> [7] 120529920 #> #> [[5]] #> [1] 10407 67254529 1072668082 1044634035 -767520122 -2092261128 #> [7] -2098310082 #> #> [[6]] #> [1] 10407 520068102 -1461006084 -1895507394 1038491043 168622900 #> [7] -1147596130 #> #> [[7]] #> [1] 10407 1582176084 -312633288 -1779753696 -1577380533 -1192542975 #> [7] 626958622 #> #> [[8]] #> [1] 10407 1481948650 -1459016231 -1192828505 957288152 959321237 #> [7] 791951982 #> #> [[9]] #> [1] 10407 -249485550 -1214357330 -2043404912 1753729054 -154685538 #> [7] 1051389303 #> #> attr(,\"iseed\") #> [1] 65720644 # expand number of unique seeds given iseed (e.g., in case more replications # are required at a later date) seed_list_tmp <- genSeeds(nrow(design)*2, iseed=iseed) str(seed_list_tmp) # first 9 seeds identical to seed_list #> List of 18 #> $ : int [1:7] 10407 379889883 659117664 -2055402783 1234182958 1566802391 1478353964 #> $ : int [1:7] 10407 -1996363988 993937887 1454248844 -1855571605 1976941944 -1338551852 #> $ : int [1:7] 10407 -711630543 1740484843 1339480259 -153618978 -1565624872 1457706395 #> $ : int [1:7] 10407 -103215903 -1613204698 379013845 -1130009889 244099792 120529920 #> $ : int [1:7] 10407 67254529 1072668082 1044634035 -767520122 -2092261128 -2098310082 #> $ : int [1:7] 10407 520068102 -1461006084 -1895507394 1038491043 168622900 -1147596130 #> $ : int [1:7] 10407 1582176084 -312633288 -1779753696 -1577380533 -1192542975 626958622 #> $ : int [1:7] 10407 1481948650 -1459016231 -1192828505 957288152 959321237 791951982 #> $ : int [1:7] 10407 -249485550 -1214357330 -2043404912 1753729054 -154685538 1051389303 #> $ : int [1:7] 10407 835808458 196552927 2117642140 149314186 338790750 1112575824 #> $ : int [1:7] 10407 -1450304404 -2045409317 -179263388 -889219308 1191986309 -566625213 #> $ : int [1:7] 10407 -688910850 1160691062 956749691 -422978313 2137821656 -29065073 #> $ : int [1:7] 10407 1786743736 -1039513504 224186150 1629560031 503129039 1663133997 #> $ : int [1:7] 10407 790977206 -2137758289 -566189168 229908201 -1823406113 -2016953102 #> $ : int [1:7] 10407 -842368572 -658180239 -1250210600 1910856618 1975438294 2111869219 #> $ : int [1:7] 10407 -1304052592 -41233767 800361080 -1382067640 -1349055971 2091568457 #> $ : int [1:7] 10407 -450434814 307478060 1866031563 874517892 -825396283 2097801285 #> $ : int [1:7] 10407 109698022 -156901408 144592825 2009040097 -1489056783 -200703622 #> - attr(*, \"iseed\")= int 65720644 # more usefully for HPC, extract only the seed associated with an arrayID arraySeed.15 <- genSeeds(nrow(design)*2, iseed=iseed, arrayID=15) arraySeed.15 #> [[1]] #> [1] 10407 -842368572 -658180239 -1250210600 1910856618 1975438294 #> [7] 2111869219 #> #> attr(,\"arrayID\") #> [1] 15 #> attr(,\"iseed\") #> [1] 65720644"},{"path":"http://philchalmers.github.io/SimDesign/reference/getArrayID.html","id":null,"dir":"Reference","previous_headings":"","what":"Get job array ID (e.g., from SLURM or other HPC array distributions) — getArrayID","title":"Get job array ID (e.g., from SLURM or other HPC array distributions) — getArrayID","text":"Get array ID HPC array distribution job (e.g., SLURM optional command line arguments). array ID used index rows design object runArraySimulation. instance, SLURM array 10 independent jobs might following shell instructions.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/getArrayID.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get job array ID (e.g., from SLURM or other HPC array distributions) — getArrayID","text":"","code":"getArrayID(type = \"slurm\", trailingOnly = TRUE, ID.shift = 0L)"},{"path":"http://philchalmers.github.io/SimDesign/reference/getArrayID.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get job array ID (e.g., from SLURM or other HPC array distributions) — getArrayID","text":"type integer indicating element result commandArgs extract, character specifying type . Default 'slurm' trailingOnly logical value passed commandArgs. used type integer ID.shift single integer value used shift array ID constant. Useful array range limitation must specified shell files (e.g., array can 10000 rows design object). example, array ID 10000 12000, cluster computer enviroment allow indices, including arrange range 1-2000 shell file shift=9999 add constant detected arrayID, thereby indexing remaining row elements design object","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/getArrayID.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Get job array ID (e.g., from SLURM or other HPC array distributions) — getArrayID","text":"#!/bin/bash -l #SBATCH --time=00:01:00 #SBATCH --array=1-10 names associated jobs numbers 1 10. getArrayID() extracts information per array, used runArraySimulation(design, ..., arrayID = getArrayID()) pass specific rows design object.","code":""},{"path":[]},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/manageMessages.html","id":null,"dir":"Reference","previous_headings":"","what":"Increase the intensity or suppress the output of an observed message — manageMessages","title":"Increase the intensity or suppress the output of an observed message — manageMessages","text":"Function provides nuanced management known message outputs messages appear function calls outside front-end users control (e.g., functions written third-party packages). Specifically, function provides less nuclear approach quiet friends, suppresses cat messages raised, instead allows specific messages raised either warnings , even extremely, errors. Note messages suppressed order output message calls appear original function retained.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/manageMessages.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Increase the intensity or suppress the output of an observed message — manageMessages","text":"","code":"manageMessages( expr, allow = NULL, message2warning = NULL, message2error = NULL, ... )"},{"path":"http://philchalmers.github.io/SimDesign/reference/manageMessages.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Increase the intensity or suppress the output of an observed message — manageMessages","text":"expr expression evaluated (e.g., ret <- myfun(args)). Function either used wrapper, manageMassages(ret <- myfun(args), ...) ret <- manageMassages(myfun(args), ...), readably pipe, ret <- myfun(args) |> manageMassages(...) allow (optional) character vector indicating messages still appear, messages remain suppressed. supplied message matched using grepl expression, partial matching supported (though specific messages less likely throw false positives). NULL, messages suppressed unless appear message2error message2warning message2warning (optional) Input can character vector containing messages probably considered warning messages current application instead. supplied character vector element matched using grepl expression, partial matching supported (though specific messages less likely throw false positives). message2error (optional) Input can character vector containing known---severe messages converted errors current application. See message2warning details. ... additional arguments passed grepl","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/manageMessages.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Increase the intensity or suppress the output of an observed message — manageMessages","text":"returns original result eval(expr), warning messages either left , increased errors, suppressed (depending input specifications)","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/manageMessages.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Increase the intensity or suppress the output of an observed message — manageMessages","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/manageMessages.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Increase the intensity or suppress the output of an observed message — manageMessages","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/manageMessages.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Increase the intensity or suppress the output of an observed message — manageMessages","text":"","code":"if (FALSE) { # \\dontrun{ myfun <- function(x, warn=FALSE){ message('This function is rather chatty') cat(\"It even prints in different output forms!\\n\") message('And even at different ') cat(\" many times!\\n\") cat(\"Too many messages can be annoying \\n\") if(warn) warning('It may even throw warnings ') x } out <- myfun(1) out # tell the function to shhhh out <- quiet(myfun(1)) out # same default behaviour as quiet(), but potential for nuance out2 <- manageMessages(myfun(1)) identical(out, out2) # allow some messages to still get printed out2 <- manageMessages(myfun(1), allow = \"many times!\") out2 <- manageMessages(myfun(1), allow = \"This function is rather chatty\") # note: . matches single character (regex) out2 <- manageMessages(myfun(1), allow = c(\"many times.\", \"This function is rather chatty\")) # convert specific message to warning out3 <- manageMessages(myfun(1), message2warning = \"many times!\") identical(out, out3) # other warnings also get through out3 <- manageMessages(myfun(1, warn=TRUE), message2warning = \"times!\") identical(out, out3) # convert message to error manageMessages(myfun(1), message2error = \"m... times!\") # multiple message intensity changes manageMessages(myfun(1), message2warning = \"It even prints in different output forms\", message2error = \"many times!\") manageMessages(myfun(1), allow = c(\"This function is rather chatty\", \"Too many messages can be annoying\"), message2warning = \"It even prints in different output forms\", message2error = \"many times!\") } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/manageWarnings.html","id":null,"dir":"Reference","previous_headings":"","what":"Manage specific warning messages — manageWarnings","title":"Manage specific warning messages — manageWarnings","text":"Function provides nuanced management known warning messages appear function calls outside front-end users control (e.g., functions written third-party packages). Specifically, function provides less nuclear approach suppressWarnings, suppresses warning messages rather known innocuous current application, globally setting options(warn=2), opposite effect treating warnings messages errors function executions. avoid two extreme behaviors, character vectors can instead supplied function either leave raised warnings -(default behaviour), raise specific warning messages errors, specify specific warning messages can generally ignored (therefore suppressed) allowing new yet discovered warnings still raised.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/manageWarnings.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Manage specific warning messages — manageWarnings","text":"","code":"manageWarnings(expr, warning2error = FALSE, suppress = NULL, ...)"},{"path":"http://philchalmers.github.io/SimDesign/reference/manageWarnings.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Manage specific warning messages — manageWarnings","text":"expr expression evaluated (e.g., ret <- myfun(args)). Function either used wrapper, manageWarnings(ret <- myfun(args), ...) ret <- manageWarnings(myfun(args), ...), readably pipe, ret <- myfun(args) |> manageWarnings(...) warning2error logical character vector control conversion warnings errors. Setting input TRUE treat observed warning messages errors (behavior options(warn=2), though defined per expression basis rather globally), setting FALSE (default) leave warning messages -, retaining default behavior Alternatively, useful specificity reasons, input can character vector containing known---severe warning messages converted errors. supplied character vector element matched using grepl expression, partial matching supported (though specific messages less likely throw false positives). suppress character vector indicating warning messages known innocuous priori can therefore suppressed. supplied warning message matched using grepl expression, partial matching supported (though specific messages less likely throw false positives). NULL, warning message suppressed ... additional arguments passed grepl","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/manageWarnings.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Manage specific warning messages — manageWarnings","text":"returns original result eval(expr), warning messages either left , increased errors, suppressed (depending input specifications)","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/manageWarnings.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Manage specific warning messages — manageWarnings","text":"general, global/nuclear behaviour warning messages avoided generally bad practice. one extreme, suppressing warning messages using suppressWarnings, potentially important warning messages become muffled, can problematic code developer become aware (now muffled) warnings. Moreover, can become long-term sustainability issue third-party functions developer's code depends upon throw new warnings future code developer less likely become aware newly implemented warnings. extreme, warning messages turned errors using options(warn=2), innocuous warning messages can (unwantingly) raised error. negatively affects logical workflow developer's functions, error messages must now manually managed (e.g., via tryCatch), including known innocuous warning messages now considered errors. avoid extremes, front-end users first make note warning messages raised prior executions, organized messages vectors ignorable warnings (least severe), known/unknown warnings remain warnings (even known code developer yet), explicit warnings considered errors current application (severe). collected, can passed respective warning2error argument increase intensity specific warning raised, suppress argument suppress messages deemed ignorable priori (therefore allowing warning messages raised).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/manageWarnings.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Manage specific warning messages — manageWarnings","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/manageWarnings.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Manage specific warning messages — manageWarnings","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/manageWarnings.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Manage specific warning messages — manageWarnings","text":"","code":"if (FALSE) { # \\dontrun{ fun <- function(warn1=FALSE, warn2=FALSE, warn3=FALSE, warn_trailing = FALSE, error=FALSE){ if(warn1) warning('Message one') if(warn2) warning('Message two') if(warn3) warning('Message three') if(warn_trailing) warning(sprintf('Message with lots of random trailings: %s', paste0(sample(letters, sample(1:20, 1)), collapse=','))) if(error) stop('terminate function call') return('Returned from fun()') } # normal run (no warnings or errors) out <- fun() out # these are all the same manageWarnings(out <- fun()) out <- manageWarnings(fun()) out <- fun() |> manageWarnings() # errors treated normally fun(error=TRUE) fun(error=TRUE) |> manageWarnings() # all warnings/returns treated normally by default ret1 <- fun(warn1=TRUE) ret2 <- fun(warn1=TRUE) |> manageWarnings() identical(ret1, ret2) # all warnings converted to errors (similar to options(warn=2), but local) fun(warn1=TRUE) |> manageWarnings(warning2error=TRUE) fun(warn2=TRUE) |> manageWarnings(warning2error=TRUE) # Specific warnings treated as errors (others stay as warnings) # Here, treat first warning message as error but not the second or third ret <- fun(warn1=TRUE) # warning ret <- fun(warn1=TRUE) |> manageWarnings(\"Message one\") # now error ret <- fun(warn2=TRUE) |> manageWarnings(\"Message one\") # still a warning # multiple warnings raised but not converted as they do not match criteria fun(warn2=TRUE, warn3=TRUE) fun(warn2=TRUE, warn3=TRUE) |> manageWarnings(\"Message one\") # Explicitly convert multiple warning messages, allowing others through. # This is generally the best use of the function's specificity fun(warn1=TRUE, warn2=TRUE) fun(warn1=TRUE) |> # error given either message manageWarnings(c(\"Message one\", \"Message two\")) fun(warn2=TRUE) |> manageWarnings(c(\"Message one\", \"Message two\")) # last warning gets through (left as valid warning) ret <- fun(warn3=TRUE) |> manageWarnings(c(\"Message one\", \"Message two\")) ret # suppress warnings that have only partial matching fun(warn_trailing=TRUE) fun(warn_trailing=TRUE) fun(warn_trailing=TRUE) # partial match, therefore suppressed fun(warn_trailing=TRUE) |> manageWarnings(suppress=\"Message with lots of random trailings: \") # multiple suppress strings fun(warn_trailing=TRUE) |> manageWarnings(suppress=c(\"Message with lots of random trailings: \", \"Suppress this too\")) # could also use .* to catch all remaining characters (finer regex control) fun(warn_trailing=TRUE) |> manageWarnings(suppress=\"Message with lots of random trailings: .*\") ########### # Combine with quiet() and suppress argument to suppress innocuous messages fun <- function(warn1=FALSE, warn2=FALSE, warn3=FALSE, error=FALSE){ message('This function is rather chatty') cat(\"It even prints in different output forms!\\n\") if(warn1) warning('Message one') if(warn2) warning('Message two') if(warn3) warning('Message three') if(error) stop('terminate function call') return('Returned from fun()') } # normal run (no warnings or errors, but messages) out <- fun() out <- quiet(fun()) # using \"indoor voice\" # suppress all print messages and warnings (not recommended) fun(warn2=TRUE) |> quiet() fun(warn2=TRUE) |> quiet() |> suppressWarnings() # convert all warning to errors, and keep suppressing messages via quiet() fun(warn2=TRUE) |> quiet() |> manageWarnings(warning2error=TRUE) # define tolerable warning messages (only warn1 deemed ignorable) ret <- fun(warn1=TRUE) |> quiet() |> manageWarnings(suppress = 'Message one') # all other warnings raised to an error except ignorable ones fun(warn1=TRUE, warn2=TRUE) |> quiet() |> manageWarnings(warning2error=TRUE, suppress = 'Message one') # only warn2 raised to an error explicitly (warn3 remains as warning) ret <- fun(warn1=TRUE, warn3=TRUE) |> quiet() |> manageWarnings(warning2error = 'Message two', suppress = 'Message one') fun(warn1=TRUE, warn2 = TRUE, warn3=TRUE) |> quiet() |> manageWarnings(warning2error = 'Message two', suppress = 'Message one') ########################### # Practical example, converting warning into error for model that # failed to converged normally library(lavaan) ## The industrialization and Political Democracy Example ## Bollen (1989), page 332 model <- ' # latent variable definitions ind60 =~ x1 + x2 + x3 dem60 =~ y1 + a*y2 + b*y3 + c*y4 dem65 =~ y5 + a*y6 + b*y7 + c*y8 # regressions dem60 ~ ind60 dem65 ~ ind60 + dem60 # residual correlations y1 ~~ y5 y2 ~~ y4 + y6 y3 ~~ y7 y4 ~~ y8 y6 ~~ y8 ' # throws a warning fit <- sem(model, data = PoliticalDemocracy, control=list(iter.max=60)) # for a simulation study, often better to treat this as an error fit <- sem(model, data = PoliticalDemocracy, control=list(iter.max=60)) |> manageWarnings(warning2error = \"the optimizer warns that a solution has NOT been found!\") } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/nc.html","id":null,"dir":"Reference","previous_headings":"","what":"Auto-named Concatenation of Vector or List — nc","title":"Auto-named Concatenation of Vector or List — nc","text":"wrapper function c, however names respective elements according input object name. reason, nesting nc() calls recommended (joining independent nc() calls via c() however reasonable).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/nc.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Auto-named Concatenation of Vector or List — nc","text":"","code":"nc(..., use.names = FALSE, error.on.duplicate = TRUE)"},{"path":"http://philchalmers.github.io/SimDesign/reference/nc.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Auto-named Concatenation of Vector or List — nc","text":"... objects concatenated use.names logical indicating names preserved (unlike c, default FALSE) error..duplicate logical; object name appears returning object error thrown? Default TRUE","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/nc.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Auto-named Concatenation of Vector or List — nc","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/nc.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Auto-named Concatenation of Vector or List — nc","text":"","code":"A <- 1 B <- 2 C <- 3 names(C) <- 'LetterC' # compare the following c(A, B, C) # unnamed #> LetterC #> 1 2 3 nc(A, B, C) # named #> A B C #> 1 2 3 nc(this=A, B, C) # respects override named (same as c() ) #> this B C #> 1 2 3 nc(this=A, B, C, use.names = TRUE) # preserve original name #> this B LetterC #> 1 2 3 if (FALSE) { # \\dontrun{ # throws errors if names not unique nc(this=A, this=B, C) nc(LetterC=A, B, C, use.names=TRUE) } # } # poor input choice names nc(t.test(c(1:2))$p.value, t.test(c(3:4))$p.value) #> t.test(c(1:2))_p.value t.test(c(3:4))_p.value #> 0.20483276 0.09033447 # better to explicitly provide name nc(T1 = t.test(c(1:2))$p.value, T2 = t.test(c(3:4))$p.value) #> T1 T2 #> 0.20483276 0.09033447 # vector of unnamed inputs A <- c(5,4,3,2,1) B <- c(100, 200) nc(A, B, C) # A's and B's numbered uniquely #> A1 A2 A3 A4 A5 B1 B2 C #> 5 4 3 2 1 100 200 3 c(A, B, C) # compare #> LetterC #> 5 4 3 2 1 100 200 3 nc(beta=A, B, C) # replacement of object name #> beta1 beta2 beta3 beta4 beta5 B1 B2 C #> 5 4 3 2 1 100 200 3 # retain names attributes (but append object name, when appropriate) names(A) <- letters[1:5] nc(A, B, C) #> A.a A.b A.c A.d A.e B1 B2 C #> 5 4 3 2 1 100 200 3 nc(beta=A, B, C) #> beta.a beta.b beta.c beta.d beta.e B1 B2 C #> 5 4 3 2 1 100 200 3 nc(A, B, C, use.names=TRUE) #> a b c d e B1 B2 LetterC #> 5 4 3 2 1 100 200 3 # mix and match if some named elements work while others do not c( nc(A, B, use.names=TRUE), nc(C)) #> a b c d e B1 B2 C #> 5 4 3 2 1 100 200 3 if (FALSE) { # \\dontrun{ # error, 'b' appears twice names(B) <- c('b', 'b2') nc(A, B, C, use.names=TRUE) } # } # List input A <- list(1) B <- list(2:3) C <- list('C') names(C) <- 'LetterC' # compare the following c(A, B, C) # unnamed #> [[1]] #> [1] 1 #> #> [[2]] #> [1] 2 3 #> #> $LetterC #> [1] \"C\" #> nc(A, B, C) # named #> $A #> [1] 1 #> #> $B #> [1] 2 3 #> #> $C #> [1] \"C\" #> nc(this=A, B, C) # respects override named (same as c() and list() ) #> $this #> [1] 1 #> #> $B #> [1] 2 3 #> #> $C #> [1] \"C\" #> nc(this=A, B, C, use.names = TRUE) # preserve original name #> $this #> [1] 1 #> #> $B #> [1] 2 3 #> #> $LetterC #> [1] \"C\" #>"},{"path":"http://philchalmers.github.io/SimDesign/reference/quiet.html","id":null,"dir":"Reference","previous_headings":"","what":"Suppress verbose function messages — quiet","title":"Suppress verbose function messages — quiet","text":"function used suppress information printed external functions make internal use message cat, provide information interactive R sessions. simulations, session interactive, therefore type output suppressed. similar behaviour suppressing warning messages, see manageWarnings.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/quiet.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Suppress verbose function messages — quiet","text":"","code":"quiet(..., cat = TRUE, keep = FALSE, attr.name = \"quiet.messages\")"},{"path":"http://philchalmers.github.io/SimDesign/reference/quiet.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Suppress verbose function messages — quiet","text":"... functional expression evaluated cat logical; also capture calls cat? FALSE message suppressed keep logical; return character vector messages/concatenate print strings attribute resulting object expr(...)? attr.name attribute name use keep = TRUE","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/quiet.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Suppress verbose function messages — quiet","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/quiet.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Suppress verbose function messages — quiet","text":"","code":"myfun <- function(x, warn=FALSE){ message('This function is rather chatty') cat(\"It even prints in different output forms!\\n\") message('And even at different....') cat(\"...times!\\n\") if(warn) warning('It may even throw warnings!') x } out <- myfun(1) #> This function is rather chatty #> It even prints in different output forms! #> And even at different.... #> ...times! out #> [1] 1 # tell the function to shhhh out <- quiet(myfun(1)) out #> [1] 1 # which messages are suppressed? Extract stored attribute out <- quiet(myfun(1), keep = TRUE) attr(out, 'quiet.messages') #> message.1 #> \"This function is rather chatty\\n\" #> message.2 #> \"And even at different....\\n\" #> cat.1 #> \"It even prints in different output forms!\" #> cat.2 #> \"...times!\" # Warning messages still get through (see manageWarnings(suppress) # for better alternative than using suppressWarnings()) out2 <- myfun(2, warn=TRUE) |> quiet() # warning gets through #> Warning: It may even throw warnings! out2 #> [1] 2 # suppress warning message explicitly, allowing others to be raised if present myfun(2, warn=TRUE) |> quiet() |> manageWarnings(suppress='It may even throw warnings!') #> [1] 2"},{"path":"http://philchalmers.github.io/SimDesign/reference/rHeadrick.html","id":null,"dir":"Reference","previous_headings":"","what":"Generate non-normal data with Headrick's (2002) method — rHeadrick","title":"Generate non-normal data with Headrick's (2002) method — rHeadrick","text":"Generate multivariate non-normal distributions using fifth-order polynomial method described Headrick (2002).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rHeadrick.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Generate non-normal data with Headrick's (2002) method — rHeadrick","text":"","code":"rHeadrick( n, mean = rep(0, nrow(sigma)), sigma = diag(length(mean)), skew = rep(0, nrow(sigma)), kurt = rep(0, nrow(sigma)), gam3 = NaN, gam4 = NaN, return_coefs = FALSE, coefs = NULL, control = list(trace = FALSE, max.ntry = 15, obj.tol = 1e-10, n.valid.sol = 1) )"},{"path":"http://philchalmers.github.io/SimDesign/reference/rHeadrick.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Generate non-normal data with Headrick's (2002) method — rHeadrick","text":"n number samples draw mean vector k elements mean variables sigma desired k x k covariance matrix bivariate non-normal variables skew vector k elements skewness variables kurt vector k elements kurtosis variables gam3 (optional) explicitly supply gamma 3 value? Default computes internally gam4 (optional) explicitly supply gamma 4 value? Default computes internally return_coefs logical; return estimated coefficients ? See regarding useful. coefs (optional) supply previously estimated coefficients? useful must multiple data sets drawn avoid repetitive computations. Must object returned passing return_coefs = TRUE control list control parameters locating polynomial coefficients","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rHeadrick.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Generate non-normal data with Headrick's (2002) method — rHeadrick","text":"function primarily wrapper code written Oscar L. Olvera Astivia (last edited Feb 26, 2015) modifications (e.g., better starting values Newton optimizer, passing previously saved coefs, etc).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rHeadrick.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Generate non-normal data with Headrick's (2002) method — rHeadrick","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953 Headrick, T. C. (2002). Fast fifth-order polynomial transforms generating univariate multivariate nonnormal distributions. Computational Statistics & Data Analysis, 40, 685-711. Olvera Astivia, O. L., & Zumbo, B. D. (2015). Cautionary Note Use Vale Maurelli Method Generate Multivariate, Nonnormal Data Simulation Purposes. Educational Psychological Measurement, 75, 541-567.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rHeadrick.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Generate non-normal data with Headrick's (2002) method — rHeadrick","text":"Oscar L. Olvera Astivia Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rHeadrick.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Generate non-normal data with Headrick's (2002) method — rHeadrick","text":"","code":"if (FALSE) { # \\dontrun{ set.seed(1) N <- 200 mean <- c(rep(0,4)) Sigma <- matrix(.49, 4, 4) diag(Sigma) <- 1 skewness <- c(rep(1,4)) kurtosis <- c(rep(2,4)) nonnormal <- rHeadrick(N, mean, Sigma, skewness, kurtosis) # cor(nonnormal) # psych::describe(nonnormal) #----------- # compute the coefficients, then supply them back to the function to avoid # extra computations cfs <- rHeadrick(N, mean, Sigma, skewness, kurtosis, return_coefs = TRUE) cfs # compare system.time(nonnormal <- rHeadrick(N, mean, Sigma, skewness, kurtosis)) system.time(nonnormal <- rHeadrick(N, mean, Sigma, skewness, kurtosis, coefs=cfs)) } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/rValeMaurelli.html","id":null,"dir":"Reference","previous_headings":"","what":"Generate non-normal data with Vale & Maurelli's (1983) method — rValeMaurelli","title":"Generate non-normal data with Vale & Maurelli's (1983) method — rValeMaurelli","text":"Generate multivariate non-normal distributions using third-order polynomial method described Vale & Maurelli (1983). single variable generated function equivalent method described Fleishman (1978).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rValeMaurelli.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Generate non-normal data with Vale & Maurelli's (1983) method — rValeMaurelli","text":"","code":"rValeMaurelli( n, mean = rep(0, nrow(sigma)), sigma = diag(length(mean)), skew = rep(0, nrow(sigma)), kurt = rep(0, nrow(sigma)) )"},{"path":"http://philchalmers.github.io/SimDesign/reference/rValeMaurelli.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Generate non-normal data with Vale & Maurelli's (1983) method — rValeMaurelli","text":"n number samples draw mean vector k elements mean variables sigma desired k x k covariance matrix bivariate non-normal variables skew vector k elements skewness variables kurt vector k elements kurtosis variables","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rValeMaurelli.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Generate non-normal data with Vale & Maurelli's (1983) method — rValeMaurelli","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953 Fleishman, . . (1978). method simulating non-normal distributions. Psychometrika, 43, 521-532. Vale, C. & Maurelli, V. (1983). Simulating multivariate nonnormal distributions. Psychometrika, 48(3), 465-471.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rValeMaurelli.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Generate non-normal data with Vale & Maurelli's (1983) method — rValeMaurelli","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rValeMaurelli.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Generate non-normal data with Vale & Maurelli's (1983) method — rValeMaurelli","text":"","code":"set.seed(1) # univariate with skew nonnormal <- rValeMaurelli(10000, mean=10, sigma=5, skew=1, kurt=3) # psych::describe(nonnormal) # multivariate with skew and kurtosis n <- 10000 r12 <- .4 r13 <- .9 r23 <- .1 cor <- matrix(c(1,r12,r13,r12,1,r23,r13,r23,1),3,3) sk <- c(1.5,1.5,0.5) ku <- c(3.75,3.5,0.5) nonnormal <- rValeMaurelli(n, sigma=cor, skew=sk, kurt=ku) # cor(nonnormal) # psych::describe(nonnormal)"},{"path":"http://philchalmers.github.io/SimDesign/reference/rbind.SimDesign.html","id":null,"dir":"Reference","previous_headings":"","what":"Combine two separate SimDesign objects by row — rbind.SimDesign","title":"Combine two separate SimDesign objects by row — rbind.SimDesign","text":"function combines two Monte Carlo simulations executed SimDesign's runSimulation function , intents purposes, executed single run. situation arises simulation completed, however Design object later modified include levels defined simulation factors. Rather re-executing previously completed simulation combinations, new combinations need evaluated different object rbind together create complete object combinations.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rbind.SimDesign.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Combine two separate SimDesign objects by row — rbind.SimDesign","text":"","code":"# S3 method for class 'SimDesign' rbind(...)"},{"path":"http://philchalmers.github.io/SimDesign/reference/rbind.SimDesign.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Combine two separate SimDesign objects by row — rbind.SimDesign","text":"... two SimDesign objects combined rows","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rbind.SimDesign.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Combine two separate SimDesign objects by row — rbind.SimDesign","text":"object returned runSimulation","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rbind.SimDesign.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Combine two separate SimDesign objects by row — rbind.SimDesign","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rbind.SimDesign.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Combine two separate SimDesign objects by row — rbind.SimDesign","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rbind.SimDesign.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Combine two separate SimDesign objects by row — rbind.SimDesign","text":"","code":"if (FALSE) { # \\dontrun{ # modified example from runSimulation() Design <- createDesign(N = c(10, 20), SD = c(1, 2)) Generate <- function(condition, fixed_objects) { dat <- with(condition, rnorm(N, 10, sd=SD)) dat } Analyse <- function(condition, dat, fixed_objects) { ret <- mean(dat) # mean of the sample data vector ret } Summarise <- function(condition, results, fixed_objects) { ret <- c(mu=mean(results), SE=sd(results)) # mean and SD summary of the sample means ret } Final1 <- runSimulation(design=Design, replications=1000, generate=Generate, analyse=Analyse, summarise=Summarise) Final1 ### # later decide that N = 30 should have also been investigated. Rather than # running the following object .... newDesign <- createDesign(N = c(10, 20, 30), SD = c(1, 2)) # ... only the new subset levels are executed to save time subDesign <- subset(newDesign, N == 30) subDesign Final2 <- runSimulation(design=subDesign, replications=1000, generate=Generate, analyse=Analyse, summarise=Summarise) Final2 # glue results together by row into one object as though the complete 'Design' # object were run all at once Final <- rbind(Final1, Final2) Final summary(Final) } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/reSummarise.html","id":null,"dir":"Reference","previous_headings":"","what":"Run a summarise step for results that have been saved to the hard drive — reSummarise","title":"Run a summarise step for results that have been saved to the hard drive — reSummarise","text":"runSimulation() uses option save_results = TRUE R replication results Generate-Analyse functions stored hard drive. , additional summarise components may required later time, whereby respective .rds files must read back R summarised. function performs reading files, application provided summarise function, final collection respective results.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/reSummarise.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Run a summarise step for results that have been saved to the hard drive — reSummarise","text":"","code":"reSummarise( summarise, dir = NULL, files = NULL, results = NULL, Design = NULL, fixed_objects = NULL, boot_method = \"none\", boot_draws = 1000L, CI = 0.95, prefix = \"results-row\" )"},{"path":"http://philchalmers.github.io/SimDesign/reference/reSummarise.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Run a summarise step for results that have been saved to the hard drive — reSummarise","text":"summarise summarise function apply read-files. See runSimulation details dir directory pointing .rds files read-saved runSimulation(..., save_results=TRUE). NULL, assumed current working directory contains .rds files files (optional) names files read-. NULL files located within dir used results (optional) results runSimulation summarise function provided. Can either tibble matrix (indicating exactly one design condition evaluated), list matrix/tibble objects indicating multiple conditions performed summarise evaluation. Alternatively, store_results = TRUE runSimulation() execution final SimDesign object may passed, generate-analyse information extracted object instead Design (optional) results input used, design condition information important summarise step, original design object runSimulation included fixed_objects (optional) see runSimulation details boot_method method performing non-parametric bootstrap confidence intervals respective meta-statistics computed Summarise function. See runSimulation details boot_draws number non-parametric bootstrap draws sample summarise function generate-analyse replications collected. Default 1000 CI bootstrap confidence interval level (default 95%) prefix character indicating prefix used stored files","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/reSummarise.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Run a summarise step for results that have been saved to the hard drive — reSummarise","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/reSummarise.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Run a summarise step for results that have been saved to the hard drive — reSummarise","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/reSummarise.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Run a summarise step for results that have been saved to the hard drive — reSummarise","text":"","code":"Design <- createDesign(N = c(10, 20, 30)) Generate <- function(condition, fixed_objects) { dat <- with(condition, rnorm(N, 10, 5)) # distributed N(10, 5) dat } Analyse <- function(condition, dat, fixed_objects) { ret <- c(mean=mean(dat), median=median(dat)) # mean/median of sample data ret } Summarise <- function(condition, results, fixed_objects){ colMeans(results) } if (FALSE) { # \\dontrun{ # run the simulation runSimulation(design=Design, replications=50, generate=Generate, analyse=Analyse, summarise=Summarise, save_results=TRUE, save_details = list(save_results_dirname='simresults')) res <- reSummarise(Summarise, dir = 'simresults/') res Summarise2 <- function(condition, results, fixed_objects){ ret <- c(mean_ests=colMeans(results), SE=colSDs(results)) ret } res2 <- reSummarise(Summarise2, dir = 'simresults/') res2 SimClean(dir='simresults/') } # } ### # Similar, but with results stored within the final object res <- runSimulation(design=Design, replications=50, store_results = TRUE, generate=Generate, analyse=Analyse, summarise=Summarise) #> #> Design: 1/3; Replications: 50; RAM Used: 159.3 Mb; Total Time: 0.00s #> Conditions: N=10 #> #> Design: 2/3; Replications: 50; RAM Used: 159.3 Mb; Total Time: 0.01s #> Conditions: N=20 #> #> Design: 3/3; Replications: 50; RAM Used: 159.3 Mb; Total Time: 0.02s #> Conditions: N=30 #> #> #> Simulation complete. Total execution time: 0.03s res #> # A tibble: 3 × 8 #> N mean median REPLICATIONS SIM_TIME RAM_USED SEED COMPLETED #> #> 1 10 10.192 10.230 50 0.01s 159.3 Mb 1771457935 Wed Dec 4 05:3… #> 2 20 10.146 10.084 50 0.01s 159.3 Mb 590590962 Wed Dec 4 05:3… #> 3 30 10.257 10.328 50 0.01s 159.3 Mb 273992574 Wed Dec 4 05:3… # same summarise but with bootstrapping res2 <- reSummarise(Summarise, results = res, boot_method = 'basic') res2 #> # A tibble: 3 × 7 #> N mean median BOOT_mean_2.5 BOOT_mean_97.5 BOOT_median_2.5 #> #> 1 10 10.2 10.2 9.76 10.6 9.69 #> 2 20 10.1 10.1 9.87 10.4 9.75 #> 3 30 10.3 10.3 9.99 10.5 10.1 #> # ℹ 1 more variable: BOOT_median_97.5 "},{"path":"http://philchalmers.github.io/SimDesign/reference/rejectionSampling.html","id":null,"dir":"Reference","previous_headings":"","what":"Rejection sampling (i.e., accept-reject method) — rejectionSampling","title":"Rejection sampling (i.e., accept-reject method) — rejectionSampling","text":"function supports rejection sampling (.e., accept-reject) approach drawing values seemingly difficult (probability) density functions sampling values manageable proxy distributions.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rejectionSampling.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Rejection sampling (i.e., accept-reject method) — rejectionSampling","text":"","code":"rejectionSampling( n, df, dg, rg, M, method = \"optimize\", interval = NULL, logfuns = FALSE, maxM = 1e+05, parstart = rg(1L), ESRS_Mstart = 1.0001 )"},{"path":"http://philchalmers.github.io/SimDesign/reference/rejectionSampling.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Rejection sampling (i.e., accept-reject method) — rejectionSampling","text":"n number samples draw df desired (potentially un-normed) density function draw independent samples . Must form function single input corresponding values sampled rg. Function assumed vectorized (, see Vectorize) dg proxy (potentially un-normed) density function draw samples lieu drawing samples df. support density function df (.e., df(x) > 0 dg(x) > 0). Must form function single input corresponding values sampled rg. Function assumed vectorized (, see Vectorize) rg proxy random number generation function, associated dg, used draw proposal samples . Must form function single input corresponding number values draw, output can either vector matrix (matrix, independent observation must stored unique row). Function assumed vectorized (, see Vectorize) M upper-bound ratio probability density functions help minimize number discarded draws define corresponding rescaled proposal envelope. missing, M computed internally finding reasonable maximum log(df(x)) - log(dg(x)), value returned console. df dg true probability density functions (.e., integrate 1) acceptance probability equal 1/M method M missing, optimization M done either finding mode log-density values (\"optimize\") using \"Empirical Supremum Rejection Sampling\" method (\"ESRS\") interval M missing, univariate density function draws, interval search within via optimize. specified, sample 5000 values rg function definition collected, min/max obtained via random sample logfuns logical; df dg function written return log-densities instead original densities? FALSE default assumes original densities returned (use TRUE higher accuracy required generating density definition) maxM logical; optimizing M value greater cut-stop; ampler likelihood efficient, optimization failing parstart starting value vector optimization M multidimensional distributions ESRS_Mstart starting M value ESRS algorithm","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rejectionSampling.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Rejection sampling (i.e., accept-reject method) — rejectionSampling","text":"returns vector matrix draws (corresponding output class rg) desired df","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rejectionSampling.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Rejection sampling (i.e., accept-reject method) — rejectionSampling","text":"accept-reject algorithm flexible approach obtaining ..d.'s difficult sample (probability) density function either transformation method fails inverse transform method difficult manage. algorithm sampling \"well-behaved\" proxy distribution (identical support, proportionality constant M reshapes proposal density envelope target density), accepts draws likely within target density. Hence, closer shape dg(x) desired df(x), likely draws accepted; otherwise, many iterations accept-reject algorithm may required, decreases computational efficiency.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rejectionSampling.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Rejection sampling (i.e., accept-reject method) — rejectionSampling","text":"Caffo, B. S., Booth, J. G., Davison, . C. (2002). Empirical supremum rejection sampling. Biometrika, 89, 745–754. Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rejectionSampling.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Rejection sampling (i.e., accept-reject method) — rejectionSampling","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rejectionSampling.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Rejection sampling (i.e., accept-reject method) — rejectionSampling","text":"","code":"if (FALSE) { # \\dontrun{ # Generate X ~ beta(a,b), where a and b are a = 2.7 and b = 6.3, # and the support is Y ~ Unif(0,1) dfn <- function(x) dbeta(x, shape1 = 2.7, shape2 = 6.3) dgn <- function(x) dunif(x, min = 0, max = 1) rgn <- function(n) runif(n, min = 0, max = 1) # when df and dg both integrate to 1, acceptance probability = 1/M M <- rejectionSampling(df=dfn, dg=dgn, rg=rgn) M dat <- rejectionSampling(10000, df=dfn, dg=dgn, rg=rgn, M=M) hist(dat, 100) hist(rbeta(10000, 2.7, 6.3), 100) # compare # obtain empirical estimate of M via ESRS method M <- rejectionSampling(1000, df=dfn, dg=dgn, rg=rgn, method='ESRS') M # generate using better support function (here, Y ~ beta(2,6)), # and use log setup in initial calls (more numerically accurate) dfn <- function(x) dbeta(x, shape1 = 2.7, shape2 = 6.3, log = TRUE) dgn <- function(x) dbeta(x, shape1 = 2, shape2 = 6, log = TRUE) rgn <- function(n) rbeta(n, shape1 = 2, shape2 = 6) M <- rejectionSampling(df=dfn, dg=dgn, rg=rgn, logfuns=TRUE) # better M M ## Alternative estimation of M ## M <- rejectionSampling(10000, df=dfn, dg=dgn, rg=rgn, logfuns=TRUE, ## method='ESRS') dat <- rejectionSampling(10000, df=dfn, dg=dgn, rg=rgn, M=M, logfuns=TRUE) hist(dat, 100) #------------------------------------------------------ # sample from wonky (and non-normalized) density function, like below dfn <- function(x){ ret <- numeric(length(x)) ret[x <= .5] <- dnorm(x[x <= .5]) ret[x > .5] <- dnorm(x[x > .5]) + dchisq(x[x > .5], df = 2) ret } y <- seq(-5,5, length.out = 1000) plot(y, dfn(y), type = 'l', main = \"Function to sample from\") # choose dg/rg functions that have support within the range [-inf, inf] rgn <- function(n) rnorm(n, sd=4) dgn <- function(x) dnorm(x, sd=4) ## example M height from above graphic ## (M selected using ESRS to help stochastically avoid local mins) M <- rejectionSampling(10000, df=dfn, dg=dgn, rg=rgn, method='ESRS') M lines(y, dgn(y)*M, lty = 2) dat <- rejectionSampling(10000, df=dfn, dg=dgn, rg=rgn, M=M) hist(dat, 100, prob=TRUE) # true density (normalized) C <- integrate(dfn, -Inf, Inf)$value ndfn <- function(x) dfn(x) / C curve(ndfn, col='red', lwd=2, add=TRUE) #----------------------------------------------------- # multivariate distribution dfn <- function(x) sum(log(c(dnorm(x[1]) + dchisq(x[1], df = 5), dnorm(x[2], -1, 2)))) rgn <- function(n) c(rnorm(n, sd=3), rnorm(n, sd=3)) dgn <- function(x) sum(log(c(dnorm(x[1], sd=3), dnorm(x[1], sd=3)))) # M <- rejectionSampling(df=dfn, dg=dgn, rg=rgn, logfuns=TRUE) dat <- rejectionSampling(5000, df=dfn, dg=dgn, rg=rgn, M=4.6, logfuns=TRUE) hist(dat[,1], 30) hist(dat[,2], 30) plot(dat) } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/rint.html","id":null,"dir":"Reference","previous_headings":"","what":"Generate integer values within specified range — rint","title":"Generate integer values within specified range — rint","text":"Efficiently generate positive negative integer values (default) without replacement. function mainly wrapper sample.int function (much efficient integer sampler general sample), however intended work positive negative integer ranges since sample.int returns positive integer values must begin 1L.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rint.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Generate integer values within specified range — rint","text":"","code":"rint(n, min, max, replace = TRUE, prob = NULL)"},{"path":"http://philchalmers.github.io/SimDesign/reference/rint.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Generate integer values within specified range — rint","text":"n number samples draw min lower limit distribution. Must finite max upper limit distribution. Must finite replace sampling replacement? prob vector probability weights obtaining elements vector sampled","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rint.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Generate integer values within specified range — rint","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rint.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Generate integer values within specified range — rint","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rint.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Generate integer values within specified range — rint","text":"","code":"set.seed(1) # sample 1000 integer values within 20 to 100 x <- rint(1000, min = 20, max = 100) summary(x) #> Min. 1st Qu. Median Mean 3rd Qu. Max. #> 20.00 40.00 59.00 59.55 79.25 100.00 # sample 1000 integer values within 100 to 10 billion x <- rint(1000, min = 100, max = 1e8) summary(x) #> Min. 1st Qu. Median Mean 3rd Qu. Max. #> 2499 25538779 48425070 49782329 75851586 99994040 # compare speed to sample() system.time(x <- rint(1000, min = 100, max = 1e8)) #> user system elapsed #> 0.001 0.000 0.000 system.time(x2 <- sample(100:1e8, 1000, replace = TRUE)) #> user system elapsed #> 0 0 0 # sample 1000 integer values within -20 to 20 x <- rint(1000, min = -20, max = 20) summary(x) #> Min. 1st Qu. Median Mean 3rd Qu. Max. #> -20.000 -10.250 0.000 0.067 11.000 20.000"},{"path":"http://philchalmers.github.io/SimDesign/reference/rinvWishart.html","id":null,"dir":"Reference","previous_headings":"","what":"Generate data with the inverse Wishart distribution — rinvWishart","title":"Generate data with the inverse Wishart distribution — rinvWishart","text":"Function generates data form symmetric matrices inverse Wishart distribution given covariance matrix degrees freedom.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rinvWishart.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Generate data with the inverse Wishart distribution — rinvWishart","text":"","code":"rinvWishart(n = 1, df, sigma)"},{"path":"http://philchalmers.github.io/SimDesign/reference/rinvWishart.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Generate data with the inverse Wishart distribution — rinvWishart","text":"n number matrix observations generate. default n = 1, returns single symmetric matrix. n > 1 list n symmetric matrices returned instead df degrees freedom sigma positive definite covariance matrix","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rinvWishart.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Generate data with the inverse Wishart distribution — rinvWishart","text":"numeric matrix columns equal ncol(sigma) n = 1, list n matrices properties","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rinvWishart.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Generate data with the inverse Wishart distribution — rinvWishart","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/rinvWishart.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Generate data with the inverse Wishart distribution — rinvWishart","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rinvWishart.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Generate data with the inverse Wishart distribution — rinvWishart","text":"","code":"# random inverse Wishart matrix given variances [3,6], covariance 2, and df=15 sigma <- matrix(c(3,2,2,6), 2, 2) x <- rinvWishart(sigma = sigma, df = 15) x #> [,1] [,2] #> [1,] 0.12592879 0.06824857 #> [2,] 0.06824857 0.31885404 # list of matrices x <- rinvWishart(20, sigma = sigma, df = 15) x #> [[1]] #> [,1] [,2] #> [1,] 0.14125605 0.06475135 #> [2,] 0.06475135 0.38464374 #> #> [[2]] #> [,1] [,2] #> [1,] 0.30345161 -0.03541925 #> [2,] -0.03541925 0.26979829 #> #> [[3]] #> [,1] [,2] #> [1,] 0.2574038 0.1281387 #> [2,] 0.1281387 0.3872453 #> #> [[4]] #> [,1] [,2] #> [1,] 0.2364292 0.1184846 #> [2,] 0.1184846 0.2543226 #> #> [[5]] #> [,1] [,2] #> [1,] 0.20712354 0.08245683 #> [2,] 0.08245683 0.39818001 #> #> [[6]] #> [,1] [,2] #> [1,] 0.1253428 0.1237987 #> [2,] 0.1237987 0.4092931 #> #> [[7]] #> [,1] [,2] #> [1,] 0.17509635 0.03126645 #> [2,] 0.03126645 0.27069185 #> #> [[8]] #> [,1] [,2] #> [1,] 0.5432412 0.4282939 #> [2,] 0.4282939 0.7519928 #> #> [[9]] #> [,1] [,2] #> [1,] 0.11311930 0.05432718 #> [2,] 0.05432718 0.39363138 #> #> [[10]] #> [,1] [,2] #> [1,] 0.2130090 0.1274582 #> [2,] 0.1274582 0.3060374 #> #> [[11]] #> [,1] [,2] #> [1,] 0.3098325 0.2499661 #> [2,] 0.2499661 0.6363222 #> #> [[12]] #> [,1] [,2] #> [1,] 0.1242213 0.0631768 #> [2,] 0.0631768 0.3443811 #> #> [[13]] #> [,1] [,2] #> [1,] 0.4275060 0.2879198 #> [2,] 0.2879198 0.5902773 #> #> [[14]] #> [,1] [,2] #> [1,] 0.3093765 0.1630539 #> [2,] 0.1630539 0.2796718 #> #> [[15]] #> [,1] [,2] #> [1,] 0.11046516 0.02646219 #> [2,] 0.02646219 0.34872737 #> #> [[16]] #> [,1] [,2] #> [1,] 0.11632594 0.02074906 #> [2,] 0.02074906 0.30345385 #> #> [[17]] #> [,1] [,2] #> [1,] 0.2470951 0.1609995 #> [2,] 0.1609995 0.6436081 #> #> [[18]] #> [,1] [,2] #> [1,] 0.1766255 0.2190138 #> [2,] 0.2190138 0.6149394 #> #> [[19]] #> [,1] [,2] #> [1,] 0.1951444 0.0698632 #> [2,] 0.0698632 0.3893907 #> #> [[20]] #> [,1] [,2] #> [1,] 0.258277 0.1546960 #> [2,] 0.154696 0.4691242 #>"},{"path":"http://philchalmers.github.io/SimDesign/reference/rmgh.html","id":null,"dir":"Reference","previous_headings":"","what":"Generate data with the multivariate g-and-h distribution — rmgh","title":"Generate data with the multivariate g-and-h distribution — rmgh","text":"Generate non-normal distributions using multivariate g--h distribution. Can used generate several different classes univariate multivariate distributions.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rmgh.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Generate data with the multivariate g-and-h distribution — rmgh","text":"","code":"rmgh(n, g, h, mean = rep(0, length(g)), sigma = diag(length(mean)))"},{"path":"http://philchalmers.github.io/SimDesign/reference/rmgh.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Generate data with the multivariate g-and-h distribution — rmgh","text":"n number samples draw g g parameter(s) control skew distribution terms direction magnitude h h parameter(s) control tail weight elongation distribution positively related kurtosis mean vector k elements mean variables sigma desired k x k covariance matrix bivariate non-normal variables","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rmgh.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Generate data with the multivariate g-and-h distribution — rmgh","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rmgh.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Generate data with the multivariate g-and-h distribution — rmgh","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rmgh.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Generate data with the multivariate g-and-h distribution — rmgh","text":"","code":"set.seed(1) # univariate norm <- rmgh(10000,1e-5,0) hist(norm) skew <- rmgh(10000,1/2,0) hist(skew) neg_skew_platykurtic <- rmgh(10000,-1,-1/2) hist(neg_skew_platykurtic) # multivariate sigma <- matrix(c(2,1,1,4), 2) mean <- c(-1, 1) twovar <- rmgh(10000, c(-1/2, 1/2), c(0,0), mean=mean, sigma=sigma) hist(twovar[,1]) hist(twovar[,2]) plot(twovar)"},{"path":"http://philchalmers.github.io/SimDesign/reference/rmvnorm.html","id":null,"dir":"Reference","previous_headings":"","what":"Generate data with the multivariate normal (i.e., Gaussian) distribution — rmvnorm","title":"Generate data with the multivariate normal (i.e., Gaussian) distribution — rmvnorm","text":"Function generates data multivariate normal distribution given mean vector /covariance matrix.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rmvnorm.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Generate data with the multivariate normal (i.e., Gaussian) distribution — rmvnorm","text":"","code":"rmvnorm(n, mean = rep(0, nrow(sigma)), sigma = diag(length(mean)))"},{"path":"http://philchalmers.github.io/SimDesign/reference/rmvnorm.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Generate data with the multivariate normal (i.e., Gaussian) distribution — rmvnorm","text":"n number observations generate mean mean vector, default rep(0, length = ncol(sigma)) sigma positive definite covariance matrix, default diag(length(mean))","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rmvnorm.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Generate data with the multivariate normal (i.e., Gaussian) distribution — rmvnorm","text":"numeric matrix columns equal length(mean)","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rmvnorm.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Generate data with the multivariate normal (i.e., Gaussian) distribution — rmvnorm","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/rmvnorm.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Generate data with the multivariate normal (i.e., Gaussian) distribution — rmvnorm","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rmvnorm.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Generate data with the multivariate normal (i.e., Gaussian) distribution — rmvnorm","text":"","code":"# random normal values with mean [5, 10] and variances [3,6], and covariance 2 sigma <- matrix(c(3,2,2,6), 2, 2) mu <- c(5,10) x <- rmvnorm(1000, mean = mu, sigma = sigma) head(x) #> [,1] [,2] #> [1,] 6.060771 11.172431 #> [2,] 4.222922 9.918127 #> [3,] 6.189192 7.146768 #> [4,] 6.328861 7.243104 #> [5,] 4.874275 12.614418 #> [6,] 2.573901 9.202745 summary(x) #> V1 V2 #> Min. : 0.008016 Min. : 1.091 #> 1st Qu.: 3.779530 1st Qu.: 8.364 #> Median : 5.018941 Median :10.009 #> Mean : 4.927963 Mean : 9.975 #> 3rd Qu.: 6.062034 3rd Qu.:11.546 #> Max. :10.018291 Max. :17.717 plot(x[,1], x[,2])"},{"path":"http://philchalmers.github.io/SimDesign/reference/rmvt.html","id":null,"dir":"Reference","previous_headings":"","what":"Generate data with the multivariate t distribution — rmvt","title":"Generate data with the multivariate t distribution — rmvt","text":"Function generates data multivariate t distribution given covariance matrix, non-centrality parameter (mode), degrees freedom.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rmvt.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Generate data with the multivariate t distribution — rmvt","text":"","code":"rmvt(n, sigma, df, delta = rep(0, nrow(sigma)), Kshirsagar = FALSE)"},{"path":"http://philchalmers.github.io/SimDesign/reference/rmvt.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Generate data with the multivariate t distribution — rmvt","text":"n number observations generate sigma positive definite covariance matrix df degrees freedom. df = 0 df = Inf corresponds multivariate normal distribution delta vector non-centrality parameters length n specifies either modes (default) non-centrality parameters Kshirsagar logical; triggers whether generate data non-centrality parameters adjust simulated data mode distribution. default uses mode","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rmvt.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Generate data with the multivariate t distribution — rmvt","text":"numeric matrix columns equal ncol(sigma)","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rmvt.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Generate data with the multivariate t distribution — rmvt","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/rmvt.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Generate data with the multivariate t distribution — rmvt","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rmvt.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Generate data with the multivariate t distribution — rmvt","text":"","code":"# random t values given variances [3,6], covariance 2, and df = 15 sigma <- matrix(c(3,2,2,6), 2, 2) x <- rmvt(1000, sigma = sigma, df = 15) head(x) #> [,1] [,2] #> [1,] 1.1355823 -1.511125 #> [2,] 4.2293566 3.520368 #> [3,] 1.4742611 -2.296661 #> [4,] 0.4845494 -0.982761 #> [5,] -0.4792007 -3.010312 #> [6,] -1.8363903 -1.736213 summary(x) #> V1 V2 #> Min. :-6.15834 Min. :-10.22541 #> 1st Qu.:-1.27204 1st Qu.: -1.86951 #> Median :-0.06571 Median : -0.13726 #> Mean :-0.05165 Mean : -0.06365 #> 3rd Qu.: 1.23916 3rd Qu.: 1.80329 #> Max. : 8.84541 Max. : 10.35493 plot(x[,1], x[,2])"},{"path":"http://philchalmers.github.io/SimDesign/reference/rtruncate.html","id":null,"dir":"Reference","previous_headings":"","what":"Generate a random set of values within a truncated range — rtruncate","title":"Generate a random set of values within a truncated range — rtruncate","text":"Function generates data given supplied random number generating function constructed fall within particular range. Sampled values outside range discarded re-sampled desired criteria met.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rtruncate.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Generate a random set of values within a truncated range — rtruncate","text":"","code":"rtruncate(n, rfun, range, ..., redraws = 100L)"},{"path":"http://philchalmers.github.io/SimDesign/reference/rtruncate.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Generate a random set of values within a truncated range — rtruncate","text":"n number observations generate. first argument passed rfun rfun function generate random values. Function can return numeric/integer vector matrix, additional arguments requred function passed argument ... range numeric vector length two, first element indicates lower bound second upper bound. values generated outside two bounds data redrawn bounded criteria met. output rfun matrix input can specified matrix two rows, first row corresponds lower bound second row upper bound generated column output ... additional arguments passed rfun redraws maximum number redraws take terminating iterative sequence. place safety case range small given random number generator, causing many consecutive rejections. Default 100","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rtruncate.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Generate a random set of values within a truncated range — rtruncate","text":"either numeric vector matrix, values within desired range","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rtruncate.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Generate a random set of values within a truncated range — rtruncate","text":"simulations often useful draw numbers truncated distributions rather across full theoretical range. instance, sampling parameters within range [-4,4] normal distribution. rtruncate function designed accept sampling function, first argument number values sample, draw values iteratively number values within specified bound obtained. situations unlikely bounds located (e.g., sampling standard normal distribution values within [-10,-6]) sampling scheme throw error many re-sampling executions required (default stop 100 calls rfun required).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rtruncate.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Generate a random set of values within a truncated range — rtruncate","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/rtruncate.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Generate a random set of values within a truncated range — rtruncate","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/rtruncate.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Generate a random set of values within a truncated range — rtruncate","text":"","code":"# n = 1000 truncated normal vector between [-2,3] vec <- rtruncate(1000, rnorm, c(-2,3)) summary(vec) #> Min. 1st Qu. Median Mean 3rd Qu. Max. #> -1.96441 -0.59106 0.06295 0.05083 0.64803 2.94607 # truncated correlated multivariate normal between [-1,4] mat <- rtruncate(1000, rmvnorm, c(-1,4), sigma = matrix(c(2,1,1,1),2)) summary(mat) #> V1 V2 #> Min. :-0.9951 Min. :-0.9982 #> 1st Qu.:-0.1587 1st Qu.:-0.1852 #> Median : 0.5094 Median : 0.3215 #> Mean : 0.6414 Mean : 0.4022 #> 3rd Qu.: 1.3206 3rd Qu.: 0.9156 #> Max. : 3.9608 Max. : 3.0498 # truncated correlated multivariate normal between [-1,4] for the # first column and [0,3] for the second column mat <- rtruncate(1000, rmvnorm, cbind(c(-1,4), c(0,3)), sigma = matrix(c(2,1,1,1),2)) summary(mat) #> V1 V2 #> Min. :-0.99785 Min. :0.002002 #> 1st Qu.: 0.07791 1st Qu.:0.316850 #> Median : 0.84390 Median :0.707062 #> Mean : 0.88020 Mean :0.792454 #> 3rd Qu.: 1.58988 3rd Qu.:1.147544 #> Max. : 3.89832 Max. :2.964880 # truncated chi-square with df = 4 between [2,6] vec <- rtruncate(1000, rchisq, c(2,6), df = 4) summary(vec) #> Min. 1st Qu. Median Mean 3rd Qu. Max. #> 2.000 2.688 3.500 3.637 4.437 5.999"},{"path":"http://philchalmers.github.io/SimDesign/reference/runArraySimulation.html","id":null,"dir":"Reference","previous_headings":"","what":"Run a Monte Carlo simulation using array job submissions per condition — runArraySimulation","title":"Run a Monte Carlo simulation using array job submissions per condition — runArraySimulation","text":"function purpose runSimulation, however rather evaluating row design object (potentially parallel computing architecture) function evaluates simulation per independent row condition. mainly useful distributing jobs HPC clusters job array number available (e.g., via SLURM), simulation results must saved independent files complete. Use expandDesign useful distributing replications different jobs, genSeeds required ensure high-quality random number generation across array submissions. See associated vignette brief tutorial setup.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/runArraySimulation.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Run a Monte Carlo simulation using array job submissions per condition — runArraySimulation","text":"","code":"runArraySimulation( design, ..., replications, iseed, filename, dirname = NULL, arrayID = getArrayID(), array2row = function(arrayID) arrayID, addArrayInfo = TRUE, parallel = FALSE, cl = NULL, ncores = parallelly::availableCores(omit = 1L), save_details = list(), control = list(), verbose = ifelse(interactive(), FALSE, TRUE) )"},{"path":"http://philchalmers.github.io/SimDesign/reference/runArraySimulation.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Run a Monte Carlo simulation using array job submissions per condition — runArraySimulation","text":"design design object containing simulation conditions per row basis. function design submit row independent job HPC cluster. See runSimulation details ... additional arguments passed runSimulation replications number independent replications perform per condition (.e., row design). See runSimulation details iseed initial seed passed genSeeds's argument name, along supplied arrayID filename file name save simulation files (need specify extension). However, array ID appended filename. example, filename = 'mysim' files stored 'mysim-1.rds', 'mysim-2.rds', row ID design dirname directory save files associated filename . omitted files stored working directory script submitted arrayID array identifier scheduler. Must number 1 nrow(design). specified getArrayID called automatically, assumes environmental variables available according SLURM scheduler array2row user defined function single argument arrayID. Used convert detected arrayID suitable row index design object input. default arrayID associated respective row design. example, arrayID evaluate 10 rows design object function function(arrayID){1:10 + 10 * (arrayID-1)} can passed array2row addArrayInfo logical; array ID original design row number added SimResults(...) output? parallel logical; use parallel computations via \"SOCK\" cluster? useful instruction shell file requires 1 core (number cores detected via ncores). application random seeds distributed using nextRNGSubStream cl cluster definition. omitted \"SOCK\" cluster defined ncores number cores use parallel=TRUE. Note default uses 1 minus number available cores, therefore useful ncores > 2 defined shell instruction file save_details optional list extra file saving details. See runSimulation control control list passed runSimulation. addition original control elements two additional arguments added: max_time max_RAM, specified character vectors one element. max_time specifies maximum time allowed single simulation condition execute (default set time limits), formatted according specification timeFormater. primarily useful HPC cluster time known elapsed time. general, input set somewhere around 80-90 cluster terminated can saved. Default applies time limit Similarly, max_RAM controls (approximate) maximum size simulation storage objects can grow RAM becomes issue. can specified either terms megabytes (MB), gigabytes (GB), terabytes (TB). example, max_RAM = \"4GB\" indicates simulation storage objects larger 4GB workflow terminate early, returning successful results point). Useful larger HPC cluster jobs RAM constraints terminate abruptly. rule thumb set around 90 available. Default applies memory limit verbose logical; pass verbose flag runSimulation. Unlike runSimulation set FALSE interactive sessions, though set TRUE non-interactive information session stored (e.g., SLURM .files)","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/runArraySimulation.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Run a Monte Carlo simulation using array job submissions per condition — runArraySimulation","text":"Due nature replication split important L'Ecuyer-CMRG (2002) method random seeds used across array ID submissions (cf. runSimulation's parallel approach, uses method distribute random seeds within isolated condition rather conditions). , function requires seeds generated using genSeeds iseed arrayID inputs ensure job analyzing high-quality set random numbers via L'Ecuyer-CMRG's (2002) method, incremented using nextRNGStream. Additionally, timed simulations HPC clusters also recommended pass control = list(max_time) value avoid discarding conditions require specified time shell script. max_time value less maximum time allocated HPC cluster (e.g., approximately 90 depends long, variable, replication ). Simulations missing replication information submit new set jobs later time collect missing replication information.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/runArraySimulation.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Run a Monte Carlo simulation using array job submissions per condition — runArraySimulation","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/runArraySimulation.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Run a Monte Carlo simulation using array job submissions per condition — runArraySimulation","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/runArraySimulation.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Run a Monte Carlo simulation using array job submissions per condition — runArraySimulation","text":"","code":"library(SimDesign) Design <- createDesign(N = c(10, 20, 30)) Generate <- function(condition, fixed_objects) { dat <- with(condition, rnorm(N, 10, 5)) # distributed N(10, 5) dat } Analyse <- function(condition, dat, fixed_objects) { ret <- c(mean=mean(dat), median=median(dat)) # mean/median of sample data ret } Summarise <- function(condition, results, fixed_objects){ colMeans(results) } if (FALSE) { # \\dontrun{ # define initial seed (do this only once to keep it constant!) # iseed <- genSeeds() iseed <- 554184288 ### On cluster submission, the active array ID is obtained via getArrayID(), ### and therefore should be used in real SLURM submissions arrayID <- getArrayID(type = 'slurm') # However, the following example arrayID is set to # the first row only for testing purposes arrayID <- 1L # run the simulation (results not caught on job submission, only files saved) res <- runArraySimulation(design=Design, replications=50, generate=Generate, analyse=Analyse, summarise=Summarise, arrayID=arrayID, iseed=iseed, filename='mysim') # saved as 'mysim-1.rds' res SimResults(res) # condition and replication count stored # same, but evaluated with multiple cores res <- runArraySimulation(design=Design, replications=50, generate=Generate, analyse=Analyse, summarise=Summarise, arrayID=arrayID, parallel=TRUE, ncores=3, iseed=iseed, filename='myparsim') res SimResults(res) # condition and replication count stored dir() SimClean(c('mysim-1.rds', 'myparsim-1.rds')) ######################## # Same submission job as above, however split the replications over multiple # evaluations and combine when complete Design5 <- expandDesign(Design, 5) Design5 # iseed <- genSeeds() iseed <- 554184288 # arrayID <- getArrayID(type = 'slurm') arrayID <- 14L # run the simulation (replications reduced per row, but same in total) runArraySimulation(design=Design5, replications=10, generate=Generate, analyse=Analyse, summarise=Summarise, iseed=iseed, filename='mylongsim', arrayID=arrayID) res <- readRDS('mylongsim-14.rds') res SimResults(res) # condition and replication count stored SimClean('mylongsim-14.rds') ### # Emulate the arrayID distribution, storing all results in a 'sim/' folder # (if 'sim/' does not exist in runArraySimulation() it will be # created automatically) dir.create('sim/') # Emulate distribution to nrow(Design5) = 15 independent job arrays ## (just used for presentation purposes on local computer) sapply(1:nrow(Design5), \\(arrayID) runArraySimulation(design=Design5, replications=10, generate=Generate, analyse=Analyse, summarise=Summarise, iseed=iseed, arrayID=arrayID, filename='condition', dirname='sim', # files: \"sim/condition-#.rds\" control = list(max_time=\"04:00:00\", max_RAM=\"4GB\"))) |> invisible() # If necessary, conditions above will manually terminate before # 4 hours and 4GB of RAM are used, returning any # successfully completed results before the HPC session times # out (provided .slurm script specified more than 4 hours) # list saved files dir('sim/') # check that all files saved (warnings will be raised if missing files) SimCheck('sim/') |> isTRUE() condition14 <- readRDS('sim/condition-14.rds') condition14 SimResults(condition14) # aggregate simulation results into single file final <- SimCollect('sim/') final # clean simulation directory SimClean(dirs='sim/') ############ # same as above, however passing different amounts of information depending # on the array ID array2row <- function(arrayID){ switch(arrayID, \"1\"=1:8, \"2\"=9:14, \"3\"=15) } # arrayID 1 does row 1 though 8, arrayID 2 does 9 to 14 array2row(1) array2row(2) array2row(3) # arrayID 3 does 15 only # emulate remote array distribution with only 3 arrays sapply(1:3, \\(arrayID) runArraySimulation(design=Design5, replications=10, generate=Generate, analyse=Analyse, summarise=Summarise, iseed=iseed, arrayID=arrayID, filename='condition', dirname='sim', array2row=array2row)) |> invisible() # list saved files dir('sim/') # note that all row conditions are still stored separately, though note that # arrayID is now 2 instead condition14 <- readRDS('sim/condition-14.rds') condition14 SimResults(condition14) # aggregate simulation results into single file final <- SimCollect('sim/') final # clean simulation directory SimClean(dirs='sim/') } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/runSimulation.html","id":null,"dir":"Reference","previous_headings":"","what":"Run a Monte Carlo simulation given conditions and simulation functions — runSimulation","title":"Run a Monte Carlo simulation given conditions and simulation functions — runSimulation","text":"function runs Monte Carlo simulation study given set predefined simulation functions, design conditions, number replications. Results can saved temporary files case interruptions may restored re-running runSimulation, provided respective temp file can found working directory. runSimulation supports parallel cluster computing (parallel future packages; see also runArraySimulation submitting array jobs HPC clusters), global local debugging, error handling (including fail-safe stopping functions fail often, even across nodes), provides bootstrap estimates sampling variability (optional), automatic tracking error warning messages associated .Random.seed states. convenience, functions available R work-space exported across nodes easily accessible (however, R objects , therefore must passed fixed_objects input become available across nodes).","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/runSimulation.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Run a Monte Carlo simulation given conditions and simulation functions — runSimulation","text":"","code":"runSimulation( design, replications, generate, analyse, summarise, fixed_objects = NULL, packages = NULL, filename = NULL, debug = \"none\", load_seed = NULL, save = any(replications > 2), store_results = TRUE, save_results = FALSE, parallel = FALSE, ncores = parallelly::availableCores(omit = 1L), cl = NULL, notification = \"none\", beep = FALSE, sound = 1, CI = 0.95, seed = NULL, boot_method = \"none\", boot_draws = 1000L, max_errors = 50L, resume = TRUE, save_details = list(), control = list(), progress = TRUE, verbose = TRUE ) # S3 method for class 'SimDesign' summary(object, ...) # S3 method for class 'SimDesign' print(x, list2char = TRUE, ...)"},{"path":"http://philchalmers.github.io/SimDesign/reference/runSimulation.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Run a Monte Carlo simulation given conditions and simulation functions — runSimulation","text":"design tibble data.frame object containing Monte Carlo simulation conditions studied, row represents unique condition column factor varied. See createDesign standard approach create simulation design object replications number independent replications perform per condition (.e., row design). Can single number, used design condition, integer vector length equal nrow(design). inputs must greater 0, though setting less 3 (initial testing purpose) disable save control$stop_on_fatal flags generate user-defined data parameter generating function (named list functions). See Generate details. Note argument may omitted user wish generate data analyse step, real-world simulations generally recommended. multiple generate functions provided list list generate functions executed order first valid generate function executed, subsequent generation functions ignored (see GenerateIf apply data generation specific conditions). analyse user-defined analysis function (named list functions) acts data generated Generate (, generate omitted, can used generate analyses simulated data). See Analyse details summarise optional (strongly recommended) user-defined summary function Summarise used compute meta-statistical summary information replications completed within design condition. Return function, order increasing complexity, : named numeric vector data.frame one row, matrix data.frame one row, , failing atomic types, named list. summary objects easily appended original design object use SimExtract option = 'summarise'. Note unlike Generate Analyse steps, Summarise portion important perfectly organize results can summarised later using built- reSummarise function (provided either store_results = TRUE save_results = TRUE included). Omitting function return tibble Design associated results information nrow(Design) * repliations evaluations results Analyse() call one-dimensional vector. general objects returned Analyse() (lists), list containing results returned form Analyse. generally recommended didactic purposes results leave large amount information (e.g., try-errors, warning messages, saving files, etc), can witness memory related issues Analyse function returns larger objects, generally flexible internally. However, may useful replications expensive ANOVA-based decompositions involving within-condition replication information interest, though course can circumvented using store_results = TRUE save_results = TRUE without supplied summarise definition. fixed_objects (optional) object (usually named list) containing additional user-defined objects remain fixed across conditions. useful including large vectors/matrices population parameters, fixed data information used across conditions replications (e.g., including common design matrix linear regression models), simply control constant global elements (e.g., constant sample size) packages character vector external packages used simulation (e.g., c('MASS', 'extraDistr', 'simsem') ). Use input running code parallel use non-standard functions additional packages, otherwise functions must made available using explicit library require calls within provided simulation functions. Alternatively, functions can called explicitly without attaching package :: operator (e.g., extraDistr::rgumbel()) filename (optional) name .rds file save final simulation results . extension .rds included file name (e.g. \"mysimulation\" versus \"mysimulation.rds\") .rds extension automatically added file name ensure file extension correct. Note file name already exists working directly time saving new file generated instead warning thrown. helps avoid accidentally overwriting existing files. Default NULL, indicating file saved default debug string indicating initiate browser() call editing debugging, pairs particularly well load_seed argument precise debugging. General options 'none' (default; debugging), 'error', starts debugger error code detected one three generate-analyse-summarise functions, '', debugs user defined functions regardless whether error thrown . Specific options include: 'generate' debug data simulation function, 'analyse' debug computational function, 'summarise' debug aggregation function. Analyse argument supplied named list functions also possible debug specific function interest passing name respective function list. instance, analyse = list(A1=Analyse.A1, A2=Analyse.A2) passing debug = 'A1' debug first function list, remaining analysis functions ignored. debugging specific rows Design input (e.g., number initial rows successfully complete kth row fails) row number can appended standard debug input using '-' separator. instance, debugging whenever error raised second row Design can declared via debug = 'error-2'. Finally, users may place browser calls within respective functions debugging specific lines, useful debugging based conditional evaluations (e.g., (== 'problem') browser()). Note parallel computation flags automatically disabled browser() detected debugging argument 'none' supplied load_seed used replicate exact simulation state, primarily useful debugging purposes. Input can character object indicating file load .Random.seeds saved (call save_seeds = TRUE), integer vector indicating actual .Random.seed values (e.g., extracted using store_seeds). E.g., load_seed = 'design-row-2/seed-1' load first seed second row design input, explicitly passing elements .Random.seed (see SimExtract extract seeds associated explicitly errors simulation, column represents unique seed). input character vector important modify design input object, otherwise path may point correct saved location, input integer vector (single column tbl object) important modify design input order load exact seed corresponding design row. Default NULL save logical; save temporary simulation state hard-drive? useful simulations require extended amount time, though shorter simulations can disabled slightly improve computational efficiency. TRUE, default evaluating replications > 2, temp file created working directory allows simulation state saved recovered (case power outages, crashes, etc). well, triggering flag save fatal .Random.seed states conditions unexpectedly crash (seed stored row-wise external .rds file), provides much easier mechanism debug issues (see load_seed details). Upon completion, temp file removed. recover simulation last known location (patched issues previous execution code) simply re-run code used initially define simulation external file automatically detected read-. Default TRUE replications > 10 FALSE otherwise store_results logical; store complete tables simulation results returned object? TRUE default, though RAM anticipated issue see save_results instead. Note Design object omitted call runSimulation(), number rows Design exactly 1, argument automatically set TRUE RAM storage longer issue. extract results pass returned object either SimResults SimExtract = 'results', return named list simulation results condition nrow(Design) > 1; otherwise, nrow(Design) == 1 Design missing results object stored -save_results logical; save results returned Analyse external .rds files located defined save_results_dirname directory/folder? Use like keep track individual parameters returned analysis function. saved object contain list three elements containing condition (row design), results (list matrix), try-errors. See SimResults example read .rds files back R simulation complete. Default FALSE. WARNING: saving results hard-drive can fill space quickly larger simulations. sure test option using smaller number replications full Monte Carlo simulation performed. See also reSummarise applying summarise functions saved simulation results parallel logical; use parallel processing parallel package unique condition? distributes independent replications across defined nodes, repeated row condition design input. Alternatively, future package approach desired passing parallel = 'future' runSimulation() use defined plan execution. allows greater flexibility specifying general computing plan (e.g., plan(multisession)) parallel computing machine, plan(future.batchtools::batchtools_torque) plan(future.batchtools::batchtools_slurm) common MPI/Slurm schedulers, etc). However, responsibility user use plan(sequential) reset computing plan jobs completed ncores number cores used parallel execution (ignored using future package approach). Default uses available minus 1 cl cluster object defined makeCluster used run code parallel (ignored using future package approach). NULL parallel = TRUE, local cluster object defined selects maximum number cores available stopped simulation complete. Note supplying cl object automatically set parallel argument TRUE. Define supply cluster object whenever multiple nodes can link together manually future package attached prior executing runSimulation() associated plan() followed instead notification optional character vector input can used send Pushbullet notifications configured computer. reports information total execution time, condition completed, error/warning messages recorded. arguments assumes users already ) registered Pushbullet account, B) installed application mobile device computer, C) created associated JSON file form ~/.rpushbullet.json using RPushbullet::pbSetup()). utilize RPushbullet SimDesign first call library(RPushbullet running runSimulation() read-default JSON file. Next, pass one following supported options: 'none' (default; send notification), 'condition' send notification condition completed, 'complete' send notification simulation finished. beep logical; call beepr package simulation completed? sound sound argument passed beepr::beep() CI bootstrap confidence interval level (default 95%) seed vector list integers used reproducibility. length vector must equal number rows design. input vector set.seed clusterSetRNGStream condition called, respectively. list provided numbers must generated gen_seeds argument CMRG.seed used specify initial. list approach ensures random number generation independence across conditions replications, vector input ensures independence within replications per conditions necessarily across conditions. Default randomly generates seeds within range 1 2147483647 condition via gen_seeds boot_method method performing non-parametric bootstrap confidence intervals respective meta-statistics computed Summarise function. Can 'basic' empirical bootstrap CI, 'percentile' percentile CIs, 'norm' normal approximations CIs, 'studentized' Studentized CIs (used simulations lower replications due computational intensity). Alternatively, CIs can constructed using argument 'CLT', computes intervals according large-sample standard error approximation \\(SD(results)/\\sqrt{R}\\). Default 'none', performs CI computations boot_draws number non-parametric bootstrap draws sample summarise function generate-analyse replications collected. Default 1000 max_errors simulation terminate number consecutive errors thrown given condition, causing simulation continue next unique design condition. included avoid getting stuck infinite re-draws, indicate something fatally problematic going wrong generate-analyse phases. Default 50 resume logical; temporary SimDesign file detected simulation resume location? Keeping TRUE generally recommended, however disabled using runSimulation within runSimulation avoid reading improper save states. Alternatively, integer supplied simulation continue associated row location design (e.g., resume=10). useful overwrite previously evaluate element temporary files detected contain fatal errors require re-evaluation without discarding originally valid rows simulation save_details list pertaining information regarding files saved save save_results flags triggered. safe logical; trigger whether safe-saving performed. TRUE files never overwritten accidentally, appropriate program either stop generate new files unique names. Default TRUE compname name computer running simulation. Normally need modified, event manual node breaks running simulation results temp files may resumed another computer changing name node match broken computer. Default result evaluating unname(Sys.info()['nodename']) out_rootdir root directory save files . Default uses current working directory save_results_dirname string indicating name folder save result objects save_results = TRUE. directory/folder exist current working directory unique one created automatically. Default 'SimDesign-results_' associated compname appended filename defined, otherwise filename used replace 'SimDesign' string save_results_filename string indicating name file store, Design row ID appended ensure uniqueness across rows. Specifying input disable checking uniqueness file folder, thereby allowing independent runSimulation calls write save_results_dirname. Useful files stored working directory, however rows Design evaluated isolation (e.g., HPC structures allow asynchronous file storage). WARNING: uniqueness file names checked using approach, therefore please ensure generated name unique priori, naming file based supplied row condition information save_seeds_dirname string indicating name folder save .Random.seed objects save_seeds = TRUE. directory/folder exist current working directory one created automatically. Default 'SimDesign-seeds_' associated compname appended filename defined, otherwise filename used replace 'SimDesign' string tmpfilename string indicating temporary file name save provisional information . specified following used: paste0('SIMDESIGN-TEMPFILE_', compname, '.rds') control list extra information flags controlling less commonly used features. include stop_on_fatal logical (default FALSE); simulation terminated immediately maximum number consecutive errors (max_errors) reached? FALSE, simulation continue though errors occur, however column FATAL_TERMINATION included resulting object indicating final error message observed, NA placeholders placed row-elements. Default FALSE, though automatically set TRUE replications < 3 purpose debugging warnings_as_errors logical (default FALSE); treat warning messages error messages simulation? Default FALSE, therefore warnings collected used restart data generation step, seeds associated warning message conditions stored within final simulation object. Note argument generally intended debugging/early planning stages designing simulation experiment. specific warnings known problematic treated errors please use manageWarnings instead save_seeds logical; save .Random.seed states prior performing replication plain text files located defined save_seeds_dirname directory/folder? Use like keep track every simulation state within replication design condition. can used completely replicate cell simulation need . well, see load_seed input load given .Random.seed exactly replicate generated data analysis state (mostly useful debugging). TRUE, temporary files also saved working directory (way save = TRUE). Default FALSE Note, however, option typically necessary recommended since .Random.seed states simulation replications throw errors execution automatically stored within final simulation object, can extracted investigated using SimExtract. Hence, option interest replications must reproducible (occurs rarely), otherwise defaults runSimulation sufficient store_Random.seeds logical; store complete .Random.seed states simulation replicate? Default FALSE can take great deal unnecessary RAM (see related save_seeds), however may useful used runArraySimulation. extract use SimExtract(..., = 'stored_Random.seeds') store_warning_seeds logical (default FALSE); addition storing .Random.seed states whenever error messages raised, also store .Random.seed states warnings raised? disabled default since warnings generally less problematic errors, many warnings messages may raised throughout simulation (potentially causing RAM related issues constructing final simulation object given simulation replicate generate numerous warnings, storing seeds states add quickly). Set TRUE replicating warning messages important, however aware many warnings messages raised simulation implementation cause RAM related issues. include_replication_index include_reps logical (default FALSE); REPLICATION element added condition object performing simulation track specific replication experiment evaluated? useful , instance, attempting run external software programs (e.g., Mplus) require saving temporary data sets hard-drive (see Wiki examples) try_all_analyse logical; analyse list, every generated data set analyzed function definition analyse list? Default TRUE. Note TRUE default can computationally demanding analysis functions require computational resources others, data discarded early invalid candidate (e.g., estimating model via maximum-likelihood analyze component, estimating model using MCMC estimation another). Hence, main benefit using FALSE instead data set may rejected earlier, easier/faster estimate analyse definitions placed earlier list functions evaluated sequence (e.g., Analyse = list(MLE=MLE_definition, MCMC=MCMC_definition)) allow_na logical (default FALSE); NAs allowed analyse step valid result simulation analysis? allow_nan logical (default FALSE); NaNs allowed analyse step valid result simulation analysis? type default type cluster create cl object supplied. Windows OS defaults \"PSOCK\", otherwise \"SOCK\" selected (suitable Linux Mac OSX). ignored user specifies cl object print_RAM logical (default TRUE); print amount RAM used throughout simulation? Set FALSE unnecessary call gc unnecessarily time consuming max_time Similar runArraySimulation, specifies (approximate) maximum time simulation allowed executed. However, unlike implementation runArraySimulation evaluated per condition basis, max_time evaluated every row design object completed (hence, notably approximate potential overshoot wider margin). Default sets time limit. See timeFormater input specifications. max_RAM Similar runArraySimulation, specifies (approximate) maximum RAM simulation allowed occupy. However, unlike implementation runArraySimulation evaluated per condition basis, max_RAM evaluated every row design object completed (hence, notably approximate potential overshoot wider margin). Default sets RAM limit. See runArraySimulation input specifications. progress logical; display progress bar (using pbapply package) simulation condition? useful simulations conditions take long time run (see also notifications argument). Default TRUE verbose logical; print messages R console? Default TRUE object SimDesign object returned runSimulation ... additional arguments x SimDesign object returned runSimulation list2char logical; tibble object re-evaluate list elements character vectors better printing levels? Note change original classes object, just printed. Default TRUE","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/runSimulation.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Run a Monte Carlo simulation given conditions and simulation functions — runSimulation","text":"tibble dplyr package (also class 'SimDesign') original design conditions left-columns, simulation results middle columns, additional information right-columns (see ). right-column information condition : REPLICATIONS indicate number Monte Carlo replications, SIM_TIME indicate long (seconds) took complete Monte Carlo replications respective design condition, RAM_USED amount RAM use time completing simulation condition, COMPLETED indicate date given simulation condition completed, SEED integer values seed argument (applicable; relevant L'Ecuyer-CMRG method used), , applicable, ERRORS WARNINGS contain counts number error warning messages caught (errors/warnings observed columns omitted). Note extract specific error warnings messages see SimExtract. Finally, boot_method valid input 'none' final right-columns contain labels BOOT_ followed name associated meta-statistic defined summarise() bootstrapped confidence interval location meta-statistics.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/runSimulation.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Run a Monte Carlo simulation given conditions and simulation functions — runSimulation","text":"-depth tutorial package please refer Chalmers Adkins (2020; doi:10.20982/tqmp.16.4.p248 ). earlier didactic presentation package refer Sigal Chalmers (2016; doi:10.1080/10691898.2016.1246953 ). Finally, see associated wiki Github (https://github.com/philchalmers/SimDesign/wiki) tutorial material, examples, applications SimDesign real-world simulation experiments, well various vignette files associated package. strategy organizing Monte Carlo simulation work-flow 1) Define suitable Design object (tibble data.frame) containing fixed conditional information Monte Carlo simulations. row design object pertains unique set simulation study, column simulation factor investigation (e.g., sample size, distribution types, etc). often expedited using createDesign function, necessary argument subset can used remove redundant non-applicable rows 2) Define three step functions generate data (Generate; see also https://CRAN.R-project.org/view=Distributions list distributions R), analyse generated data computing respective parameter estimates, detection rates, etc (Analyse), finally summarise results across total number replications (Summarise). 3) Pass design object three defined R functions runSimulation, declare number replications perform replications input. function return suitable tibble object complete simulation results execution details 4) Analyze output runSimulation, possibly using ANOVA techniques (SimAnova) generating suitable plots tables Expressing succinctly, functions called following form, exact functional arguments listed: Design <- createDesign(...) Generate <- function(condition, fixed_objects) {...} Analyse <- function(condition, dat, fixed_objects) {...} Summarise <- function(condition, results, fixed_objects) {...} res <- runSimulation(design=Design, replications, generate=Generate, analyse=Analyse, summarise=Summarise) condition object represents single row design object, indicating unique Monte Carlo simulation condition. condition object also contains two additional elements help track simulation's state: ID variable, indicating respective row number design object, REPLICATION element indicating replication iteration number (integer value 1 replication). setup allows users easily locate rth replication (e.g., REPLICATION == 500) within jth row simulation design (e.g., ID == 2). REPLICATION input also useful temporarily saving files hard-drive calling external command line utilities (see examples wiki). template-based version work-flow, often useful initially defining simulation, use SimFunctions function. function write template simulation one/two files modifying required functions objects can begin immediately. means users can focus Monte Carlo simulation details right away rather worrying repetitive administrative code-work required organize simulation's execution flow. Finally, examples, presentation files, tutorials can found package wiki located https://github.com/philchalmers/SimDesign/wiki.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/runSimulation.html","id":"saving-data-results-seeds-and-the-simulation-state","dir":"Reference","previous_headings":"","what":"Saving data, results, seeds, and the simulation state","title":"Run a Monte Carlo simulation given conditions and simulation functions — runSimulation","text":"conserve RAM, temporary objects (data generated across conditions replications) discarded; however, can saved hard-disk passing appropriate flags. longer simulations recommended use save_results flag write analysis results hard-drive. use store_seeds save_seeds options can evoked save R's .Random.seed state allow complete reproducibility replication within condition. individual .Random.seed terms can read load_seed input reproduce exact simulation state given replication. often though, saving complete list seeds unnecessary problematic seeds automatically stored final simulation object allow easier replicability potentially problematic errors (incidentally can extracted using SimExtract(res, 'error_seeds') passed load_seed argument). Finally, providing vector seeds also possible ensure simulation condition macro reproducible single/multi-core method selected. Finally, Monte Carlo simulation complete recommended write results hard-drive safe keeping, particularly filename argument provided (reasons obvious parallel computation descriptions ). Using filename argument supplied safer using, instance, saveRDS directly files never accidentally overwritten, instead new file name created conflict arises; type implementation safety prevalent many locations package help avoid unrecoverable (yet surprisingly common) mistakes process designing executing Monte Carlo simulations.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/runSimulation.html","id":"resuming-temporary-results","dir":"Reference","previous_headings":"","what":"Resuming temporary results","title":"Run a Monte Carlo simulation given conditions and simulation functions — runSimulation","text":"event computer crash, power outage, etc, save = TRUE used (default) original code used execute runSimulation() need re-run resume simulation. saved temp file read function automatically, simulation continue one condition left simulation state terminated. users wish remove temporary simulation state entirely start anew simply pass SimClean(temp = TRUE) R console remove previously saved temporary objects.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/runSimulation.html","id":"a-note-on-parallel-computing","dir":"Reference","previous_headings":"","what":"A note on parallel computing","title":"Run a Monte Carlo simulation given conditions and simulation functions — runSimulation","text":"running simulations parallel (either parallel = TRUE using future approach plan() sequential) R objects defined global environment generally visible across nodes. Hence, may see errors Error: object 'something' found try use object defined work space passed runSimulation. avoid type error, simply pass additional objects fixed_objects input (usually convenient supply named list objects). Fortunately, however, custom functions defined global environment exported across nodes automatically. makes convenient writing code custom functions always available across nodes visible R work space. well, note packages input declare packages must loaded via library() order make specific non-standard R functions available across nodes.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/runSimulation.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Run a Monte Carlo simulation given conditions and simulation functions — runSimulation","text":"Chalmers, R. P., & Adkins, M. C. (2020). Writing Effective Reliable Monte Carlo Simulations SimDesign Package. Quantitative Methods Psychology, 16(4), 248-280. doi:10.20982/tqmp.16.4.p248 Sigal, M. J., & Chalmers, R. P. (2016). Play : Teaching statistics Monte Carlo simulation. Journal Statistics Education, 24(3), 136-156. doi:10.1080/10691898.2016.1246953","code":""},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/reference/runSimulation.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Run a Monte Carlo simulation given conditions and simulation functions — runSimulation","text":"Phil Chalmers rphilip.chalmers@gmail.com","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/runSimulation.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Run a Monte Carlo simulation given conditions and simulation functions — runSimulation","text":"","code":"#------------------------------------------------------------------------------- # Example 1: Sampling distribution of mean # This example demonstrate some of the simpler uses of SimDesign, # particularly for classroom settings. The only factor varied in this simulation # is sample size. # skeleton functions to be saved and edited SimFunctions() #> #------------------------------------------------------------------- #> #> library(SimDesign) #> #> Design <- createDesign(factor1 = NA, #> factor2 = NA) #> #> #------------------------------------------------------------------- #> #> Generate <- function(condition, fixed_objects) { #> dat <- data.frame() #> dat #> } #> #> Analyse <- function(condition, dat, fixed_objects) { #> ret <- nc(stat1 = NaN, stat2 = NaN) #> ret #> } #> #> Summarise <- function(condition, results, fixed_objects) { #> ret <- c(bias = NaN, RMSE = NaN) #> ret #> } #> #> #------------------------------------------------------------------- #> #> res <- runSimulation(design=Design, replications=2, generate=Generate, #> analyse=Analyse, summarise=Summarise) #> res #> #### Step 1 --- Define your conditions under study and create design data.frame Design <- createDesign(N = c(10, 20, 30)) #~~~~~~~~~~~~~~~~~~~~~~~~ #### Step 2 --- Define generate, analyse, and summarise functions # help(Generate) Generate <- function(condition, fixed_objects) { dat <- with(condition, rnorm(N, 10, 5)) # distributed N(10, 5) dat } # help(Analyse) Analyse <- function(condition, dat, fixed_objects) { ret <- c(mean=mean(dat)) # mean of the sample data vector ret } # help(Summarise) Summarise <- function(condition, results, fixed_objects) { # mean and SD summary of the sample means ret <- c(mu=mean(results$mean), SE=sd(results$mean)) ret } #~~~~~~~~~~~~~~~~~~~~~~~~ #### Step 3 --- Collect results by looping over the rows in design # run the simulation in testing mode (replications = 2) Final <- runSimulation(design=Design, replications=2, generate=Generate, analyse=Analyse, summarise=Summarise) #> save, stop_on_fatal, and print_RAM flags disabled for testing purposes #> #> Design: 1/3; Replications: 2 Total Time: 0.00s #> Conditions: N=10 #> #> Design: 2/3; Replications: 2 Total Time: 0.00s #> Conditions: N=20 #> #> Design: 3/3; Replications: 2 Total Time: 0.00s #> Conditions: N=30 #> #> #> Simulation complete. Total execution time: 0.01s Final #> # A tibble: 3 × 7 #> N mu SE REPLICATIONS SIM_TIME SEED COMPLETED #> #> 1 10 10.202 1.4729 2 0.00s 533810122 Wed Dec 4 05:32:34 20… #> 2 20 10.885 0.31864 2 0.00s 1340659367 Wed Dec 4 05:32:34 20… #> 3 30 9.6268 1.2548 2 0.00s 881068069 Wed Dec 4 05:32:34 20… SimResults(Final) #> # A tibble: 6 × 2 #> N mean #> #> 1 10 11.2 #> 2 10 9.16 #> 3 20 10.7 #> 4 20 11.1 #> 5 30 10.5 #> 6 30 8.74 # reproduce exact simulation Final_rep <- runSimulation(design=Design, replications=2, seed=Final$SEED, generate=Generate, analyse=Analyse, summarise=Summarise) #> save, stop_on_fatal, and print_RAM flags disabled for testing purposes #> #> Design: 1/3; Replications: 2 Total Time: 0.00s #> Conditions: N=10 #> #> Design: 2/3; Replications: 2 Total Time: 0.00s #> Conditions: N=20 #> #> Design: 3/3; Replications: 2 Total Time: 0.00s #> Conditions: N=30 #> #> #> Simulation complete. Total execution time: 0.00s Final_rep #> # A tibble: 3 × 7 #> N mu SE REPLICATIONS SIM_TIME SEED COMPLETED #> #> 1 10 10.202 1.4729 2 0.00s 533810122 Wed Dec 4 05:32:34 20… #> 2 20 10.885 0.31864 2 0.00s 1340659367 Wed Dec 4 05:32:34 20… #> 3 30 9.6268 1.2548 2 0.00s 881068069 Wed Dec 4 05:32:34 20… SimResults(Final_rep) #> # A tibble: 6 × 2 #> N mean #> #> 1 10 11.2 #> 2 10 9.16 #> 3 20 10.7 #> 4 20 11.1 #> 5 30 10.5 #> 6 30 8.74 if (FALSE) { # \\dontrun{ # run with more standard number of replications Final <- runSimulation(design=Design, replications=1000, generate=Generate, analyse=Analyse, summarise=Summarise) Final SimResults(Final) #~~~~~~~~~~~~~~~~~~~~~~~~ #### Extras # compare SEs estimates to the true SEs from the formula sigma/sqrt(N) 5 / sqrt(Design$N) # To store the results from the analyse function either # a) omit a definition of summarise() to return all results, # b) use store_results = TRUE (default) to store results internally and later # extract with SimResults(), or # c) pass save_results = TRUE to runSimulation() and read the results in with SimResults() # # Note that method c) should be adopted for larger simulations, particularly # if RAM storage could be an issue and error/warning message information is important. # a) approach res <- runSimulation(design=Design, replications=100, generate=Generate, analyse=Analyse) res # b) approach (store_results = TRUE by default) res <- runSimulation(design=Design, replications=100, generate=Generate, analyse=Analyse, summarise=Summarise) res SimResults(res) # c) approach Final <- runSimulation(design=Design, replications=100, save_results=TRUE, generate=Generate, analyse=Analyse, summarise=Summarise) # read-in all conditions (can be memory heavy) res <- SimResults(Final) res head(res[[1]]$results) # just first condition res <- SimResults(Final, which=1) head(res$results) dplyr::tibble(res$condition, res$results) # obtain empirical bootstrapped CIs during an initial run # the simulation was completed (necessarily requires save_results = TRUE) res <- runSimulation(design=Design, replications=1000, boot_method = 'basic', generate=Generate, analyse=Analyse, summarise=Summarise) res # alternative bootstrapped CIs that uses saved results via reSummarise(). # Default directory save to: dirname <- paste0('SimDesign-results_', unname(Sys.info()['nodename']), \"/\") res <- reSummarise(summarise=Summarise, dir=dirname, boot_method = 'basic') res # remove the saved results from the hard-drive if you no longer want them SimClean(results = TRUE) } # } #------------------------------------------------------------------------------- # Example 2: t-test and Welch test when varying sample size, group sizes, and SDs # skeleton functions to be saved and edited SimFunctions() #> #------------------------------------------------------------------- #> #> library(SimDesign) #> #> Design <- createDesign(factor1 = NA, #> factor2 = NA) #> #> #------------------------------------------------------------------- #> #> Generate <- function(condition, fixed_objects) { #> dat <- data.frame() #> dat #> } #> #> Analyse <- function(condition, dat, fixed_objects) { #> ret <- nc(stat1 = NaN, stat2 = NaN) #> ret #> } #> #> Summarise <- function(condition, results, fixed_objects) { #> ret <- c(bias = NaN, RMSE = NaN) #> ret #> } #> #> #------------------------------------------------------------------- #> #> res <- runSimulation(design=Design, replications=2, generate=Generate, #> analyse=Analyse, summarise=Summarise) #> res #> if (FALSE) { # \\dontrun{ # in real-world simulations it's often better/easier to save # these functions directly to your hard-drive with SimFunctions('my-simulation') } # } #### Step 1 --- Define your conditions under study and create design data.frame Design <- createDesign(sample_size = c(30, 60, 90, 120), group_size_ratio = c(1, 4, 8), standard_deviation_ratio = c(.5, 1, 2)) Design #> # A tibble: 36 × 3 #> sample_size group_size_ratio standard_deviation_ratio #> #> 1 30 1 0.5 #> 2 60 1 0.5 #> 3 90 1 0.5 #> 4 120 1 0.5 #> 5 30 4 0.5 #> 6 60 4 0.5 #> 7 90 4 0.5 #> 8 120 4 0.5 #> 9 30 8 0.5 #> 10 60 8 0.5 #> # ℹ 26 more rows #~~~~~~~~~~~~~~~~~~~~~~~~ #### Step 2 --- Define generate, analyse, and summarise functions Generate <- function(condition, fixed_objects) { N <- condition$sample_size # could use Attach() to make objects available grs <- condition$group_size_ratio sd <- condition$standard_deviation_ratio if(grs < 1){ N2 <- N / (1/grs + 1) N1 <- N - N2 } else { N1 <- N / (grs + 1) N2 <- N - N1 } group1 <- rnorm(N1) group2 <- rnorm(N2, sd=sd) dat <- data.frame(group = c(rep('g1', N1), rep('g2', N2)), DV = c(group1, group2)) dat } Analyse <- function(condition, dat, fixed_objects) { welch <- t.test(DV ~ group, dat)$p.value independent <- t.test(DV ~ group, dat, var.equal=TRUE)$p.value # In this function the p values for the t-tests are returned, # and make sure to name each element, for future reference ret <- nc(welch, independent) ret } Summarise <- function(condition, results, fixed_objects) { #find results of interest here (e.g., alpha < .1, .05, .01) ret <- EDR(results, alpha = .05) ret } #~~~~~~~~~~~~~~~~~~~~~~~~ #### Step 3 --- Collect results by looping over the rows in design # first, test to see if it works res <- runSimulation(design=Design, replications=2, generate=Generate, analyse=Analyse, summarise=Summarise) #> save, stop_on_fatal, and print_RAM flags disabled for testing purposes #> #> Design: 1/36; Replications: 2 Total Time: 0.00s #> Conditions: sample_size=30, group_size_ratio=1, standard_deviation_ratio=0.5 #> #> Design: 2/36; Replications: 2 Total Time: 0.01s #> Conditions: sample_size=60, group_size_ratio=1, standard_deviation_ratio=0.5 #> #> Design: 3/36; Replications: 2 Total Time: 0.01s #> Conditions: sample_size=90, group_size_ratio=1, standard_deviation_ratio=0.5 #> #> Design: 4/36; Replications: 2 Total Time: 0.01s #> Conditions: sample_size=120, group_size_ratio=1, standard_deviation_ratio=0.5 #> #> Design: 5/36; Replications: 2 Total Time: 0.02s #> Conditions: sample_size=30, group_size_ratio=4, standard_deviation_ratio=0.5 #> #> Design: 6/36; Replications: 2 Total Time: 0.02s #> Conditions: sample_size=60, group_size_ratio=4, standard_deviation_ratio=0.5 #> #> Design: 7/36; Replications: 2 Total Time: 0.03s #> Conditions: sample_size=90, group_size_ratio=4, standard_deviation_ratio=0.5 #> #> Design: 8/36; Replications: 2 Total Time: 0.03s #> Conditions: sample_size=120, group_size_ratio=4, standard_deviation_ratio=0.5 #> #> Design: 9/36; Replications: 2 Total Time: 0.04s #> Conditions: sample_size=30, group_size_ratio=8, standard_deviation_ratio=0.5 #> #> Design: 10/36; Replications: 2 Total Time: 0.04s #> Conditions: sample_size=60, group_size_ratio=8, standard_deviation_ratio=0.5 #> #> Design: 11/36; Replications: 2 Total Time: 0.04s #> Conditions: sample_size=90, group_size_ratio=8, standard_deviation_ratio=0.5 #> #> Design: 12/36; Replications: 2 Total Time: 0.05s #> Conditions: sample_size=120, group_size_ratio=8, standard_deviation_ratio=0.5 #> #> Design: 13/36; Replications: 2 Total Time: 0.05s #> Conditions: sample_size=30, group_size_ratio=1, standard_deviation_ratio=1 #> #> Design: 14/36; Replications: 2 Total Time: 0.06s #> Conditions: sample_size=60, group_size_ratio=1, standard_deviation_ratio=1 #> #> Design: 15/36; Replications: 2 Total Time: 0.06s #> Conditions: sample_size=90, group_size_ratio=1, standard_deviation_ratio=1 #> #> Design: 16/36; Replications: 2 Total Time: 0.07s #> Conditions: sample_size=120, group_size_ratio=1, standard_deviation_ratio=1 #> #> Design: 17/36; Replications: 2 Total Time: 0.07s #> Conditions: sample_size=30, group_size_ratio=4, standard_deviation_ratio=1 #> #> Design: 18/36; Replications: 2 Total Time: 0.08s #> Conditions: sample_size=60, group_size_ratio=4, standard_deviation_ratio=1 #> #> Design: 19/36; Replications: 2 Total Time: 0.08s #> Conditions: sample_size=90, group_size_ratio=4, standard_deviation_ratio=1 #> #> Design: 20/36; Replications: 2 Total Time: 0.08s #> Conditions: sample_size=120, group_size_ratio=4, standard_deviation_ratio=1 #> #> Design: 21/36; Replications: 2 Total Time: 0.09s #> Conditions: sample_size=30, group_size_ratio=8, standard_deviation_ratio=1 #> #> Design: 22/36; Replications: 2 Total Time: 0.09s #> Conditions: sample_size=60, group_size_ratio=8, standard_deviation_ratio=1 #> #> Design: 23/36; Replications: 2 Total Time: 0.10s #> Conditions: sample_size=90, group_size_ratio=8, standard_deviation_ratio=1 #> #> Design: 24/36; Replications: 2 Total Time: 0.10s #> Conditions: sample_size=120, group_size_ratio=8, standard_deviation_ratio=1 #> #> Design: 25/36; Replications: 2 Total Time: 0.11s #> Conditions: sample_size=30, group_size_ratio=1, standard_deviation_ratio=2 #> #> Design: 26/36; Replications: 2 Total Time: 0.11s #> Conditions: sample_size=60, group_size_ratio=1, standard_deviation_ratio=2 #> #> Design: 27/36; Replications: 2 Total Time: 0.13s #> Conditions: sample_size=90, group_size_ratio=1, standard_deviation_ratio=2 #> #> Design: 28/36; Replications: 2 Total Time: 0.14s #> Conditions: sample_size=120, group_size_ratio=1, standard_deviation_ratio=2 #> #> Design: 29/36; Replications: 2 Total Time: 0.14s #> Conditions: sample_size=30, group_size_ratio=4, standard_deviation_ratio=2 #> #> Design: 30/36; Replications: 2 Total Time: 0.15s #> Conditions: sample_size=60, group_size_ratio=4, standard_deviation_ratio=2 #> #> Design: 31/36; Replications: 2 Total Time: 0.15s #> Conditions: sample_size=90, group_size_ratio=4, standard_deviation_ratio=2 #> #> Design: 32/36; Replications: 2 Total Time: 0.15s #> Conditions: sample_size=120, group_size_ratio=4, standard_deviation_ratio=2 #> #> Design: 33/36; Replications: 2 Total Time: 0.16s #> Conditions: sample_size=30, group_size_ratio=8, standard_deviation_ratio=2 #> #> Design: 34/36; Replications: 2 Total Time: 0.16s #> Conditions: sample_size=60, group_size_ratio=8, standard_deviation_ratio=2 #> #> Design: 35/36; Replications: 2 Total Time: 0.17s #> Conditions: sample_size=90, group_size_ratio=8, standard_deviation_ratio=2 #> #> Design: 36/36; Replications: 2 Total Time: 0.17s #> Conditions: sample_size=120, group_size_ratio=8, standard_deviation_ratio=2 #> #> #> Simulation complete. Total execution time: 0.18s res #> # A tibble: 36 × 9 #> sample_size group_size_ratio standard_deviation_ratio welch independent #> #> 1 30 1 0.5 0 0 #> 2 60 1 0.5 0 0 #> 3 90 1 0.5 0 0 #> 4 120 1 0.5 0 0 #> 5 30 4 0.5 0.5 0.5 #> 6 60 4 0.5 0 0.5 #> 7 90 4 0.5 0 0 #> 8 120 4 0.5 0 0 #> 9 30 8 0.5 0 0.5 #> 10 60 8 0.5 0 0.5 #> # ℹ 26 more rows #> # ℹ 4 more variables: REPLICATIONS , SIM_TIME , SEED , #> # COMPLETED if (FALSE) { # \\dontrun{ # complete run with 1000 replications per condition res <- runSimulation(design=Design, replications=1000, parallel=TRUE, generate=Generate, analyse=Analyse, summarise=Summarise) res View(res) ## save final results to a file upon completion, and play a beep when done runSimulation(design=Design, replications=1000, parallel=TRUE, filename = 'mysim', generate=Generate, analyse=Analyse, summarise=Summarise, beep=TRUE) ## same as above, but send a notification via Pushbullet upon completion library(RPushbullet) # read-in default JSON file runSimulation(design=Design, replications=1000, parallel=TRUE, filename = 'mysim', generate=Generate, analyse=Analyse, summarise=Summarise, notification = 'complete') ## Submit as RStudio job (requires job package and active RStudio session) job::job({ res <- runSimulation(design=Design, replications=100, generate=Generate, analyse=Analyse, summarise=Summarise) }, title='t-test simulation') res # object res returned to console when completed ## Debug the generate function. See ?browser for help on debugging ## Type help to see available commands (e.g., n, c, where, ...), ## ls() to see what has been defined, and type Q to quit the debugger runSimulation(design=Design, replications=1000, generate=Generate, analyse=Analyse, summarise=Summarise, parallel=TRUE, debug='generate') ## Alternatively, place a browser() within the desired function line to ## jump to a specific location Summarise <- function(condition, results, fixed_objects) { #find results of interest here (e.g., alpha < .1, .05, .01) browser() ret <- EDR(results[,nms], alpha = .05) ret } ## The following debugs the analyse function for the ## second row of the Design input runSimulation(design=Design, replications=1000, generate=Generate, analyse=Analyse, summarise=Summarise, parallel=TRUE, debug='analyse-2') #################################### ## EXTRA: To run the simulation on a user-define cluster, use the following setup (not run) ## Network linked via ssh (two way ssh key-paired connection must be ## possible between master and slave nodes) ## ## Define IP addresses, including primary IP primary <- '192.168.2.20' IPs <- list( list(host=primary, user='phil', ncore=8), list(host='192.168.2.17', user='phil', ncore=8) ) spec <- lapply(IPs, function(IP) rep(list(list(host=IP$host, user=IP$user)), IP$ncore)) spec <- unlist(spec, recursive=FALSE) cl <- parallel::makeCluster(type='PSOCK', master=primary, spec=spec) res <- runSimulation(design=Design, replications=1000, parallel = TRUE, generate=Generate, analyse=Analyse, summarise=Summarise, cl=cl) ## Using parallel='future' to allow the future framework to be used instead library(future) # future structure to be used internally plan(multisession) # specify different plan (default is sequential) res <- runSimulation(design=Design, replications=100, parallel='future', generate=Generate, analyse=Analyse, summarise=Summarise) head(res) # The progressr package is used for progress reporting with futures. To redefine # use progressr::handlers() (see below) library(progressr) with_progress(res <- runSimulation(design=Design, replications=100, parallel='future', generate=Generate, analyse=Analyse, summarise=Summarise)) head(res) # re-define progressr's bar (below requires cli) handlers(handler_pbcol( adjust = 1.0, complete = function(s) cli::bg_red(cli::col_black(s)), incomplete = function(s) cli::bg_cyan(cli::col_black(s)) )) with_progress(res <- runSimulation(design=Design, replications=100, parallel='future', generate=Generate, analyse=Analyse, summarise=Summarise)) # reset future computing plan when complete (good practice) plan(sequential) #################################### ###### Post-analysis: Analyze the results via functions like lm() or SimAnova(), and create ###### tables(dplyr) or plots (ggplot2) to help visualize the results. ###### This is where you get to be a data analyst! library(dplyr) res %>% summarise(mean(welch), mean(independent)) res %>% group_by(standard_deviation_ratio, group_size_ratio) %>% summarise(mean(welch), mean(independent)) # quick ANOVA analysis method with all two-way interactions SimAnova( ~ (sample_size + group_size_ratio + standard_deviation_ratio)^2, res, rates = TRUE) # or more specific ANOVAs SimAnova(independent ~ (group_size_ratio + standard_deviation_ratio)^2, res, rates = TRUE) # make some plots library(ggplot2) library(tidyr) dd <- res %>% select(group_size_ratio, standard_deviation_ratio, welch, independent) %>% pivot_longer(cols=c('welch', 'independent'), names_to = 'stats') dd ggplot(dd, aes(factor(group_size_ratio), value)) + geom_boxplot() + geom_abline(intercept=0.05, slope=0, col = 'red') + geom_abline(intercept=0.075, slope=0, col = 'red', linetype='dotted') + geom_abline(intercept=0.025, slope=0, col = 'red', linetype='dotted') + facet_wrap(~stats) ggplot(dd, aes(factor(group_size_ratio), value, fill = factor(standard_deviation_ratio))) + geom_boxplot() + geom_abline(intercept=0.05, slope=0, col = 'red') + geom_abline(intercept=0.075, slope=0, col = 'red', linetype='dotted') + geom_abline(intercept=0.025, slope=0, col = 'red', linetype='dotted') + facet_grid(stats~standard_deviation_ratio) + theme(legend.position = 'none') } # }"},{"path":"http://philchalmers.github.io/SimDesign/reference/timeFormater.html","id":null,"dir":"Reference","previous_headings":"","what":"Format time string to suitable numeric output — timeFormater","title":"Format time string to suitable numeric output — timeFormater","text":"Format time input string suitable numeric output metric (e.g., seconds). Input follows SBATCH utility specifications. Accepted time formats include \"minutes\", \"minutes:seconds\", \"hours:minutes:seconds\", \"days-hours\", \"days-hours:minutes\" \"days-hours:minutes:seconds\".","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/timeFormater.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Format time string to suitable numeric output — timeFormater","text":"","code":"timeFormater(time, output = \"sec\")"},{"path":"http://philchalmers.github.io/SimDesign/reference/timeFormater.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Format time string to suitable numeric output — timeFormater","text":"time character string formatted. numeric vector supplied interpreted seconds. output type numeric output convert time . Currently supported 'sec' seconds (default), 'min' minutes, 'hour', 'day'","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/timeFormater.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Format time string to suitable numeric output — timeFormater","text":"example, max_time = \"60\" indicates maximum time 60 minutes, max_time = \"03:00:00\" maximum time 3 hours, max_time = \"4-12\" maximum 4 days 12 hours, max_time = \"2-02:30:00\" maximum 2 days, 2 hours 30 minutes.","code":""},{"path":"http://philchalmers.github.io/SimDesign/reference/timeFormater.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Format time string to suitable numeric output — timeFormater","text":"","code":"# Test cases (outputs in seconds) timeFormater(\"4-12\") # day-hours #> [1] 388800 timeFormater(\"4-12:15\") # day-hours:minutes #> [1] 389700 timeFormater(\"4-12:15:30\") # day-hours:minutes:seconds #> [1] 389730 timeFormater(\"30\") # minutes #> [1] 1800 timeFormater(\"30:30\") # minutes:seconds #> [1] 1830 timeFormater(\"4:30:30\") # hours:minutes:seconds #> [1] 16230 # output in hours timeFormater(\"4-12\", output = 'hour') #> [1] 108 timeFormater(\"4-12:15\", output = 'hour') #> [1] 108.25 timeFormater(\"4-12:15:30\", output = 'hour') #> [1] 108.2583 timeFormater(\"30\", output = 'hour') #> [1] 0.5 timeFormater(\"30:30\", output = 'hour') #> [1] 0.5083333 timeFormater(\"4:30:30\", output = 'hour') #> [1] 4.508333"}] +[{"path":"http://philchalmers.github.io/SimDesign/articles/Catch_errors.html","id":"define-the-functions","dir":"Articles","previous_headings":"","what":"Define the functions","title":"Managing warning and error messages","text":"usual, define functions interest. simulation just example errors tracked SimDesign, well throw manual error case data re-drawn based user’s decision (e.g., model converges, fails number predefined iterations).","code":"library(SimDesign) # SimFunctions(comments=FALSE) Design <- createDesign(N = c(10,20,30)) Generate <- function(condition, fixed_objects) { ret <- with(condition, rnorm(N)) ret } Analyse <- function(condition, dat, fixed_objects) { whc <- sample(c(0,1,2,3), 1, prob = c(.7, .20, .05, .05)) if(whc == 0){ ret <- mean(dat) } else if(whc == 1){ ret <- t.test() # missing arguments } else if(whc == 2){ ret <- t.test('invalid') # invalid arguments } else if(whc == 3){ # throw error manually stop('Manual error thrown') } # manual warnings if(sample(c(TRUE, FALSE), 1, prob = c(.1, .9))) warning('This warning happens rarely') if(sample(c(TRUE, FALSE), 1, prob = c(.5, .5))) warning('This warning happens much more often') ret } Summarise <- function(condition, results, fixed_objects) { ret <- c(bias = bias(results, 0)) ret }"},{"path":"http://philchalmers.github.io/SimDesign/articles/Catch_errors.html","id":"run-the-simulation","dir":"Articles","previous_headings":"","what":"Run the simulation","title":"Managing warning and error messages","text":"’ll immediately notice output object counts error warning messages appended result object. useful determine just problematic errors warnings based frequency alone. Furthermore, specific frequency errors/warnings occurred also included design condition (t.test.default() error, inputs supplied, occurred often manually thrown error well invalid-input error) extracting inspecting SimExtract(results, = 'errors') SimExtract(results, = 'warnings'). Finally, SimDesign built-safety feature controlled max_errors argument avoid getting stuck infinite redrawing loops. default, 50 errors consecutively returned simulation condition halted, warning message printed console indicating last observed fatal error. safety features built-many consecutive stop() calls generally indicates major problem simulation code fixed continuing. However, encountering fatal errors given simulation condition remainder simulation experiment still executed normal, problematic conditions combinations NA placeholders assigned rows final output object. entire experiment unexpectedly terminate due one problematic row conditions Design, instead conditions can inspected debugged later time. course, inspecting code directly, simulation manually halted terminal errors can attended immediately (e.g., using Ctrl + c, clicking ‘Stop’ icon Rstudio).","code":"result <- runSimulation(Design, replications = 100, generate=Generate, analyse=Analyse, summarise=Summarise) ## ## Design: 1/3; Replications: 100; RAM Used: 67.9 Mb; Total Time: 0.00s ## Conditions: N=10 ## ## Design: 2/3; Replications: 100; RAM Used: 69.1 Mb; Total Time: 0.05s ## Conditions: N=20 ## ## Design: 3/3; Replications: 100; RAM Used: 69.2 Mb; Total Time: 0.09s ## Conditions: N=30 ## print(result) ## # A tibble: 3 × 9 ## N bias REPLICATIONS SIM_TIME RAM_USED SEED COMPLETED ERRORS ## ## 1 10 0.061138 100 0.05s 69.1 Mb 1140350788 Wed Dec 4 05… 53 ## 2 20 0.014295 100 0.05s 69.2 Mb 312928385 Wed Dec 4 05… 52 ## 3 30 0.017927 100 0.04s 69.3 Mb 866248189 Wed Dec 4 05… 42 ## # ℹ 1 more variable: WARNINGS SimExtract(result, what = 'errors') ## N ERROR: Error in t.test.default(\"invalid\") : not enough 'x' observations\\n ## 1 10 12 ## 2 20 9 ## 3 30 10 ## ERROR: Error in t.test.default() : argument \"x\" is missing, with no default\\n ## 1 31 ## 2 38 ## 3 25 ## ERROR: Manual error thrown\\n ## 1 10 ## 2 5 ## 3 7"},{"path":"http://philchalmers.github.io/SimDesign/articles/Catch_errors.html","id":"what-to-do-explicit-debugging","dir":"Articles","previous_headings":"","what":"What to do (explicit debugging)","title":"Managing warning and error messages","text":"errors occur often (fatal way) respective design conditions either extracted simulation inspected determine can fixed (e.g., providing better starting values, increasing convergence criteria/number iterations, etc). instance, say fourth row design object raised number error messages inspected . One useful approach debug 4th row instance error raised, can achieved using following: error flag used enter R’s debugger first instance error, -4 indicates 4th row design evaluated. also one instance changing warning messages error messages (.e., runSimulation(..., extra_options = list(warnings_as_errors=TRUE))) particularly useful state generated warning can inspected directly. Note similar arguments can made explicitly debugging functions generate-analyse-summarise chain (e.g., debug = 'analyse-4'), though less useful debugging (useful initial code design).","code":"runSimulation(..., debug = 'error-4')"},{"path":"http://philchalmers.github.io/SimDesign/articles/Catch_errors.html","id":"manual-debugging-via-try","dir":"Articles","previous_headings":"What to do (explicit debugging)","what":"Manual debugging via try()","title":"Managing warning and error messages","text":"Failing approach, manually wrapping problematic functions try() call. Adding line ((object, 'try-error')) browser() jump location/replication object unexpectedly witnessed, though admittedly clunky approach using debug. Nevertheless, jumping exact location error occurred, particularly case analyse() function throwing multiple error messages, greatly help determine exactly went wrong simulation state, allowing quickly locate fix issue.","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/Catch_errors.html","id":"extracting-error-seeds-for-hard-to-find-bugs","dir":"Articles","previous_headings":"What to do (explicit debugging)","what":"Extracting error seeds for hard-to-find bugs","title":"Managing warning and error messages","text":"alternative approach locating errors general use information stored within SimDesign objects time completion. default, .Random.seed states associated errors stored within final object, can extracted using SimExtract(..., ='error_seeds') option. function returns data.frame object seed stored column-wise, associated error message contained column name (allowed coerced valid column name make easier use $ operator). example, Given seeds, replicating exact error can achieved ) extracting single column integer vector, b) passing vector load_seed input. example, replicating first error message can achieved follows, makes sense immediately go debugging mode via debug inputs. Note: important manually select correct Design row using error extraction approach; otherwise, seed clearly replicate exact problem state. .Random.seed state loaded exact state, always related state well (case c typed debugger, somehow error harder find walking debug mode). Hence, users must type Q exit debugger better understood nature error message first-hand.","code":"seeds <- SimExtract(result, what = 'error_seeds') head(seeds[,1:3]) ## # A tibble: 6 × 3 ## Design_row_1.1..Error.in.t.tes…¹ Design_row_1.2..Manu…² Design_row_1.3..Erro…³ ## ## 1 10403 10403 10403 ## 2 624 21 65 ## 3 -159038368 -230613222 -230613222 ## 4 1905303777 203707493 203707493 ## 5 -371375826 1161141503 1161141503 ## 6 -1012234281 549195142 549195142 ## # ℹ abbreviated names: ## # ¹Design_row_1.1..Error.in.t.test.default.....argument..x..is.missing..with.no.default., ## # ²Design_row_1.2..Manual.error.thrown., ## # ³Design_row_1.3..Error.in.t.test.default.....argument..x..is.missing..with.no.default. picked_seed <- seeds$Design_row_1.1..Error.in.t.test.default..invalid.....not.enough..x..observations. # debug analyse() for first row of Design object via debug='analyse-1' runSimulation(Design, replications = 100, load_seed=picked_seed, debug='analyse-1', generate=Generate, analyse=Analyse, summarise=Summarise)"},{"path":"http://philchalmers.github.io/SimDesign/articles/Catch_errors.html","id":"converting-warings-to-errors-explicitly","dir":"Articles","previous_headings":"","what":"Converting warings to errors explicitly","title":"Managing warning and error messages","text":"occasion functions return warning message either border (treated ) error messages influence veracity simulation results. examples may include models appear ‘converge’ non-nonsensical parameter estimates (e.g., negative variances, non-positive definite correlation matrices, maximum iterations reached numerical searching algorithm, etc). However, issues non-fatal third-party software (.e., functions written developer simulation) may simply raise warning message inspection, whereas Monte Carlo simulation experiment issues generally treated invalid (though frequency occurrence still tracked, default SimDesign). resolve issue, avoid using nuclear option setting option(warn=2) treat warnings errors simulation, function manageWarnings() can used explicit convert known warning message strings errors disrupt code execution allowing warning messages continue raised. example, function utilized simulation whenever serious warning message raised explicitly converted error using internal grepl() test. now converts previous warning message error message, thereby correctly disrupting flow Monte Carlo simulation experiment prompting new call Generate(). course, warning error messages tallied resulting runSimulation() object, though now serious warnings tallied disruptive errors instead warnings continued normally.","code":"myfun <- function() { if(sample(c(TRUE, FALSE), 1, prob = c(.1, .9))) warning('This warning is serious') if(sample(c(TRUE, FALSE), 1, prob = c(.5, .5))) warning('This warning is no big deal') return(1) } set.seed(1) out <- myfun() set.seed(2) out <- myfun() ## Warning in myfun(): This warning is no big deal set.seed(7) out <- myfun() ## Warning in myfun(): This warning is serious set.seed(1) out1 <- manageWarnings(myfun(), warning2error='This warning is serious') out1 ## [1] 1 set.seed(2) out2 <- manageWarnings(myfun(), warning2error='This warning is serious') ## Warning in myfun(): This warning is no big deal out2 ## [1] 1 set.seed(7) out3 <- manageWarnings(myfun(), warning2error='This warning is serious') ## Error: This warning is serious"},{"path":"http://philchalmers.github.io/SimDesign/articles/Fixed_obj_fun.html","id":"including-fixed-objects","dir":"Articles","previous_headings":"","what":"Including fixed objects","title":"Exporting objects and functions from the workspace","text":"R fun language computer programming statistics, ’s without ’s quirks. instance, R generally recursive strategy attempting find objects within functions. object can’t found, R start look outside function’s environment see object can located , , look within even higher-level environments… recursive search continues searches object user workspace/Global environment, object can’t found error thrown. strange feature programmers come languages, writing simulations may cause severely unwanted issues. tutorial demonstrates make sure required user-defined objects visible SimDesign.","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/Fixed_obj_fun.html","id":"scoping","dir":"Articles","previous_headings":"","what":"Scoping","title":"Exporting objects and functions from the workspace","text":"demonstrate issue, let’s define two objects function uses objects. evaluated, objects visible user, can seen typing R console typing ls(). Functions define objects name also able locate values. behavior indeed bit strange, ’s one R’s quirks. Unfortunately, running code parallel across different cores objects visible, therefore must exported using methods (e.g., parallel package done clusterExport()). Exporting objects cluster fixes issue. reasoning applies functions defined R workspace well, including functions defined within external R packages. Hence, order use functions packages must either explicitly loaded require() library() within distributed code, referenced via Namespace :: operator (e.g., mvtnorm::rmvtnorm()).","code":"obj1 <- 10 obj2 <- 20 myfun <- function(x) obj1 + obj2 myfun(1) ## [1] 30 library(parallel) cl <- makeCluster(2) res <- try(parSapply(cl=cl, 1:4, myfun)) res ## Error in checkForRemoteErrors(val) : ## 2 nodes produced errors; first error: object 'obj1' not found clusterExport(cl=cl, c('obj1', 'obj2')) parSapply(cl=cl, 1:4, myfun) ## [1] 30 30 30 30"},{"path":"http://philchalmers.github.io/SimDesign/articles/Fixed_obj_fun.html","id":"exporting-objects-example","dir":"Articles","previous_headings":"","what":"Exporting objects example","title":"Exporting objects and functions from the workspace","text":"order make objects safely visible SimDesign strategy simple: wrap desired objects named list (object), pass fixed_objects argument. , elements can indexed using $ operator () function, whatever method may convenient. Note, however, required defined objects functions — SimDesign automatically makes user-defined functions available across nodes. aside, alternative approach simply define/source objects within respective SimDesign functions; way clearly visible runtime. following fixed_objects approach really useful defined objects contain large amount code. placing objects list passing fixed_objects, objects safely exported relevant functions. Furthermore, running code parallel also valid consequence (see ) objects properly exported core. , remember required R objects, user-defined functions!","code":"library(SimDesign) #SimFunctions(comments = FALSE) ### Define design conditions and number of replications Design <- createDesign(N = c(10, 20, 30)) replications <- 1000 # define custom functions and objects (or use source() to read these in from an external file) SD <- 2 my_gen_fun <- function(n, sd) rnorm(n, sd = sd) my_analyse_fun <- function(x) c(p = t.test(x)$p.value) fixed_objects <- list(SD=SD) #--------------------------------------------------------------------------- Generate <- function(condition, fixed_objects) { Attach(condition) # make condition names available (e.g., N) # further, can use with() to use 'SD' directly instead of 'fixed_objects$SD' ret <- with(fixed_objects, my_gen_fun(N, sd=SD)) ret } Analyse <- function(condition, dat, fixed_objects) { ret <- my_analyse_fun(dat) ret } Summarise <- function(condition, results, fixed_objects) { ret <- EDR(results, alpha = .05) ret } #--------------------------------------------------------------------------- ### Run the simulation res <- runSimulation(Design, replications, verbose=FALSE, fixed_objects=fixed_objects, generate=Generate, analyse=Analyse, summarise=Summarise, debug='none') res ## # A tibble: 3 × 6 ## N p REPLICATIONS SIM_TIME SEED COMPLETED ## ## 1 10 0.038 1000 0.22s 525310970 Wed Dec 4 05:41:15 2024 ## 2 20 0.05 1000 0.22s 529914981 Wed Dec 4 05:41:15 2024 ## 3 30 0.056 1000 0.24s 602604143 Wed Dec 4 05:41:15 2024 res <- runSimulation(Design, replications, verbose=FALSE, fixed_objects=fixed_objects, generate=Generate, analyse=Analyse, summarise=Summarise, debug='none', parallel = TRUE)"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"introduction","dir":"Articles","previous_headings":"","what":"Introduction","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"purpose vignette demonstrate utilize SimDesign context distributing many jobs across independent computing environments, high-performance computing (HPC) clusters, way allows reproducibility simulation conditions, resubmission jobs case incomplete collection results within specified HPC execution time budget, ensure random number generation across entire simulation (subsequent resubmissions, required) properly manged throughout given batch nature job. following text examples primarily managing larger simulations, often thousands replications many simulation conditions (.e., rows defined design object) generally require non-trivial amounts computing resources execute (hence, need super-computing resources job schedulers like Slurm, TORQUE, MAUI, among others). information Slurm’s Job Array support particular, vignette uses example, see https://slurm.schedmd.com/job_array.html","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"standard-setup-on-hpc-cluster","dir":"Articles","previous_headings":"","what":"Standard setup on HPC cluster","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"start, structure simulation code used later distribute jobs HPC scheduler effectively usual generate-analyse-summarise workflow described runSimulation(), organizational exceptions. , always good place start designing, testing, debugging simulation experiment submitting HPC clusters. IMPORTANT: vast majority bugs coding logic work consider moving next step involving HPC clusters. code well vetted step later jobs evaluated HPC cluster waste time resources (garbage-, garbage-).","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"example","dir":"Articles","previous_headings":"Standard setup on HPC cluster","what":"Example","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"Suppose following simulation evaluated, though time constraint reasons possible execute single computer (smaller network computers) therefore submitted HPC cluster. following script (hypothetically written file called SimDesign_simulation.R) contains simulation experiment whose instructions submitted Slurm scheduler. , sbatch utility used along set instructions specifying type hardware required file slurmInstructions.slurm. R side simulation, defined code must grab available cores (minus 1) detectable via parallelly::availableCores(), occurs automatically using runSimulation(..., parallel=TRUE). standard runSimulation(..., parallel=TRUE) setup 10,000 replications distributed available computing cores evaluated independently across three row conditions design object. However, process executed sequence: design[1, ] evaluated first , 10,000 replications collected, design[2, ] evaluated complete, design[3, ], . well, order approach optimal HPC cluster must assign job containing large number resources; specifically, higher RAM CPUs. demonstrate, following slurmInstructions.slurm file larger number CPUs requested allocating computational structure/cluster associated job, well larger amounts RAM. job request computing cluster built 192 GB RAM 96 CPUs (across whatever computing nodes available; likely 2 ), SimDesign_simulation.R evaluated , submitted scheduler via sbatch slurmInstructions.slurm.","code":"# SimDesign::SimFunctions() library(SimDesign) Design <- createDesign(N = c(10, 20, 30)) Generate <- function(condition, fixed_objects) { dat <- with(condition, rnorm(N, 10, 5)) # distributed N(10, 5) dat } Analyse <- function(condition, dat, fixed_objects) { ret <- c(mean=mean(dat), median=median(dat)) # mean/median of sample data ret } Summarise <- function(condition, results, fixed_objects){ colMeans(results) } # standard setup (not ideal for HPC clusters as parallelization # occurs within the design conditions, not across) res <- runSimulation(design=Design, replications=10000, generate=Generate, analyse=Analyse, summarise=Summarise, parallel=TRUE, filename='mysim') #!/bin/bash #SBATCH --job-name=\"My simulation (multiple CPUs)\" #SBATCH --mail-type=ALL #SBATCH --mail-user=somewhere@out.there #SBATCH --output=/dev/null ## (optional) delete .out files #SBATCH --time=12:00:00 ## HH:MM:SS #SBATCH --cpus-per-task=96 ## Build a computer with 96 cores #SBATCH --mem-per-cpu=2G ## Build a computer with 192GB of RAM module load r Rscript --vanilla SimDesign_simulation.R"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"limitations","dir":"Articles","previous_headings":"Standard setup on HPC cluster","what":"Limitations","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"generally effective distributing computational load, limitations approach: Due row-evaluation nature design conditions, computing cores point sit idle waiting remaining experiments complete job. occurs row design input (.e., per simulation condition) , simulation experiments many conditions evaluate suffer due rolling overhead, resulting wasted resource management (kind users HPC cluster) ultimately results jobs take longer complete Problematic fewer resource jobs given higher priority scheduler, may taking resources away researchers cores sit idle Using 96 CPU cluster result 96x speedup 1 CPU job. fact, larger allocated cluster, worse performance efficiency request 96 CPUs 192 GB RAM take considerably longer allocate compared requesting 96 independent computing arrays 1 CPU 2 GB RAM (latter approach described next section) concert point 3), results jobs ) take longer get started sit longer queue b) may distribute load efficiently enough, thereby resulting larger wall time necessary Setting seed condition ensure within design condition random numbers high quality, however guarantee repeated use set.seed() result high-quality random numbers (see next section example) Hence, repeated job submissions type, even unique seeds per condition, may generate high quality numbers repeated many times (alternative isolate design row submit row unique job, demonstrated near end vignette) Note avoid wasting time swapping/paging, schedulers never allocate jobs whose memory requirements exceed amount available memory address computational inefficiencies/wasted resources, one can instead switch cluster-based approach array distribution approach, discussed next section.","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"array-jobs","dir":"Articles","previous_headings":"","what":"Array jobs","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"HPC computing often optimal distribute replications conditions simultaneously unique computing nodes (termed arrays) effectively break problem several mini-batches (e.g., split simulation 1000+ independent pieces, collect results later). , design object runSimulation() structure readily lend optimal distribution array scheduler manage. Nevertheless, core components still useful initial code design, testing, debugging, therefore serve necessary first step writing simulation experiment code prior submitting HPC cluster. defining testing simulation ensure works expected, now comes time setup components required organizing HPC cluster submission using runArraySimulation() function.","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"converting-runsimulation-workflow-to-one-for-runarraysimulation","dir":"Articles","previous_headings":"Array jobs","what":"Converting runSimulation() workflow to one for runArraySimulation()","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"job runArraySimulation() utilize relevant information defined .sh .slurm script. done extracting information provided scheduler (specifically, via arrayID), used select specific subsets design rows. However, unlike runSimulation() function runArraySimulation() designed control important information pertaining .Random.seeds relevant distribution information allow rows design object contain repeated experimental condition information. allows design rows replication information optimally distributed HPC cluster. following example presents essential modifications required move single runSimulation() workflow batch workflow suitable runArraySimulation() Slurm scheduler.","code":""},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"expand-the-standard-simulation-design-object-for-each-array-id","dir":"Articles","previous_headings":"Array jobs","what":"Expand the standard simulation design object for each array ID","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"Suppose 300 computing cores independently available HPC cluster, though availability cores trickle function schedulers decided availability. case, strategy split 3 * 10000 = 30000 condition--replications experiments across gradually available resources, jobs evaluated parallel different times. Given specifications, may decide 300 computing nodes requested scheduler evaluate exactly 100 replications (nrow(design) * 10000 / 300 = 100). case, expandDesign() used repeat row condition 100 times, creating new expanded design object 300 rows instead 3. accommodate new rows, associated replications now defined according many replications used within --distributed conditions, need equal (see ). approach assumes design condition equally balanced terms computing time resources, though case (e.g., last condition contains notably higher computing times first two conditions) repeat_conditions can specified vector instead, repeat_conditions = c(100, 100, 1000), latter portion associated 10 replications per distributed node instead 100. Regardless whether expanded design balanced unbalanced row resulting expanded design object assigned unique computing array node, identified according Slurm assigned array ID.","code":"rc <- 100 # number of times the design row was repeated Design300 <- expandDesign(Design, repeat_conditions = rc) Design300 ## # A tibble: 300 × 1 ## N ## ## 1 10 ## 2 10 ## 3 10 ## 4 10 ## 5 10 ## 6 10 ## 7 10 ## 8 10 ## 9 10 ## 10 10 ## # ℹ 290 more rows # target replication number for each condition rep_target <- 10000 # replications per row in Design300 replications <- rep(rep_target / rc, nrow(Design300)) rc <- c(100, 100, 1000) DesignUnbalanced <- expandDesign(Design, repeat_conditions = rc) DesignUnbalanced ## # A tibble: 1,200 × 1 ## N ## ## 1 10 ## 2 10 ## 3 10 ## 4 10 ## 5 10 ## 6 10 ## 7 10 ## 8 10 ## 9 10 ## 10 10 ## # ℹ 1,190 more rows rep_target <- 10000 replicationsUnbalanced <- rep(rep_target / rc, times = rc) head(replicationsUnbalanced) ## [1] 100 100 100 100 100 100 table(replicationsUnbalanced) ## replicationsUnbalanced ## 10 100 ## 1000 200"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"construct-and-record-proper-random-seeds","dir":"Articles","previous_headings":"Array jobs","what":"Construct and record proper random seeds","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"principle, expanded Design300 object passed runSimulation(Design300, replications=100, ...) evaluate repeated conditions, row now replicated 100 times. However, now issue respect random seed management use functions set.seed() friends longer viable. repeated use set.seed() guarantee independent high-quality random numbers different instances. example, issue generally problem standard runSimulation() approach within design condition high quality random numbers used default, potentially repeated number sequences across conditions highly unlikely affect quality overall simulation experiment (conditions typically generate manage random numbers different ways due varying simulation factors, sample sizes, variance conditions, fitted models, number variables, type probability distributions use, ). However, expandDesign() setup likelihood witnessing correlated/redundant random samples increases quickly, particularly problematic within distributed replication set; hence, special care must taken ensure proper seeds distributed job array. Fortunately, seeds easy manage genSeeds() function using two-step approach, internally managed runArraySimulation() supplying initial seed (iseed) value associated array ID (arrayID). utilize L’Ecuyer’s (1999) method, constructs sequentially computed .Random.seed states ensure independence across replications conditions. Note iseed must constant across job arrays, make sure define iseed ! discussed FAQ section bottom, associated value also allow generation new .Random.seed elements () second third set simulation jobs submitted HPC cluster later time must also generate simulated data independent initial submission(s).","code":"set.seed(0) x <- runif(100) set.seed(1) y <- runif(100) plot(x, y) ## seemingly independent plot(x[-1], y[-100]) ## subsets perfectly correlated # genSeeds() # do this once on the main node/home computer and store the number! iseed <- 1276149341"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"extract-array-id-information-from-the--slurm-script","dir":"Articles","previous_headings":"Array jobs","what":"Extract array ID information from the .slurm script","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"submitting HPC cluster ’ll need include information scheduler distribute simulation experiment workers. Slurm systems, may script following, stored suitable .slurm file: reference later, label file simulation.slurm file must submitted scheduler ’s time. top part .slurm file provides BASH instructions Slurm scheduler via #SBATCH statements. case, many array jobs queue (1 300), much memory use per job (2GB), time limits (12 hours), ; see SBATCH details. important input focus context #SBATCH –array=1-300 used Slurm scheduler assign unique ID array job. scheduler take defined mySimDesignScript.R script send 300 independent resources (1 CPU 4GB RAM, case), independent jobs assigned unique array ID number within --array=1-300 range (e.g., distribution first computing resource assigned arrayID=1, second resource arrayID=2, ). runArraySimulation() function used subset Design300 object row; hence, array range must correspond row identifiers design object proper subsetting! Collecting single number assigned Slurm scheduler also easy. Just include obtain associated array ID, example single integer value 1 300. value used final execution step via runArraySimulation(..., arrayID=arrayID), finally turn .","code":"#!/bin/bash #SBATCH --job-name=\"My simulation (array jobs, distributing conditions + replications)\" #SBATCH --mail-type=ALL #SBATCH --mail-user=somewhere@out.there #SBATCH --output=/dev/null ## (optional) delete .out files #SBATCH --time=12:00:00 ## HH:MM:SS #SBATCH --mem-per-cpu=4G ## 4GB of RAM per cpu #SBATCH --cpus-per-task=1 #SBATCH --array=1-300 ## Slurm schedulers often allow up to 10,000 arrays module load r Rscript --vanilla mySimDesignScript.R # get assigned array ID (default uses type = 'slurm') arrayID <- getArrayID()"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"organize-information-for-runarraysimulation","dir":"Articles","previous_headings":"Array jobs","what":"Organize information for runArraySimulation()","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"prerequisite steps place ’re finally ready pass information runArraySimulation(), effectively wrapper runSimulation() suppresses verbose outputs, takes subsets Design300 object given supplied arrayID (objects, replications, seeds, etc), forces evaluation single CPU (hence, #SBATCH --cpus-per-task=1 used default, unless parallelization occur within replications, via OpenMP), manages random number generation seeds tractable way, saves SimDesign results file names based filename argument suffixes associated arrayID (e.g., filename='mysim' save files mysim-1.rds array 1, mysim-2.rds array 2, …, mysim-300.rds array 300). ’s ! store mysim-#.rds files directory job submitted, somewhat messy side, may also want specify directory name store simulation files . Hence, main (.e., landing) location associated ssh account create directory, using something like mkdir mysimfiles (R, dir.create('mysimfiles')) location .R .slurm files stored. following can used store 300 collected .rds files, making use dirname argument. Regardless, hard part done , though information included way control list input necessary, including explicit time/resource limits R executions within array jobs (see FAQ section information).","code":"# run the simulation on subset based on arrayID subset information runArraySimulation(design=Design300, replications=replications, generate=Generate, analyse=Analyse, summarise=Summarise, iseed=iseed, arrayID=arrayID, filename='mysim') # run the simulation on subset based on arrayID subset information runArraySimulation(design=Design300, replications=replications, generate=Generate, analyse=Analyse, summarise=Summarise, iseed=iseed, arrayID=arrayID, dirname='mysimfiles', filename='mysim')"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"putting-it-all-together","dir":"Articles","previous_headings":"Array jobs","what":"Putting it all together","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"complete submission script collecting everything presented . assumes .R file simulation code stored file mySimDesignScript.R, suitable Slurm instruction file created file simulation.slurm, points mySimDesignScript.R includes relevant R modules, directory called mysimfiles/ created storing files computer used submit array job file submitted job scheduler via sbatch, pointing .slurm instructions. complete can now go get beer, coffee, whatever else tickles fancy celebrate hard part .","code":"library(SimDesign) Design <- createDesign(N = c(10, 20, 30)) Generate <- function(condition, fixed_objects) { dat <- with(condition, rnorm(N, 10, 5)) # distributed N(10, 5) dat } Analyse <- function(condition, dat, fixed_objects) { ret <- c(mean=mean(dat), median=median(dat)) # mean/median of sample data ret } Summarise <- function(condition, results, fixed_objects){ colMeans(results) } # expand the design to create 300 rows with associated replications rc <- 100 Design300 <- expandDesign(Design, repeat_conditions = rc) rep_target <- 10000 replications <- rep(rep_target / rc, nrow(Design300)) # genSeeds() # do this once on the main node/home computer, and store the number! iseed <- 1276149341 # get assigned array ID (default uses type = 'slurm') arrayID <- getArrayID() # run the simulation on subset based on arrayID subset information runArraySimulation(design=Design300, replications=replications, generate=Generate, analyse=Analyse, summarise=Summarise, iseed=iseed, arrayID=arrayID, dirname='mysimfiles', filename='mysim') sbatch simulation.slurm"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"post-evaluation-combine-the-files","dir":"Articles","previous_headings":"","what":"Post-evaluation: Combine the files","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"time elapsed, job evaluation now complete, ’ll access complete set simulation files store file names mysim-#.rds. final step process collect independent results simulation object resembles returned canonical runSimulation() function. Fortunately, easy SimCollect(). must point point working directory containing simulation files use SimCollect(): function detects Design300 rows belong original Design object, collapse meta-statistic results, stored results information accordingly. fuss, mess. course, ’ll want store object later use complete collection results 300 array jobs, organized one neat little (object) package. now consider moving \"final_sim.rds\" Slurm landing node onto home computer via scp favourite method (e.g., using WinSCP Windows). also move saved *.rds files ssh landing case need inspect files (e.g., debugging purposes).","code":"library(SimDesign) # automatically checks whether all saved files are present via SimCheck() Final <- SimCollect('mysimfiles/') Final # A tibble: 3 × 8 N mean median REPLICATIONS SIM_TIME COMPLETED 1 10 9.9973 9.9934 10000 23.42s Thu Apr 4 11:50:11 2024 2 20 10.007 10.015 10000 24.24s Thu Apr 4 11:50:35 2024 3 30 10.003 10.007 10000 24.39s Thu Apr 4 11:51:00 2024 # save the aggregated simulation object for subsequent analyses saveRDS(Final, \"../final_sim.rds\")"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"array-jobs-and-multicore-computing-simultaneously","dir":"Articles","previous_headings":"","what":"Array jobs and multicore computing simultaneously","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"course, nothing really stops mixing matching ideas related multicore computing array jobs Slurm HPC clusters. example, wanted take original design object submit batches instead (e.g., submit one rows design object array job), within array multicore processing requested, something like following work: associated .R file containing, case, nine simulation conditions across rows Design. Depending intensity conditions, may choose distribute one row Design object array (multirow=TRUE following), otherwise natural choice distribute row Design object assigned array. complete, function SimCollect() can used put simulation results together given nine saved files (nine files also saved multirow set TRUE #SBATCH --array=1-3 used instead stored per-row basis). type hybrid approach middle ground submitting complete job (top vignette) condition + replication distributed load previous section, though similar overhead + inefficiency issues (though less , array jobs evaluated independently). Note row’s take different amounts time evaluate strategy can prove less efficient (e.g., first two rows may take 2 hours complete, third row may take 12 hours complete), case nuanced array2row() function defined help explicit balance load computing cluster.","code":"#!/bin/bash #SBATCH --job-name=\"My simulation (arrays + multiple CPUs)\" #SBATCH --mail-type=ALL #SBATCH --mail-user=somewhere@out.there #SBATCH --output=/dev/null ## (optional) delete .out files #SBATCH --time=04:00:00 ## HH:MM:SS #SBATCH --mem-per-cpu=4G ## Build a computing cluster with 64GB of RAM #SBATCH --cpus-per-task=16 ## 16 CPUs per array, likely built from 1 node #SBATCH --array=1-9 ## 9 array jobs module load r Rscript --vanilla mySimDesignScript.R library(SimDesign) Design <- createDesign(N = c(10, 20, 30), SD = c(1,2,3)) Generate <- function(condition, fixed_objects) { dat <- with(condition, rnorm(N, 10, sd=SD)) # distributed N(10, 5) dat } Analyse <- function(condition, dat, fixed_objects) { ret <- c(mean=mean(dat), median=median(dat)) # mean/median of sample data ret } Summarise <- function(condition, results, fixed_objects){ colMeans(results) } Design ## # A tibble: 9 × 2 ## N SD ## ## 1 10 1 ## 2 20 1 ## 3 30 1 ## 4 10 2 ## 5 20 2 ## 6 30 2 ## 7 10 3 ## 8 20 3 ## 9 30 3 # get array ID arrayID <- getArrayID() multirow <- FALSE # submit multiple rows of Design object to array? if(multirow){ # If selecting multiple design rows per array, such as the first 3 rows, # then next 3 rows, and so on, something like the following would work ## For arrayID=1, rows 1 through 3 are evaluated ## For arrayID=2, rows 4 through 6 are evaluated ## For arrayID=3, rows 7 through 9 are evaluated array2row <- function(arrayID) 1:3 + 3 * (arrayID-1) } else { # otherwise, use one row per respective arrayID array2row <- function(arrayID) arrayID } # Make sure parallel=TRUE flag is on to use all available cores! runArraySimulation(design=Design, replications=10000, generate=Generate, analyse=Analyse, summarise=Summarise, iseed=iseed, dirname='mysimfiles', filename='mysim', parallel=TRUE, arrayID=arrayID, array2row=array2row)"},{"path":[]},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"helpful-slurm-commands","dir":"Articles","previous_headings":"Extra information (FAQs)","what":"Helpful Slurm commands","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"addition using sbatch submit jobs, following contains useful Slurm commands.","code":"sbatch # submit job file to Slurm scheduler squeue -u # what jobs are currently queued/running for a specific user sshare -U # check the share usage for a specific user scancel # cancel a specific job scancel -u # cancel all queued and running jobs for a specific user"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"my-hpc-cluster-excution-timeram-is-limited-and-terminates-before-the-simulation-is-complete","dir":"Articles","previous_headings":"Extra information (FAQs)","what":"My HPC cluster excution time/RAM is limited and terminates before the simulation is complete","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"issue important whenever HPC cluster mandatory time/RAM limits job submissions, array job may complete within assigned resources — hence, properly managed, discard valid replication information abruptly terminated. Unfortunately, likely occurrence, largely function unsure long simulation condition/replication take complete distributed across arrays (conditions/replications take longer others, difficult perfectly knowledgeable information beforehand) large final objects grow simulation progresses. avoid time/resource waste strongly recommended add max_time /max_RAM argument control list (see help(runArraySimulation) supported specifications), less Slurm specifications. control flags halt runArraySimulation() executions early return complete simulation results point. However, work arguments non-trivially less allocated Slurm resources; otherwise, ’ll run risk job terminates SimDesign functions chance store successfully completed replications. Setting around 90-95% respective #SBATCH --time= #SBATCH --mem-per-cpu= inputs , however, sufficient cases. course, session time early implies target replications missed first job submission batch. Therefore, covered next section, new job must submitted scheduler mindful initial simulation history (particularly, .Random.seed states).","code":"# Return successful results up to the 11 hour mark, and terminate early # if more than 3.5 GB of RAM are required to store the internal results runArraySimulation(design=Design300, replications=replications, generate=Generate, analyse=Analyse, summarise=Summarise, iseed=iseed, arrayID=arrayID, dirname='mysimfiles', filename='mysim', control=list(max_time=\"11:00:00\", max_RAM=\"3.5GB\"))"},{"path":"http://philchalmers.github.io/SimDesign/articles/HPC-computing.html","id":"uploading-array-jobs-related-to-previous-array-submissions","dir":"Articles","previous_headings":"Extra information (FAQs)","what":"Uploading array jobs related to previous array submissions","title":"Distributing jobs for high-performance computing (HPC) clusters (e.g., via Slurm)","text":"Related early termination issue missing replication information event complete set replication information collected. Obtaining missing information clearly requires second (third) submission simulation job, though obviously conditions collected replication results problematic. Moreover, performed care avoid redundant random data generation strings, ultimately resulting sub-optimal results. start, locate simulation conditions aggregated result meet target replication criteria. obtained via inspection aggregated results via informative (less memory intensive) SimCollect(..., check.=TRUE) flag. Next, build new simulation structure containing missing information components. point, can return logic organizing simulation script job, distributing information across many array jobs necessary fill missing information. However, must careful random number generators per row subDesign original submission job. fix case straightforward well: simply create continuation previous logic, new elements treated additional rows resulting object though part initial job submission. , now just glue new subDesign information original expanded version, though telling scheduler evaluate new rows #SBATCH --array specification (technically unnecessary, conceptually clear keeps simulation files array IDs consistent). , approach simply expands original simulation 300 array jobs one 400 array jobs though added structure intended part initial design (obviously wasn’t, organized ). Finally, .slurm submission file longer want evaluate first 1-300 cases, .rds files already evaluated, instead want change --array line Submit job compute missing replication information, stores files working directory new information stored mysim-301.rds mysim-400.rds. example, now total 400 files saved. complete, run one last time, now reads complete set 400 stored files instead previous 300, thereby obtaining complete set high-quality simulation results. Rinse repeat issue appears yet second submission.","code":"Final <- SimCollect('mysimfiles/') Final # A tibble: 3 × 8 N mean median REPLICATIONS SIM_TIME COMPLETED 1 10 9.9973 9.9934 9000 23.42s Thu Apr 4 11:50:11 2024 2 20 10.007 10.015 10000 24.24s Thu Apr 4 11:50:35 2024 3 30 10.003 10.007 8000 24.39s Thu Apr 4 11:51:00 2024 Missed <- aggregate_simulations(files=dir(), check.only=TRUE) Missed # A tibble: 4 × 3 N MISSED_REPLICATIONS TARGET_REPLICATIONS 1 10 1000 10000 2 30 2000 10000 subDesign <- subset(Missed, select=N) replications_missed <- subset(Missed, select=MISSED_REPLICATIONS) subDesign ## # A tibble: 2 × 1 ## N ## ## 1 10 ## 2 30 replications_missed ## [1] 1000 2000 rc <- 50 Design_left <- expandDesign(subDesign, rc) # smaller number of reps per array Design_left ## # A tibble: 100 × 1 ## N ## ## 1 10 ## 2 10 ## 3 10 ## 4 10 ## 5 10 ## 6 10 ## 7 10 ## 8 10 ## 9 10 ## 10 10 ## # ℹ 90 more rows replications_left <- rep(replications_missed/rc, each=rc) table(replications_left) ## replications_left ## 20 40 ## 50 50 # new total design and replication objects Design_total <- rbind(Design300, Design_left) nrow(Design_total) ## [1] 400 replications_total <- c(replications, replications_left) table(replications_total) ## replications_total ## 20 40 100 ## 50 50 300 # this *must* be the same as the original submission! iseed <- 1276149341 #SBATCH --array=1-300 #SBATCH --array=301-400 # See if any missing still SimCollect('mysimfiles', check.only=TRUE) # Obtain complete simulation results Final <- SimCollect('mysimfiles')"},{"path":"http://philchalmers.github.io/SimDesign/articles/MultipleAnalyses.html","id":"description-of-structure","dir":"Articles","previous_headings":"","what":"Description of structure","title":"Multiple analysis functions","text":"usual work-flow SimDesign requires first calling SimFunctions() generate working template, following. uses default nAnalyses=1 generate single Analyse() function. context multiple analysis functions, however, users may interested passing number analysis functions believe need simulation (e.g., analyzing tt-test setup compare Welch versus independent samples t-test, two analysis functions used). Passing nAnalyses=2 SimFunctions() creates following template: Notice case two Analyse.#() definitions constructed, passed runSimulation() organized named list. names list ultimately attached names analysis objects ambiguity outputted information. However, inputs Analyse() functions always , dat object formed Generate() call passed Analyse definitions (hence, generate data held constant across respective analyses). template course modified replace less useful names Analyse.#() components. default users want change something like Analyse.some_statistic, Analyse.some_other_statistic, …, Analyse.some_other_other_statistic, , number Analyse.# function definitions ultimately end runSimulation(..., Analyse=list()) input. Supplying better names named list component also recommended used name associated results Summarise() step. Finally, note rules objects object naming typical single Analyse function still apply properly checked internally suitable names consistency. independently defined Analyse functions also interchangable removable/replaceable, makes structure Generate-Analyse-Summarise setup modular respect analysis components.","code":"SimDesign::SimFunctions() ## #------------------------------------------------------------------- ## ## library(SimDesign) ## ## Design <- createDesign(factor1 = NA, ## factor2 = NA) ## ## #------------------------------------------------------------------- ## ## Generate <- function(condition, fixed_objects) { ## dat <- data.frame() ## dat ## } ## ## Analyse <- function(condition, dat, fixed_objects) { ## ret <- nc(stat1 = NaN, stat2 = NaN) ## ret ## } ## ## Summarise <- function(condition, results, fixed_objects) { ## ret <- c(bias = NaN, RMSE = NaN) ## ret ## } ## ## #------------------------------------------------------------------- ## ## res <- runSimulation(design=Design, replications=2, generate=Generate, ## analyse=Analyse, summarise=Summarise) ## res SimDesign::SimFunctions(nAnalyses = 2) ## #------------------------------------------------------------------- ## ## library(SimDesign) ## ## Design <- createDesign(factor1 = NA, ## factor2 = NA) ## ## #------------------------------------------------------------------- ## ## Generate <- function(condition, fixed_objects) { ## dat <- data.frame() ## dat ## } ## ## Analyse.A1 <- function(condition, dat, fixed_objects) { ## ret <- nc(stat1 = NaN, stat2 = NaN) ## ret ## } ## ## Analyse.A2 <- function(condition, dat, fixed_objects) { ## ret <- nc(stat1 = NaN, stat2 = NaN) ## ret ## } ## ## #------------------------------------------------------------------- ## ## Summarise <- function(condition, results, fixed_objects) { ## ret <- c(bias = NaN, RMSE = NaN) ## ret ## } ## ## #------------------------------------------------------------------- ## ## res <- runSimulation(design=Design, replications=2,generate=Generate, ## analyse=list(A1=Analyse.A1, A2=Analyse.A2), ## summarise=Summarise) ## res"},{"path":"http://philchalmers.github.io/SimDesign/articles/MultipleAnalyses.html","id":"an-example","dir":"Articles","previous_headings":"Description of structure","what":"An example","title":"Multiple analysis functions","text":"following code adopted Wiki, details simulation obtained source. particular formulation mirt lavaan package analyses completely isolated respective functions, principle therefore analyzed independently future simulation studies. adds nicer layer potential modularity Analyse portion SimDesign framework, re-using modifying previous SimDesign code less painful.","code":"library(SimDesign) # SimFunctions(nAnalyses = 2) sample_sizes <- c(250, 500, 1000) nitems <- c(10, 20) Design <- createDesign(sample_size = sample_sizes, nitems = nitems) # create list of additional parameters which are fixed across conditions set.seed(1) pars_10 <- rbind(a = round(rlnorm(10, .3, .5)/1.702, 2), d = round(rnorm(10, 0, .5)/1.702, 2)) pars_20 <- rbind(a = round(rlnorm(20, .3, .5)/1.702, 2), d = round(rnorm(20, 0, .5)/1.702, 2)) pars <- list(ten=pars_10, twenty=pars_20) P_logit <- function(a, d, Theta) exp(a * Theta + d) / (1 + exp(a * Theta + d)) P_ogive <- function(a, d, Theta) pnorm(a * Theta + d) Generate <- function(condition, fixed_objects) { N <- condition$sample_size nitems <- condition$nitems nitems_name <- ifelse(nitems == 10, 'ten', 'twenty') #extract objects from fixed_objects a <- fixed_objects[[nitems_name]]['a', ] d <- fixed_objects[[nitems_name]]['d', ] dat <- matrix(NA, N, nitems) colnames(dat) <- paste0('item_', 1:nitems) Theta <- rnorm(N) for(j in 1:nitems){ p <- P_ogive(a[j], d[j], Theta) for(i in 1:N) dat[i,j] <- sample(c(1,0), 1, prob = c(p[i], 1 - p[i])) } as.data.frame(dat) #data.frame works nicer with lavaan } Analyse.FIML <- function(condition, dat, fixed_objects) { mod <- mirt(dat, 1L, verbose=FALSE) if(!extract.mirt(mod, 'converged')) stop('mirt did not converge') cfs <- mirt::coef(mod, simplify = TRUE, digits = Inf) FIML_as <- cfs$items[,1L] / 1.702 ret <- c(as=unname(FIML_as)) ret } Analyse.DWLS <- function(condition, dat, fixed_objects) { nitems <- condition$nitems lavmod <- paste0('F =~ ', paste0('NA*', colnames(dat)[1L], ' + '), paste0(colnames(dat)[-1L], collapse = ' + '), '\\nF ~~ 1*F') lmod <- sem(lavmod, dat, ordered = colnames(dat)) if(!lavInspect(lmod, 'converged')) stop('lavaan did not converge') cfs2 <- lavaan::coef(lmod) DWLS_alpha <- cfs2[1L:nitems] const <- sqrt(1 - DWLS_alpha^2) DWLS_as <- DWLS_alpha / const ret <- c(as=unname(DWLS_as)) ret } Summarise <- function(condition, results, fixed_objects) { nitems <- condition$nitems nitems_name <- ifelse(nitems == 10, 'ten', 'twenty') #extract objects from fixed_objects a <- fixed_objects[[nitems_name]]['a', ] pop <- c(a, a) obt_bias <- bias(results, pop) obt_RMSE <- RMSE(results, pop) ret <- c(bias=obt_bias, RMSE=obt_RMSE) ret } res <- runSimulation(Design, replications=100, verbose=FALSE, parallel=TRUE, generate=Generate, analyse=list(FIML=Analyse.FIML, DWLS=Analyse.DWLS), summarise=Summarise, filename = 'mirt_lavaan', packages=c('mirt', 'lavaan'), fixed_objects=pars) res ## # A tibble: 6 × 86 ## sample_size nitems bias.FIML.as1 bias.FIML.as2 bias.FIML.as3 bias.FIML.as4 ## ## 1 250 10 0.0099887 0.013457 -0.020161 0.18616 ## 2 500 10 -0.021136 0.018720 -0.018173 0.10424 ## 3 1000 10 -0.0039473 -0.0014778 -0.013393 0.077890 ## 4 250 20 0.060647 0.011787 0.064688 -0.026314 ## 5 500 20 0.019202 0.045376 0.021837 -0.0063587 ## 6 1000 20 0.019976 0.018608 0.0079817 -0.011498 ## # ℹ 80 more variables: bias.FIML.as5 , bias.FIML.as6 , ## # bias.FIML.as7 , bias.FIML.as8 , bias.FIML.as9 , ## # bias.FIML.as10 , bias.DWLS.as1 , bias.DWLS.as2 , ## # bias.DWLS.as3 , bias.DWLS.as4 , bias.DWLS.as5 , ## # bias.DWLS.as6 , bias.DWLS.as7 , bias.DWLS.as8 , ## # bias.DWLS.as9 , bias.DWLS.as10