-
Notifications
You must be signed in to change notification settings - Fork 9
/
arcade.html
267 lines (241 loc) · 14.6 KB
/
arcade.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<title>So You Think You Can Test? - Arcade-mode</title>
<link rel="stylesheet" href="bootstrap/css/bootstrap.min.css">
<link rel="stylesheet" type="text/css" href="confidence.css">
</head>
<body>
<div id="app-confidence-arcade" class="container">
<!-- In game feedback -->
<div class="modal" id="feedback" tabindex="-1" role="dialog" data-backdrop="static" data-keyboard="false">
<div class="modal-dialog" role="document">
<div class="modal-content">
<div class="modal-body" v-if="winner == selected">
<h1 class="text-center">Correct!</h1>
<p class="text-center lead">Experiment {{selected}} was indeed not an A/A.</p>
<experiment-table v-if="selected >= 0" :e="summary_statistics[selected]"></experiment-table>
<p class="text-center ">The simulated effect was {{effect.toLocaleString('en', {style: 'percent'})}} relative uplift.</p>
<p class="text-center lead">You now have {{score}} points. Get ready for the next round!</p>
</div>
<div class="modal-body" v-if="winner != selected">
<h1 class="text-center">Oh noes!</h1>
<p class="text-center lead">Experiment {{selected}} was in fact an A/A.</p>
<experiment-table v-if="selected >= 0" :e="summary_statistics[selected]"></experiment-table>
<p class="text-center" v-if="selected >= 0 && summary_statistics[selected].is_significant()">Although the observed effect was significant, the simulated effect was in fact zero.</p>
<p class="text-center" v-if="selected >= 0 && !summary_statistics[selected].is_significant()">There was no simulated effect, and the observed result was not significant.</p>
<p class="text-center lead">The real A/B was experiment {{winner}}.</p>
<experiment-table :e="summary_statistics[winner]"></experiment-table>
<p class="text-center">The simulated effect was {{effect.toLocaleString('en', {style: 'percent'})}} relative uplift.</p>
<p class="text-center lead">You now have {{score}} points. Get ready for the next round!</p>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-lg btn-primary btn-block" v-on:click="next_round_arcade"><strong>Next</strong><br /><small>press space to continue</small></button>
</div>
</div>
</div>
</div>
<div v-if="game_state != 'play' && game_state != 'break'" class="row">
<!-- A general introduction to the game. -->
<div class="col-md-offset-2 col-md-8" v-if="game_state == ''">
<h1>So You Think You Can Test?</h1>
<p class="text-justify lead">Decision making under uncertainty is complicated business. This game aims to make decision makers more aware of the complex trade off between indecision and acting on insufficient information.</p>
<h2>Arcade-mode</h2>
<p class="text-justify">You will be presented with a series of ten independently simulated experiments. Only one of these is simulating a scenario where there is a difference in effect between base and variant. The other nine are so-called A/A experiments, with absolutely no difference between the two treatment groups. Because of random fluctuations, combined with the fact that we observe the experiments as they are simulated, all ten result will seem to change over time.</p>
<p class="text-justify lead">Your job is to identify which of these experiments is not an A/A.</p>
<p class="text-justify">Once you are certain you've identified the right experiment, make your choice known by clicking (or by using the numeric keys "0" through "9"). With every subsequent level, the size of the simulated effect will diminish, making your task progressively more difficult. A point is awarded for correct decisions; mistakes will cost a point. When visitors reaches zero, the game is over.</p>
<p class="text-justify lead">This is a game of chance, but not of chance alone. Good luck!</p>
<p><button type="button" class="btn btn-lg btn-primary btn-block" v-on:click="start_arcade_game"><strong>Start arcade-mode</strong><br /><small>fast-paced time-based action</small></button></p>
<p><a href="index.html" class="btn btn-lg btn-default btn-block"><strong>Switch to simulation-mode</strong><br /><small>realistic turn-based simulation</small></a></p>
<p class="text-center"><small>Created by <a href="http://www.lukasvermeer.nl">Lukas Vermeer</a> — Make this more awesome on <a href="https://github.com/lukasvermeer/confidence">GitHub</a></small></p>
</div>
<!-- End game feedback. -->
<div class="col-md-offset-3 col-md-6" v-if="game_state == 'eval'">
<h1>
<span v-if="level == 1">Paradox Of Choice, hmm?</span>
<span v-if="score < 0">Worse Than Random</span>
<span v-if="level > 1 && score == 0">You Win Some, You Lose Some</span>
<span v-if="score > 0 && score <= 2">Okay, I Guess</span>
<span v-if="score > 2 && score <= 4">Great Job, No Really</span>
<span v-if="score == 5 && level < 7">That's Awesome! Can You Do Better?</span>
<span v-if="score == 5 && level >= 7">Easy Does It</span>
<span v-if="score == 6">Best Probable Score. Seriously.</span>
<span v-if="score > 6">You Can See The Matrix!</span>
</h1>
<p class="lead">You made {{level-1}} decisions and scored a total of {{score}} points. <span v-if="score>4">That's awesome!</span></p>
<p>
<span v-if="level == 1">Just remember that not making any decisions is also a choice. You might not have made any mistakes, but you've also not really added any value to the company. Perhaps you were sleeping?</span>
<span v-if="level > 1 && score <= 0">Sure, you were decisive, but sometimes it is better to wait a bit before you make a decision. Maybe try playing another round and this time stick to making good decisions only, hmkay?</span>
<span v-if="score > 0 && score <= 2">Keep in mind that this game is all about making trade offs. You cannot win, but you can probably do better than this. You can probably make a few more good decisions, or a few less mistakes.</span>
<span v-if="score > 2 && score <= 4">This is not an easy game. You should be proud you managed to get a positive score. While it is possible to do better than this, it will not be easy. Are you up for the challenge?</span>
<span v-if="score == 5 && level < 7">This is a very good score! Looks like you made all the right choices, but perhaps you can do even better than this if you make all the right choices just a little bit faster?</span>
<span v-if="score == 5 && level >= 7">This is a very good score! Looks like you made a few costly mistakes along the way. Perhaps you can do better if you wait just a little bit longer when you\'re not quite confident you're making the right decision?</span>
<span v-if="score == 6">We've done the math, and the odds of getting a score even higher than this are absolutely astronomically small. This is as close as you'll ever be to beating this game, but perhaps you'd like to try just one more time?</span>
<span v-if="score > 6">What you've achieved is statistically very improbable. You probably cheated, but you might also simply be The One. If you manage to do this a second time, we'd be very impressed.</span>
</p>
<p><button type="button" class="btn btn-lg btn-primary btn-block" v-on:click="start_arcade_game"><strong>Thanks!</strong><br /><small>let me try that one more time</small></button></p>
<p><button type="button" class="btn btn-lg btn-default btn-block" v-on:click="game_state = '';"><strong>Hm. Okay.</strong><br /><small>maybe I should read the instructions again</small></button></p>
<p class="text-center"><small>Created by <a href="http://www.lukasvermeer.nl">Lukas Vermeer</a> — Make this more awesome on <a href="https://github.com/lukasvermeer/confidence">GitHub</a></small></p>
</div>
</div>
<!-- Scores header for Arcade-mode. -->
<div class="row" v-if="game_state == 'play' || game_state == 'break'">
<div class="col-md-4 text-center"><h2>Effect {{(1/Math.pow(2,level-1)*100).toFixed(2) + '%'}}</h2></div>
<div class="col-md-4 text-center"><h2>Score {{score}}</h2></div>
<div class="col-md-4 text-center"><h2>Visitors {{time.toLocaleString()}}</h2></div>
</div>
<div class="row" v-if="game_state == 'play' || game_state == 'break'">
<div class="progress">
<div class="progress-bar" role="progressbar" :aria-valuenow="progressNumRounded" aria-valuemin="0" aria-valuemax="100" :style="progressStyle"></div>
</div>
</div>
<!-- Experiments display for Arcade-mode. -->
<div v-if="game_state == 'play' || game_state == 'break'">
<div class="col-md-6" v-for="(exp,i) in summary_statistics">
<div class="panel panel-default panel-hover" v-on:click="choose_exp(i)">
<div class="panel-heading">
<strong>Experiment {{exp.experiment_id}}</strong>
</div>
<experiment-table :e="exp"></experiment-table>
</div>
</div>
</div>
</div>
<script type="text/javascript" src="jquery/jquery-1.9.1.min.js"></script>
<script type="text/javascript" src="bootstrap/js/bootstrap.min.js"></script>
<script type="text/javascript" src="vue/vue.js"></script>
<script type="text/javascript" src="jstat/jstat.min.js"></script>
<script type="text/javascript" src="experiment.js"></script>
<script type="text/javascript">
//
// Decision making under uncertainty is complicated business. This game aims to make
// decision makers more aware of the complex trade off between indecision and acting
// on insufficient information.
//
var e = [];
confidence = new Vue({
el: '#app-confidence-arcade',
data: {
EXPERIMENTS: 8,
VISITORS_PER_STEP: 500,
TOTAL_VISITORS: 500000,
CONVERSION_RATE: 0.05,
summary_statistics: [],
score: 0,
level: 0,
time: 0,
winner: -1,
selected: -1,
game_state: '',
},
computed: {
progressNum: function () {
return (this.time/this.TOTAL_VISITORS*100);
},
progressNumRounded: function () {
return this.progressNum.toFixed(0);
},
progressStyle: function () {
return {
width: this.progressNumRounded+'%'
}
},
effect: function () {
return 1/Math.pow(2,this.level-1);
},
},
methods: {
update_summary_statistics: function() {
this.summary_statistics = e;
// This is super nasty. I wish there was a nicer way to ensure Vue updates.
for (var i = 0; i < this.summary_statistics.length; ++i) {
this.summary_statistics[i].visits.splice(0,1,this.summary_statistics[i].visits[0]);
}
},
start_arcade_game: function() {
this.game_state = 'break';
// Initialise experiment data.
e = new Array();
for (var i = 0; i < this.EXPERIMENTS; ++i) { e.push(new Experiment(i)); }
this.score = 0; this.level = 0; this.time = this.TOTAL_VISITORS;
this.next_round_arcade();
this.recursive_experiment_loop();
},
next_round_arcade: function() {
if (this.game_state != 'break') return;
this.level++;
// Pick a winner.
this.winner = Math.round(Math.random()*(this.EXPERIMENTS-1));
// Reset experiments.
for (var i = 0; i < this.EXPERIMENTS; ++i) { e[i].reset(this.winner == i ? 1+this.effect : 1); }
$('#feedback').modal('hide');
this.game_state = 'play';
},
sim_visitor: function() {
var c = this.CONVERSION_RATE;
// assign to treatment for each exp and calculate aggregate effect size.
var a = [];
for (var i = 0; i < e.length; ++i) {
a[i] = e[i].assign_variant();
c = c * e[i].effect[a[i]];
}
// converted or not.
b = false; if (Math.random() <= c) { b = true; }
// update exps to reflect.
for (var i = 0; i < e.length; ++i) {
if (e[i].active) {
e[i].visits[a[i]]++;
if (b) e[i].conversions[a[i]]++;
}
}
},
run_experiments: function(v) {
for (var i = 0; i < v; ++i) { this.sim_visitor(); }
this.update_summary_statistics();
},
recursive_experiment_loop: function() {
if (this.game_state == 'play') {
this.run_experiments(this.VISITORS_PER_STEP);
this.time -= this.VISITORS_PER_STEP;
if (this.time <= 0) {
this.game_state = 'eval';
}
}
if (this.time > 0) {
// This feels kinda nasty. Would be nice to use relative instead of global reference.
this.timeoutID = window.setTimeout(function(){ confidence.recursive_experiment_loop(); }, 10);
}
},
choose_exp: function(e) {
if (this.time > 0) {
this.selected = e;
if(this.winner==e) {
this.score++;
}
else {
this.score--;
}
this.game_state = 'break';
$('#feedback').modal('show');
}
},
}
});
$(document).ready(function() {
document.addEventListener('keydown', function(event) {
if(event.keyCode >= 48 && event.keyCode <= 57) {
confidence.choose_exp(event.keyCode-48);
}
if(event.keyCode == 32) {
confidence.next_round_arcade();
}
});
});
</script>
<script async defer src="https://scripts.simpleanalyticscdn.com/latest.js"></script>
<noscript><img src="https://queue.simpleanalyticscdn.com/noscript.gif" alt="" referrerpolicy="no-referrer-when-downgrade" /></noscript>
</body>
</html>