MDP solution via linear programming for explicit engine.

.classpath

@@ -15,5 +15,6 @@
 	<classpathentry kind="lib" path="prism/lib/log4j-api.jar"/>
 	<classpathentry kind="lib" path="prism/lib/log4j-core.jar"/>
 	<classpathentry kind="lib" path="prism/lib/nailgun-server.jar"/>
+	<classpathentry kind="lib" path="prism/lib/gurobi.jar"/>
 	<classpathentry kind="output" path="prism/classes"/>
 </classpath>

ac-imdp.prism

@@ -0,0 +1,54 @@
+mdp
+
+
+const int maxX = 8;//10;
+const int maxY = 5;//4;
+const double r=0.8; //= 0.820795462532;// = 0.5; // pilot response factor
+const double e1 = 0.1;
+const double p=0.3;// = 0.19; // prob of adversary moving up or down, ensure 2p <= 1
+const double e2=0.01;
+
+
+formula xclose = (x-ax >= 0 & x-ax < 3) | (ax-x >= 0 & ax-x < 3);
+formula yclose = (y-ay >= 0 & y-ay < 2) | (ay-y >= 0 & ay-y < 2);
+formula collision = xclose & yclose;
+
+module aircraft
+
+	x : [0..maxX] init 0;
+	y : [0..maxY] init 1;
+
+
+	[down] x < maxX -> [r-e1,r+e1] : (y'=max(y-1,0)) & (x'=x+1) + [1-r-e1,1-r+e1] : (y'=y) & (x'=x+1);
+	[up] x < maxX -> [r-e1,r+e1] : (y'=min(y+1,maxY)) & (x'=x+1)  + [1-r-e1,1-r+e1]  : (y'=y) & (x'=x+1);
+	[stay] x < maxX -> (y'=y) & (x'=x+1);
+
+	[down] x = maxX -> true;
+	[up] x = maxX -> true;
+	[stay] x = maxX -> true;
+
+endmodule
+
+
+module adversary
+
+	ax : [0..maxX] init maxX;
+	ay : [0..maxY] init 1;
+
+	[up] ax > 0 -> p : (ay'=max(ay-1,0)) & (ax'=ax-1) + p : (ay'=min(ay+1,maxY)) & (ax'=ax-1) + 1-2*p : (ay'=ay) & (ax'=ax-1);
+	[down] ax > 0 -> p : (ay'=max(ay-1,0)) & (ax'=ax-1) + p : (ay'=min(ay+1,maxY)) & (ax'=ax-1) + 1-2*p : (ay'=ay) & (ax'=ax-1);
+	[stay] ax > 0 -> p : (ay'=max(ay-1,0)) & (ax'=ax-1) + p : (ay'=min(ay+1,maxY)) & (ax'=ax-1) + 1-2*p : (ay'=ay) & (ax'=ax-1);
+	[up] ax = 0 -> true;
+	[down] ax = 0 -> true;
+	[stay] ax = 0 -> true;
+
+endmodule
+
+rewards
+	[up] true : 1;
+	[down] true : 1;
+	[stay] true : 1;
+endrewards
+
+label "goal" = x=maxX;
+label "Crash" = collision;

ad-6x5.prism

@@ -0,0 +1,51 @@
+mdp
+
+// Model parameters 
+const double p = 0.1;  // Mean disturbance probability
+const double e = 0.05;        // Epistemic uncertainty 
+formula a = x * e;     // Epistemic uncertainty grows with x-coordinate
+
+const int M = 5; // Environment width
+const int N = 6; // Environment height
+
+const double eps = 0.0001; // Lower bound for probability ranges
+
+// Goal
+formula goal = x = 1 & y = N; 
+
+// Obstacles 
+formula crashed = (x = 2 & y = 3) | (x = 3 & y = 3) | (x = 3 & y = 4);
+
+// UAV behaviour 
+module env
+
+	x : [1..M] init 1;
+	y : [1..N] init 1;
+
+	[up]    !crashed & y < N -> [max(eps,1-3*(p+a)), min(1-3*(p-a),1)] : (y'=min(y+1,N)) // Probability to move in intended direction
+			  + [max(eps,p-a), min(p+a,1)] : (y'=min(y+1,N)) & (x'=max(x-1,1))   // Probabilities to get pushed by wind
+			  + [max(eps,p-a), min(p+a,1)] : (y'=min(y+1,N)) & (x'=min(x+1,M))
+			  + [max(eps,p-a), min(p+a,1)] : true;
+
+	[right] !crashed & x < M -> [max(eps,1-3*(p+a)), min(1-3*(p-a),1)] : (x'=min(x+1,M)) // Probability to move in intended direction
+			  + [max(eps,p-a), min(p+a,1)] : (x'=min(x+1,M)) & (y'=min(y+1,N))   // Probabilities to get pushed by wind
+			  + [max(eps,p-a), min(p+a,1)] : (x'=max(x-1,1)) 		     
+			  + [max(eps,p-a), min(p+a,1)] : true;
+
+	[left] 	!crashed & x > 1 -> [max(eps,1-3*(p+a)), min(1-3*(p-a),1)] : (x'=max(x-1,1)) // Probability to move in intended direction
+		   	  + [max(eps,p-a), min(p+a,1)] : (x'=max(x-1,1)) & (y'=min(y+1,N))   // Probabilities to get pushed by wind
+			  + [max(eps,p-a), min(p+a,1)] : (x'=min(x+1,M)) 		     
+			  + [max(eps,p-a), min(p+a,1)] : true;
+
+endmodule
+
+// Your handcrafted policy
+module policy
+	[up]    x < 3  -> true;
+	[left]  x >= 3 -> true;
+	[right] false -> true;
+endmodule
+
+// Labelling
+label "Crash" = crashed;
+label "goal" = goal;

bg-imdp.prism

@@ -0,0 +1,28 @@
+mdp
+
+
+const int n = 8;
+const double p=0.5;
+const double e=0.1;
+module bet
+
+	money : [0..(n+1)*10] init 10;
+	steps : [0..n+1] init 0; // +1 for terminal state
+
+	[bet0]  steps < n & money >= 0 -> 1 : (money'=money) & (steps'=steps+1);
+	[bet1]  steps < n & money >= 1 & money <= n*10 -> [p-e,p+e] : (money'=money+1) & (steps'=steps+1) + [1-p-e,1-p+e] : (money'=money-1) & (steps'=steps+1);
+	[bet2]  steps < n & money >= 2 & money <= n*10 -> [p-e,p+e] : (money'=money+2) & (steps'=steps+1) + [1-p-e,1-p+e] : (money'=money-2) & (steps'=steps+1);
+	[bet5]  steps < n & money >= 5 & money <= n*10 -> [p-e,p+e] : (money'=money+5) & (steps'=steps+1) + [1-p-e,1-p+e] : (money'=money-5) & (steps'=steps+1);
+	[bet10] steps < n & money >=10 & money <= n*10 -> [p-e,p+e] : (money'=money+10) & (steps'=steps+1) + [1-p-e,1-p+e] : (money'=money-10) & (steps'=steps+1);
+
+	[end] steps = n -> 1: (steps'=n+1);
+	[stop] steps = n+1 -> 1: true;
+
+endmodule
+
+rewards
+    [] steps < n : money;
+	[end] true : money;
+endrewards
+
+label "goal" = steps = n+1;

chain-imdp-reverse.prism

@@ -0,0 +1,23 @@
+mdp
+
+const double p=0.4;
+const double uncertainty=0.05;
+const int H = 6;
+const double m=200.0;
+
+module chain
+
+	s : [0..H] init 0;
+
+	[a] true -> [p-uncertainty, p+uncertainty]: (s'=min(s+1,H)) + [1-p-uncertainty,1-p+uncertainty]: (s'=0);
+	[b] true -> [1-p-uncertainty, 1-p+uncertainty] : (s'=min(s+1,H)) + [p-uncertainty, p+uncertainty] : (s'=0);
+    [end] (s=H) -> true;
+
+endmodule
+
+rewards
+	[a] true : 1;
+	[b] true : 1;
+endrewards
+
+label "goal" = s=H;

chain-imdp.prism

@@ -0,0 +1,22 @@
+mdp
+
+const double p=0.4;
+const double uncertainty=0.05;
+const int H = 6;
+
+module chain
+
+	s : [0..H] init 0;
+
+	[a] true -> [p-uncertainty, p+uncertainty]: (s'=min(s+1,H)) + [1-p-uncertainty,1-p+uncertainty]: (s'=0);
+	[b] true -> [1-p-uncertainty, 1-p+uncertainty] : (s'=min(s+1,H)) + [p-uncertainty, p+uncertainty] : (s'=0);
+    [end] (s=H) -> true;
+
+endmodule
+
+rewards
+	[a] true : 1;
+	[b] true : 1;
+endrewards
+
+label "goal" = s=H;

drone-imdp.prism

@@ -0,0 +1,66 @@
+mdp
+
+// Environment Dimensions
+const int MAXX = 8;
+const int MAXY = 8;
+const int MAXZ = 8;
+
+// Disturbance Probability Parameter
+const double p=0.2; // must be less than 1/3
+const double e=0.05;
+
+// Regions
+
+// Target Region
+formula target = x > MAXX - 2 & y < 2 & z > MAXZ - 2;
+
+// Obstacle Regions
+formula crash = obstacle1 | obstacle2 | obstacle3 | obstacle4;
+formula obstacle1 = x >= 3 & x <= 4 & z >= 1 & z <= 2;
+formula obstacle2 = x >= 3 & x <= 4 & y >= 1 & y <= 2;
+formula obstacle3 = x >= 3 & x <= 4 & y >= 3 & y <= 5;
+formula obstacle4 = x >= 3 & x <= 4 & z >= 7 & z <= 8;
+
+module drone
+	x : [1..MAXX] init 1;
+	y : [1..MAXY] init 1;
+	z : [1..MAXZ] init 1;
+
+	[front] true -> [1 - 3 * p-e,1 - 3 * p+e]: (x' = min(x + 1, MAXX))
+				+ [p-e,p+e] : (x' = min(x + 1, MAXX)) & (y' = max(y - 1,1))
+				+ [p-e,p+e] : (x' = min(x + 1, MAXX)) & (y' = min(y + 1, MAXY))
+				+ [p-e,p+e] : (x' = x) & (y' = max(y - 2, 1));
+
+	[back] true ->[1 - 3 * p-e,1 - 3 * p+e]: (x' = max(x - 1, 1))
+				+ [p-e,p+e] : (x' = max(x - 1, 1)) & (y' = max(y - 1,1))
+				+ [p-e,p+e] : (x' = max(x - 1, 1)) & (y' = min(y + 1, MAXY))
+				+ [p-e,p+e] : (x' = x) & (y' = max(y - 2, 1));
+
+	[right] true -> [1 - 2 * p-e,1-2*p+e]: (y' = min(y + 1, MAXY))
+				+ [p-e,p+e]: (y' = min(y + 2, MAXY))
+				+ [p-e,p+e]: (y' = max(y - 1, 1));
+
+	[left] true -> [1 - 2 * p-e,1-2*p+e]: (y' = max(y - 1, 1))
+				+ [p-e,p+e]: (y' = max(y - 2, 1))
+				+ [p-e,p+e]: (y' = min(y + 1, MAXY));
+
+	[up] true -> [1 - 3 * p-e,1 - 3 * p+e]: (z' = min(z + 1, MAXZ))
+				+ [p-e,p+e] : (z' = min(z + 1, MAXZ)) & (y' = max(y - 1,1))
+				+ [p-e,p+e] : (z' = min(z + 1, MAXZ)) & (y' = min(y + 1, MAXY))
+				+ [p-e,p+e] : (z' = z) & (y' = max(y - 2, 1));
+
+	[down] true ->[1 - 3 * p-e,1 - 3 * p+e]: (z' = max(z - 1,1))
+				+ [p-e,p+e] : (z' = max(z - 1,1)) & (y' = max(y - 1,1))
+				+ [p-e,p+e] : (z' = max(z - 1,1)) & (y' = min(y + 1, MAXY))
+				+ [p-e,p+e] : (z' = z) & (y' = max(y - 2, 1));
+    [done] target -> 1:true;
+    [cry] crash -> 1:true;
+
+endmodule 
+
+rewards
+    [] true : 1;
+endrewards
+
+label "goal" = target;
+label "Crash" = crash;

ds-imdp.prism

@@ -0,0 +1,74 @@
+mdp
+
+const int M = 15;
+const int N = 15;
+const double p=0.4; // with probability p robot moves to an adjacent location instead of desired
+const double e = 0.02;
+const int MAX_STEPS=16; // time cut off
+const int TERM_COST = 500;
+const int STEP_COST= 5; // cost for every time step
+
+formula treasure = (c=1 & r=4)|(c=2 & r=6)|(c=3 & r=6)| (c=4 & r=7)|(c=5 & r=8)|(c=6 & r=8)|(c=7 & r=9)|(c=8 & r=10)|(c=9 & r=10)|(c=10 & r=11)|(c=11 & r=12)|(c=12 & r=12)|(c=13 & r=13)|(c=14 & r=14)|(c=15 & r=14);
+formula wall = (c=1 & r>4) | ((c=2 | c=3) & r>6) | (c=4 & r>7) | ((c=5 | c=6) & r>8) | (c=7 & r>9) | ((c=8 | c=9) & r>10)| (c=10 & r>11) | ((c=11 | c=12) & r>12)| (c=13 & r>13) | ((c=14 | c=15) & r>14);
+
+// action guards i.e. if south wall is true -> south action should not be allowed
+formula west_wall = (c=2 & r=5);
+formula south_west_wall = (c=2 & (r=4 | r=5)) | (c=4 & r=6) | (c=5 & r=7) | (c=7 & r=8) | (c=8 & r=9)| (c=10 & r=10) | (c=11 & r=11)| (c=13 & r=12) | (c=14 & r=13);
+
+module robot
+	//define robot position
+	r:[1..N] init 1; //grid row
+	c:[1..M] init 1; //grid column
+    t:[1..MAX_STEPS+1] init 1; // time step
+
+	// transitions
+	[end] treasure -> (t'=MAX_STEPS+1);
+	[end] (t=MAX_STEPS)  & !treasure -> (t'=t+1);
+	[east]  (c<M | c=M) & !treasure & (t<MAX_STEPS)& !treasure-> [p/2-e,p/2+e]:(r'=min(r+1, N))&(c'=min(c+1, M))&(t'=min(t+1, MAX_STEPS))+ [p/2-e,p/2+e]: (r'=max(r-1, 1))&(c'=min(c+1, M))&(t'=min(t+1, MAX_STEPS)) + [1-p-e,1-p+e]: (c'=min(c+1, M))&(t'=min(t+1, MAX_STEPS));
+	[south] (r<N | r=N) & !south_west_wall & (t<MAX_STEPS) & !treasure-> [1-p-e,1-p+e]:(r'=min(r+1, N))&(t'=min(t+1, MAX_STEPS)) + [p/2-e,p/2+e]: (c'=min(c+1, M))&(r'=min(r+1, N))&(t'=min(t+1, MAX_STEPS))+ [p/2-e,p/2+e]: (c'=max(c-1, 1))&(r'=min(r+1, N))&(t'=min(t+1, MAX_STEPS));
+	[west]  (c>1 | c=1) & !west_wall & !south_west_wall & (t<MAX_STEPS)& !treasure-> [p/2-e,p/2+e]:(r'=max(r-1, 1))&(c'=max(c-1, 1))&(t'=min(t+1, MAX_STEPS))+ [p/2-e,p/2+e]:(r'=min(r+1, N))&(c'=max(c-1, 1))&(t'=min(t+1, MAX_STEPS)) + [1-p-e,1-p+e]: (c'=max(c-1, 1))&(t'=min(t+1, MAX_STEPS));
+	[north] (r=1 | r>1) & (t<MAX_STEPS)& !treasure-> [1-p-e,1-p+e]:(r'=max(r-1, 1))&(t'=min(t+1, MAX_STEPS)) + [p/2-e,p/2+e]: (c'=max(c-1, 1))&(r'=max(r-1, 1))&(t'=min(t+1, MAX_STEPS)) + [p/2-e,p/2+e]: (c'=min(c+1, M))&(r'=max(r-1, 1))&(t'=min(t+1, MAX_STEPS)) ;
+
+	[south] (r<N | r=N) &  south_west_wall & (t<MAX_STEPS)& !treasure-> [1-p-e,1-p+e]:(r'=min(r+1, N))&(t'=min(t+1, MAX_STEPS)) + [p-e,p+e]: (c'=min(c+1, M))&(r'=min(r+1, N))&(t'=min(t+1, MAX_STEPS));
+    [west]  (c>1 | c=1) & !west_wall & south_west_wall & (t<MAX_STEPS)& !treasure-> [p-e,p+e]:(r'=max(r-1, 1))&(c'=max(c-1, 1))&(t'=min(t+1, MAX_STEPS)) + [1-p-e,1-p+e]: (c'=max(c-1, 1))&(t'=min(t+1, MAX_STEPS));
+
+    // diagonal transitions
+    [north_east] (c<M | c=M) & (r=1 | r>1)  & (t<MAX_STEPS) & !treasure-> p/2:(c'=min(c+1, M))&(t'=min(t+1, MAX_STEPS))+ p/2: (r'=max(r-1, 1))&(t'=min(t+1, MAX_STEPS)) + (1-p): (c'=min(c+1, M))&(r'=max(r-1, 1))&(t'=min(t+1, MAX_STEPS));
+    [north_west] (c>1 | c=1) & (r=1 | r>1) & !west_wall & (t<MAX_STEPS) & !treasure-> (1-p):(r'=max(r-1, 1))&(c'=max(c-1, 1))&(t'=min(t+1, MAX_STEPS)) + p/2: (r'=max(r-1, 1))&(t'=min(t+1, MAX_STEPS))+ p/2: (c'=max(c-1, 1))&(t'=min(t+1, MAX_STEPS));
+    [south_east] (c<M | c=M) & (r<N | r=N) & (t<MAX_STEPS) & !treasure-> p/2:(c'=min(c+1, M))&(t'=min(t+1, MAX_STEPS))+ p/2:(r'=min(r+1, N))&(t'=min(t+1, MAX_STEPS)) + (1-p): (r'=min(r+1, N))&(c'=min(c+1, M))&(t'=min(t+1, MAX_STEPS));
+    [south_west] (c>1 | c=1) & (r<N | r=N) & !south_west_wall & !west_wall & (t<MAX_STEPS) & !treasure-> (1-p):(r'=min(r+1, N))&(c'=max(c-1, 1))&(t'=min(t+1, MAX_STEPS)) + p/2: (c'=max(c-1, 1))&(t'=min(t+1, MAX_STEPS)) + p/2: (r'=min(r+1, N))&(t'=min(t+1, MAX_STEPS)) ;
+
+    [north_west] (c>1 | c=1) & (r=1 | r>1) & west_wall & (t<MAX_STEPS) & !treasure-> (1-p):(r'=max(r-1, 1))&(c'=max(c-1, 1))&(t'=min(t+1, MAX_STEPS)) + p: (r'=max(r-1, 1))&(t'=min(t+1, MAX_STEPS));
+    [south_west] (c>1 | c=1) & (r<N | r=N) & !south_west_wall & west_wall & (t<MAX_STEPS) & !treasure-> (1-p):(r'=min(r+1, N))&(c'=max(c-1, 1))&(t'=min(t+1, MAX_STEPS)) + p: (r'=min(r+1, N))&(t'=min(t+1, MAX_STEPS)) ;
+
+endmodule
+
+label "goal" = (t=MAX_STEPS+1);
+label "obs" = (wall);
+
+rewards
+    [east] (t<MAX_STEPS): STEP_COST;
+    [south] (t<MAX_STEPS): STEP_COST;
+    [west] (t<MAX_STEPS): STEP_COST;
+    [north] (t<MAX_STEPS): STEP_COST;
+    [north_east] (t<MAX_STEPS): STEP_COST;
+    [north_west] (t<MAX_STEPS): STEP_COST;
+    [south_east] (t<MAX_STEPS): STEP_COST;
+    [south_west] (t<MAX_STEPS): STEP_COST;
+    [end] (t=MAX_STEPS) : TERM_COST;
+    [end] (c=1 & r=4): TERM_COST-10;
+    [end] (c=2 & r=6): TERM_COST-15;
+    [end] (c=3 & r=6): TERM_COST-25;
+    [end] (c=4 & r=7): TERM_COST-30;
+    [end] (c=5 & r=8): TERM_COST-40;
+    [end] (c=6 & r=8): TERM_COST-55;
+    [end] (c=7 & r=9): TERM_COST-75;
+    [end] (c=8 & r=10): TERM_COST-100;
+    [end] (c=9 & r=10): TERM_COST-140;
+    [end] (c=10 & r=11): TERM_COST-190;
+    [end] (c=11 & r=12): TERM_COST-240;
+    [end] (c=12 & r=12): TERM_COST-300;
+    [end] (c=13 & r=13): TERM_COST-360;
+    [end] (c=14 & r=14): TERM_COST-430;
+    [end] (c=15 & r=14): TERM_COST-500;
+endrewards