Lecture 8: Scripts

.gitignore

@@ -4,7 +4,6 @@
 
 Manifest.toml
 
-/scripts/
 /_documents/
 /_tmp/
 /_tikz/

scripts/lecture_08/Project.toml

@@ -0,0 +1,14 @@
+[deps]
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
+Query = "1a8c2f83-1ff3-5112-b086-8aa67b057ba1"
+RDatasets = "ce6b1742-4840-55fa-b093-852dadbb1d8b"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+StatsPlots = "f3b207a7-027a-5e70-b257-86293d7955fd"
+
+[comp]
+Plots = "v1.10.3"
+Query = "v1.0.0"
+RDatasets = "v0.7.4"
+StatsPlots = "v0.14.18"
+julia = "1.5"

scripts/lecture_08/script.jl

@@ -0,0 +1,121 @@
+###########################################
+# From last lecture
+###########################################
+
+abstract type Step end
+
+struct GD <: Step
+    α::Real
+end
+
+optim_step(s::GD, f, g, x) = -s.α*g(x)
+
+function optim(f, g, x, s::Step; max_iter=100)
+    for i in 1:max_iter
+        x += optim_step(s, f, g, x)
+    end
+    return x
+end
+
+###########################################
+# Linear regression
+###########################################
+
+# Load data
+
+using Plots
+using StatsPlots
+using RDatasets
+using LinearAlgebra
+using Query
+using Statistics
+
+iris = dataset("datasets", "iris")
+
+iris[1:5,:]
+
+# Exercise
+
+
+
+# Exercise
+
+
+
+norm(w-w2)
+
+# Exercise
+
+
+
+
+###########################################
+# Logistic regression
+###########################################
+
+# Exercise
+
+
+
+
+
+
+# Extract data
+
+X = Matrix(iris_reduced[:, 1:3])
+y = iris_reduced.label
+
+# Exercise
+
+@df iris_reduced scatter(
+    :PetalLength,
+    :PetalWidth;
+    group = :Species,
+    xlabel = "Petal length",
+    ylabel = "Petal width",
+    legend = :topleft,
+)
+
+σ(z) = 1/(1+exp(z))
+
+# Exercise
+
+
+
+
+
+
+
+w = log_reg(X, y, zeros(size(X,2)))
+
+# Plot the separating hyperplane
+
+separ(x, w) = (-w[3]-w[1]*x)/w[2]
+
+xlims = extrema(iris_reduced.PetalLength) .+ [-0.1, 0.1]
+ylims = extrema(iris_reduced.PetalWidth) .+ [-0.1, 0.1]
+
+@df iris_reduced scatter(
+    :PetalLength,
+    :PetalWidth;
+    group = :Species,
+    xlabel = "Petal length",
+    ylabel = "Petal width",
+    legend = :topleft,
+    xlims,
+    ylims,
+)
+
+plot!(xlims, x -> separ(x,w); label = "Separation", line = (:black,3))
+
+# Check optimality
+
+y_hat = σ.(-X*w)
+grad = X'*(y_hat.-y) / size(X,1)
+norm(grad)
+
+# Exercise
+
+
+
+

docs/src/lecture_08/script_init.jl → scripts/lecture_08/script_init.jl

@@ -1,3 +1,7 @@
+using Pkg
+Pkg.activate(".")
+Pkg.instantiate()
+
 using Plots
 using StatsPlots
 using RDatasets

scripts/lecture_08/script_sol.jl

@@ -0,0 +1,165 @@
+###########################################
+# From last lecture
+###########################################
+
+abstract type Step end
+
+struct GD <: Step
+    α::Real
+end
+
+optim_step(s::GD, f, g, x) = -s.α*g(x)
+
+function optim(f, g, x, s::Step; max_iter=100)
+    for i in 1:max_iter
+        x += optim_step(s, f, g, x)
+    end
+    return x
+end
+
+###########################################
+# Linear regression
+###########################################
+
+# Load data
+
+using Plots
+using StatsPlots
+using RDatasets
+using LinearAlgebra
+using Query
+using Statistics
+
+iris = dataset("datasets", "iris")
+
+iris[1:5,:]
+
+# Exercise
+
+y = iris.PetalWidth
+X = hcat(iris.PetalLength, ones(length(y)))
+
+@df iris scatter(
+    :PetalLength,
+    :PetalWidth;
+    label="",
+    xlabel = "Petal length",
+    ylabel = "Petal width"
+)    
+
+# Exercise
+
+w = (X'*X) \ (X'*y)
+
+g(w) = X'*(X*w-y)
+w2 = optim([], g, zeros(size(X,2)), GD(1e-4); max_iter=10000)
+
+norm(w-w2)
+
+# Exercise
+
+f_pred(x, w) = w[1]*x + w[2]
+
+x_lims = extrema(iris.PetalLength) .+ [-0.1, 0.1]
+
+@df iris scatter(
+    :PetalLength,
+    :PetalWidth;
+    xlabel = "Petal length",
+    ylabel = "Petal width",
+    label = "",
+    legend = :topleft,
+)
+
+plot!(x_lims, x -> f_pred(x,w); label = "Prediction", line = (:black,3))
+
+###########################################
+# Logistic regression
+###########################################
+
+# Exercise
+
+iris_reduced = @from i in iris begin
+    @where i.Species != "setosa"
+    @select {
+        i.PetalLength,
+        i.PetalWidth,
+        intercept = 1,
+        i.Species,
+        label = i.Species == "virginica",
+    }
+    @collect DataFrame
+ end
+
+iris_reduced2 = iris[iris.Species .!= "setosa", :]
+iris_reduced2 = iris_reduced2[:,[3;4;5]]
+
+insertcols!(iris_reduced2, 3, :intercept => 1)
+insertcols!(iris_reduced2, 5, :label => iris_reduced2.Species .== "virginica")
+
+isequal(iris_reduced, iris_reduced2)
+
+# Extract data
+
+X = Matrix(iris_reduced[:, 1:3])
+y = iris_reduced.label
+
+# Exercise
+
+@df iris_reduced scatter(
+    :PetalLength,
+    :PetalWidth;
+    group = :Species,
+    xlabel = "Petal length",
+    ylabel = "Petal width",
+    legend = :topleft,
+)
+
+σ(z) = 1/(1+exp(z))
+
+# Exercise
+
+function log_reg(X, y, w; max_iter=100, tol=1e-6)
+    X_mult = [row*row' for row in eachrow(X)]
+    for i in 1:max_iter
+        y_hat = σ.(-X*w)
+        grad = X'*(y_hat.-y) / size(X,1)
+        hess = y_hat.*(1 .-y_hat).*X_mult |> mean
+        w -= hess \ grad
+    end
+    return w
+end
+
+w = log_reg(X, y, zeros(size(X,2)))
+
+# Plot the separating hyperplane
+
+separ(x, w) = (-w[3]-w[1]*x)/w[2]
+
+xlims = extrema(iris_reduced.PetalLength) .+ [-0.1, 0.1]
+ylims = extrema(iris_reduced.PetalWidth) .+ [-0.1, 0.1]
+
+@df iris_reduced scatter(
+    :PetalLength,
+    :PetalWidth;
+    group = :Species,
+    xlabel = "Petal length",
+    ylabel = "Petal width",
+    legend = :topleft,
+    xlims,
+    ylims,
+)
+
+plot!(xlims, x -> separ(x,w); label = "Separation", line = (:black,3))
+
+# Check optimality
+
+y_hat = σ.(-X*w)
+grad = X'*(y_hat.-y) / size(X,1)
+norm(grad)
+
+# Exercise
+
+pred = y_hat .>= 0.5
+"Correct number of predictions: " * string(sum(pred .== y))
+"Wrong   number of predictions: " * string(sum(pred .!= y))