using Random, Statistics, StatsPlots
Random.seed!(4)
topfeatures(X, y, m) = partialsortperm(vec(abs.(cor(X, y))), 1:m, rev = true)
function cv_accuracy(X, y; k = 5, nfeat = 20, select_inside = true)
n = size(X, 1)
folds = shuffle(repeat(1:k, outer = cld(n, k))[1:n])
sel_all = topfeatures(X, y, nfeat) # uses the test folds too: leakage
correct = 0
for f in 1:k
tr, te = folds .!= f, folds .== f
sel = select_inside ? topfeatures(X[tr, :], y[tr], nfeat) : sel_all
w = hcat(ones(sum(tr)), X[tr, sel]) \ y[tr] # least-squares linear classifier
correct += sum(sign.(hcat(ones(sum(te)), X[te, sel]) * w) .== y[te])
end
correct / n
end
reps = 100
acc_wrong, acc_right = Float64[], Float64[]
for r in 1:reps
X = randn(50, 2000)
y = rand([-1.0, 1.0], 50)
push!(acc_wrong, cv_accuracy(X, y; select_inside = false))
push!(acc_right, cv_accuracy(X, y; select_inside = true))
end
boxplot(["Wrong: select, then CV"], acc_wrong, label = "", color = :darkred, fillalpha = 0.6)
boxplot!(["Right: select inside CV"], acc_right, label = "", color = :steelblue, fillalpha = 0.6)
hline!([0.5], color = :black, linestyle = :dash, linewidth = 3, label = "Chance (true accuracy)", legend = :bottomleft)
plot!(ylabel = "Cross-validated accuracy", ylims = (0, 1), size = (1000, 420), margin = 5mm,
title = "Mean CV accuracy on pure noise: wrong = $(round(mean(acc_wrong), digits = 2)), right = $(round(mean(acc_right), digits = 2))")