From 04d63d59c438c87b54d3dfee5494a477cc52c1c8 Mon Sep 17 00:00:00 2001 From: PyDataBlog Date: Wed, 31 Mar 2021 12:17:12 +0200 Subject: [PATCH 1/3] Added create_containers to minibatch --- src/lloyd.jl | 2 +- src/mini_batch.jl | 66 ++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 57 insertions(+), 11 deletions(-) diff --git a/src/lloyd.jl b/src/lloyd.jl index 693534d..47b624f 100644 --- a/src/lloyd.jl +++ b/src/lloyd.jl @@ -6,7 +6,7 @@ Basic algorithm for k-means calculation. struct Lloyd <: AbstractKMeansAlg end """ - Kmeans!(alg::AbstractKMeansAlg, containers, design_matrix, k; n_threads = nthreads(), k_init="k-means++", max_iters=300, tol=1e-6, verbose=true) + kmeans!(alg::AbstractKMeansAlg, containers, design_matrix, k; n_threads = nthreads(), k_init="k-means++", max_iters=300, tol=1e-6, verbose=true) Mutable version of `kmeans` function. Definition of arguments and results can be found in `kmeans`. diff --git a/src/mini_batch.jl b/src/mini_batch.jl index 5f179f7..6e46b8c 100644 --- a/src/mini_batch.jl +++ b/src/mini_batch.jl @@ -2,6 +2,12 @@ MiniBatch(b::Int) Sculley et al. 2007 Mini batch k-means algorithm implementation. + +```julia +X = rand(30, 100_000) # 100_000 random points in 30 dimensions + +kmeans(MiniBatch(100), X, 3) # 3 clusters, MiniBatch algorithm with 100 batch samples at each iteration +``` """ struct MiniBatch <: AbstractKMeansAlg b::Int # batch size @@ -39,35 +45,34 @@ function kmeans!(alg::MiniBatch, X, k; # b examples picked randomly from X (Step 5 in paper) batch_rand_idx = isnothing(weights) ? rand(rng, 1:ncol, alg.b) : wsample(rng, 1:ncol, weights, alg.b) - batch_sample = X[:, batch_rand_idx] # Cache/label the batch samples nearest to the centers (Step 6 & 7) - @inbounds for i in axes(batch_sample, 2) - min_dist = distance(metric, batch_sample, centroids, i, 1) + @inbounds for i in batch_rand_idx + min_dist = distance(metric, X, centroids, i, 1) label = 1 for j in 2:size(centroids, 2) - dist = distance(metric, batch_sample, centroids, i, j) + dist = distance(metric, X, centroids, i, j) label = dist < min_dist ? j : label min_dist = dist < min_dist ? dist : min_dist end - final_labels[batch_rand_idx[i]] = label + final_labels[i] = label end # TODO: Batch gradient step - @inbounds for j in axes(batch_sample, 2) # iterate over examples (Step 9) + @inbounds for j in batch_rand_idx # iterate over examples (Step 9) - # Get cached center/label for this x => labels[batch_rand_idx[j]] (Step 10) - label = final_labels[batch_rand_idx[j]] + # Get cached center/label for this x => (Step 10) + label = final_labels[j] # Update per-center counts N[label] += isnothing(weights) ? 1 : weights[j] # verify (Step 11) # Get per-center learning rate (Step 12) lr = 1 / N[label] - # Take gradient step (Step 13) # TODO: Replace with an allocation-less loop. - centroids[:, label] .= (1 - lr) .* centroids[:, label] .+ (lr .* batch_sample[:, j]) + # Take gradient step (Step 13) # TODO: Replace with faster loop? + @views centroids[:, label] .= (1 - lr) .* centroids[:, label] .+ (lr .* X[:, j]) end # TODO: Reassign all labels based on new centres generated from the latest sample @@ -97,7 +102,17 @@ function kmeans!(alg::MiniBatch, X, k; end else counter = 0 + end + + # TODO: Warn users if model doesn't converge at max iterations + if (niters > max_iters) & (!converged) + + println("Clustering model failed to converge. Labelling data with latest centroids.") + final_labels = reassign_labels(X, metric, final_labels, centroids) + # TODO: Compute totalcost for unconverged model + J = sum_of_squares(X, final_labels, centroids) + break end J_previous = J @@ -135,3 +150,34 @@ function reassign_labels(DMatrix, metric, labels, centres) end return labels end + +""" + create_containers(::MiniBatch, k, nrow, ncol, n_threads) + +Internal function for the creation of all necessary intermidiate structures. + +- `centroids_new` - container which holds new positions of centroids +- `centroids_cnt` - container which holds number of points for each centroid +- `labels` - vector which holds labels of corresponding points +""" +function create_containers(::MiniBatch, X, k, nrow, ncol, n_threads) + T = eltype(X) + lng = n_threads + 1 + centroids_new = Vector{Matrix{T}}(undef, lng) + centroids_cnt = Vector{Vector{T}}(undef, lng) + + for i in 1:lng + centroids_new[i] = Matrix{T}(undef, nrow, k) + centroids_cnt[i] = Vector{Int}(undef, k) + end + + labels = Vector{Int}(undef, ncol) + + J = Vector{T}(undef, n_threads) + + # total_sum_calculation + sum_of_squares = Vector{T}(undef, n_threads) + + return (centroids_new = centroids_new, centroids_cnt = centroids_cnt, + labels = labels, J = J, sum_of_squares = sum_of_squares) +end From 17214f1f31ad25459e69c10a3f7d4aaa33193f02 Mon Sep 17 00:00:00 2001 From: PyDataBlog Date: Wed, 31 Mar 2021 23:23:12 +0200 Subject: [PATCH 2/3] Mini batch now fully compliant with API. Optimizations left --- src/kmeans.jl | 22 +++++++--- src/mini_batch.jl | 88 ++++++++++++++++------------------------ test/test90_minibatch.jl | 16 +++----- 3 files changed, 58 insertions(+), 68 deletions(-) diff --git a/src/kmeans.jl b/src/kmeans.jl index 1961e32..5b59376 100644 --- a/src/kmeans.jl +++ b/src/kmeans.jl @@ -170,11 +170,16 @@ alternatively one can use `rand` to choose random points for init. A `KmeansResult` structure representing labels, centroids, and sum_squares is returned. """ -function kmeans(alg::AbstractKMeansAlg, design_matrix, k; weights = nothing, +function kmeans(alg::AbstractKMeansAlg, design_matrix, k; + weights = nothing, n_threads = Threads.nthreads(), - k_init = "k-means++", max_iters = 300, - tol = eltype(design_matrix)(1e-6), verbose = false, - init = nothing, rng = Random.GLOBAL_RNG, metric = Euclidean()) + k_init = "k-means++", + max_iters = 300, + tol = eltype(design_matrix)(1e-6), + verbose = false, + init = nothing, + rng = Random.GLOBAL_RNG, + metric = Euclidean()) nrow, ncol = size(design_matrix) @@ -182,8 +187,13 @@ function kmeans(alg::AbstractKMeansAlg, design_matrix, k; weights = nothing, containers = create_containers(alg, design_matrix, k, nrow, ncol, n_threads) return kmeans!(alg, containers, design_matrix, k, weights, metric; - n_threads = n_threads, k_init = k_init, max_iters = max_iters, - tol = tol, verbose = verbose, init = init, rng = rng) + n_threads = n_threads, + k_init = k_init, + max_iters = max_iters, + tol = tol, + verbose = verbose, + init = init, + rng = rng) end diff --git a/src/mini_batch.jl b/src/mini_batch.jl index 6e46b8c..f589605 100644 --- a/src/mini_batch.jl +++ b/src/mini_batch.jl @@ -1,5 +1,6 @@ """ MiniBatch(b::Int) + `b` represents the size of the batch which should be sampled. Sculley et al. 2007 Mini batch k-means algorithm implementation. @@ -16,8 +17,8 @@ end MiniBatch() = MiniBatch(100) -function kmeans!(alg::MiniBatch, X, k; - weights = nothing, metric = Euclidean(), n_threads = Threads.nthreads(), +function kmeans!(alg::MiniBatch, containers, X, k, + weights = nothing, metric = Euclidean(); n_threads = Threads.nthreads(), k_init = "k-means++", init = nothing, max_iters = 300, tol = eltype(X)(1e-6), max_no_improvement = 10, verbose = false, rng = Random.GLOBAL_RNG) @@ -32,15 +33,14 @@ function kmeans!(alg::MiniBatch, X, k; N = zeros(T, k) # Initialize nearest centers for both batch and whole dataset labels - final_labels = Vector{Int}(undef, ncol) # dataset labels - converged = false niters = 0 counter = 0 J_previous = zero(T) J = zero(T) + totalcost = zero(T) - # TODO: Main Steps. Batch update centroids until convergence + # Main Steps. Batch update centroids until convergence while niters <= max_iters # Step 4 in paper # b examples picked randomly from X (Step 5 in paper) @@ -57,16 +57,17 @@ function kmeans!(alg::MiniBatch, X, k; min_dist = dist < min_dist ? dist : min_dist end - final_labels[i] = label + containers.labels[i] = label end - # TODO: Batch gradient step + # Batch gradient step @inbounds for j in batch_rand_idx # iterate over examples (Step 9) # Get cached center/label for this x => (Step 10) - label = final_labels[j] + label = containers.labels[j] + # Update per-center counts - N[label] += isnothing(weights) ? 1 : weights[j] # verify (Step 11) + N[label] += isnothing(weights) ? 1 : weights[j] # (Step 11) # Get per-center learning rate (Step 12) lr = 1 / N[label] @@ -75,43 +76,46 @@ function kmeans!(alg::MiniBatch, X, k; @views centroids[:, label] .= (1 - lr) .* centroids[:, label] .+ (lr .* X[:, j]) end - # TODO: Reassign all labels based on new centres generated from the latest sample - final_labels = reassign_labels(X, metric, final_labels, centroids) + # Reassign all labels based on new centres generated from the latest sample + containers.labels .= reassign_labels(X, metric, containers.labels, centroids) - # TODO: Calculate cost on whole dataset after reassignment and check for convergence - J = sum_of_squares(X, final_labels, centroids) # just a placeholder for now + # Calculate cost on whole dataset after reassignment and check for convergence + @parallelize 1 ncol sum_of_squares(containers, X, containers.labels, centroids, weights, metric) + J = sum(containers.sum_of_squares) if verbose # Show progress and terminate if J stopped decreasing. println("Iteration $niters: Jclust = $J") end - # TODO: Check for early stopping convergence + # Check for early stopping convergence if (niters > 1) & (abs(J - J_previous) < (tol * J)) counter += 1 # Declare convergence if max_no_improvement criterion is met if counter >= max_no_improvement converged = true - # TODO: Compute label assignment for the complete dataset - final_labels = reassign_labels(X, metric, final_labels, centroids) + # Compute label assignment for the complete dataset + containers.labels .= reassign_labels(X, metric, containers.labels, centroids) - # TODO: Compute totalcost for the complete dataset - J = sum_of_squares(X, final_labels, centroids) # just a placeholder for now + # Compute totalcost for the complete dataset + @parallelize 1 ncol sum_of_squares(containers, X, containers.labels, centroids, weights, metric) + totalcost = sum(containers.sum_of_squares) break end else counter = 0 end - # TODO: Warn users if model doesn't converge at max iterations + # Warn users if model doesn't converge at max iterations if (niters > max_iters) & (!converged) println("Clustering model failed to converge. Labelling data with latest centroids.") - final_labels = reassign_labels(X, metric, final_labels, centroids) + containers.labels = reassign_labels(X, metric, containers.labels, centroids) - # TODO: Compute totalcost for unconverged model - J = sum_of_squares(X, final_labels, centroids) + # Compute totalcost for unconverged model + @parallelize 1 ncol sum_of_squares(containers, X, containers.labels, centroids, weights, metric) + totalcost = sum(containers.sum_of_squares) break end @@ -119,21 +123,10 @@ function kmeans!(alg::MiniBatch, X, k; niters += 1 end - return centroids, niters, converged, final_labels, J # TODO: push learned artifacts to KmeansResult - #return KmeansResult(centroids, containers.labels, T[], Int[], T[], totalcost, niters, converged) + # Push learned artifacts to KmeansResult + return KmeansResult(centroids, containers.labels, T[], Int[], T[], totalcost, niters, converged) end -# TODO: Only being used to test generic implementation. Get rid off after! -function sum_of_squares(x, labels, centre) - s = 0.0 - - for i in axes(x, 2) - for j in axes(x, 1) - s += (x[j, i] - centre[j, labels[i]])^2 - end - end - return s -end function reassign_labels(DMatrix, metric, labels, centres) @inbounds for i in axes(DMatrix, 2) @@ -159,25 +152,16 @@ Internal function for the creation of all necessary intermidiate structures. - `centroids_new` - container which holds new positions of centroids - `centroids_cnt` - container which holds number of points for each centroid - `labels` - vector which holds labels of corresponding points +- `sum_of_squares` - vector which holds the sum of squares values for each thread """ function create_containers(::MiniBatch, X, k, nrow, ncol, n_threads) - T = eltype(X) - lng = n_threads + 1 - centroids_new = Vector{Matrix{T}}(undef, lng) - centroids_cnt = Vector{Vector{T}}(undef, lng) - - for i in 1:lng - centroids_new[i] = Matrix{T}(undef, nrow, k) - centroids_cnt[i] = Vector{Int}(undef, k) - end - - labels = Vector{Int}(undef, ncol) - - J = Vector{T}(undef, n_threads) - - # total_sum_calculation - sum_of_squares = Vector{T}(undef, n_threads) + # Initiate placeholders to avoid allocations + T = eltype(X) + centroids_new = Matrix{T}(undef, nrow, k) # main centroids + centroids_cnt = Vector{T}(undef, k) # centroids counter + labels = Vector{Int}(undef, ncol) # labels vector + sum_of_squares = Vector{T}(undef, 1) # total_sum_calculation return (centroids_new = centroids_new, centroids_cnt = centroids_cnt, - labels = labels, J = J, sum_of_squares = sum_of_squares) + labels = labels, sum_of_squares = sum_of_squares) end diff --git a/test/test90_minibatch.jl b/test/test90_minibatch.jl index c0767d0..54dc11a 100644 --- a/test/test90_minibatch.jl +++ b/test/test90_minibatch.jl @@ -16,9 +16,9 @@ end rng = StableRNG(2020) X = rand(rng, 3, 100) - baseline = [kmeans(Lloyd(), X, 2).totalcost for i in 1:1_000] |> mean |> round - # TODO: Switch to kmeans after full implementation - res = [ParallelKMeans.kmeans!(MiniBatch(50), X, 2)[end] for i in 1:1_000] |> mean |> round + baseline = [kmeans(Lloyd(), X, 2; max_iters=100_000).totalcost for i in 1:200] |> mean |> round + + res = [kmeans(MiniBatch(10), X, 2; max_iters=100_000).totalcost for i in 1:200] |> mean |> round @test baseline == res end @@ -28,13 +28,9 @@ end rng = StableRNG(2020) X = rand(rng, 3, 100) - baseline = [kmeans(Lloyd(), X, 2; - tol=1e-6, metric=Cityblock(), - max_iters=500).totalcost for i in 1:1000] |> mean |> floor - # TODO: Switch to kmeans after full implementation - res = [ParallelKMeans.kmeans!(MiniBatch(), X, 2; - metric=Cityblock(), tol=1e-6, - max_iters=500)[end] for i in 1:1000] |> mean |> floor + baseline = [kmeans(Lloyd(), X, 2; metric=Cityblock(), max_iters=100_000).totalcost for i in 1:200] |> mean |> round + + res = [kmeans(MiniBatch(10), X, 2; metric=Cityblock(), max_iters=100_000).totalcost for i in 1:200] |> mean |> round @test baseline == res end From afae56aa395bb102656f213b2fe3311676957516 Mon Sep 17 00:00:00 2001 From: PyDataBlog Date: Fri, 2 Apr 2021 00:51:55 +0200 Subject: [PATCH 3/3] Optimised implementation based on initial feedback. Allocations down & speed gains --- .travis.yml | 3 ++- docs/src/index.md | 2 +- src/mini_batch.jl | 29 ++++++++++++++--------------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 82ec229..97ede4e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ julia: - 1.3 - 1.4 - 1.5 + - 1.6 - nightly after_success: - julia -e 'using Pkg; Pkg.add("Coverage"); using Coverage; Coveralls.submit(process_folder())' @@ -16,7 +17,7 @@ jobs: fast_finish: true include: - stage: Documentation - julia: 1.5 + julia: 1.6 script: julia --project=docs -e ' using Pkg; Pkg.develop(PackageSpec(path=pwd())); diff --git a/docs/src/index.md b/docs/src/index.md index 5839a67..9b203f7 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -127,8 +127,8 @@ r.converged # whether the procedure converged - [Elkan()](https://www.aaai.org/Papers/ICML/2003/ICML03-022.pdf) - Recommended for high dimensional data. - [Yinyang()](https://www.microsoft.com/en-us/research/wp-content/uploads/2016/02/ding15.pdf) - Recommended for large dimensions and/or large number of clusters. - [Coreset()](http://proceedings.mlr.press/v51/lucic16-supp.pdf) - Recommended for very fast clustering of very large datasets, when extreme accuracy is not important. +- [MiniBatch()](https://www.eecs.tufts.edu/~dsculley/papers/fastkmeans.pdf) - Recommended for extremely large datasets. - [Geometric()](http://cs.baylor.edu/~hamerly/papers/sdm2016_rysavy_hamerly.pdf) - (Coming soon) -- [MiniBatch()](https://www.eecs.tufts.edu/~dsculley/papers/fastkmeans.pdf) - (Coming soon) ### Practical Usage Examples diff --git a/src/mini_batch.jl b/src/mini_batch.jl index f589605..b785f8e 100644 --- a/src/mini_batch.jl +++ b/src/mini_batch.jl @@ -39,12 +39,13 @@ function kmeans!(alg::MiniBatch, containers, X, k, J_previous = zero(T) J = zero(T) totalcost = zero(T) - + batch_rand_idx = containers.batch_rand_idx + # Main Steps. Batch update centroids until convergence while niters <= max_iters # Step 4 in paper # b examples picked randomly from X (Step 5 in paper) - batch_rand_idx = isnothing(weights) ? rand(rng, 1:ncol, alg.b) : wsample(rng, 1:ncol, weights, alg.b) + batch_rand_idx = isnothing(weights) ? rand!(rng, batch_rand_idx, 1:ncol) : wsample!(rng, 1:ncol, weights, batch_rand_idx) # Cache/label the batch samples nearest to the centers (Step 6 & 7) @inbounds for i in batch_rand_idx @@ -58,22 +59,19 @@ function kmeans!(alg::MiniBatch, containers, X, k, end containers.labels[i] = label - end - # Batch gradient step - @inbounds for j in batch_rand_idx # iterate over examples (Step 9) - - # Get cached center/label for this x => (Step 10) - label = containers.labels[j] + ##### Batch gradient step ##### + # iterate over examples (each column) ==> (Step 9) + # Get cached center/label for each example label = labels[i] => (Step 10) # Update per-center counts - N[label] += isnothing(weights) ? 1 : weights[j] # (Step 11) + N[label] += isnothing(weights) ? 1 : weights[i] # (Step 11) # Get per-center learning rate (Step 12) lr = 1 / N[label] # Take gradient step (Step 13) # TODO: Replace with faster loop? - @views centroids[:, label] .= (1 - lr) .* centroids[:, label] .+ (lr .* X[:, j]) + @views centroids[:, label] .= (1 - lr) .* centroids[:, label] .+ (lr .* X[:, i]) end # Reassign all labels based on new centres generated from the latest sample @@ -110,7 +108,9 @@ function kmeans!(alg::MiniBatch, containers, X, k, # Warn users if model doesn't converge at max iterations if (niters > max_iters) & (!converged) - println("Clustering model failed to converge. Labelling data with latest centroids.") + if verbose + println("Clustering model failed to converge. Labelling data with latest centroids.") + end containers.labels = reassign_labels(X, metric, containers.labels, centroids) # Compute totalcost for unconverged model @@ -154,14 +154,13 @@ Internal function for the creation of all necessary intermidiate structures. - `labels` - vector which holds labels of corresponding points - `sum_of_squares` - vector which holds the sum of squares values for each thread """ -function create_containers(::MiniBatch, X, k, nrow, ncol, n_threads) +function create_containers(alg::MiniBatch, X, k, nrow, ncol, n_threads) # Initiate placeholders to avoid allocations T = eltype(X) - centroids_new = Matrix{T}(undef, nrow, k) # main centroids - centroids_cnt = Vector{T}(undef, k) # centroids counter labels = Vector{Int}(undef, ncol) # labels vector sum_of_squares = Vector{T}(undef, 1) # total_sum_calculation + batch_rand_idx = Vector{Int}(undef, alg.b) - return (centroids_new = centroids_new, centroids_cnt = centroids_cnt, + return (batch_rand_idx = batch_rand_idx, labels = labels, sum_of_squares = sum_of_squares) end