Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/components/WasmImageProcessor.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -89,14 +89,14 @@ const WasmImageProcessor = () => {
const thresholded = await blackThreshold({
...fileData,
pixels: imgBilateralFiltered,
num_colors: 8,
num_colors: 16,
});

step(70);
const { pixels: kmeansed, labels } = await kmeans({
...fileData,
pixels: thresholded,
num_colors: 8,
num_colors: 16,

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't recommend using 16 for this right now because it really slows down larger images and will make it difficult for mobile users to use the app.

My proposed reversion
Suggested change
num_colors: 16,
});
step(70);
const { pixels: kmeansed, labels } = await kmeans({
...fileData,
pixels: thresholded,
num_colors: 8,
num_colors: 16,
num_colors: 8,
});
step(70);
const { pixels: kmeansed, labels } = await kmeans({
...fileData,
pixels: thresholded,
num_colors: 8,

Try running this image through it:

Image

Unless you have a strong PC, it'll take a while to process and then lag when you try to color it in or interact with the viewport on the editor page.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is in my set of test images. with multithreading it processes fast on my machine which is using an i7-13700H cpu

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Try processing the image on a cellphone.

In the future, we will add a config panel for the web app, but we don't have that right now, so this could be a problem for some users.

});
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

const contours = await findContours({
Expand Down
8 changes: 6 additions & 2 deletions src/hooks/useWasmWorker.js
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,12 @@ export function useWasmWorker() {
sigma_spatial = 3.0,
sigma_range = 50.0,
color_space = 0,
n_threads = 8,
}) => {
return (
await call('bilateral_filter', { pixels, width, height, sigma_spatial, sigma_range, color_space }, ['pixels'])
await call('bilateral_filter', { pixels, width, height, sigma_spatial, sigma_range, color_space, n_threads }, [
'pixels',
])
).output.pixels;
};
const blackThreshold = async ({ pixels, width, height, num_colors }) => {
Expand All @@ -58,9 +61,10 @@ export function useWasmWorker() {
out_labels = new Int32Array(width * height),
num_colors,
max_iter = 100,
n_threads = 8,
}) => {
const result = (
await call('kmeans', { pixels, out_pixels, out_labels, width, height, num_colors, max_iter }, [
await call('kmeans', { pixels, out_pixels, out_labels, width, height, num_colors, max_iter, n_threads }, [
'pixels',
'out_pixels',
'out_labels',
Expand Down
8 changes: 5 additions & 3 deletions src/wasm/modules/image/CMakeLists.txt
Comment thread
Krasner marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
Expand Up @@ -42,20 +42,22 @@ set(COMMON_FLAGS
"SHELL:-s MAXIMUM_MEMORY=2048MB"
"SHELL:-s ALLOW_MEMORY_GROWTH=1"
"SHELL:-s EXPORT_NAME=create${CAP_MODULE_NAME}Module"
"SHELL:-s USE_PTHREADS=1"
"SHELL:-s PTHREAD_POOL_SIZE=8"
)

# Apply common flags
target_link_options(${MODULE_NAME}_wasm PRIVATE ${COMMON_FLAGS})
target_link_options(${MODULE_NAME}_wasm PRIVATE ${COMMON_FLAGS} -pthread)

# Build-type specific flags
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
target_compile_options(${MODULE_NAME}_wasm PRIVATE -O0 -g4)
target_compile_options(${MODULE_NAME}_wasm PRIVATE -O0 -g4 -pthread)
target_link_options(${MODULE_NAME}_wasm PRIVATE
"SHELL:-s ASSERTIONS=2"
-g4
)
else()
target_compile_options(${MODULE_NAME}_wasm PRIVATE -O3)
target_compile_options(${MODULE_NAME}_wasm PRIVATE -O3 -pthread)
target_link_options(${MODULE_NAME}_wasm PRIVATE
"SHELL:-s SINGLE_FILE=0"
)
Expand Down
2 changes: 1 addition & 1 deletion src/wasm/modules/image/include/bilateral_filter.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ namespace bilateral {
// (radiometric decay)
void bilateral_filter(uint8_t *image, size_t width, size_t height,
double sigma_spatial, double sigma_range,
uint8_t color_space);
uint8_t color_space, int n_threads = 8);

} // namespace bilateral

Expand Down
2 changes: 1 addition & 1 deletion src/wasm/modules/image/include/kmeans.h
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
EXPORTED void kmeans(const uint8_t *data, uint8_t *out_data,
int32_t *out_labels, const int32_t width,
const int32_t height, const int32_t k,
const int32_t max_iter);
const int32_t max_iter, const int n_threads = 8);
Comment thread
Krasner marked this conversation as resolved.
Resolved

EXPORTED void kmeans_clustering_spatial(uint8_t *data, int32_t width,
int32_t height, int32_t k,
Expand Down
179 changes: 111 additions & 68 deletions src/wasm/modules/image/src/bilateral_filter.cpp

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the mutex is slowing down. We don't need to stop the threads from modifying the same data point (pixel) because they never will since each thread gets a piece of the the image to work on.

Additionally, the thread count should be clamped and should be an unsigned tiny type, like a uint8_t, since we don't want billions:

$$ \text{If} \quad n_\text{threads} > \text{height}, \quad \quad \left\lfloor \frac{\text{height}}{n_\text{threads}} \right\rfloor = 0 $$

n_threads = std::min(n_threads, height);

Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@
#include <climits>
#include <cmath>
#include <cstring>
#include <functional>
#include <mutex>
#include <thread>
#include <vector>

namespace bilateral {
Expand All @@ -23,6 +26,8 @@ inline double gaussian(double x, double sigma) {
return std::exp(-(x * x) / (2.0 * sigma * sigma));
}

std::mutex write_mutex;

/*
The Bilateral Filter applies a composite weight based on both spatial distance
and radiometric difference (intensity) to return an image that is smoothed while
Expand All @@ -40,74 +45,18 @@ decay)
├── 0: CIELAB
└── 1: RGB
*/
void bilateral_filter(uint8_t *image, size_t width, size_t height,
double sigma_spatial, double sigma_range,
uint8_t color_space) {
// bad data -> return
if (sigma_spatial <= 0.0 || sigma_range <= 0.0 || width <= 0 || height <= 0)
return;
if (color_space != COLOR_SPACE_OPTION_CIELAB &&
color_space != COLOR_SPACE_OPTION_RGB)
return;

const int raw_radius{
static_cast<int>(std::ceil(SIGMA_RADIUS_FACTOR * sigma_spatial))};
const int radius{std::min(raw_radius, MAX_KERNEL_RADIUS)};
const int kernel_diameter{2 * radius + 1};

std::vector<uint8_t> result(width * height * 4);

std::vector<double> spatial_weights(kernel_diameter * kernel_diameter);

// Precompute Spatial Weights (Gaussian Kernel)
for (int ky{-radius}; ky <= radius; ++ky) {
for (int kx{-radius}; kx <= radius; ++kx) {
const double dist{static_cast<double>(std::sqrt(kx * kx + ky * ky))};
spatial_weights[(ky + radius) * kernel_diameter + (kx + radius)] =
gaussian(dist, sigma_spatial);
}
}

// ========= RGB-only section start =========
// Precompute Range Weights
std::vector<double> range_lut;
if (color_space == COLOR_SPACE_OPTION_RGB) {
range_lut.resize(MAX_RGB_DIST_SQ + 1);
for (int i{0}; i <= MAX_RGB_DIST_SQ; ++i) {
range_lut[i] = gaussian(static_cast<double>(std::sqrt(i)), sigma_range);
}
}
// ========= RGB-only section end =========

// ========= CIELAB section start =========
// Compute full image RGB - CIELAB conversion
std::vector<double> cie_image;
if (color_space == COLOR_SPACE_OPTION_CIELAB) {
cie_image.resize(width * height * 4);

for (int y{0}; y < height; y++) {
for (int x{0}; x < width; x++) {
int center_idx{(y * static_cast<int>(width) + x) * 4};
uint8_t r0{image[center_idx]};
uint8_t g0{image[center_idx + 1]};
uint8_t b0{image[center_idx + 2]};
uint8_t a0{image[center_idx + 3]};
double L0, A0, B0;
rgb_to_lab(r0, g0, b0, L0, A0, B0);

cie_image[center_idx] = L0;
cie_image[center_idx + 1] = A0;
cie_image[center_idx + 2] = B0;
cie_image[center_idx + 3] =
0.0; // unused but keep for indexing purposes
}
}
}
// ========= CIELAB section end =========

void _process(const uint8_t *image, const std::vector<double> &cie_image,
std::vector<uint8_t> &result,
const std::vector<double> &spatial_weights,
const std::vector<double> &range_lut, int radius,
double sigma_range, int start_row, int end_row, size_t height,
size_t width, uint8_t color_space, int n_threads) {
int h{static_cast<int>(height)};
int w{static_cast<int>(width)};
for (int y{0}; y < h; ++y) {
const int kernel_diameter{2 * radius + 1};

for (int y{start_row}; y < end_row; ++y) {
for (int x{0}; x < w; ++x) {
size_t center_idx{(y * width + x) * 4};

Expand Down Expand Up @@ -184,6 +133,8 @@ void bilateral_filter(uint8_t *image, size_t width, size_t height,
weight_acc += w_space * w_range;
}
}
// writing - must grab mutex
std::unique_lock<std::mutex> lock(write_mutex);
Comment thread
Krasner marked this conversation as resolved.
Resolved

switch (color_space) {
case COLOR_SPACE_OPTION_RGB: {
Expand All @@ -209,7 +160,99 @@ void bilateral_filter(uint8_t *image, size_t width, size_t height,
break;
}
}
// done writing
lock.unlock();
}
}
}

void bilateral_filter(uint8_t *image, size_t width, size_t height,
double sigma_spatial, double sigma_range,
uint8_t color_space, int n_threads) {
// bad data -> return
if (sigma_spatial <= 0.0 || sigma_range <= 0.0 || width <= 0 || height <= 0)
return;
if (color_space != COLOR_SPACE_OPTION_CIELAB &&
color_space != COLOR_SPACE_OPTION_RGB)
return;

std::vector<std::thread> threads;
Comment thread
Krasner marked this conversation as resolved.
Resolved

int rows_per_thread = static_cast<int>(height) / n_threads;

const int raw_radius{
static_cast<int>(std::ceil(SIGMA_RADIUS_FACTOR * sigma_spatial))};
const int radius{std::min(raw_radius, MAX_KERNEL_RADIUS)};
const int kernel_diameter{2 * radius + 1};

std::vector<uint8_t> result(width * height * 4);

std::vector<double> spatial_weights(kernel_diameter * kernel_diameter);
Comment thread
Krasner marked this conversation as resolved.
Resolved

// Precompute Spatial Weights (Gaussian Kernel)
for (int ky{-radius}; ky <= radius; ++ky) {
for (int kx{-radius}; kx <= radius; ++kx) {
const double dist{static_cast<double>(std::sqrt(kx * kx + ky * ky))};
spatial_weights[(ky + radius) * kernel_diameter + (kx + radius)] =
gaussian(dist, sigma_spatial);
}
}

// ========= RGB-only section start =========
// Precompute Range Weights
std::vector<double> range_lut;
if (color_space == COLOR_SPACE_OPTION_RGB) {
range_lut.resize(MAX_RGB_DIST_SQ + 1);
for (int i{0}; i <= MAX_RGB_DIST_SQ; ++i) {
range_lut[i] = gaussian(static_cast<double>(std::sqrt(i)), sigma_range);
}
}
// ========= RGB-only section end =========

// ========= CIELAB section start =========
// Compute full image RGB - CIELAB conversion
std::vector<double> cie_image;
if (color_space == COLOR_SPACE_OPTION_CIELAB) {
cie_image.resize(width * height * 4);

for (int y{0}; y < height; y++) {
for (int x{0}; x < width; x++) {
int center_idx{(y * static_cast<int>(width) + x) * 4};
uint8_t r0{image[center_idx]};
uint8_t g0{image[center_idx + 1]};
uint8_t b0{image[center_idx + 2]};
uint8_t a0{image[center_idx + 3]};
double L0, A0, B0;
rgb_to_lab(r0, g0, b0, L0, A0, B0);

cie_image[center_idx] = L0;
cie_image[center_idx + 1] = A0;
cie_image[center_idx + 2] = B0;
cie_image[center_idx + 3] =
0.0; // unused but keep for indexing purposes
}
}
}
// ========= CIELAB section end =========
if (n_threads > 1) {
for (unsigned int i = 0; i < n_threads; ++i) {
int start_row = i * rows_per_thread;
int end_row = (i == n_threads - 1) ? height : (i + 1) * rows_per_thread;
Comment thread
Krasner marked this conversation as resolved.
Resolved
// Launch a thread and add to vector
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
threads.emplace_back(_process, std::cref(image), std::cref(cie_image),
std::ref(result), std::cref(spatial_weights),
std::cref(range_lut), radius, sigma_range, start_row,
end_row, height, width, color_space, n_threads);
}

// wait for threads to finish
for (auto &thread : threads) {
thread.join();
}
Comment thread
Krasner marked this conversation as resolved.
Resolved
} else {
_process(image, cie_image, result, spatial_weights, range_lut, radius,
sigma_range, 0, static_cast<int>(height), height, width,
color_space, n_threads);
}

std::memcpy(image, result.data(), result.size());
Expand All @@ -220,7 +263,7 @@ void bilateral_filter(uint8_t *image, size_t width, size_t height,
// Global wrapper for WASM export
EXPORTED void bilateral_filter(uint8_t *image, size_t width, size_t height,
double sigma_spatial, double sigma_range,
uint8_t color_space) {
uint8_t color_space, int n_threads) {
bilateral::bilateral_filter(image, width, height, sigma_spatial, sigma_range,
color_space);
}
color_space, n_threads);
}
Loading
Loading