From e7d4d9b0baeb9a9d5efe8fdd35db99fd452a0b9b Mon Sep 17 00:00:00 2001 From: DerrickUnleashed Date: Wed, 27 Aug 2025 14:29:51 +0530 Subject: [PATCH 1/8] Adding vignettes for visualisation utils #113 --- vignettes/examples/oxfordiiitpet-fcnresnet.R | 89 +++++++++++++++++++ .../examples/oxfordiiitpet-fcnresnet.Rmd | 9 ++ 2 files changed, 98 insertions(+) create mode 100644 vignettes/examples/oxfordiiitpet-fcnresnet.R create mode 100644 vignettes/examples/oxfordiiitpet-fcnresnet.Rmd diff --git a/vignettes/examples/oxfordiiitpet-fcnresnet.R b/vignettes/examples/oxfordiiitpet-fcnresnet.R new file mode 100644 index 00000000..ca8e27b7 --- /dev/null +++ b/vignettes/examples/oxfordiiitpet-fcnresnet.R @@ -0,0 +1,89 @@ +# Loading Images --------------------------------------------------- + + +read_to_tensor <- function(url) { + tmp <- download_and_cache(url) + arr <- jpeg::readJPEG(tmp) + arr <- as.array(arr) + torch_tensor(aperm(arr, c(3,1,2)), dtype = torch_float()) +} + +url1 <- "https://raw.githubusercontent.com/pytorch/vision/main/gallery/assets/dog1.jpg" +url2 <- "https://raw.githubusercontent.com/pytorch/vision/main/gallery/assets/dog2.jpg" + +dog1 <- read_to_tensor(url1) +dog2 <- read_to_tensor(url2) + + +# Visualizing a grid of images ------------------------------------- + + +dogs <- torch_stack(list(dog1, dog2)) +grid <- vision_make_grid(dogs, scale = TRUE, num_rows = 2) +grid_arr <- as.array(grid$permute(c(2,3,1))) +plot(c(0, dim(grid_arr)[2]), c(0, dim(grid_arr)[1]), type = "n", ann = FALSE, axes = FALSE, asp = 1) +rasterImage(grid_arr, 0, 0, w, h) + + +# Preprocessing the data ------------------------------------- + + +norm_mean <- c(0.485, 0.456, 0.406) +norm_std <- c(0.229, 0.224, 0.225) + +preprocess <- function(img) { + resized <- nnf_interpolate( + img$unsqueeze(1), size = c(520, 520), mode = "bilinear", align_corners = FALSE + )$squeeze(1) + normed <- (resized - torch_tensor(norm_mean)$unsqueeze(2)$unsqueeze(3)) / + torch_tensor(norm_std)$unsqueeze(2)$unsqueeze(3) + list(resized = resized, normed = normed) +} + +dog1_prep <- preprocess(dog1) +dog2_prep <- preprocess(dog2) + +# make batch (2,3,520,520) +input <- torch_stack(list(dog1_prep$normed, dog2_prep$normed)) + + +# Loading Model ------------------------------------- + + +model <- model_fcn_resnet50(pretrained = TRUE) +model$eval() + +# run model +output <- model(input) + + +# Processing the Output ------------------------------ + + +# argmax over classes -> (1,H,W) +mask_id <- output$out$argmax(dim = 2) + +make_masks <- function(mask_id_img, num_classes = 21) { + torch_stack(lapply(0:(num_classes-1), function(cls) mask_id_img$eq(cls)), dim = 1) +} +mask_bool1 <- make_masks(mask_id[1,..]) +mask_bool2 <- make_masks(mask_id[2,..]) + + +# Visualizing the Output ------------------------------ + + +segmented1 <- draw_segmentation_masks( + (dog1_prep$resized * 255)$to(torch_uint8()), + masks = mask_bool1, + alpha = 0.6 +) + +segmented2 <- draw_segmentation_masks( + (dog2_prep$resized * 255)$to(torch_uint8()), + masks = mask_bool2, + alpha = 0.6 +) + +tensor_image_browse(segmented1) +tensor_image_browse(segmented2) \ No newline at end of file diff --git a/vignettes/examples/oxfordiiitpet-fcnresnet.Rmd b/vignettes/examples/oxfordiiitpet-fcnresnet.Rmd new file mode 100644 index 00000000..118812f4 --- /dev/null +++ b/vignettes/examples/oxfordiiitpet-fcnresnet.Rmd @@ -0,0 +1,9 @@ +--- +title: oxfordiiitpet-fcnresnet +type: docs +--- + +```{r, echo = FALSE} +knitr::opts_chunk$set(eval = FALSE) +knitr::spin_child(paste0(rmarkdown::metadata$title, ".R")) +``` \ No newline at end of file From 06c7d5b740b31a02bf90506068c9cd12db5c4483 Mon Sep 17 00:00:00 2001 From: DerrickUnleashed Date: Wed, 27 Aug 2025 14:30:05 +0530 Subject: [PATCH 2/8] Adding NEWS.md #113 --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index a3dc9801..17c1cd56 100644 --- a/NEWS.md +++ b/NEWS.md @@ -14,6 +14,8 @@ ## New features +* Added vignette showcasing `model_fcn_resnet50()` with visualization utilities `draw_segmentation_masks()` and `make_vision_grid()` (@DerrickUnleashed, #244). + ## Bug fixes and improvements * Switch pre 0.5.0 models to their `/v2/` URL in torch-cdn.mlverse.org. (#215) From 9aba253ba55506300018241ad9f6681c7f8eaa53 Mon Sep 17 00:00:00 2001 From: DerrickUnleashed Date: Wed, 27 Aug 2025 14:31:23 +0530 Subject: [PATCH 3/8] Naming the vignettes appropiately --- vignettes/examples/{oxfordiiitpet-fcnresnet.R => fcnresnet.R} | 0 vignettes/examples/{oxfordiiitpet-fcnresnet.Rmd => fcnresnet.Rmd} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename vignettes/examples/{oxfordiiitpet-fcnresnet.R => fcnresnet.R} (100%) rename vignettes/examples/{oxfordiiitpet-fcnresnet.Rmd => fcnresnet.Rmd} (100%) diff --git a/vignettes/examples/oxfordiiitpet-fcnresnet.R b/vignettes/examples/fcnresnet.R similarity index 100% rename from vignettes/examples/oxfordiiitpet-fcnresnet.R rename to vignettes/examples/fcnresnet.R diff --git a/vignettes/examples/oxfordiiitpet-fcnresnet.Rmd b/vignettes/examples/fcnresnet.Rmd similarity index 100% rename from vignettes/examples/oxfordiiitpet-fcnresnet.Rmd rename to vignettes/examples/fcnresnet.Rmd From e35235864e5cbe1038c9988ad70bb9b78fd1fa3c Mon Sep 17 00:00:00 2001 From: DerrickUnleashed Date: Wed, 27 Aug 2025 14:32:32 +0530 Subject: [PATCH 4/8] Updating pkgdown.yml #113 --- _pkgdown.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/_pkgdown.yml b/_pkgdown.yml index 550a46e6..607aeca9 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -27,6 +27,8 @@ navbar: href: articles/examples/style-transfer.html - text: texture-nca href: articles/examples/texture-nca.html + - text: fcnresnet + href: articles/examples/fcnresnet.html reference: - title: Transforms From 8cf4ba9ead824aaf7c7ab9c887a35387a8263301 Mon Sep 17 00:00:00 2001 From: "C. Regouby" Date: Wed, 27 Aug 2025 20:11:02 +0200 Subject: [PATCH 5/8] few fix (WiP) --- vignettes/examples/fcnresnet.R | 17 ++++++++--------- vignettes/examples/fcnresnet.Rmd | 6 +++--- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/vignettes/examples/fcnresnet.R b/vignettes/examples/fcnresnet.R index ca8e27b7..ead73ee6 100644 --- a/vignettes/examples/fcnresnet.R +++ b/vignettes/examples/fcnresnet.R @@ -1,11 +1,10 @@ # Loading Images --------------------------------------------------- - +library(torchvision) +library(torch) read_to_tensor <- function(url) { - tmp <- download_and_cache(url) - arr <- jpeg::readJPEG(tmp) - arr <- as.array(arr) - torch_tensor(aperm(arr, c(3,1,2)), dtype = torch_float()) + arr <- magick_loader(url) + torch_tensor(arr, dtype = torch_float())$permute(c(3,1,2)) } url1 <- "https://raw.githubusercontent.com/pytorch/vision/main/gallery/assets/dog1.jpg" @@ -20,7 +19,7 @@ dog2 <- read_to_tensor(url2) dogs <- torch_stack(list(dog1, dog2)) grid <- vision_make_grid(dogs, scale = TRUE, num_rows = 2) -grid_arr <- as.array(grid$permute(c(2,3,1))) +grid_arr <- as.array(grid$permute(c(2,3,1))) plot(c(0, dim(grid_arr)[2]), c(0, dim(grid_arr)[1]), type = "n", ann = FALSE, axes = FALSE, asp = 1) rasterImage(grid_arr, 0, 0, w, h) @@ -74,16 +73,16 @@ mask_bool2 <- make_masks(mask_id[2,..]) segmented1 <- draw_segmentation_masks( - (dog1_prep$resized * 255)$to(torch_uint8()), + (dog1_prep$resized * 255)$to(torch_uint8()), masks = mask_bool1, alpha = 0.6 ) segmented2 <- draw_segmentation_masks( - (dog2_prep$resized * 255)$to(torch_uint8()), + (dog2_prep$resized * 255)$to(torch_uint8()), masks = mask_bool2, alpha = 0.6 ) tensor_image_browse(segmented1) -tensor_image_browse(segmented2) \ No newline at end of file +tensor_image_browse(segmented2) diff --git a/vignettes/examples/fcnresnet.Rmd b/vignettes/examples/fcnresnet.Rmd index 118812f4..237fde8d 100644 --- a/vignettes/examples/fcnresnet.Rmd +++ b/vignettes/examples/fcnresnet.Rmd @@ -1,9 +1,9 @@ --- -title: oxfordiiitpet-fcnresnet +title: "fcnresnet" type: docs --- ```{r, echo = FALSE} -knitr::opts_chunk$set(eval = FALSE) +knitr::opts_chunk$set(eval = TRUE) knitr::spin_child(paste0(rmarkdown::metadata$title, ".R")) -``` \ No newline at end of file +``` From e84b67cf1c8c99956acfe3c3489ebb02b7c3ffb5 Mon Sep 17 00:00:00 2001 From: "C. Regouby" Date: Sun, 1 Feb 2026 15:33:56 +0100 Subject: [PATCH 6/8] simplify article to use most of the integrated functions --- NEWS.md | 2 +- tests/testthat/test-dataset-coco.R | 14 +++---- vignettes/examples/fcnresnet.R | 59 ++++++++++-------------------- 3 files changed, 28 insertions(+), 47 deletions(-) diff --git a/NEWS.md b/NEWS.md index d8ff6de7..4657e0b2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,7 +2,7 @@ ## New features -* Added vignette showcasing `model_fcn_resnet50()` with visualization utilities `draw_segmentation_masks()` and `make_vision_grid()` (@DerrickUnleashed, #244). +* Added segmentation article on `model_fcn_resnet50()` with two images (@DerrickUnleashed, #244). * Added collection dataset catalog with `search_collection()`, `get_collection_catalog()`, and `list_collection_datasets()` functions for discovering and exploring collections (#271, @ANAMASGARD). * Added `target_transform_coco_masks()` and `target_transform_trimap_masks()` transformation functions for explicit segmentation mask generation (@ANAMASGARD). diff --git a/tests/testthat/test-dataset-coco.R b/tests/testthat/test-dataset-coco.R index 9257a27b..cbb20b58 100644 --- a/tests/testthat/test-dataset-coco.R +++ b/tests/testthat/test-dataset-coco.R @@ -16,7 +16,7 @@ test_that("coco_detection_dataset handles missing files gracefully", { }) test_that("coco_detection_dataset loads a single example correctly", { - skip_if(Sys.getenv("TEST_LARGE_DATASETS", unset = 0) < 1, + skip_if(Sys.getenv("TEST_LARGE_DATASETS", unset = 0) != 1, "Skipping test: set TEST_LARGE_DATASETS=1 to enable tests requiring large downloads.") ds <- coco_detection_dataset(root = tmp, train = FALSE, year = "2017", download = TRUE) @@ -47,7 +47,7 @@ test_that("coco_detection_dataset loads a single example correctly", { }) test_that("coco_ dataset loads a single segmentation example correctly", { - skip_if(Sys.getenv("TEST_LARGE_DATASETS", unset = 0) < 1, + skip_if(Sys.getenv("TEST_LARGE_DATASETS", unset = 0) != 1, "Skipping test: set TEST_LARGE_DATASETS=1 to enable tests requiring large downloads.") ds <- coco_detection_dataset(root = tmp, train = FALSE, year = "2017", download = TRUE, @@ -71,7 +71,7 @@ test_that("coco_ dataset loads a single segmentation example correctly", { }) test_that("coco_detection_dataset batches correctly using dataloader", { - skip_if(Sys.getenv("TEST_LARGE_DATASETS", unset = 0) < 1, + skip_if(Sys.getenv("TEST_LARGE_DATASETS", unset = 0) != 1, "Skipping test: set TEST_LARGE_DATASETS=1 to enable tests requiring large downloads.") @@ -99,8 +99,8 @@ test_that("coco_caption_dataset handles missing files gracefully", { }) test_that("coco_caption_dataset loads a single example correctly", { - skip_if(Sys.getenv("TEST_LARGE_DATASETS", unset = 0) < 2, - "Skipping test: set TEST_LARGE_DATASETS=2 to enable tests requiring huge downloads.") + skip_if(Sys.getenv("TEST_HUGE_DATASETS", unset = 0) != 1, + "Skipping test: set TEST_HUGE_DATASETS=1 to enable tests requiring huge downloads.") ds <- coco_caption_dataset(root = tmp, train = FALSE, download = TRUE) @@ -120,8 +120,8 @@ test_that("coco_caption_dataset loads a single example correctly", { }) test_that("coco_caption_dataset batches correctly using dataloader", { - skip_if(Sys.getenv("TEST_LARGE_DATASETS", unset = 0) < 2, - "Skipping test: set TEST_LARGE_DATASETS=2 to enable tests requiring huge downloads.") + skip_if(Sys.getenv("TEST_HUGE_DATASETS", unset = 0) != 1, + "Skipping test: set TEST_HUGE_DATASETS=1 to enable tests requiring huge downloads.") ds <- coco_caption_dataset(root = tmp, train = FALSE, download = TRUE) diff --git a/vignettes/examples/fcnresnet.R b/vignettes/examples/fcnresnet.R index ead73ee6..ae456f74 100644 --- a/vignettes/examples/fcnresnet.R +++ b/vignettes/examples/fcnresnet.R @@ -2,16 +2,11 @@ library(torchvision) library(torch) -read_to_tensor <- function(url) { - arr <- magick_loader(url) - torch_tensor(arr, dtype = torch_float())$permute(c(3,1,2)) -} - url1 <- "https://raw.githubusercontent.com/pytorch/vision/main/gallery/assets/dog1.jpg" url2 <- "https://raw.githubusercontent.com/pytorch/vision/main/gallery/assets/dog2.jpg" -dog1 <- read_to_tensor(url1) -dog2 <- read_to_tensor(url2) +dog1 <- magick_loader(url1) |> transform_to_tensor() +dog2 <- magick_loader(url2) |> transform_to_tensor() # Visualizing a grid of images ------------------------------------- @@ -19,9 +14,7 @@ dog2 <- read_to_tensor(url2) dogs <- torch_stack(list(dog1, dog2)) grid <- vision_make_grid(dogs, scale = TRUE, num_rows = 2) -grid_arr <- as.array(grid$permute(c(2,3,1))) -plot(c(0, dim(grid_arr)[2]), c(0, dim(grid_arr)[1]), type = "n", ann = FALSE, axes = FALSE, asp = 1) -rasterImage(grid_arr, 0, 0, w, h) +tensor_image_browse(grid) # Preprocessing the data ------------------------------------- @@ -30,20 +23,15 @@ rasterImage(grid_arr, 0, 0, w, h) norm_mean <- c(0.485, 0.456, 0.406) norm_std <- c(0.229, 0.224, 0.225) -preprocess <- function(img) { - resized <- nnf_interpolate( - img$unsqueeze(1), size = c(520, 520), mode = "bilinear", align_corners = FALSE - )$squeeze(1) - normed <- (resized - torch_tensor(norm_mean)$unsqueeze(2)$unsqueeze(3)) / - torch_tensor(norm_std)$unsqueeze(2)$unsqueeze(3) - list(resized = resized, normed = normed) -} - -dog1_prep <- preprocess(dog1) -dog2_prep <- preprocess(dog2) +dog1_prep <- dog1 |> + transform_resize(c(520,520)) |> + transform_normalize(mean = norm_mean, std = norm_std) +dog2_prep <- dog2 |> + transform_resize(c(520,520)) |> + transform_normalize(mean = norm_mean, std = norm_std) # make batch (2,3,520,520) -input <- torch_stack(list(dog1_prep$normed, dog2_prep$normed)) +dog_batch <- torch_stack(list(dog1_prep, dog2_prep)) # Loading Model ------------------------------------- @@ -53,35 +41,28 @@ model <- model_fcn_resnet50(pretrained = TRUE) model$eval() # run model -output <- model(input) +output <- model(dog_batch) # Processing the Output ------------------------------ - -# argmax over classes -> (1,H,W) -mask_id <- output$out$argmax(dim = 2) - -make_masks <- function(mask_id_img, num_classes = 21) { - torch_stack(lapply(0:(num_classes-1), function(cls) mask_id_img$eq(cls)), dim = 1) -} -mask_bool1 <- make_masks(mask_id[1,..]) -mask_bool2 <- make_masks(mask_id[2,..]) - +mask <- output$out +mask$shape +mask$dtype # Visualizing the Output ------------------------------ segmented1 <- draw_segmentation_masks( - (dog1_prep$resized * 255)$to(torch_uint8()), - masks = mask_bool1, - alpha = 0.6 + dog1 |> transform_resize(c(520,520)), + masks = mask[1,, ], + alpha = 0.5 ) segmented2 <- draw_segmentation_masks( - (dog2_prep$resized * 255)$to(torch_uint8()), - masks = mask_bool2, - alpha = 0.6 + dog2 |> transform_resize(c(520,520)), + masks = mask[2,, ], + alpha = 0.5 ) tensor_image_browse(segmented1) From 894b1a10b31d180af515d4a1f9deac65b1e62766 Mon Sep 17 00:00:00 2001 From: "C. Regouby" Date: Sun, 1 Feb 2026 15:51:48 +0100 Subject: [PATCH 7/8] fix P.R. number --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 4657e0b2..5177adc4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,7 +2,7 @@ ## New features -* Added segmentation article on `model_fcn_resnet50()` with two images (@DerrickUnleashed, #244). +* Added segmentation article on `model_fcn_resnet50()` with two images (@DerrickUnleashed, #281). * Added collection dataset catalog with `search_collection()`, `get_collection_catalog()`, and `list_collection_datasets()` functions for discovering and exploring collections (#271, @ANAMASGARD). * Added `target_transform_coco_masks()` and `target_transform_trimap_masks()` transformation functions for explicit segmentation mask generation (@ANAMASGARD). From dca0919c0092a99f1aaa37cd477a33cb7b930e53 Mon Sep 17 00:00:00 2001 From: "C. Regouby" Date: Sun, 1 Feb 2026 15:55:58 +0100 Subject: [PATCH 8/8] make mask shape alignment more robust --- R/vision_utils.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/vision_utils.R b/R/vision_utils.R index 801ab697..b4515cd4 100644 --- a/R/vision_utils.R +++ b/R/vision_utils.R @@ -346,7 +346,7 @@ draw_segmentation_masks.torch_tensor <- function(x, if (masks$dtype != torch::torch_bool() && masks$dtype != torch::torch_float() ) { type_error("`masks` is expected to be of dtype torch_bool() or torch_float()") } - if (any(masks$shape[2:3] != img_to_draw$shape[2:3])) { + if (any(masks$shape[-2:-1] != img_to_draw$shape[-2:-1])) { value_error("`masks` and `image` must have the same height and width") } # if mask is a model inference output, we need to convert float mask to boolean mask