diff --git a/large_image/tilesource/jupyter.py b/large_image/tilesource/jupyter.py index 889611ef9..5b552e203 100644 --- a/large_image/tilesource/jupyter.py +++ b/large_image/tilesource/jupyter.py @@ -20,21 +20,41 @@ import importlib.util import json import os +import re import threading +import time import weakref from typing import Any, cast from urllib.parse import parse_qs, quote, urlencode, urlparse, urlunparse import numpy as np +import yaml import large_image from large_image.exceptions import TileSourceError, TileSourceXYZRangeError from large_image.tilesource.utilities import JSONDict +from large_image.widgets.components import FrameSelector ipyleafletPresent = importlib.util.find_spec('ipyleaflet') is not None ipyvuePresent = importlib.util.find_spec('ipyvue') is not None aiohttpPresent = importlib.util.find_spec('aiohttp') is not None +skimage_transform = None + + +def _lazyImportSkimageTransform(): + """ + Import the skimage.transform module. This is only needed when `editWarp=True` is used. + """ + global skimage_transform + + if skimage_transform is None: + try: + import skimage.transform as skimage_transform + except ImportError: + msg = 'scikit-image transform module not found.' + raise TileSourceError(msg) + class IPyLeafletMixin: """Mixin class to support interactive visualization in JupyterLab. @@ -92,7 +112,11 @@ class IPyLeafletMixin: def __init__(self, *args, **kwargs) -> None: self._jupyter_server_manager = None - self._map = Map(ts=self) + self._map = Map( + ts=self, + editWarp=kwargs.get('editWarp', False), + reference=kwargs.get('reference'), + ) if ipyleafletPresent: self.to_map = self._map.to_map self.from_map = self._map.from_map @@ -158,6 +182,10 @@ def iplmap(self) -> Any: """ return self._map.map + @property + def warp_points(self): + return self._map.warp_points + class Map: """ @@ -168,7 +196,9 @@ def __init__( self, *, ts: IPyLeafletMixin | None = None, metadata: dict | None = None, url: str | None = None, gc: Any | None = None, id: str | None = None, - resource: str | None = None) -> None: + resource: str | None = None, + editWarp: bool = False, reference: IPyLeafletMixin | None = None, + ) -> None: """ Specify the large image to be used with the IPyLeaflet Map. One of (a) a tile source, (b) metadata dictionary and tile url, (c) girder client @@ -186,8 +216,17 @@ def __init__( on the girder client. """ self._layer = self._map = self._metadata = self._frame_slider = None + self.frame_selector: FrameSelector | None = None self._frame_histograms: dict[int, Any] | None = None self._ts = ts + self._edit_warp = editWarp + self._reference = reference + self._reference_layer = None + if self._edit_warp: + self.warp_points: dict[str, list[list[int] | None]] = dict(src=[], dst=[]) + self._warp_widgets: dict = {} + self._warp_markers: dict = {'src': [], 'dst': []} + self._dragging_marker_id: str | None = None if (not url or not metadata) and gc and (id or resource): fileId = None if id is None: @@ -281,8 +320,8 @@ def make_map( Create an ipyleaflet map given large_image metadata, an optional ipyleaflet layer, and the center of the tile source. """ - from ipyleaflet import Map, basemaps, projections - from ipywidgets import VBox + from ipyleaflet import FullScreenControl, Map, basemaps, projections + from ipywidgets import FloatSlider, VBox try: default_zoom = metadata['levels'] - metadata['sourceLevels'] @@ -324,8 +363,6 @@ def make_map( children: list[Any] = [] frames = metadata.get('frames') if frames is not None and ipyvuePresent and aiohttpPresent: - from large_image.widgets.components import FrameSelector - self.frame_selector = FrameSelector() self.frame_selector.imageMetadata = metadata self.frame_selector.updateFrameCallback = self.update_frame @@ -347,14 +384,273 @@ def make_map( m.fit_bounds(bounds=[[0, 0], [metadata['sizeY'], metadata['sizeX']]]) if self._geospatial: m.add_layer(layer) + + if self._reference is not None: + default_opacity = 0.5 + self._reference_layer = self._reference.as_leaflet_layer() + if self._reference_layer is not None: + self._reference_layer.opacity = default_opacity + m.add_layer(self._reference_layer) + + def update_reference_opacity(event): + if self._reference_layer is not None: + self._reference_layer.opacity = event.get('new', default_opacity) + + reference_slider = FloatSlider( + description='Reference Opacity', + value=default_opacity, step=0.1, + min=0, max=1, + readout_format='.1f', + style={'description_width': 'initial'}, + ) + reference_slider.observe(update_reference_opacity, names=['value']) + children.append(reference_slider) + self._map = m children.append(m) - self.add_region_indicator() + if self._edit_warp: + children.append(self.add_warp_editor()) + else: + # Only add region indicator if not using warp editor so that + # the map doesn't have conflicting on_interaction callbacks + self.add_region_indicator() + self._map.add(FullScreenControl()) return VBox(children) + def warp_editor_validate_source(self): + if self._ts is None: + msg = 'Warp editor mode not allowed; source is not defined.' + raise TileSourceError(msg) + + if not os.path.exists(str(self._ts.largeImagePath)): # type: ignore[attr-defined] + msg = 'Warp editor mode not allowed; source file does not exist.' + raise TileSourceError(msg) + + def convert_coordinate_map_to_warp(self, map_coord): + y, x = map_coord + if self._ts is not None: + y = self._ts.sizeY - y # type: ignore[attr-defined] + return [int(x), int(y)] + + def convert_coordinate_warp_to_map(self, warp_coord): + x, y = warp_coord + if self._ts is not None: + y = self._ts.sizeY - y # type: ignore[attr-defined] + return [int(y), int(x)] + + def toggle_warp_transform(self, event): + self.update_warp(event.get('new')) + + def inverse_warp(self, coord): + _lazyImportSkimageTransform() + + warp_src = np.array(self.warp_points['src']) + warp_dst = np.array(self.warp_points['dst']) + n_points = warp_src.shape[0] + inverse_coord = None + if n_points == 0: + return coord + if n_points == 1: + if warp_src[0] is not None and warp_dst[0] is not None: + inverse_coord = [ + v + warp_src[0][i] - warp_dst[0][i] + for i, v in enumerate(coord) + ] + elif skimage_transform is not None: + srcsvd = np.linalg.svd(warp_src - warp_src.mean(axis=0), compute_uv=False) + dstsvd = np.linalg.svd(warp_dst - warp_dst.mean(axis=0), compute_uv=False) + useSimilarity = n_points < 3 or min( + srcsvd[1] / (srcsvd[0] or 1), dstsvd[1] / (dstsvd[0] or 1)) < 1e-3 + + if useSimilarity: + transformer = skimage_transform.SimilarityTransform() + elif n_points <= 3: + transformer = skimage_transform.AffineTransform() + else: + transformer = skimage_transform.ThinPlateSplineTransform() + transformer.estimate(warp_dst, warp_src) + inverse_coord = transformer([coord])[0] + if inverse_coord is not None: + return [int(v) for v in inverse_coord] + + def get_warp_schema(self): + if self._ts is None: + return None + schema = dict(sources=[ + dict( + path=str(self._ts.largeImagePath), # type: ignore[attr-defined] + z=0, position=dict(x=0, y=0, warp=self.warp_points), + ), + ]) + json_content = json.dumps(schema, indent=4) + # convert from json to avoid aliases + yaml_content = yaml.dump(json.loads(json_content)) + return (json_content, yaml_content) + + def copy_warp_schema(self, button): + from IPython.display import Javascript + + schema_copy_output = self._warp_widgets.get('copy_output') + if schema_copy_output is not None: + content = '' + json_content, yaml_content = self.get_warp_schema() + desc = button.description + if 'YAML' in desc: + content = yaml_content + elif 'JSON' in desc: + content = json_content + content = content.replace('\n', '\\n') + command = f"navigator.clipboard.writeText(unescape('{content}'))" + schema_copy_output.clear_output() + schema_copy_output.append_display_data(Javascript(command)) + button.description = 'Copied!' + button.icon = 'fa-check' + time.sleep(2) + button.description = desc + button.icon = 'fa-copy' + + def update_warp_schemas(self): + yaml_schema = self._warp_widgets.get('yaml') + json_schema = self._warp_widgets.get('json') + schema_accordion = self._warp_widgets.get('accordion') + transform_checkbox = self._warp_widgets.get('transform') + help_text = self._warp_widgets.get('help_text') + if ( + self._ts is None or + yaml_schema is None or + json_schema is None or + schema_accordion is None or + transform_checkbox is None or + help_text is None + ): + return + json_content, yaml_content = self.get_warp_schema() + yaml_schema.value = f'
{yaml_content}'
+ json_schema.value = f'{json_content}'
+ schema_accordion.layout.display = 'block'
+ transform_checkbox.layout.display = 'block'
+ help_text.value = (
+ 'Reference the schemas below to use this warp with '
+ 'the MultiFileTileSource (either as YAML or JSON).'
+ )
+ self.update_warp(transform_checkbox.value)
+
+ def start_drag(self, marker):
+ self._dragging_marker_id = marker.title
+
+ def handle_drag(self, coords):
+ if self._dragging_marker_id is not None:
+ marker_title = self._dragging_marker_id
+ group_name = marker_title[:3]
+ index = int(marker_title[3:])
+ self.warp_points[group_name][index] = self.convert_coordinate_map_to_warp(coords)
+ self.update_warp_schemas()
+
+ def end_drag(self):
+ self._dragging_marker_id = None
+
+ def create_warp_reference_point_pair(self, coord):
+ from ipyleaflet import DivIcon, Marker
+
+ marker_style = (
+ 'border-radius: 50%; position: relative;'
+ 'height: 16px; width: 16px; top: -8px; left: -8px;'
+ 'text-align: center; font-size: 11px;'
+ )
+ converted = self.convert_coordinate_map_to_warp(coord)
+ locations = dict(src=converted, dst=converted)
+ transform_checkbox = self._warp_widgets.get('transform')
+ transform_enabled = transform_checkbox.value if transform_checkbox is not None else True
+ if transform_enabled:
+ locations['src'] = self.inverse_warp(locations['src'])
+ index = len(self.warp_points['src'])
+ self.warp_points['src'].append(locations['src'])
+ self.warp_points['dst'].append(locations['dst'])
+
+ for group_name, color in [
+ ('src', '#ff6a5e'), ('dst', '#19a7ff'),
+ ]:
+ html = f'{json.dumps(expected, indent=4)}'
+ yaml_schema = sourceMap._warp_widgets.get('yaml')
+ assert yaml_schema.value == f'{yaml.dump(expected)}'
+
+ assert len(display.children) == 2
+ vbox = display.children[1]
+ assert len(vbox.children) == 4
+ copy_output = vbox.children[3]
+ accordion = vbox.children[2]
+ yaml_section = accordion.children[0]
+ copy_yaml_button = yaml_section.children[0]
+ sourceMap.copy_warp_schema(copy_yaml_button)
+ assert len(copy_output.outputs) == 1
+ output = copy_output.outputs[0]
+ assert output['data']['application/javascript'] == (
+ "navigator.clipboard.writeText(unescape('%s'))"
+ % yaml.dump(expected).replace('\n', '\\n')
+ )
+
+
+def testJupyterEditWarpInverseWarp():
+ sourceMap, _ = initEditWarp()
+ # inverse from single ref point
+ sourceMap.warp_points = dict(
+ src=[[10, 10]],
+ dst=[[15, 15]],
+ )
+ assert sourceMap.inverse_warp([5, 5]) == [0, 0]
+ # inverse from two ref points
+ sourceMap.warp_points = dict(
+ src=[[10, 10], [5, 5]],
+ dst=[[15, 15], [3, 4]],
+ )
+ assert sourceMap.inverse_warp([2, 4]) == [4, 4]
+ # inverse from three ref points
+ sourceMap.warp_points = dict(
+ src=[[10, 10], [5, 5], [2, 4]],
+ dst=[[15, 15], [3, 4], [3, 6]],
+ )
+ assert sourceMap.inverse_warp([6, 9]) == [2, 5]
+ # inverse from four ref points
+ sourceMap.warp_points = dict(
+ src=[[10, 10], [5, 5], [2, 4], [6, 9]],
+ dst=[[15, 15], [3, 4], [3, 6], [3, 5]],
+ )
+ assert sourceMap.inverse_warp([8, 8]) == [7, 6]