From 5a63a8ce250557a1fc5b6b6b7e47c9de608454f2 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Blum de Aguiar Date: Mon, 31 Jan 2022 20:20:17 +0100 Subject: [PATCH 01/11] opencv_engine: Reformat files with Black --- Makefile | 3 + integration_tests/__init__.py | 25 +++-- integration_tests/pil_test.py | 2 +- integration_tests/urls_helpers.py | 172 +++++++++++++++--------------- opencv_engine/__init__.py | 6 +- opencv_engine/engine.py | 68 +++++++----- opencv_engine/engine_cv3.py | 76 ++++++++----- pylintrc | 30 ++++++ setup.py | 69 ++++++------ tests/integration/opencv_test.py | 2 +- 10 files changed, 262 insertions(+), 191 deletions(-) create mode 100644 pylintrc diff --git a/Makefile b/Makefile index 9b15e9b..7e3066d 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,6 @@ +black: + @black . + test: unit integration unit: diff --git a/integration_tests/__init__.py b/integration_tests/__init__.py index c8f32bb..27761d7 100644 --- a/integration_tests/__init__.py +++ b/integration_tests/__init__.py @@ -12,27 +12,26 @@ class EngineCase(AsyncHTTPTestCase): - def get_app(self): - cfg = Config(SECURITY_KEY='ACME-SEC') + cfg = Config(SECURITY_KEY="ACME-SEC") server_params = ServerParameters(None, None, None, None, None, None) - server_params.gifsicle_path = which('gifsicle') + server_params.gifsicle_path = which("gifsicle") cfg.DETECTORS = [ - 'thumbor.detectors.face_detector', - 'thumbor.detectors.profile_detector', - 'thumbor.detectors.glasses_detector', - 'thumbor.detectors.feature_detector', + "thumbor.detectors.face_detector", + "thumbor.detectors.profile_detector", + "thumbor.detectors.glasses_detector", + "thumbor.detectors.feature_detector", ] - cfg.STORAGE = 'thumbor.storages.no_storage' - cfg.LOADER = 'thumbor.loaders.file_loader' - cfg.FILE_LOADER_ROOT_PATH = os.path.join(os.path.dirname(__file__), 'imgs') - cfg.ENGINE = getattr(self, 'engine', None) + cfg.STORAGE = "thumbor.storages.no_storage" + cfg.LOADER = "thumbor.loaders.file_loader" + cfg.FILE_LOADER_ROOT_PATH = os.path.join(os.path.dirname(__file__), "imgs") + cfg.ENGINE = getattr(self, "engine", None) cfg.USE_GIFSICLE_ENGINE = True - cfg.FFMPEG_PATH = which('ffmpeg') + cfg.FFMPEG_PATH = which("ffmpeg") cfg.ENGINE_THREADPOOL_SIZE = 10 cfg.OPTIMIZERS = [ - 'thumbor.optimizers.gifv', + "thumbor.optimizers.gifv", ] if not cfg.ENGINE: return None diff --git a/integration_tests/pil_test.py b/integration_tests/pil_test.py index a1ec963..17bbcb5 100644 --- a/integration_tests/pil_test.py +++ b/integration_tests/pil_test.py @@ -2,7 +2,7 @@ class PILTest(EngineCase): - engine = 'thumbor.engines.pil' + engine = "thumbor.engines.pil" def test_single_params(self): self.exec_single_params() diff --git a/integration_tests/urls_helpers.py b/integration_tests/urls_helpers.py index 55f50d6..0712718 100644 --- a/integration_tests/urls_helpers.py +++ b/integration_tests/urls_helpers.py @@ -8,123 +8,111 @@ from colorama import Fore -debugs = [ - '', - 'debug' -] +debugs = ["", "debug"] -metas = [ - 'meta' -] +metas = ["meta"] trims = [ - 'trim', - 'trim:top-left', - 'trim:bottom-right', - 'trim:top-left:10', - 'trim:bottom-right:20', + "trim", + "trim:top-left", + "trim:bottom-right", + "trim:top-left:10", + "trim:bottom-right:20", ] -crops = [ - '10x10:100x100' -] +crops = ["10x10:100x100"] -fitins = [ - 'fit-in', - 'adaptive-fit-in', - 'full-fit-in', - 'adaptive-full-fit-in' -] +fitins = ["fit-in", "adaptive-fit-in", "full-fit-in", "adaptive-full-fit-in"] sizes = [ - '200x200', - '-300x100', - '100x-300', - '-100x-300', - 'origx300', - '200xorig', - 'origxorig', + "200x200", + "-300x100", + "100x-300", + "-100x-300", + "origx300", + "200xorig", + "origxorig", ] haligns = [ - 'left', - 'right', - 'center', + "left", + "right", + "center", ] valigns = [ - 'top', - 'bottom', - 'middle', + "top", + "bottom", + "middle", ] smarts = [ - 'smart', + "smart", ] filters = [ - 'filters:brightness(10)', - 'filters:contrast(10)', - 'filters:equalize()', - 'filters:grayscale()', - 'filters:rotate(90)', - 'filters:noise(10)', - 'filters:quality(5)', - 'filters:redeye()', - 'filters:rgb(10,-10,20)', - 'filters:round_corner(20,255,255,100)', - 'filters:sharpen(6,2.5,false)', - 'filters:sharpen(6,2.5,true)', - 'filters:strip_icc()', - 'filters:watermark(rgba-interlaced.png,10,10,50)', - 'filters:watermark(rgba-interlaced.png,center,center,50)', - 'filters:watermark(rgba-interlaced.png,repeat,repeat,50)', - 'filters:frame(rgba.png)', - 'filters:fill(ff0000)', - 'filters:fill(auto)', - 'filters:fill(ff0000,true)', - 'filters:fill(transparent)', - 'filters:fill(transparent,true)', - 'filters:blur(2)', - 'filters:extract_focal()', - 'filters:focal()', - 'filters:focal(0x0:1x1)', - 'filters:no_upscale()', - 'filters:gifv()', - 'filters:gifv(webm)', - 'filters:gifv(mp4)', - 'filters:max_age(600)', - + "filters:brightness(10)", + "filters:contrast(10)", + "filters:equalize()", + "filters:grayscale()", + "filters:rotate(90)", + "filters:noise(10)", + "filters:quality(5)", + "filters:redeye()", + "filters:rgb(10,-10,20)", + "filters:round_corner(20,255,255,100)", + "filters:sharpen(6,2.5,false)", + "filters:sharpen(6,2.5,true)", + "filters:strip_icc()", + "filters:watermark(rgba-interlaced.png,10,10,50)", + "filters:watermark(rgba-interlaced.png,center,center,50)", + "filters:watermark(rgba-interlaced.png,repeat,repeat,50)", + "filters:frame(rgba.png)", + "filters:fill(ff0000)", + "filters:fill(auto)", + "filters:fill(ff0000,true)", + "filters:fill(transparent)", + "filters:fill(transparent,true)", + "filters:blur(2)", + "filters:extract_focal()", + "filters:focal()", + "filters:focal(0x0:1x1)", + "filters:no_upscale()", + "filters:gifv()", + "filters:gifv(webm)", + "filters:gifv(mp4)", + "filters:max_age(600)", # one big filter 4-line string - 'filters:curve([(0,0),(255,255)],[(0,50),(16,51),(32,69),(58,85),(92,120),(128,170),(140,186),(167,225),' - '(192,245),(225,255),(244,255),(255,254)],[(0,0),(16,2),(32,18),(64,59),(92,116),(128,182),(167,211),(192,227)' - ',(224,240),(244,247),(255,252)],[(0,48),(16,50),(62,77),(92,110),(128,144),(140,153),(167,180),(192,192),' - '(224,217),(244,225),(255,225)])', + "filters:curve([(0,0),(255,255)],[(0,50),(16,51),(32,69),(58,85),(92,120),(128,170),(140,186),(167,225)," + "(192,245),(225,255),(244,255),(255,254)],[(0,0),(16,2),(32,18),(64,59),(92,116),(128,182),(167,211),(192,227)" + ",(224,240),(244,247),(255,252)],[(0,48),(16,50),(62,77),(92,110),(128,144),(140,153),(167,180),(192,192)," + "(224,217),(244,225),(255,225)])", ] original_images_base = [ - 'gradient.jpg', - 'cmyk.jpg', - 'rgba.png', - 'grayscale.jpg', - '16bit.png', + "gradient.jpg", + "cmyk.jpg", + "rgba.png", + "grayscale.jpg", + "16bit.png", ] original_images_gif_webp = [ - 'gradient.webp', - 'gradient.gif', - 'animated.gif', + "gradient.webp", + "gradient.gif", + "animated.gif", ] class UrlsTester(object): - def __init__(self, fetcher, group): self.failed_items = [] self.test_group(fetcher, group) def report(self): - assert len(self.failed_items) == 0, "Failed urls:\n%s" % '\n'.join(self.failed_items) + assert len(self.failed_items) == 0, "Failed urls:\n%s" % "\n".join( + self.failed_items + ) def try_url(self, fetcher, url): result = None @@ -133,7 +121,7 @@ def try_url(self, fetcher, url): try: result = fetcher("/%s" % url) except Exception: - logging.exception('Error in %s' % url) + logging.exception("Error in %s" % url) failed = True if result is not None and result.code == 200 and not failed: @@ -141,7 +129,11 @@ def try_url(self, fetcher, url): return self.failed_items.append(url) - print("{0.RED} FAILED ({1}) - ERR({2}) {0.RESET}".format(Fore, url, result and result.code)) + print( + "{0.RED} FAILED ({1}) - ERR({2}) {0.RESET}".format( + Fore, url, result and result.code + ) + ) def test_group(self, fetcher, group): group = list(group) @@ -160,7 +152,9 @@ def single_dataset(fetcher, with_gif=True): images = original_images_base[:] if with_gif: images += original_images_gif_webp - all_options = metas + trims + crops + fitins + sizes + haligns + valigns + smarts + filters + all_options = ( + metas + trims + crops + fitins + sizes + haligns + valigns + smarts + filters + ) UrlsTester(fetcher, product(all_options, images)) @@ -169,6 +163,14 @@ def combined_dataset(fetcher, with_gif=True): if with_gif: images += original_images_gif_webp combined_options = product( - trims[:2], crops[:2], fitins[:2], sizes[:2], haligns[:2], valigns[:2], smarts[:2], filters[:2], images + trims[:2], + crops[:2], + fitins[:2], + sizes[:2], + haligns[:2], + valigns[:2], + smarts[:2], + filters[:2], + images, ) UrlsTester(fetcher, combined_options) diff --git a/opencv_engine/__init__.py b/opencv_engine/__init__.py index dfb5a57..172bcb9 100644 --- a/opencv_engine/__init__.py +++ b/opencv_engine/__init__.py @@ -3,9 +3,11 @@ import logging -__version__ = '1.0.1' +__version__ = "1.0.1" try: from opencv_engine.engine_cv3 import Engine # NOQA except ImportError: - logging.exception('Could not import opencv_engine. Probably due to setup.py installing it.') + logging.exception( + "Could not import opencv_engine. Probably due to setup.py installing it." + ) diff --git a/opencv_engine/engine.py b/opencv_engine/engine.py index 813abdc..18eda7d 100644 --- a/opencv_engine/engine.py +++ b/opencv_engine/engine.py @@ -20,20 +20,15 @@ try: from thumbor.ext.filters import _composite + FILTERS_AVAILABLE = True except ImportError: FILTERS_AVAILABLE = False -FORMATS = { - '.jpg': 'JPEG', - '.jpeg': 'JPEG', - '.gif': 'GIF', - '.png': 'PNG' -} +FORMATS = {".jpg": "JPEG", ".jpeg": "JPEG", ".gif": "GIF", ".png": "PNG"} class Engine(BaseEngine): - @property def image_depth(self): if self.image is None: @@ -56,12 +51,12 @@ def parse_hex_color(cls, color): def gen_image(self, size, color_value): img0 = cv.CreateImage(size, self.image_depth, self.image_channels) - if color_value == 'transparent': + if color_value == "transparent": color = (255, 255, 255, 255) else: color = self.parse_hex_color(color_value) if not color: - raise ValueError('Color %s is not valid.' % color_value) + raise ValueError("Color %s is not valid." % color_value) cv.Set(img0, color) return img0 @@ -70,7 +65,7 @@ def create_image(self, buffer): # segfaults when trying to decoding a gif. An exception is a # less drastic measure. try: - if FORMATS[self.extension] == 'GIF': + if FORMATS[self.extension] == "GIF": raise ValueError("opencv doesn't support gifs") except KeyError: pass @@ -79,7 +74,7 @@ def create_image(self, buffer): cv.SetData(imagefiledata, buffer, len(buffer)) img0 = cv.DecodeImageM(imagefiledata, cv.CV_LOAD_IMAGE_UNCHANGED) - if FORMATS[self.extension] == 'JPEG': + if FORMATS[self.extension] == "JPEG": try: info = JpegFile.fromString(buffer).get_exif() if info: @@ -101,7 +96,7 @@ def resize(self, width, height): thumbnail = cv.CreateImage( (int(round(width, 0)), int(round(height, 0))), self.image_depth, - self.image_channels + self.image_channels, ) cv.Resize(self.image, thumbnail, cv.CV_INTER_AREA) self.image = thumbnail @@ -118,7 +113,7 @@ def crop(self, left, top, right, bottom): self.image = cropped def rotate(self, degrees): - if (degrees > 180): + if degrees > 180: # Flip around both axes cv.Flip(self.image, None, -1) degrees = degrees - 180 @@ -126,7 +121,7 @@ def rotate(self, degrees): img = self.image size = cv.GetSize(img) - if (degrees / 90 % 2): + if degrees / 90 % 2: new_size = (size[1], size[0]) center = ((size[0] - 1) * 0.5, (size[0] - 1) * 0.5) else: @@ -153,7 +148,7 @@ def read(self, extension=None, quality=None): options = None extension = extension or self.extension try: - if FORMATS[extension] == 'JPEG': + if FORMATS[extension] == "JPEG": options = [cv.CV_IMWRITE_JPEG_QUALITY, quality] except KeyError: # default is JPEG so @@ -161,10 +156,12 @@ def read(self, extension=None, quality=None): data = cv.EncodeImage(extension, self.image, options or []).tostring() - if FORMATS[extension] == 'JPEG' and self.context.config.PRESERVE_EXIF_INFO: - if hasattr(self, 'exif'): + if FORMATS[extension] == "JPEG" and self.context.config.PRESERVE_EXIF_INFO: + if hasattr(self, "exif"): img = JpegFile.fromString(data) - img._segments.insert(0, ExifSegment(self.exif_marker, None, self.exif, 'rw')) + img._segments.insert( + 0, ExifSegment(self.exif_marker, None, self.exif, "rw") + ) data = img.writeString() return data @@ -175,31 +172,39 @@ def set_image_data(self, data): def image_data_as_rgb(self, update_image=True): # TODO: Handle other formats if self.image_channels == 4: - mode = 'BGRA' + mode = "BGRA" elif self.image_channels == 3: - mode = 'BGR' + mode = "BGR" else: - mode = 'BGR' + mode = "BGR" rgb_copy = cv.CreateImage((self.image.width, self.image.height), 8, 3) cv.CvtColor(self.image, rgb_copy, cv.CV_GRAY2BGR) self.image = rgb_copy return mode, self.image.tostring() def draw_rectangle(self, x, y, width, height): - cv.Rectangle(self.image, (int(x), int(y)), (int(x + width), int(y + height)), cv.Scalar(255, 255, 255, 1.0)) + cv.Rectangle( + self.image, + (int(x), int(y)), + (int(x + width), int(y + height)), + cv.Scalar(255, 255, 255, 1.0), + ) def convert_to_grayscale(self): if self.image_channels >= 3: # FIXME: OpenCV does not support grayscale with alpha channel? - grayscaled = cv.CreateImage((self.image.width, self.image.height), self.image_depth, 1) + grayscaled = cv.CreateImage( + (self.image.width, self.image.height), self.image_depth, 1 + ) cv.CvtColor(self.image, grayscaled, cv.CV_BGRA2GRAY) self.image = grayscaled def paste(self, other_engine, pos, merge=True): if merge and not FILTERS_AVAILABLE: raise RuntimeError( - 'You need filters enabled to use paste with merge. Please reinstall ' + - 'thumbor with proper compilation of its filters.') + "You need filters enabled to use paste with merge. Please reinstall " + + "thumbor with proper compilation of its filters." + ) self.enable_alpha() other_engine.enable_alpha() @@ -211,8 +216,17 @@ def paste(self, other_engine, pos, merge=True): other_mode, other_data = other_engine.image_data_as_rgb() imgdata = _composite.apply( - mode, data, sz[0], sz[1], - other_data, other_size[0], other_size[1], pos[0], pos[1], merge) + mode, + data, + sz[0], + sz[1], + other_data, + other_size[0], + other_size[1], + pos[0], + pos[1], + merge, + ) self.set_image_data(imgdata) diff --git a/opencv_engine/engine_cv3.py b/opencv_engine/engine_cv3.py index 1d22b9e..08a91df 100644 --- a/opencv_engine/engine_cv3.py +++ b/opencv_engine/engine_cv3.py @@ -14,16 +14,17 @@ try: from thumbor.ext.filters import _composite + FILTERS_AVAILABLE = True except ImportError: FILTERS_AVAILABLE = False FORMATS = { - '.jpg': 'JPEG', - '.jpeg': 'JPEG', - '.gif': 'GIF', - '.png': 'PNG', - '.webp': 'WEBP' + ".jpg": "JPEG", + ".jpeg": "JPEG", + ".gif": "GIF", + ".png": "PNG", + ".webp": "WEBP", } @@ -53,14 +54,14 @@ def parse_hex_color(cls, color): return None def gen_image(self, size, color_value): - if color_value == 'transparent': + if color_value == "transparent": color = (255, 255, 255, 255) img = np.zeros((size[1], size[0], 4), self.image_depth) else: img = np.zeros((size[1], size[0], self.image_channels), self.image_depth) color = self.parse_hex_color(color_value) if not color: - raise ValueError('Color %s is not valid.' % color_value) + raise ValueError("Color %s is not valid." % color_value) img[:] = color return img @@ -69,13 +70,13 @@ def create_image(self, buffer): # segfaults when trying to decoding a gif. An exception is a # less drastic measure. try: - if FORMATS[self.extension] == 'GIF': + if FORMATS[self.extension] == "GIF": raise ValueError("opencv doesn't support gifs") except KeyError: pass img = cv2.imdecode(np.frombuffer(buffer, np.uint8), -1) - if FORMATS[self.extension] == 'JPEG': + if FORMATS[self.extension] == "JPEG": self.exif = None try: info = JpegFile.fromString(buffer).get_exif() @@ -100,7 +101,7 @@ def resize(self, width, height): self.image = cv2.resize(self.image, dim, interpolation=cv2.INTER_AREA) def crop(self, left, top, right, bottom): - self.image = self.image[top: bottom, left: right] + self.image = self.image[top:bottom, left:right] def rotate(self, degrees): # see http://stackoverflow.com/a/23990392 @@ -128,8 +129,8 @@ def rotate(self, degrees): bound_w = int((height * abs_sin) + (width * abs_cos)) bound_h = int((height * abs_cos) + (width * abs_sin)) - rot_mat[0, 2] += ((bound_w / 2) - image_center[0]) - rot_mat[1, 2] += ((bound_h / 2) - image_center[1]) + rot_mat[0, 2] += (bound_w / 2) - image_center[0] + rot_mat[1, 2] += (bound_h / 2) - image_center[1] self.image = cv2.warpAffine(self.image, rot_mat, (bound_w, bound_h)) @@ -146,14 +147,14 @@ def read(self, extension=None, quality=None): options = None extension = extension or self.extension try: - if FORMATS[extension] == 'JPEG': + if FORMATS[extension] == "JPEG": options = [cv2.IMWRITE_JPEG_QUALITY, quality] except KeyError: # default is JPEG so options = [cv2.IMWRITE_JPEG_QUALITY, quality] try: - if FORMATS[extension] == 'WEBP': + if FORMATS[extension] == "WEBP": options = [cv2.IMWRITE_WEBP_QUALITY, quality] except KeyError: options = [cv2.IMWRITE_JPEG_QUALITY, quality] @@ -161,31 +162,40 @@ def read(self, extension=None, quality=None): success, buf = cv2.imencode(extension, self.image, options or []) data = buf.tostring() - if FORMATS[extension] == 'JPEG' and self.context.config.PRESERVE_EXIF_INFO: - if hasattr(self, 'exif') and self.exif != None: + if FORMATS[extension] == "JPEG" and self.context.config.PRESERVE_EXIF_INFO: + if hasattr(self, "exif") and self.exif != None: img = JpegFile.fromString(data) - img._segments.insert(0, ExifSegment(self.exif_marker, None, self.exif, 'rw')) + img._segments.insert( + 0, ExifSegment(self.exif_marker, None, self.exif, "rw") + ) data = img.writeString() return data def set_image_data(self, data): - self.image = np.frombuffer(data, dtype=self.image.dtype).reshape(self.image.shape) + self.image = np.frombuffer(data, dtype=self.image.dtype).reshape( + self.image.shape + ) def image_data_as_rgb(self, update_image=True): if self.image_channels == 4: - mode = 'BGRA' + mode = "BGRA" elif self.image_channels == 3: - mode = 'BGR' + mode = "BGR" else: - mode = 'BGR' + mode = "BGR" rgb_copy = np.zeros((self.size[1], self.size[0], 3), self.image.dtype) cv2.cvtColor(self.image, cv2.COLOR_GRAY2BGR, rgb_copy) self.image = rgb_copy return mode, self.image.tostring() def draw_rectangle(self, x, y, width, height): - cv2.rectangle(self.image, (int(x), int(y)), (int(x + width), int(y + height)), (255, 255, 255)) + cv2.rectangle( + self.image, + (int(x), int(y)), + (int(x + width), int(y + height)), + (255, 255, 255), + ) def convert_to_grayscale(self, update_image=True, with_alpha=True): image = None @@ -199,15 +209,16 @@ def convert_to_grayscale(self, update_image=True, with_alpha=True): if update_image: self.image = image elif self.image_depth == np.uint16: - #Feature detector reqiures uint8 images - image = np.array(image, dtype='uint8') + # Feature detector reqiures uint8 images + image = np.array(image, dtype="uint8") return image def paste(self, other_engine, pos, merge=True): if merge and not FILTERS_AVAILABLE: raise RuntimeError( - 'You need filters enabled to use paste with merge. Please reinstall ' + - 'thumbor with proper compilation of its filters.') + "You need filters enabled to use paste with merge. Please reinstall " + + "thumbor with proper compilation of its filters." + ) self.enable_alpha() other_engine.enable_alpha() @@ -219,8 +230,17 @@ def paste(self, other_engine, pos, merge=True): other_mode, other_data = other_engine.image_data_as_rgb() imgdata = _composite.apply( - mode, data, sz[0], sz[1], - other_data, other_size[0], other_size[1], pos[0], pos[1], merge) + mode, + data, + sz[0], + sz[1], + other_data, + other_size[0], + other_size[1], + pos[0], + pos[1], + merge, + ) self.set_image_data(imgdata) diff --git a/pylintrc b/pylintrc new file mode 100644 index 0000000..b245bc9 --- /dev/null +++ b/pylintrc @@ -0,0 +1,30 @@ +[MESSAGES CONTROL] + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable= + missing-function-docstring, + missing-module-docstring, + missing-class-docstring, + bad-continuation, + c-extension-no-member, + too-many-arguments, + assignment-from-none, + no-self-use, + too-few-public-methods, + attribute-defined-outside-init, + abstract-method, + too-many-instance-attributes, + broad-except, + too-many-locals, + too-many-public-methods, + fixme, + R0801, + deprecated-module, diff --git a/setup.py b/setup.py index 7bba119..3f3478c 100644 --- a/setup.py +++ b/setup.py @@ -5,50 +5,51 @@ from opencv_engine import __version__ tests_require = [ - 'mock', - 'nose', - 'coverage', - 'yanc', - 'colorama', - 'preggy', - 'ipdb', - 'coveralls', - 'numpy', - 'colour', + "black", + "mock", + "nose", + "coverage", + "yanc", + "colorama", + "preggy", + "ipdb", + "coveralls", + "numpy", + "colour", ] setup( - name='opencv_engine', + name="opencv_engine", version=__version__, - description='OpenCV imaging engine for thumbor.', - long_description=''' + description="OpenCV imaging engine for thumbor.", + long_description=""" OpenCV imaging engine for thumbor. -''', - keywords='thumbor imaging opencv', - author='Globo.com', - author_email='timehome@corp.globo.com', - url='', - license='MIT', +""", + keywords="thumbor imaging opencv", + author="Globo.com", + author_email="timehome@corp.globo.com", + url="", + license="MIT", classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Natural Language :: English', - 'Operating System :: MacOS', - 'Operating System :: POSIX', - 'Operating System :: Unix', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2.7', + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Operating System :: MacOS", + "Operating System :: POSIX", + "Operating System :: Unix", + "Operating System :: OS Independent", + "Programming Language :: Python :: 2.7", ], packages=find_packages(), include_package_data=True, install_requires=[ - 'colour', - 'numpy', - 'thumbor', - 'opencv-python', + "colour", + "numpy", + "thumbor", + "opencv-python", ], extras_require={ - 'tests': tests_require, - } + "tests": tests_require, + }, ) diff --git a/tests/integration/opencv_test.py b/tests/integration/opencv_test.py index 5d24589..782e678 100644 --- a/tests/integration/opencv_test.py +++ b/tests/integration/opencv_test.py @@ -3,7 +3,7 @@ class OpenCVTest(EngineCase): - engine = 'opencv_engine' + engine = "opencv_engine" def test_single_params(self): single_dataset(self.retrieve, with_gif=False) From d8ed9c4f9cdb42f74542277b7c7456a5ee069a80 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Blum de Aguiar Date: Mon, 31 Jan 2022 20:38:50 +0100 Subject: [PATCH 02/11] opencv_engine: Reorder imports with isort --- Makefile | 3 +++ integration_tests/__init__.py | 10 +++++----- integration_tests/urls_helpers.py | 3 +-- opencv_engine/engine.py | 3 +-- opencv_engine/engine_cv3.py | 3 +-- setup.py | 4 +++- 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 7e3066d..ddc9598 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ black: @black . +isort: + @isort . + test: unit integration unit: diff --git a/integration_tests/__init__.py b/integration_tests/__init__.py index 27761d7..8a3d2b6 100644 --- a/integration_tests/__init__.py +++ b/integration_tests/__init__.py @@ -1,14 +1,14 @@ import os.path -from tornado.testing import AsyncHTTPTestCase -from tornado.ioloop import IOLoop - from thumbor.app import ThumborServiceApp -from thumbor.importer import Importer from thumbor.config import Config from thumbor.context import Context, ServerParameters -from .urls_helpers import single_dataset # , combined_dataset +from thumbor.importer import Importer from thumbor.utils import which +from tornado.ioloop import IOLoop +from tornado.testing import AsyncHTTPTestCase + +from .urls_helpers import single_dataset # , combined_dataset class EngineCase(AsyncHTTPTestCase): diff --git a/integration_tests/urls_helpers.py b/integration_tests/urls_helpers.py index 0712718..4e08b9e 100644 --- a/integration_tests/urls_helpers.py +++ b/integration_tests/urls_helpers.py @@ -2,12 +2,11 @@ # -*- coding: utf-8 -*- import logging -from os.path import join from itertools import product +from os.path import join from colorama import Fore - debugs = ["", "debug"] metas = ["meta"] diff --git a/opencv_engine/engine.py b/opencv_engine/engine.py index 18eda7d..2406912 100644 --- a/opencv_engine/engine.py +++ b/opencv_engine/engine.py @@ -14,9 +14,8 @@ import cv2.cv as cv from colour import Color - +from pexif import ExifSegment, JpegFile from thumbor.engines import BaseEngine -from pexif import JpegFile, ExifSegment try: from thumbor.ext.filters import _composite diff --git a/opencv_engine/engine_cv3.py b/opencv_engine/engine_cv3.py index 08a91df..d05de36 100644 --- a/opencv_engine/engine_cv3.py +++ b/opencv_engine/engine_cv3.py @@ -7,10 +7,9 @@ import cv2 import numpy as np - from colour import Color +from pexif import ExifSegment, JpegFile from thumbor.engines import BaseEngine -from pexif import JpegFile, ExifSegment try: from thumbor.ext.filters import _composite diff --git a/setup.py b/setup.py index 3f3478c..7825b04 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from setuptools import setup, find_packages +from setuptools import find_packages, setup + from opencv_engine import __version__ tests_require = [ @@ -13,6 +14,7 @@ "colorama", "preggy", "ipdb", + "isort", "coveralls", "numpy", "colour", From c0dc5362271fab76cb6eb2b89aa4e18cda1b097d Mon Sep 17 00:00:00 2001 From: Pablo Santiago Blum de Aguiar Date: Mon, 31 Jan 2022 20:40:30 +0100 Subject: [PATCH 03/11] setup: Order requiremen lists --- setup.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 7825b04..5acb0fc 100644 --- a/setup.py +++ b/setup.py @@ -7,17 +7,17 @@ tests_require = [ "black", - "mock", - "nose", - "coverage", - "yanc", "colorama", - "preggy", + "colour", + "coverage", + "coveralls", "ipdb", "isort", - "coveralls", + "mock", + "nose", "numpy", - "colour", + "preggy", + "yanc", ] setup( @@ -48,8 +48,8 @@ install_requires=[ "colour", "numpy", - "thumbor", "opencv-python", + "thumbor", ], extras_require={ "tests": tests_require, From 73c7b5f7aa04dad6f391b0fcb80cbdbd46b0af08 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Blum de Aguiar Date: Mon, 31 Jan 2022 20:56:32 +0100 Subject: [PATCH 04/11] integration_tests: Import which form shutil --- integration_tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/__init__.py b/integration_tests/__init__.py index 8a3d2b6..70ddc61 100644 --- a/integration_tests/__init__.py +++ b/integration_tests/__init__.py @@ -1,10 +1,10 @@ import os.path +from shutil import which from thumbor.app import ThumborServiceApp from thumbor.config import Config from thumbor.context import Context, ServerParameters from thumbor.importer import Importer -from thumbor.utils import which from tornado.ioloop import IOLoop from tornado.testing import AsyncHTTPTestCase From 20df650af9dd771f9563ed9d2cead8dc2551cbed Mon Sep 17 00:00:00 2001 From: Pablo Santiago Blum de Aguiar Date: Mon, 31 Jan 2022 22:54:16 +0100 Subject: [PATCH 05/11] opencv_engine: Fix convert_to_grayscale along with integration tests --- integration_tests/__init__.py | 21 +----- integration_tests/urls_helpers.py | 116 +++++++++++++++--------------- opencv_engine/engine_cv3.py | 4 +- tests/integration/opencv_test.py | 22 +++++- 4 files changed, 80 insertions(+), 83 deletions(-) diff --git a/integration_tests/__init__.py b/integration_tests/__init__.py index 70ddc61..2b3fdd7 100644 --- a/integration_tests/__init__.py +++ b/integration_tests/__init__.py @@ -5,11 +5,8 @@ from thumbor.config import Config from thumbor.context import Context, ServerParameters from thumbor.importer import Importer -from tornado.ioloop import IOLoop from tornado.testing import AsyncHTTPTestCase -from .urls_helpers import single_dataset # , combined_dataset - class EngineCase(AsyncHTTPTestCase): def get_app(self): @@ -41,21 +38,5 @@ def get_app(self): ctx = Context(server_params, cfg, importer) application = ThumborServiceApp(ctx) + application.debug = True return application - - def get_new_ioloop(self): - return IOLoop.instance() - - def retrieve(self, url): - self.http_client.fetch(self.get_url(url), self.stop) - return self.wait(timeout=30) - - def exec_single_params(self): - if not self._app: - return True - single_dataset(self.retrieve) - - # def test_combined_params__with_pil(self): - # if not self._app: - # return True - # combined_dataset(self.retrieve) diff --git a/integration_tests/urls_helpers.py b/integration_tests/urls_helpers.py index 4e08b9e..ef7ed17 100644 --- a/integration_tests/urls_helpers.py +++ b/integration_tests/urls_helpers.py @@ -3,15 +3,14 @@ import logging from itertools import product -from os.path import join from colorama import Fore -debugs = ["", "debug"] +DEBUGS = ["", "debug"] -metas = ["meta"] +METAS = ["meta"] -trims = [ +TRIMS = [ "trim", "trim:top-left", "trim:bottom-right", @@ -19,11 +18,11 @@ "trim:bottom-right:20", ] -crops = ["10x10:100x100"] +CROPS = ["10x10:100x100"] -fitins = ["fit-in", "adaptive-fit-in", "full-fit-in", "adaptive-full-fit-in"] +FITINS = ["fit-in", "adaptive-fit-in", "full-fit-in", "adaptive-full-fit-in"] -sizes = [ +SIZES = [ "200x200", "-300x100", "100x-300", @@ -33,23 +32,23 @@ "origxorig", ] -haligns = [ +H_ALIGNS = [ "left", "right", "center", ] -valigns = [ +V_ALIGNS = [ "top", "bottom", "middle", ] -smarts = [ +SMARTS = [ "smart", ] -filters = [ +FILTERS = [ "filters:brightness(10)", "filters:contrast(10)", "filters:equalize()", @@ -62,6 +61,7 @@ "filters:round_corner(20,255,255,100)", "filters:sharpen(6,2.5,false)", "filters:sharpen(6,2.5,true)", + "filters:strip_exif()", "filters:strip_icc()", "filters:watermark(rgba-interlaced.png,10,10,50)", "filters:watermark(rgba-interlaced.png,center,center,50)", @@ -81,14 +81,18 @@ "filters:gifv(webm)", "filters:gifv(mp4)", "filters:max_age(600)", + "filters:upscale()", # one big filter 4-line string - "filters:curve([(0,0),(255,255)],[(0,50),(16,51),(32,69),(58,85),(92,120),(128,170),(140,186),(167,225)," - "(192,245),(225,255),(244,255),(255,254)],[(0,0),(16,2),(32,18),(64,59),(92,116),(128,182),(167,211),(192,227)" - ",(224,240),(244,247),(255,252)],[(0,48),(16,50),(62,77),(92,110),(128,144),(140,153),(167,180),(192,192)," + "filters:curve([(0,0),(255,255)],[(0,50),(16,51),(32,69)," + "(58,85),(92,120),(128,170),(140,186),(167,225)," # NOQA + "(192,245),(225,255),(244,255),(255,254)],[(0,0),(16,2)," + "(32,18),(64,59),(92,116),(128,182),(167,211),(192,227)" # NOQA + ",(224,240),(244,247),(255,252)],[(0,48),(16,50),(62,77)," + "(92,110),(128,144),(140,153),(167,180),(192,192)," # NOQA "(224,217),(244,225),(255,225)])", ] -original_images_base = [ +ORIGINAL_IMAGES_BASE = [ "gradient.jpg", "cmyk.jpg", "rgba.png", @@ -96,31 +100,42 @@ "16bit.png", ] -original_images_gif_webp = [ +ORIGINAL_IMAGES_GIF_WEBP = [ "gradient.webp", "gradient.gif", "animated.gif", ] +ALL_OPTIONS = ( + METAS + TRIMS + CROPS + FITINS + SIZES + H_ALIGNS + V_ALIGNS + SMARTS + FILTERS +) -class UrlsTester(object): - def __init__(self, fetcher, group): +MAX_DATASET_SIZE = len(ALL_OPTIONS) * ( + len(ORIGINAL_IMAGES_BASE) + len(ORIGINAL_IMAGES_GIF_WEBP) +) + + +class UrlsTester: + def __init__(self, http_client): self.failed_items = [] - self.test_group(fetcher, group) + self.http_client = http_client def report(self): - assert len(self.failed_items) == 0, "Failed urls:\n%s" % "\n".join( - self.failed_items - ) + if len(self.failed_items) == 0: + return + + raise AssertionError("Failed urls:\n%s" % "\n".join(self.failed_items)) - def try_url(self, fetcher, url): + async def try_url(self, url): result = None + error = None failed = False try: - result = fetcher("/%s" % url) - except Exception: - logging.exception("Error in %s" % url) + result = await self.http_client.fetch(url, request_timeout=60) + except Exception as err: # pylint: disable=broad-except + logging.exception("Error in %s: %s", url, err) + error = err failed = True if result is not None and result.code == 200 and not failed: @@ -130,46 +145,31 @@ def try_url(self, fetcher, url): self.failed_items.append(url) print( "{0.RED} FAILED ({1}) - ERR({2}) {0.RESET}".format( - Fore, url, result and result.code + Fore, url, result is not None and result.code or error ) ) - def test_group(self, fetcher, group): - group = list(group) - count = len(group) - print("Requests count: %d" % count) - for options in group: - joined_parts = join(*options) - url = "unsafe/%s" % joined_parts - self.try_url(fetcher, url) - - self.report() - - -def single_dataset(fetcher, with_gif=True): - images = original_images_base[:] +def single_dataset(with_gif=True): + images = ORIGINAL_IMAGES_BASE[:] if with_gif: - images += original_images_gif_webp - all_options = ( - metas + trims + crops + fitins + sizes + haligns + valigns + smarts + filters - ) - UrlsTester(fetcher, product(all_options, images)) + images += ORIGINAL_IMAGES_GIF_WEBP + return product(ALL_OPTIONS, images) -def combined_dataset(fetcher, with_gif=True): - images = original_images_base[:] +def combined_dataset(with_gif=True): + images = ORIGINAL_IMAGES_BASE[:] if with_gif: - images += original_images_gif_webp + images += ORIGINAL_IMAGES_GIF_WEBP combined_options = product( - trims[:2], - crops[:2], - fitins[:2], - sizes[:2], - haligns[:2], - valigns[:2], - smarts[:2], - filters[:2], + TRIMS[:2], + CROPS[:2], + FITINS[:2], + SIZES[:2], + H_ALIGNS[:2], + V_ALIGNS[:2], + SMARTS[:2], + FILTERS[:2], images, ) - UrlsTester(fetcher, combined_options) + return combined_options diff --git a/opencv_engine/engine_cv3.py b/opencv_engine/engine_cv3.py index d05de36..0b2e7ac 100644 --- a/opencv_engine/engine_cv3.py +++ b/opencv_engine/engine_cv3.py @@ -196,9 +196,9 @@ def draw_rectangle(self, x, y, width, height): (255, 255, 255), ) - def convert_to_grayscale(self, update_image=True, with_alpha=True): + def convert_to_grayscale(self, update_image=True, alpha=True): image = None - if self.image_channels >= 3 and with_alpha: + if self.image_channels >= 3 and alpha: image = cv2.cvtColor(self.image, cv2.COLOR_BGRA2GRAY) elif self.image_channels >= 3: image = cv2.cvtColor(self.image, cv2.COLOR_BGR2GRAY) diff --git a/tests/integration/opencv_test.py b/tests/integration/opencv_test.py index 782e678..66e6bdb 100644 --- a/tests/integration/opencv_test.py +++ b/tests/integration/opencv_test.py @@ -1,9 +1,25 @@ +from os.path import join + from integration_tests import EngineCase -from integration_tests.urls_helpers import single_dataset +from integration_tests.urls_helpers import UrlsTester, single_dataset +from tornado.testing import gen_test class OpenCVTest(EngineCase): engine = "opencv_engine" - def test_single_params(self): - single_dataset(self.retrieve, with_gif=False) + @gen_test(timeout=60) + async def test_single_params(self): + if not self._app: + return True + group = list(single_dataset(False)) # FIXME: remove False + count = len(group) + tester = UrlsTester(self.http_client) + + print("Requests count: %d" % count) + for options in group: + joined_parts = join(*options) + url = "unsafe/%s" % joined_parts + await tester.try_url(self.get_url(f"/{url}")) + + tester.report() From f127588d98ec12f1168496308b590505d1e2a46f Mon Sep 17 00:00:00 2001 From: Pablo Santiago Blum de Aguiar Date: Mon, 31 Jan 2022 23:06:52 +0100 Subject: [PATCH 06/11] opencv_engine: Rename submodule back to engine --- opencv_engine/__init__.py | 2 +- opencv_engine/engine.py | 175 +++++++++++++------------ opencv_engine/engine_cv3.py | 253 ------------------------------------ 3 files changed, 96 insertions(+), 334 deletions(-) delete mode 100644 opencv_engine/engine_cv3.py diff --git a/opencv_engine/__init__.py b/opencv_engine/__init__.py index 172bcb9..8749c01 100644 --- a/opencv_engine/__init__.py +++ b/opencv_engine/__init__.py @@ -6,7 +6,7 @@ __version__ = "1.0.1" try: - from opencv_engine.engine_cv3 import Engine # NOQA + from opencv_engine.engine import Engine # NOQA except ImportError: logging.exception( "Could not import opencv_engine. Probably due to setup.py installing it." diff --git a/opencv_engine/engine.py b/opencv_engine/engine.py index 2406912..6c7284f 100644 --- a/opencv_engine/engine.py +++ b/opencv_engine/engine.py @@ -8,11 +8,8 @@ # http://www.opensource.org/licenses/mit-license # Copyright (c) 2014 globo.com timehome@corp.globo.com -try: - import cv -except ImportError: - import cv2.cv as cv - +import cv2 +import numpy as np from colour import Color from pexif import ExifSegment, JpegFile from thumbor.engines import BaseEngine @@ -24,21 +21,31 @@ except ImportError: FILTERS_AVAILABLE = False -FORMATS = {".jpg": "JPEG", ".jpeg": "JPEG", ".gif": "GIF", ".png": "PNG"} +FORMATS = { + ".jpg": "JPEG", + ".jpeg": "JPEG", + ".gif": "GIF", + ".png": "PNG", + ".webp": "WEBP", +} class Engine(BaseEngine): @property def image_depth(self): if self.image is None: - return 8 - return cv.GetImage(self.image).depth + return np.uint8 + return self.image.dtype @property def image_channels(self): if self.image is None: return 3 - return self.image.channels + # if the image is grayscale + try: + return self.image.shape[2] + except IndexError: + return 1 @classmethod def parse_hex_color(cls, color): @@ -49,15 +56,16 @@ def parse_hex_color(cls, color): return None def gen_image(self, size, color_value): - img0 = cv.CreateImage(size, self.image_depth, self.image_channels) if color_value == "transparent": color = (255, 255, 255, 255) + img = np.zeros((size[1], size[0], 4), self.image_depth) else: + img = np.zeros((size[1], size[0], self.image_channels), self.image_depth) color = self.parse_hex_color(color_value) if not color: raise ValueError("Color %s is not valid." % color_value) - cv.Set(img0, color) - return img0 + img[:] = color + return img def create_image(self, buffer): # FIXME: opencv doesn't support gifs, even worse, the library @@ -69,11 +77,9 @@ def create_image(self, buffer): except KeyError: pass - imagefiledata = cv.CreateMatHeader(1, len(buffer), cv.CV_8UC1) - cv.SetData(imagefiledata, buffer, len(buffer)) - img0 = cv.DecodeImageM(imagefiledata, cv.CV_LOAD_IMAGE_UNCHANGED) - + img = cv2.imdecode(np.frombuffer(buffer, np.uint8), -1) if FORMATS[self.extension] == "JPEG": + self.exif = None try: info = JpegFile.fromString(buffer).get_exif() if info: @@ -81,64 +87,60 @@ def create_image(self, buffer): self.exif_marker = info.marker except Exception: pass - - return img0 + return img @property def size(self): - return cv.GetSize(self.image) + return self.image.shape[1], self.image.shape[0] def normalize(self): pass def resize(self, width, height): - thumbnail = cv.CreateImage( - (int(round(width, 0)), int(round(height, 0))), - self.image_depth, - self.image_channels, - ) - cv.Resize(self.image, thumbnail, cv.CV_INTER_AREA) - self.image = thumbnail + r = height / self.size[1] + width = int(self.size[0] * r) + dim = (int(round(width, 0)), int(round(height, 0))) + self.image = cv2.resize(self.image, dim, interpolation=cv2.INTER_AREA) def crop(self, left, top, right, bottom): - new_width = right - left - new_height = bottom - top - cropped = cv.CreateImage( - (new_width, new_height), self.image_depth, self.image_channels - ) - src_region = cv.GetSubRect(self.image, (left, top, new_width, new_height)) - cv.Copy(src_region, cropped) - - self.image = cropped + self.image = self.image[top:bottom, left:right] def rotate(self, degrees): - if degrees > 180: - # Flip around both axes - cv.Flip(self.image, None, -1) - degrees = degrees - 180 - - img = self.image - size = cv.GetSize(img) - - if degrees / 90 % 2: - new_size = (size[1], size[0]) - center = ((size[0] - 1) * 0.5, (size[0] - 1) * 0.5) + # see http://stackoverflow.com/a/23990392 + if degrees == 90: + self.image = cv2.transpose(self.image) + cv2.flip(self.image, 0, self.image) + elif degrees == 180: + cv2.flip(self.image, -1, self.image) + elif degrees == 270: + self.image = cv2.transpose(self.image) + cv2.flip(self.image, 1, self.image) else: - new_size = size - center = ((size[0] - 1) * 0.5, (size[1] - 1) * 0.5) - - mapMatrix = cv.CreateMat(2, 3, cv.CV_64F) - cv.GetRotationMatrix2D(center, degrees, 1.0, mapMatrix) - dst = cv.CreateImage(new_size, self.image_depth, self.image_channels) - cv.SetZero(dst) - cv.WarpAffine(img, dst, mapMatrix) - self.image = dst + # see http://stackoverflow.com/a/37347070 + # one pixel glitch seems to happen with 90/180/270 + # degrees pictures in this algorithm if you check + # the typical github.com/recurser/exif-orientation-examples + # but the above transpose/flip algorithm is working fine + # for those cases already + width, height = self.size + image_center = (width / 2, height / 2) + rot_mat = cv2.getRotationMatrix2D(image_center, degrees, 1.0) + + abs_cos = abs(rot_mat[0, 0]) + abs_sin = abs(rot_mat[0, 1]) + bound_w = int((height * abs_sin) + (width * abs_cos)) + bound_h = int((height * abs_cos) + (width * abs_sin)) + + rot_mat[0, 2] += (bound_w / 2) - image_center[0] + rot_mat[1, 2] += (bound_h / 2) - image_center[1] + + self.image = cv2.warpAffine(self.image, rot_mat, (bound_w, bound_h)) def flip_vertically(self): - cv.Flip(self.image, None, 1) + self.image = np.flipud(self.image) def flip_horizontally(self): - cv.Flip(self.image, None, 0) + self.image = np.fliplr(self.image) def read(self, extension=None, quality=None): if quality is None: @@ -148,15 +150,22 @@ def read(self, extension=None, quality=None): extension = extension or self.extension try: if FORMATS[extension] == "JPEG": - options = [cv.CV_IMWRITE_JPEG_QUALITY, quality] + options = [cv2.IMWRITE_JPEG_QUALITY, quality] except KeyError: # default is JPEG so - options = [cv.CV_IMWRITE_JPEG_QUALITY, quality] + options = [cv2.IMWRITE_JPEG_QUALITY, quality] - data = cv.EncodeImage(extension, self.image, options or []).tostring() + try: + if FORMATS[extension] == "WEBP": + options = [cv2.IMWRITE_WEBP_QUALITY, quality] + except KeyError: + options = [cv2.IMWRITE_JPEG_QUALITY, quality] + + success, buf = cv2.imencode(extension, self.image, options or []) + data = buf.tostring() if FORMATS[extension] == "JPEG" and self.context.config.PRESERVE_EXIF_INFO: - if hasattr(self, "exif"): + if hasattr(self, "exif") and self.exif != None: img = JpegFile.fromString(data) img._segments.insert( 0, ExifSegment(self.exif_marker, None, self.exif, "rw") @@ -166,37 +175,45 @@ def read(self, extension=None, quality=None): return data def set_image_data(self, data): - cv.SetData(self.image, data) + self.image = np.frombuffer(data, dtype=self.image.dtype).reshape( + self.image.shape + ) def image_data_as_rgb(self, update_image=True): - # TODO: Handle other formats if self.image_channels == 4: mode = "BGRA" elif self.image_channels == 3: mode = "BGR" else: mode = "BGR" - rgb_copy = cv.CreateImage((self.image.width, self.image.height), 8, 3) - cv.CvtColor(self.image, rgb_copy, cv.CV_GRAY2BGR) + rgb_copy = np.zeros((self.size[1], self.size[0], 3), self.image.dtype) + cv2.cvtColor(self.image, cv2.COLOR_GRAY2BGR, rgb_copy) self.image = rgb_copy return mode, self.image.tostring() def draw_rectangle(self, x, y, width, height): - cv.Rectangle( + cv2.rectangle( self.image, (int(x), int(y)), (int(x + width), int(y + height)), - cv.Scalar(255, 255, 255, 1.0), + (255, 255, 255), ) - def convert_to_grayscale(self): - if self.image_channels >= 3: - # FIXME: OpenCV does not support grayscale with alpha channel? - grayscaled = cv.CreateImage( - (self.image.width, self.image.height), self.image_depth, 1 - ) - cv.CvtColor(self.image, grayscaled, cv.CV_BGRA2GRAY) - self.image = grayscaled + def convert_to_grayscale(self, update_image=True, alpha=True): + image = None + if self.image_channels >= 3 and alpha: + image = cv2.cvtColor(self.image, cv2.COLOR_BGRA2GRAY) + elif self.image_channels >= 3: + image = cv2.cvtColor(self.image, cv2.COLOR_BGR2GRAY) + elif self.image_channels == 1: + # Already grayscale, + image = self.image + if update_image: + self.image = image + elif self.image_depth == np.uint16: + # Feature detector reqiures uint8 images + image = np.array(image, dtype="uint8") + return image def paste(self, other_engine, pos, merge=True): if merge and not FILTERS_AVAILABLE: @@ -231,11 +248,9 @@ def paste(self, other_engine, pos, merge=True): def enable_alpha(self): if self.image_channels < 4: - with_alpha = cv.CreateImage( - (self.image.width, self.image.height), self.image_depth, 4 - ) + with_alpha = np.zeros((self.size[1], self.size[0], 4), self.image.dtype) if self.image_channels == 3: - cv.CvtColor(self.image, with_alpha, cv.CV_BGR2BGRA) + cv2.cvtColor(self.image, cv2.COLOR_BGR2BGRA, with_alpha) else: - cv.CvtColor(self.image, with_alpha, cv.CV_GRAY2BGRA) + cv2.cvtColor(self.image, cv2.COLOR_GRAY2BGRA, with_alpha) self.image = with_alpha diff --git a/opencv_engine/engine_cv3.py b/opencv_engine/engine_cv3.py deleted file mode 100644 index 0b2e7ac..0000000 --- a/opencv_engine/engine_cv3.py +++ /dev/null @@ -1,253 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Licensed under the MIT license: -# http://www.opensource.org/licenses/mit-license -# Copyright (c) 2016 fanhero.com christian@fanhero.com - -import cv2 -import numpy as np -from colour import Color -from pexif import ExifSegment, JpegFile -from thumbor.engines import BaseEngine - -try: - from thumbor.ext.filters import _composite - - FILTERS_AVAILABLE = True -except ImportError: - FILTERS_AVAILABLE = False - -FORMATS = { - ".jpg": "JPEG", - ".jpeg": "JPEG", - ".gif": "GIF", - ".png": "PNG", - ".webp": "WEBP", -} - - -class Engine(BaseEngine): - @property - def image_depth(self): - if self.image is None: - return np.uint8 - return self.image.dtype - - @property - def image_channels(self): - if self.image is None: - return 3 - # if the image is grayscale - try: - return self.image.shape[2] - except IndexError: - return 1 - - @classmethod - def parse_hex_color(cls, color): - try: - color = Color(color).get_rgb() - return tuple(c * 255 for c in reversed(color)) - except Exception: - return None - - def gen_image(self, size, color_value): - if color_value == "transparent": - color = (255, 255, 255, 255) - img = np.zeros((size[1], size[0], 4), self.image_depth) - else: - img = np.zeros((size[1], size[0], self.image_channels), self.image_depth) - color = self.parse_hex_color(color_value) - if not color: - raise ValueError("Color %s is not valid." % color_value) - img[:] = color - return img - - def create_image(self, buffer): - # FIXME: opencv doesn't support gifs, even worse, the library - # segfaults when trying to decoding a gif. An exception is a - # less drastic measure. - try: - if FORMATS[self.extension] == "GIF": - raise ValueError("opencv doesn't support gifs") - except KeyError: - pass - - img = cv2.imdecode(np.frombuffer(buffer, np.uint8), -1) - if FORMATS[self.extension] == "JPEG": - self.exif = None - try: - info = JpegFile.fromString(buffer).get_exif() - if info: - self.exif = info.data - self.exif_marker = info.marker - except Exception: - pass - return img - - @property - def size(self): - return self.image.shape[1], self.image.shape[0] - - def normalize(self): - pass - - def resize(self, width, height): - r = height / self.size[1] - width = int(self.size[0] * r) - dim = (int(round(width, 0)), int(round(height, 0))) - self.image = cv2.resize(self.image, dim, interpolation=cv2.INTER_AREA) - - def crop(self, left, top, right, bottom): - self.image = self.image[top:bottom, left:right] - - def rotate(self, degrees): - # see http://stackoverflow.com/a/23990392 - if degrees == 90: - self.image = cv2.transpose(self.image) - cv2.flip(self.image, 0, self.image) - elif degrees == 180: - cv2.flip(self.image, -1, self.image) - elif degrees == 270: - self.image = cv2.transpose(self.image) - cv2.flip(self.image, 1, self.image) - else: - # see http://stackoverflow.com/a/37347070 - # one pixel glitch seems to happen with 90/180/270 - # degrees pictures in this algorithm if you check - # the typical github.com/recurser/exif-orientation-examples - # but the above transpose/flip algorithm is working fine - # for those cases already - width, height = self.size - image_center = (width / 2, height / 2) - rot_mat = cv2.getRotationMatrix2D(image_center, degrees, 1.0) - - abs_cos = abs(rot_mat[0, 0]) - abs_sin = abs(rot_mat[0, 1]) - bound_w = int((height * abs_sin) + (width * abs_cos)) - bound_h = int((height * abs_cos) + (width * abs_sin)) - - rot_mat[0, 2] += (bound_w / 2) - image_center[0] - rot_mat[1, 2] += (bound_h / 2) - image_center[1] - - self.image = cv2.warpAffine(self.image, rot_mat, (bound_w, bound_h)) - - def flip_vertically(self): - self.image = np.flipud(self.image) - - def flip_horizontally(self): - self.image = np.fliplr(self.image) - - def read(self, extension=None, quality=None): - if quality is None: - quality = self.context.config.QUALITY - - options = None - extension = extension or self.extension - try: - if FORMATS[extension] == "JPEG": - options = [cv2.IMWRITE_JPEG_QUALITY, quality] - except KeyError: - # default is JPEG so - options = [cv2.IMWRITE_JPEG_QUALITY, quality] - - try: - if FORMATS[extension] == "WEBP": - options = [cv2.IMWRITE_WEBP_QUALITY, quality] - except KeyError: - options = [cv2.IMWRITE_JPEG_QUALITY, quality] - - success, buf = cv2.imencode(extension, self.image, options or []) - data = buf.tostring() - - if FORMATS[extension] == "JPEG" and self.context.config.PRESERVE_EXIF_INFO: - if hasattr(self, "exif") and self.exif != None: - img = JpegFile.fromString(data) - img._segments.insert( - 0, ExifSegment(self.exif_marker, None, self.exif, "rw") - ) - data = img.writeString() - - return data - - def set_image_data(self, data): - self.image = np.frombuffer(data, dtype=self.image.dtype).reshape( - self.image.shape - ) - - def image_data_as_rgb(self, update_image=True): - if self.image_channels == 4: - mode = "BGRA" - elif self.image_channels == 3: - mode = "BGR" - else: - mode = "BGR" - rgb_copy = np.zeros((self.size[1], self.size[0], 3), self.image.dtype) - cv2.cvtColor(self.image, cv2.COLOR_GRAY2BGR, rgb_copy) - self.image = rgb_copy - return mode, self.image.tostring() - - def draw_rectangle(self, x, y, width, height): - cv2.rectangle( - self.image, - (int(x), int(y)), - (int(x + width), int(y + height)), - (255, 255, 255), - ) - - def convert_to_grayscale(self, update_image=True, alpha=True): - image = None - if self.image_channels >= 3 and alpha: - image = cv2.cvtColor(self.image, cv2.COLOR_BGRA2GRAY) - elif self.image_channels >= 3: - image = cv2.cvtColor(self.image, cv2.COLOR_BGR2GRAY) - elif self.image_channels == 1: - # Already grayscale, - image = self.image - if update_image: - self.image = image - elif self.image_depth == np.uint16: - # Feature detector reqiures uint8 images - image = np.array(image, dtype="uint8") - return image - - def paste(self, other_engine, pos, merge=True): - if merge and not FILTERS_AVAILABLE: - raise RuntimeError( - "You need filters enabled to use paste with merge. Please reinstall " - + "thumbor with proper compilation of its filters." - ) - - self.enable_alpha() - other_engine.enable_alpha() - - sz = self.size - other_size = other_engine.size - - mode, data = self.image_data_as_rgb() - other_mode, other_data = other_engine.image_data_as_rgb() - - imgdata = _composite.apply( - mode, - data, - sz[0], - sz[1], - other_data, - other_size[0], - other_size[1], - pos[0], - pos[1], - merge, - ) - - self.set_image_data(imgdata) - - def enable_alpha(self): - if self.image_channels < 4: - with_alpha = np.zeros((self.size[1], self.size[0], 4), self.image.dtype) - if self.image_channels == 3: - cv2.cvtColor(self.image, cv2.COLOR_BGR2BGRA, with_alpha) - else: - cv2.cvtColor(self.image, cv2.COLOR_GRAY2BGRA, with_alpha) - self.image = with_alpha From d415b8e5844c6b19d0bda0650f0f042f9785ace9 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Blum de Aguiar Date: Mon, 31 Jan 2022 23:11:52 +0100 Subject: [PATCH 07/11] engine: Correctly compare a variable to None --- opencv_engine/engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencv_engine/engine.py b/opencv_engine/engine.py index 6c7284f..367f81b 100644 --- a/opencv_engine/engine.py +++ b/opencv_engine/engine.py @@ -165,7 +165,7 @@ def read(self, extension=None, quality=None): data = buf.tostring() if FORMATS[extension] == "JPEG" and self.context.config.PRESERVE_EXIF_INFO: - if hasattr(self, "exif") and self.exif != None: + if hasattr(self, "exif") and self.exif is not None: img = JpegFile.fromString(data) img._segments.insert( 0, ExifSegment(self.exif_marker, None, self.exif, "rw") From 6b51efd6151b3aba0c552fc358244cd07d2acd41 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Blum de Aguiar Date: Mon, 31 Jan 2022 23:25:35 +0100 Subject: [PATCH 08/11] engine: Use piexif for EXIF if available --- opencv_engine/engine.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/opencv_engine/engine.py b/opencv_engine/engine.py index 367f81b..24adf9c 100644 --- a/opencv_engine/engine.py +++ b/opencv_engine/engine.py @@ -8,10 +8,11 @@ # http://www.opensource.org/licenses/mit-license # Copyright (c) 2014 globo.com timehome@corp.globo.com +import io + import cv2 import numpy as np from colour import Color -from pexif import ExifSegment, JpegFile from thumbor.engines import BaseEngine try: @@ -21,6 +22,13 @@ except ImportError: FILTERS_AVAILABLE = False +try: + import piexif + + PIEXIF_AVAILABLE = True +except ImportError: + PIEXIF_AVAILABLE = False + FORMATS = { ".jpg": "JPEG", ".jpeg": "JPEG", @@ -78,15 +86,11 @@ def create_image(self, buffer): pass img = cv2.imdecode(np.frombuffer(buffer, np.uint8), -1) - if FORMATS[self.extension] == "JPEG": - self.exif = None + if FORMATS[self.extension] == "JPEG" and PIEXIF_AVAILABLE: try: - info = JpegFile.fromString(buffer).get_exif() - if info: - self.exif = info.data - self.exif_marker = info.marker + self.exif = piexif.load(buffer) except Exception: - pass + self.exif = None return img @property @@ -166,11 +170,9 @@ def read(self, extension=None, quality=None): if FORMATS[extension] == "JPEG" and self.context.config.PRESERVE_EXIF_INFO: if hasattr(self, "exif") and self.exif is not None: - img = JpegFile.fromString(data) - img._segments.insert( - 0, ExifSegment(self.exif_marker, None, self.exif, "rw") - ) - data = img.writeString() + output = io.BytesIO() + piexif.insert(piexif.dump(self.exif), data, output) + data = output.getvalue() return data From e860168993299f8e27d83839a8f2fb526e67d0e8 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Blum de Aguiar Date: Mon, 31 Jan 2022 23:43:28 +0100 Subject: [PATCH 09/11] tests: Use pytest instead of nose --- Makefile | 6 +++--- setup.py | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index ddc9598..6f708d4 100644 --- a/Makefile +++ b/Makefile @@ -7,14 +7,14 @@ isort: test: unit integration unit: - @coverage run --branch `which nosetests` -vvvv --with-yanc -s tests/unit/ - @coverage report -m + @-pytest -sv --cov=opencv_engine tests/unit/ + @-coverage report -m coverage-html: unit @coverage html -d cover integration: - @`which nosetests` -vvvv --with-yanc -s tests/integration/ + @pytest -sv tests/integration/ setup: @pip install -U -e .\[tests\] diff --git a/setup.py b/setup.py index 5acb0fc..a975555 100644 --- a/setup.py +++ b/setup.py @@ -9,15 +9,13 @@ "black", "colorama", "colour", - "coverage", - "coveralls", "ipdb", "isort", "mock", - "nose", "numpy", "preggy", - "yanc", + "pytest", + "pytest-cov", ] setup( From 28742dd5df13b9611fc84ee936fa417a28c9a906 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Blum de Aguiar Date: Mon, 31 Jan 2022 23:43:54 +0100 Subject: [PATCH 10/11] setup: Remove unneeded or duplicated requirements --- setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.py b/setup.py index a975555..7bfbdfa 100644 --- a/setup.py +++ b/setup.py @@ -11,8 +11,6 @@ "colour", "ipdb", "isort", - "mock", - "numpy", "preggy", "pytest", "pytest-cov", From becb80d624d9b67e8e369f6a0a8aebddd19b4fcd Mon Sep 17 00:00:00 2001 From: Pablo Santiago Blum de Aguiar Date: Mon, 31 Jan 2022 23:45:55 +0100 Subject: [PATCH 11/11] workflows: Ditch TravisCI and embrace GitHub Actions --- .github/workflows/build.yml | 23 +++++++++++++++++++++++ .travis.yml | 19 ------------------- 2 files changed, 23 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/build.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..46bdc5d --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,23 @@ +name: build +on: [push, pull_request] +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image-tag: + - 37 + - 38 + - 39 + - 310 + container: thumbororg/thumbor-test:${{ matrix.image-tag }} + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Setup + run: make setup + - name: Run unit tests + run: make unit + - name: Run integration tests + run: make integration diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index e246cc7..0000000 --- a/.travis.yml +++ /dev/null @@ -1,19 +0,0 @@ -language: python -python: - - 2.7 - -before_install: - # update aptitude - - sudo apt-get update - - # verify all requirements were met - - pip install opencv-python - - INSTALLDIR=$(python -c "import os; import numpy; import cv2; print(os.path.dirname(cv2.__file__))") - -install: - # install python requirements - - make setup - -script: - # finally run tests - - make test