From a4309db29b27ede1ca2d32fa3f886ccb613882b0 Mon Sep 17 00:00:00 2001 From: Ram Mohan M Date: Thu, 21 Nov 2024 03:22:42 +0530 Subject: [PATCH] choose gainmap application space adaptively basing on input gamuts during generateGainMap, the color spaces of hdr intent and sdr intent are unified. if a wider gamut space is converted to narrower-gamut this may result in pixel values less than zero and/or greater than 1. choosing offsets to keep the domain of log during gainmap computation is tricky. This can be mitigated by converting narrower gamut space data to wider gamut space. This yields lesser overrange values. current change implements this. for this to work as intended writing xmp metadata in the bitstream needs to be disabled. Test: ./ultrahdr_app options Co-authored-by: Vivek R Jadhav Change-Id: I0155a1dc043f3c3e0493e04e5a6eabe160659476 --- lib/include/ultrahdr/ultrahdrcommon.h | 7 ++- lib/src/gainmapmetadata.cpp | 13 +--- lib/src/gpu/applygainmap_gl.cpp | 14 ++++- lib/src/jpegr.cpp | 87 ++++++++++++++++++++------- lib/src/jpegrutils.cpp | 1 + tests/gainmapmetadata_test.cpp | 4 ++ tests/jpegr_test.cpp | 2 + 7 files changed, 90 insertions(+), 38 deletions(-) diff --git a/lib/include/ultrahdr/ultrahdrcommon.h b/lib/include/ultrahdr/ultrahdrcommon.h index 67a3d066..582385e4 100644 --- a/lib/include/ultrahdr/ultrahdrcommon.h +++ b/lib/include/ultrahdr/ultrahdrcommon.h @@ -204,9 +204,10 @@ typedef struct uhdr_effect_desc uhdr_effect_desc_t; typedef struct uhdr_gainmap_metadata_ext : uhdr_gainmap_metadata { uhdr_gainmap_metadata_ext() {} - uhdr_gainmap_metadata_ext(std::string ver) { version = ver; } + uhdr_gainmap_metadata_ext(std::string ver) : version(ver), use_base_cg(true) {} - uhdr_gainmap_metadata_ext(uhdr_gainmap_metadata& metadata, std::string ver) { + uhdr_gainmap_metadata_ext(uhdr_gainmap_metadata& metadata, std::string ver) + : uhdr_gainmap_metadata_ext(ver) { max_content_boost = metadata.max_content_boost; min_content_boost = metadata.min_content_boost; gamma = metadata.gamma; @@ -214,10 +215,10 @@ typedef struct uhdr_gainmap_metadata_ext : uhdr_gainmap_metadata { offset_hdr = metadata.offset_hdr; hdr_capacity_min = metadata.hdr_capacity_min; hdr_capacity_max = metadata.hdr_capacity_max; - version = ver; } std::string version; /**< Ultra HDR format version */ + bool use_base_cg; /**< Is gainmap application space base color space */ } uhdr_gainmap_metadata_ext_t; /**< alias for struct uhdr_gainmap_metadata */ #ifdef UHDR_ENABLE_GLES diff --git a/lib/src/gainmapmetadata.cpp b/lib/src/gainmapmetadata.cpp index 6979c82d..2516b115 100644 --- a/lib/src/gainmapmetadata.cpp +++ b/lib/src/gainmapmetadata.cpp @@ -344,16 +344,6 @@ uhdr_error_info_t uhdr_gainmap_metadata_frac::gainmapMetadataFractionToFloat( return status; } - // TODO: parse gainmap image icc and use it for color conversion during applygainmap - if (!from->useBaseColorSpace) { - uhdr_error_info_t status; - status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE; - status.has_detail = 1; - snprintf(status.detail, sizeof status.detail, - "current implementation requires gainmap application space to match base color space"); - return status; - } - to->version = kJpegrVersion; to->max_content_boost = exp2((float)from->gainMapMaxN[0] / from->gainMapMaxD[0]); to->min_content_boost = exp2((float)from->gainMapMinN[0] / from->gainMapMinD[0]); @@ -365,6 +355,7 @@ uhdr_error_info_t uhdr_gainmap_metadata_frac::gainmapMetadataFractionToFloat( to->offset_hdr = (float)from->alternateOffsetN[0] / from->alternateOffsetD[0]; to->hdr_capacity_max = exp2((float)from->alternateHdrHeadroomN / from->alternateHdrHeadroomD); to->hdr_capacity_min = exp2((float)from->baseHdrHeadroomN / from->baseHdrHeadroomD); + to->use_base_cg = from->useBaseColorSpace; return g_no_error; } @@ -381,7 +372,7 @@ uhdr_error_info_t uhdr_gainmap_metadata_frac::gainmapMetadataFloatToFraction( } to->backwardDirection = false; - to->useBaseColorSpace = true; + to->useBaseColorSpace = from->use_base_cg; #define CONVERT_FLT_TO_UNSIGNED_FRACTION(flt, numerator, denominator) \ if (!floatToUnsignedFraction(flt, numerator, denominator)) { \ diff --git a/lib/src/gpu/applygainmap_gl.cpp b/lib/src/gpu/applygainmap_gl.cpp index 28a7ae04..6454497b 100644 --- a/lib/src/gpu/applygainmap_gl.cpp +++ b/lib/src/gpu/applygainmap_gl.cpp @@ -244,7 +244,7 @@ std::string getGamutConversionShader(uhdr_color_gamut_t src_cg, uhdr_color_gamut std::string getApplyGainMapFragmentShader(uhdr_img_fmt sdr_fmt, uhdr_img_fmt gm_fmt, uhdr_color_transfer output_ct, uhdr_color_gamut_t sdr_cg, - uhdr_color_gamut_t hdr_cg) { + uhdr_color_gamut_t hdr_cg, bool use_base_cg) { std::string shader_code = R"__SHADER__(#version 300 es precision highp float; precision highp int; @@ -278,10 +278,17 @@ std::string getApplyGainMapFragmentShader(uhdr_img_fmt sdr_fmt, uhdr_img_fmt gm_ vec3 yuv_gamma_sdr = getYUVPixel(); vec3 rgb_gamma_sdr = p3YuvToRgb(yuv_gamma_sdr); vec3 rgb_sdr = sRGBEOTF(rgb_gamma_sdr); + )__SHADER__"); + if (sdr_cg != hdr_cg && !use_base_cg) { + shader_code.append(R"__SHADER__( + rgb_sdr = gamutConversion(rgb_sdr); + )__SHADER__"); + } + shader_code.append(R"__SHADER__( vec3 gain = sampleMap(gainMapTexture); vec3 rgb_hdr = applyGain(rgb_sdr, gain); )__SHADER__"); - if (sdr_cg != hdr_cg) { + if (sdr_cg != hdr_cg && use_base_cg) { shader_code.append(R"__SHADER__( rgb_hdr = gamutConversion(rgb_hdr); )__SHADER__"); @@ -354,7 +361,8 @@ uhdr_error_info_t applyGainMapGLES(uhdr_raw_image_t* sdr_intent, uhdr_raw_image_ shaderProgram = opengl_ctxt->create_shader_program( vertex_shader.c_str(), - getApplyGainMapFragmentShader(sdr_intent->fmt, gainmap_img->fmt, output_ct, sdr_cg, hdr_cg) + getApplyGainMapFragmentShader(sdr_intent->fmt, gainmap_img->fmt, output_ct, sdr_cg, hdr_cg, + gainmap_metadata->use_base_cg) .c_str()); RET_IF_ERR() diff --git a/lib/src/jpegr.cpp b/lib/src/jpegr.cpp index 458f6082..36bdffb1 100644 --- a/lib/src/jpegr.cpp +++ b/lib/src/jpegr.cpp @@ -587,15 +587,40 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ return status; } - ColorTransformFn hdrGamutConversionFn = getGamutConversionFn(sdr_intent->cg, hdr_intent->cg); - if (hdrGamutConversionFn == nullptr) { - status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE; - status.has_detail = 1; - snprintf(status.detail, sizeof status.detail, - "No implementation available for gamut conversion from %d to %d", hdr_intent->cg, - sdr_intent->cg); - return status; + ColorTransformFn hdrGamutConversionFn; + ColorTransformFn sdrGamutConversionFn; + bool use_sdr_cg = true; + if (sdr_intent->cg != hdr_intent->cg) { + use_sdr_cg = kWriteXmpMetadata || + !(hdr_intent->cg == UHDR_CG_BT_2100 || + (hdr_intent->cg == UHDR_CG_DISPLAY_P3 && sdr_intent->cg != UHDR_CG_BT_2100)); + if (use_sdr_cg) { + hdrGamutConversionFn = getGamutConversionFn(sdr_intent->cg, hdr_intent->cg); + if (hdrGamutConversionFn == nullptr) { + status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE; + status.has_detail = 1; + snprintf(status.detail, sizeof status.detail, + "No implementation available for gamut conversion from %d to %d", hdr_intent->cg, + sdr_intent->cg); + return status; + } + sdrGamutConversionFn = identityConversion; + } else { + hdrGamutConversionFn = identityConversion; + sdrGamutConversionFn = getGamutConversionFn(hdr_intent->cg, sdr_intent->cg); + if (sdrGamutConversionFn == nullptr) { + status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE; + status.has_detail = 1; + snprintf(status.detail, sizeof status.detail, + "No implementation available for gamut conversion from %d to %d", sdr_intent->cg, + hdr_intent->cg); + return status; + } + } + } else { + hdrGamutConversionFn = sdrGamutConversionFn = identityConversion; } + gainmap_metadata->use_base_cg = use_sdr_cg; ColorTransformFn sdrYuvToRgbFn = getYuvToRgbFn(sdr_intent->cg); if (sdrYuvToRgbFn == nullptr) { @@ -678,8 +703,9 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ auto generateGainMapOnePass = [this, sdr_intent, hdr_intent, gainmap_metadata, dest, map_height, hdrInvOetf, hdrLuminanceFn, hdrOotfFn, hdrGamutConversionFn, - luminanceFn, sdrYuvToRgbFn, hdrYuvToRgbFn, sdr_sample_pixel_fn, - hdr_sample_pixel_fn, hdr_white_nits, use_luminance]() -> void { + sdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn, hdrYuvToRgbFn, + sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits, + use_luminance]() -> void { gainmap_metadata->max_content_boost = hdr_white_nits / kSdrWhiteNits; gainmap_metadata->min_content_boost = 1.0f; gainmap_metadata->gamma = mGamma; @@ -701,9 +727,9 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ JobQueue jobQueue; std::function generateMap = [this, sdr_intent, hdr_intent, gainmap_metadata, dest, hdrInvOetf, hdrLuminanceFn, - hdrOotfFn, hdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn, hdrYuvToRgbFn, - sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits, log2MinBoost, log2MaxBoost, - use_luminance, &jobQueue]() -> void { + hdrOotfFn, hdrGamutConversionFn, sdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn, + hdrYuvToRgbFn, sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits, log2MinBoost, + log2MaxBoost, use_luminance, &jobQueue]() -> void { unsigned int rowStart, rowEnd; const bool isHdrIntentRgb = isPixelFormatRgb(hdr_intent->fmt); const bool isSdrIntentRgb = isPixelFormatRgb(sdr_intent->fmt); @@ -727,6 +753,8 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ #else Color sdr_rgb = srgbInvOetf(sdr_rgb_gamma); #endif + sdr_rgb = sdrGamutConversionFn(sdr_rgb); + sdr_rgb = clipNegatives(sdr_rgb); Color hdr_rgb_gamma; @@ -791,10 +819,11 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ std::for_each(workers.begin(), workers.end(), [](std::thread& t) { t.join(); }); }; - auto generateGainMapTwoPass = - [this, sdr_intent, hdr_intent, gainmap_metadata, dest, map_width, map_height, hdrInvOetf, - hdrLuminanceFn, hdrOotfFn, hdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn, hdrYuvToRgbFn, - sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits, use_luminance]() -> void { + auto generateGainMapTwoPass = [this, sdr_intent, hdr_intent, gainmap_metadata, dest, map_width, + map_height, hdrInvOetf, hdrLuminanceFn, hdrOotfFn, + hdrGamutConversionFn, sdrGamutConversionFn, luminanceFn, + sdrYuvToRgbFn, hdrYuvToRgbFn, sdr_sample_pixel_fn, + hdr_sample_pixel_fn, hdr_white_nits, use_luminance]() -> void { uhdr_memory_block_t gainmap_mem((size_t)map_width * map_height * sizeof(float) * (mUseMultiChannelGainMap ? 3 : 1)); float* gainmap_data = reinterpret_cast(gainmap_mem.m_buffer.get()); @@ -808,9 +837,9 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ JobQueue jobQueue; std::function generateMap = [this, sdr_intent, hdr_intent, gainmap_data, map_width, hdrInvOetf, hdrLuminanceFn, - hdrOotfFn, hdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn, hdrYuvToRgbFn, - sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits, use_luminance, &gainmap_min, - &gainmap_max, &gainmap_minmax, &jobQueue]() -> void { + hdrOotfFn, hdrGamutConversionFn, sdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn, + hdrYuvToRgbFn, sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits, use_luminance, + &gainmap_min, &gainmap_max, &gainmap_minmax, &jobQueue]() -> void { unsigned int rowStart, rowEnd; const bool isHdrIntentRgb = isPixelFormatRgb(hdr_intent->fmt); const bool isSdrIntentRgb = isPixelFormatRgb(sdr_intent->fmt); @@ -837,6 +866,8 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ #else Color sdr_rgb = srgbInvOetf(sdr_rgb_gamma); #endif + sdr_rgb = sdrGamutConversionFn(sdr_rgb); + sdr_rgb = clipNegatives(sdr_rgb); Color hdr_rgb_gamma; @@ -1410,8 +1441,19 @@ uhdr_error_info_t JpegR::applyGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ima uhdr_color_gamut_t sdr_cg = sdr_intent->cg == UHDR_CG_UNSPECIFIED ? UHDR_CG_BT_709 : sdr_intent->cg; uhdr_color_gamut_t hdr_cg = gainmap_img->cg == UHDR_CG_UNSPECIFIED ? sdr_cg : gainmap_img->cg; - ColorTransformFn hdrGamutConversionFn = getGamutConversionFn(hdr_cg, sdr_cg); dest->cg = hdr_cg; + ColorTransformFn hdrGamutConversionFn = + gainmap_metadata->use_base_cg ? getGamutConversionFn(hdr_cg, sdr_cg) : identityConversion; + ColorTransformFn sdrGamutConversionFn = + gainmap_metadata->use_base_cg ? identityConversion : getGamutConversionFn(hdr_cg, sdr_cg); + if (hdrGamutConversionFn == nullptr || sdrGamutConversionFn == nullptr) { + uhdr_error_info_t status; + status.error_code = UHDR_CODEC_ERROR; + status.has_detail = 1; + snprintf(status.detail, sizeof status.detail, + "No implementation available for converting from gamut %d to %d", sdr_cg, hdr_cg); + return status; + } #ifdef UHDR_ENABLE_GLES if (mUhdrGLESCtxt != nullptr) { @@ -1485,6 +1527,7 @@ uhdr_error_info_t JpegR::applyGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ima JobQueue jobQueue; std::function applyRecMap = [sdr_intent, gainmap_img, dest, &jobQueue, &idwTable, output_ct, &gainLUT, gainmap_metadata, hdrGamutConversionFn, + sdrGamutConversionFn, #if !USE_APPLY_GAIN_LUT gainmap_weight, #endif @@ -1504,6 +1547,7 @@ uhdr_error_info_t JpegR::applyGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ima #else Color rgb_sdr = srgbInvOetf(rgb_gamma_sdr); #endif + rgb_sdr = sdrGamutConversionFn(rgb_sdr); Color rgb_hdr; if (gainmap_img->fmt == UHDR_IMG_FMT_8bppYCbCr400) { float gain; @@ -2488,6 +2532,7 @@ status_t JpegR::encodeJPEGR(jr_compressed_ptr yuv420jpg_image_ptr, uhdr_gainmap_metadata_ext_t meta; meta.version = metadata->version; + meta.use_base_cg = true; meta.hdr_capacity_max = metadata->hdrCapacityMax; meta.hdr_capacity_min = metadata->hdrCapacityMin; meta.gamma = metadata->gamma; diff --git a/lib/src/jpegrutils.cpp b/lib/src/jpegrutils.cpp index 463a3595..079be01f 100644 --- a/lib/src/jpegrutils.cpp +++ b/lib/src/jpegrutils.cpp @@ -623,6 +623,7 @@ uhdr_error_info_t getMetadataFromXMP(uint8_t* xmp_data, size_t xmp_size, snprintf(status.detail, sizeof status.detail, "hdr intent as base rendition is not supported"); return status; } + metadata->use_base_cg = true; return g_no_error; } diff --git a/tests/gainmapmetadata_test.cpp b/tests/gainmapmetadata_test.cpp index 18eb68e3..400a7bef 100644 --- a/tests/gainmapmetadata_test.cpp +++ b/tests/gainmapmetadata_test.cpp @@ -50,6 +50,7 @@ TEST_F(GainMapMetadataTest, encodeMetadataThenDecode) { expected.offset_hdr = 0.0625f; expected.hdr_capacity_min = 1.0f; expected.hdr_capacity_max = 10000.0f / 203.0f; + expected.use_base_cg = false; uhdr_gainmap_metadata_frac metadata; EXPECT_EQ( @@ -78,12 +79,14 @@ TEST_F(GainMapMetadataTest, encodeMetadataThenDecode) { EXPECT_FLOAT_EQ(expected.offset_hdr, decodedUHdrMetadata.offset_hdr); EXPECT_FLOAT_EQ(expected.hdr_capacity_min, decodedUHdrMetadata.hdr_capacity_min); EXPECT_FLOAT_EQ(expected.hdr_capacity_max, decodedUHdrMetadata.hdr_capacity_max); + EXPECT_EQ(expected.use_base_cg, decodedUHdrMetadata.use_base_cg); data.clear(); expected.min_content_boost = 0.000578369f; expected.offset_sdr = -0.0625f; expected.offset_hdr = -0.0625f; expected.hdr_capacity_max = 1000.0f / 203.0f; + expected.use_base_cg = true; EXPECT_EQ( uhdr_gainmap_metadata_frac::gainmapMetadataFloatToFraction(&expected, &metadata).error_code, @@ -104,6 +107,7 @@ TEST_F(GainMapMetadataTest, encodeMetadataThenDecode) { EXPECT_FLOAT_EQ(expected.offset_hdr, decodedUHdrMetadata.offset_hdr); EXPECT_FLOAT_EQ(expected.hdr_capacity_min, decodedUHdrMetadata.hdr_capacity_min); EXPECT_FLOAT_EQ(expected.hdr_capacity_max, decodedUHdrMetadata.hdr_capacity_max); + EXPECT_EQ(expected.use_base_cg, decodedUHdrMetadata.use_base_cg); } } // namespace ultrahdr diff --git a/tests/jpegr_test.cpp b/tests/jpegr_test.cpp index 82db6bf7..c3f09aba 100644 --- a/tests/jpegr_test.cpp +++ b/tests/jpegr_test.cpp @@ -1403,6 +1403,7 @@ TEST(JpegRTest, DecodeAPIWithInvalidArgs) { TEST(JpegRTest, writeXmpThenRead) { uhdr_gainmap_metadata_ext_t metadata_expected; metadata_expected.version = "1.0"; + metadata_expected.use_base_cg = true; metadata_expected.max_content_boost = 1.25f; metadata_expected.min_content_boost = 0.75f; metadata_expected.gamma = 1.0f; @@ -1432,6 +1433,7 @@ TEST(JpegRTest, writeXmpThenRead) { EXPECT_FLOAT_EQ(metadata_expected.offset_hdr, metadata_read.offset_hdr); EXPECT_FLOAT_EQ(metadata_expected.hdr_capacity_min, metadata_read.hdr_capacity_min); EXPECT_FLOAT_EQ(metadata_expected.hdr_capacity_max, metadata_read.hdr_capacity_max); + EXPECT_TRUE(metadata_read.use_base_cg); } class JpegRAPIEncodeAndDecodeTest