From 787c190d6c68f9dc6338d37a0c8c8972a302fb98 Mon Sep 17 00:00:00 2001 From: JohnnyChen Date: Sun, 6 Oct 2019 18:56:14 +0800 Subject: [PATCH 1/5] break `handlers.jl` into default_rendermode, default_equality and _convert To avoid code duplication, codes in `handler.jl` are broken down into three functions: * `default_rendermode` that infers the most appropriate render mode according to DataFormat and type of `actual` * `default_equality` that infers the suitable equality compare method. * `_convert` that converts `actual` so that it has the same type of `reference`. --- src/ReferenceTests.jl | 3 +- src/conversion.jl | 35 ++++++++++++++++++++++ src/core.jl | 27 +++++++++++++++-- src/equality_metrics.jl | 18 ++++++++++-- src/handlers.jl | 65 ----------------------------------------- src/render.jl | 19 ++++++++++++ src/test_reference.jl | 7 +++-- 7 files changed, 101 insertions(+), 73 deletions(-) create mode 100644 src/conversion.jl delete mode 100644 src/handlers.jl create mode 100644 src/render.jl diff --git a/src/ReferenceTests.jl b/src/ReferenceTests.jl index ebfe0f4..a9e368e 100644 --- a/src/ReferenceTests.jl +++ b/src/ReferenceTests.jl @@ -18,7 +18,8 @@ export include("utils.jl") include("test_reference.jl") include("core.jl") -include("handlers.jl") +include("conversion.jl") include("equality_metrics.jl") +include("render.jl") end # module diff --git a/src/conversion.jl b/src/conversion.jl new file mode 100644 index 0000000..87a95fb --- /dev/null +++ b/src/conversion.jl @@ -0,0 +1,35 @@ +""" + _convert(T::Type{<:DataFormat}, x; kw...) -> out + +Convert `x` to a validate content for file data format `T`. +""" +_convert(::Type{<:DataFormat}, x; kw...) = x + +# plain TXT +_convert(::Type{DataFormat{:TXT}}, x; kw...) = string(x) +_convert(::Type{DataFormat{:TXT}}, x::Number; kw...) = x +function _convert(::Type{DataFormat{:TXT}}, x::AbstractArray{<:AbstractString}; kw...) + return join(x, '\n') +end +function _convert( + ::Type{DataFormat{:TXT}}, img::AbstractArray{<:Colorant}; + size = (20,40), kw...) + + # encode image into string + strs = @withcolor ImageInTerminal.encodeimg( + ImageInTerminal.SmallBlocks(), + ImageInTerminal.TermColor256(), + img, + size...)[1] + return join(strs,'\n') +end + +# SHA256 +_convert(::Type{DataFormat{:SHA256}}, x; kw...) = bytes2hex(sha256(string(x))) +function _convert(::Type{DataFormat{:SHA256}}, img::AbstractArray{<:Colorant}; kw...) + # encode image into SHA256 + size_str = bytes2hex(sha256(reinterpret(UInt8,[map(Int64,size(img))...]))) + img_str = bytes2hex(sha256(reinterpret(UInt8,vec(rawview(channelview(img)))))) + + return size_str * img_str +end diff --git a/src/core.jl b/src/core.jl index 19c2094..430c3a5 100644 --- a/src/core.jl +++ b/src/core.jl @@ -76,14 +76,27 @@ end # all other functions should hit one of this eventually # Which handles the actual testing and user prompting -function _test_reference(equiv, rendermode, file::File, actual::T) where T +function test_reference( + file::File{F}, + raw_actual::T, + equiv=nothing, + rendermode=nothing; + kw...) where {F <: DataFormat, T} + path = file.filename dir, filename = splitdir(path) + # infer the default rendermode here + # since `nothing` is always passed to this method from + # test_reference(filename::AbstractString, raw_actual; kw...) + if rendermode === nothing + rendermode = default_rendermode(F, raw_actual) + end + # preprocessing when reference file doesn't exists if !isfile(path) println("Reference file for \"$filename\" does not exist.") - render(rendermode, actual) + render(rendermode, raw_actual) if !isinteractive() error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to create new reference images") @@ -93,14 +106,22 @@ function _test_reference(equiv, rendermode, file::File, actual::T) where T @test false else mkpath(dir) - savefile(file, actual) + savefile(file, raw_actual) @info("Please run the tests again for any changes to take effect") end return nothing # skip current test case end + # file exists + actual = _convert(F, raw_actual; kw...) reference = loadfile(T, file) + + if equiv === nothing + # generally, `reference` and `actual` are of the same type after preprocessing + equiv = default_equality(reference, actual) + end + if equiv(reference, actual) @test true # to increase test counter if reached else diff --git a/src/equality_metrics.jl b/src/equality_metrics.jl index bb7923e..c011573 100644 --- a/src/equality_metrics.jl +++ b/src/equality_metrics.jl @@ -1,8 +1,22 @@ +""" + default_equality(reference, actual) -> f + +Infer a suitable equality comparison method `f` according to input types. + +`f` is a function that satisfies signature `f(reference, actual)::Bool`. If `f` outputs +`true`, it indicates that `reference` and `actual` are "equal" in the sense of `f`. +""" +default_equality(reference, actual) = isequal +function default_equality( + reference::AbstractArray{<:Colorant}, + actual::AbstractArray{<:Colorant}) + + return psnr_equality() +end + # --------------------------------- # Image -default_image_equality(reference, actual) = psnr_equality()(reference, actual) - """ psnr_equality(threshold=25) -> f diff --git a/src/handlers.jl b/src/handlers.jl deleted file mode 100644 index f766ba8..0000000 --- a/src/handlers.jl +++ /dev/null @@ -1,65 +0,0 @@ -# -------------------------------------------------------------------- -# plain TXT - -function test_reference(file::File{format"TXT"}, actual; by = isequal, render = Diff()) - _test_reference(by, render, file, string(actual)) -end - -function test_reference(file::File{format"TXT"}, actual::Number; by = isequal, render = BeforeAfterFull()) - _test_reference(by, render, file, actual) -end - -function test_reference(file::File{format"TXT"}, actual::AbstractArray{<:AbstractString}; by = isequal, render = Diff()) - str = join(actual, '\n') - _test_reference(by, render, file, str) -end - -# --------------------------------- -# Image - -function test_reference(file::File, actual::AbstractArray{<:Colorant}; by = default_image_equality, render = BeforeAfterImage()) - _test_reference(by, render, file, actual) -end - -# Image as txt using ImageInTerminal -function test_reference(file::File{format"TXT"}, actual::AbstractArray{<:Colorant}; size = (20,40), by = isequal, render = BeforeAfterFull()) - strs = @withcolor ImageInTerminal.encodeimg(ImageInTerminal.SmallBlocks(), ImageInTerminal.TermColor256(), actual, size...)[1] - str = join(strs,'\n') - _test_reference(by, render, file, str) -end - -# -------------------------------------------------------------------- -# SHA as string - -function test_reference(file::File{format"SHA256"}, actual; by = nothing, render = BeforeAfterFull()) - test_reference(file, string(actual); render = render) -end - -function test_reference(file::File{format"SHA256"}, actual::Union{AbstractString,Vector{UInt8}}; by = nothing, render = BeforeAfterFull()) - str = bytes2hex(sha256(actual)) - _test_reference(isequal, render, file, str) -end - -function test_reference(file::File{format"SHA256"}, actual::AbstractArray{<:Colorant}; by = nothing, render = BeforeAfterFull()) - size_str = bytes2hex(sha256(reinterpret(UInt8,[map(Int64,size(actual))...]))) - img_str = bytes2hex(sha256(reinterpret(UInt8,vec(rawview(channelview(actual)))))) - _test_reference(isequal, render, file, size_str * img_str) -end - -# -------------------------------------------------------------------- - -# Fallback -function test_reference(file::File, actual; by = isequal, render = nothing) - if !(render === nothing) - _test_reference(by, render, file, actual) - else - if actual isa AbstractString - # we don't use dispatch for this as it is very ambiguous - # specialization will remove this conditional regardless - - _test_reference(by, Diff(), file, actual) - else - _test_reference(by, BeforeAfterLimited(), file, actual) - end - end -end diff --git a/src/render.jl b/src/render.jl new file mode 100644 index 0000000..15736ec --- /dev/null +++ b/src/render.jl @@ -0,0 +1,19 @@ +""" + default_rendermode(::DataFormat, actual) + +Infer the most appropriate render mode according to type of reference file and `actual`. +""" +default_rendermode(::Type{<:DataFormat}, ::Any) = BeforeAfterLimited() +default_rendermode(::Type{<:DataFormat}, ::AbstractString) = Diff() +default_rendermode(::Type{<:DataFormat}, ::AbstractArray{<:Colorant}) = BeforeAfterImage() + +# plain TXTs +default_rendermode(::Type{DataFormat{:TXT}}, ::Any) = Diff() +default_rendermode(::Type{DataFormat{:TXT}}, ::AbstractString) = Diff() +default_rendermode(::Type{DataFormat{:TXT}}, ::Number) = BeforeAfterFull() +default_rendermode(::Type{DataFormat{:TXT}}, ::AbstractArray{<:Colorant}) = BeforeAfterFull() + +# SHA256 +default_rendermode(::Type{DataFormat{:SHA256}}, ::Any) = BeforeAfterFull() +default_rendermode(::Type{DataFormat{:SHA256}}, ::AbstractString) = BeforeAfterFull() +default_rendermode(::Type{DataFormat{:SHA256}}, ::AbstractArray{<:Colorant}) = BeforeAfterFull() diff --git a/src/test_reference.jl b/src/test_reference.jl index 8bbfccd..37f4da4 100644 --- a/src/test_reference.jl +++ b/src/test_reference.jl @@ -83,8 +83,11 @@ macro test_reference(reference, actual, kws...) expr end -function test_reference(filename::AbstractString, actual; kw...) - test_reference(query_extended(filename), actual; kw...) +function test_reference( + filename::AbstractString, raw_actual; + by = nothing, render = nothing, kw...) + + test_reference(query_extended(filename), raw_actual, by, render; kw...) end function query_extended(filename) From 3a34f11c22f99216584cd7deee91ba1ba1d418ea Mon Sep 17 00:00:00 2001 From: JohnnyChen Date: Fri, 11 Oct 2019 23:46:06 +0800 Subject: [PATCH 2/5] move file related codes into fileio.jl --- src/ReferenceTests.jl | 2 +- src/core.jl | 26 --------------------- src/{conversion.jl => fileio.jl} | 40 ++++++++++++++++++++++++++++++++ src/test_reference.jl | 14 ----------- 4 files changed, 41 insertions(+), 41 deletions(-) rename src/{conversion.jl => fileio.jl} (54%) diff --git a/src/ReferenceTests.jl b/src/ReferenceTests.jl index a9e368e..c3ef4ee 100644 --- a/src/ReferenceTests.jl +++ b/src/ReferenceTests.jl @@ -18,7 +18,7 @@ export include("utils.jl") include("test_reference.jl") include("core.jl") -include("conversion.jl") +include("fileio.jl") include("equality_metrics.jl") include("render.jl") diff --git a/src/core.jl b/src/core.jl index 430c3a5..f1d4608 100644 --- a/src/core.jl +++ b/src/core.jl @@ -44,32 +44,6 @@ function render(mode::RenderMode, actual) println("-------------------------------") end -####################################### -# IO -# Right now this basically just extends FileIO to support some things as text files - -const TextFile = Union{File{format"TXT"}, File{format"SHA256"}} - -function loadfile(T, file::File) - T(load(file)) # Fallback to FileIO -end - -function loadfile(T, file::TextFile) - read(file.filename, String) -end - -function loadfile(::Type{<:Number}, file::File{format"TXT"}) - parse(Float64, loadfile(String, file)) -end - -function savefile(file::File, content) - save(file, content) # Fallback to FileIO -end - -function savefile(file::TextFile, content) - write(file.filename, content) -end - ########################################## # Final function diff --git a/src/conversion.jl b/src/fileio.jl similarity index 54% rename from src/conversion.jl rename to src/fileio.jl index 87a95fb..fe63d63 100644 --- a/src/conversion.jl +++ b/src/fileio.jl @@ -1,3 +1,43 @@ +####################################### +# IO +# Right now this basically just extends FileIO to support some things as text files + +const TextFile = Union{File{format"TXT"}, File{format"SHA256"}} + +function loadfile(T, file::File) + T(load(file)) # Fallback to FileIO +end + +function loadfile(T, file::TextFile) + read(file.filename, String) +end + +function loadfile(::Type{<:Number}, file::File{format"TXT"}) + parse(Float64, loadfile(String, file)) +end + +function savefile(file::File, content) + save(file, content) # Fallback to FileIO +end + +function savefile(file::TextFile, content) + write(file.filename, content) +end + +function query_extended(filename) + file, ext = splitext(filename) + # TODO: make this less hacky + if uppercase(ext) == ".SHA256" + res = File{format"SHA256"}(filename) + else + res = query(filename) + if res isa File{DataFormat{:UNKNOWN}} + res = File{format"TXT"}(filename) + end + end + res +end + """ _convert(T::Type{<:DataFormat}, x; kw...) -> out diff --git a/src/test_reference.jl b/src/test_reference.jl index 37f4da4..1153256 100644 --- a/src/test_reference.jl +++ b/src/test_reference.jl @@ -89,17 +89,3 @@ function test_reference( test_reference(query_extended(filename), raw_actual, by, render; kw...) end - -function query_extended(filename) - file, ext = splitext(filename) - # TODO: make this less hacky - if uppercase(ext) == ".SHA256" - res = File{format"SHA256"}(filename) - else - res = query(filename) - if res isa File{DataFormat{:UNKNOWN}} - res = File{format"TXT"}(filename) - end - end - res -end From 390fff98c7be57828a3911cfe042ad7fd3cd56fd Mon Sep 17 00:00:00 2001 From: JohnnyChen Date: Fri, 11 Oct 2019 23:46:52 +0800 Subject: [PATCH 3/5] move render related codes into render.jl --- src/core.jl | 46 ---------------------------------------------- src/render.jl | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/src/core.jl b/src/core.jl index f1d4608..dea05c3 100644 --- a/src/core.jl +++ b/src/core.jl @@ -1,49 +1,3 @@ -################################################# -# Rendering -# This controls how failures are displayed -abstract type RenderMode end -struct Diff <: RenderMode end - -abstract type BeforeAfter <: RenderMode end -struct BeforeAfterLimited <: BeforeAfter end -struct BeforeAfterFull <: BeforeAfter end -struct BeforeAfterImage <: BeforeAfter end - -render_item(::RenderMode, item) = println(item) -function render_item(::BeforeAfterLimited, item) - show(IOContext(stdout, :limit=>true, :displaysize=>(20,80)), "text/plain", item) - println() -end -function render_item(::BeforeAfterImage, item) - str_item = @withcolor ImageInTerminal.encodeimg(ImageInTerminal.SmallBlocks(), ImageInTerminal.TermColor256(), item, 20, 40)[1] - println("eltype: ", eltype(item)) - println("size: ", map(length, axes(item))) - println("thumbnail:") - println.(str_item) -end - -## 2 arg form render for comparing -function render(mode::BeforeAfter, reference, actual) - println("- REFERENCE -------------------") - render_item(mode, reference) - println("-------------------------------") - println("- ACTUAL ----------------------") - render_item(mode, actual) - println("-------------------------------") -end -function render(::Diff, reference, actual) - println("- DIFF ------------------------") - @withcolor println(deepdiff(reference, actual)) - println("-------------------------------") -end - -## 1 arg form render for new content -function render(mode::RenderMode, actual) - println("- NEW CONTENT -----------------") - render_item(mode, actual) - println("-------------------------------") -end - ########################################## # Final function diff --git a/src/render.jl b/src/render.jl index 15736ec..267d141 100644 --- a/src/render.jl +++ b/src/render.jl @@ -1,3 +1,49 @@ +################################################# +# Rendering +# This controls how failures are displayed +abstract type RenderMode end +struct Diff <: RenderMode end + +abstract type BeforeAfter <: RenderMode end +struct BeforeAfterLimited <: BeforeAfter end +struct BeforeAfterFull <: BeforeAfter end +struct BeforeAfterImage <: BeforeAfter end + +render_item(::RenderMode, item) = println(item) +function render_item(::BeforeAfterLimited, item) + show(IOContext(stdout, :limit=>true, :displaysize=>(20,80)), "text/plain", item) + println() +end +function render_item(::BeforeAfterImage, item) + str_item = @withcolor ImageInTerminal.encodeimg(ImageInTerminal.SmallBlocks(), ImageInTerminal.TermColor256(), item, 20, 40)[1] + println("eltype: ", eltype(item)) + println("size: ", map(length, axes(item))) + println("thumbnail:") + println.(str_item) +end + +## 2 arg form render for comparing +function render(mode::BeforeAfter, reference, actual) + println("- REFERENCE -------------------") + render_item(mode, reference) + println("-------------------------------") + println("- ACTUAL ----------------------") + render_item(mode, actual) + println("-------------------------------") +end +function render(::Diff, reference, actual) + println("- DIFF ------------------------") + @withcolor println(deepdiff(reference, actual)) + println("-------------------------------") +end + +## 1 arg form render for new content +function render(mode::RenderMode, actual) + println("- NEW CONTENT -----------------") + render_item(mode, actual) + println("-------------------------------") +end + """ default_rendermode(::DataFormat, actual) From 892c18170b25235691daac37290aebabb12be0e3 Mon Sep 17 00:00:00 2001 From: JohnnyChen Date: Fri, 11 Oct 2019 23:48:28 +0800 Subject: [PATCH 4/5] move implementation of test_reference into test_reference.jl --- src/ReferenceTests.jl | 1 - src/core.jl | 71 ------------------------------------------- src/test_reference.jl | 66 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 72 deletions(-) delete mode 100644 src/core.jl diff --git a/src/ReferenceTests.jl b/src/ReferenceTests.jl index c3ef4ee..ef7c750 100644 --- a/src/ReferenceTests.jl +++ b/src/ReferenceTests.jl @@ -17,7 +17,6 @@ export include("utils.jl") include("test_reference.jl") -include("core.jl") include("fileio.jl") include("equality_metrics.jl") include("render.jl") diff --git a/src/core.jl b/src/core.jl deleted file mode 100644 index dea05c3..0000000 --- a/src/core.jl +++ /dev/null @@ -1,71 +0,0 @@ -########################################## - -# Final function -# all other functions should hit one of this eventually -# Which handles the actual testing and user prompting - -function test_reference( - file::File{F}, - raw_actual::T, - equiv=nothing, - rendermode=nothing; - kw...) where {F <: DataFormat, T} - - path = file.filename - dir, filename = splitdir(path) - - # infer the default rendermode here - # since `nothing` is always passed to this method from - # test_reference(filename::AbstractString, raw_actual; kw...) - if rendermode === nothing - rendermode = default_rendermode(F, raw_actual) - end - - # preprocessing when reference file doesn't exists - if !isfile(path) - println("Reference file for \"$filename\" does not exist.") - render(rendermode, raw_actual) - - if !isinteractive() - error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to create new reference images") - end - - if !input_bool("Create reference file with above content (path: $path)?") - @test false - else - mkpath(dir) - savefile(file, raw_actual) - @info("Please run the tests again for any changes to take effect") - end - - return nothing # skip current test case - end - - # file exists - actual = _convert(F, raw_actual; kw...) - reference = loadfile(T, file) - - if equiv === nothing - # generally, `reference` and `actual` are of the same type after preprocessing - equiv = default_equality(reference, actual) - end - - if equiv(reference, actual) - @test true # to increase test counter if reached - else - # post-processing when test fails - println("Test for \"$filename\" failed.") - render(rendermode, reference, actual) - - if !isinteractive() - error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update reference images") - end - - if !input_bool("Replace reference with actual result (path: $path)?") - @test false - else - savefile(file, actual) - @info("Please run the tests again for any changes to take effect") - end - end -end diff --git a/src/test_reference.jl b/src/test_reference.jl index 1153256..d9cdb31 100644 --- a/src/test_reference.jl +++ b/src/test_reference.jl @@ -89,3 +89,69 @@ function test_reference( test_reference(query_extended(filename), raw_actual, by, render; kw...) end + +function test_reference( + file::File{F}, + raw_actual::T, + equiv=nothing, + rendermode=nothing; + kw...) where {F <: DataFormat, T} + + path = file.filename + dir, filename = splitdir(path) + + # infer the default rendermode here + # since `nothing` is always passed to this method from + # test_reference(filename::AbstractString, raw_actual; kw...) + if rendermode === nothing + rendermode = default_rendermode(F, raw_actual) + end + + # preprocessing when reference file doesn't exists + if !isfile(path) + println("Reference file for \"$filename\" does not exist.") + render(rendermode, raw_actual) + + if !isinteractive() + error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to create new reference images") + end + + if !input_bool("Create reference file with above content (path: $path)?") + @test false + else + mkpath(dir) + savefile(file, raw_actual) + @info("Please run the tests again for any changes to take effect") + end + + return nothing # skip current test case + end + + # file exists + actual = _convert(F, raw_actual; kw...) + reference = loadfile(T, file) + + if equiv === nothing + # generally, `reference` and `actual` are of the same type after preprocessing + equiv = default_equality(reference, actual) + end + + if equiv(reference, actual) + @test true # to increase test counter if reached + else + # post-processing when test fails + println("Test for \"$filename\" failed.") + render(rendermode, reference, actual) + + if !isinteractive() + error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update reference images") + end + + if !input_bool("Replace reference with actual result (path: $path)?") + @test false + else + savefile(file, actual) + @info("Please run the tests again for any changes to take effect") + end + end +end From ffa165a1d348b10ef15489fa7ae848c04cd1179e Mon Sep 17 00:00:00 2001 From: JohnnyChen Date: Sat, 12 Oct 2019 00:59:12 +0800 Subject: [PATCH 5/5] ReferenceTests v0.8.2 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index c4d6f59..81cff91 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ReferenceTests" uuid = "324d217c-45ce-50fc-942e-d289b448e8cf" authors = ["Christof Stocker ", "Lyndon White "] -version = "0.8.1" +version = "0.8.2" [deps] DeepDiffs = "ab62b9b5-e342-54a8-a765-a90f495de1a6"