From 58422bc24cf09e8f5f0f0caea6fbbddd0397d28e Mon Sep 17 00:00:00 2001 From: mxw96 <928826483@qq.com> Date: Tue, 7 May 2019 20:49:29 +0800 Subject: [PATCH 1/4] Add different padding supports in SR dnn native scripts for the padding method support in FFmpeg dnn_native --- generate_header_and_model.py | 37 ++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/generate_header_and_model.py b/generate_header_and_model.py index b82a8c3..960ff3c 100644 --- a/generate_header_and_model.py +++ b/generate_header_and_model.py @@ -8,6 +8,11 @@ from models.model_vsrnet import VSRnet from collections import OrderedDict +@enum.unique +class Padding(enum.Enum): + Valid = 0 + Same = 1 + Same_clamp_to_edge = 2 def get_arguments(): parser = argparse.ArgumentParser(description='generate c header with model weights and binary model file') @@ -64,9 +69,9 @@ def dump_to_file(file, values, name): file.write('\n};\n') -def write_conv_layer(kernel, bias, activation, model_file): +def write_conv_layer(kernel, bias, padding, activation, model_file): kernel = np.transpose(kernel, [3, 0, 1, 2]) - np.array([1, activation, kernel.shape[3], kernel.shape[0], kernel.shape[1]], dtype=np.uint32).tofile(model_file) + np.array([1, padding.value, activation, kernel.shape[3], kernel.shape[0], kernel.shape[1]], dtype=np.uint32).tofile(model_file) kernel.tofile(model_file) bias.tofile(model_file) @@ -77,34 +82,34 @@ def write_depth_to_space_layer(block_size, model_file): def prepare_native_mf_srcnn(weights, model_file): np.array([3], dtype=np.uint32).tofile(model_file) - write_conv_layer(weights['srcnn/conv1/kernel:0'], weights['srcnn/conv1/bias:0'], 0, model_file) - write_conv_layer(weights['srcnn/conv2/kernel:0'], weights['srcnn/conv2/bias:0'], 0, model_file) - write_conv_layer(weights['srcnn/conv3/kernel:0'], weights['srcnn/conv3/bias:0'], 0, model_file) + write_conv_layer(weights['srcnn/conv1/kernel:0'], weights['srcnn/conv1/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['srcnn/conv2/kernel:0'], weights['srcnn/conv2/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['srcnn/conv3/kernel:0'], weights['srcnn/conv3/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) def prepare_native_mf_espcn(weights, model_file, scale_factor): np.array([4], dtype=np.uint32).tofile(model_file) - write_conv_layer(weights['espcn/conv1/kernel:0'], weights['espcn/conv1/bias:0'], 1, model_file) - write_conv_layer(weights['espcn/conv2/kernel:0'], weights['espcn/conv2/bias:0'], 1, model_file) - write_conv_layer(weights['espcn/conv3/kernel:0'], weights['espcn/conv3/bias:0'], 2, model_file) + write_conv_layer(weights['espcn/conv1/kernel:0'], weights['espcn/conv1/bias:0'], Padding.Same_clamp_to_edge, 1, model_file) + write_conv_layer(weights['espcn/conv2/kernel:0'], weights['espcn/conv2/bias:0'], Padding.Same_clamp_to_edge, 1, model_file) + write_conv_layer(weights['espcn/conv3/kernel:0'], weights['espcn/conv3/bias:0'], Padding.Same_clamp_to_edge, 2, model_file) write_depth_to_space_layer(scale_factor, model_file) def prepare_native_mf_vespcn(weights, model_file, scale_factor): np.array([6], dtype=np.uint32).tofile(model_file) - write_conv_layer(weights['vespcn/conv1/kernel:0'], weights['vespcn/conv1/bias:0'], 0, model_file) - write_conv_layer(weights['vespcn/conv2/kernel:0'], weights['vespcn/conv2/bias:0'], 0, model_file) - write_conv_layer(weights['vespcn/conv3/kernel:0'], weights['vespcn/conv3/bias:0'], 0, model_file) - write_conv_layer(weights['vespcn/conv4/kernel:0'], weights['vespcn/conv4/bias:0'], 0, model_file) - write_conv_layer(weights['vespcn/conv5/kernel:0'], weights['vespcn/conv5/bias:0'], 0, model_file) + write_conv_layer(weights['vespcn/conv1/kernel:0'], weights['vespcn/conv1/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vespcn/conv2/kernel:0'], weights['vespcn/conv2/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vespcn/conv3/kernel:0'], weights['vespcn/conv3/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vespcn/conv4/kernel:0'], weights['vespcn/conv4/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vespcn/conv5/kernel:0'], weights['vespcn/conv5/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) write_depth_to_space_layer(scale_factor, model_file) def prepare_native_mf_vsrnet(weights, model_file): np.array([3], dtype=np.uint32).tofile(model_file) - write_conv_layer(weights['vsrnet/conv1/kernel:0'], weights['vsrnet/conv1/bias:0'], 0, model_file) - write_conv_layer(weights['vsrnet/conv2/kernel:0'], weights['vsrnet/conv2/bias:0'], 0, model_file) - write_conv_layer(weights['vsrnet/conv3/kernel:0'], weights['vsrnet/conv3/bias:0'], 0, model_file) + write_conv_layer(weights['vsrnet/conv1/kernel:0'], weights['vsrnet/conv1/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vsrnet/conv2/kernel:0'], weights['vsrnet/conv2/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vsrnet/conv3/kernel:0'], weights['vsrnet/conv3/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) def main(): From 0fa38eef1db744f890e94d61a992d261b710b289 Mon Sep 17 00:00:00 2001 From: mxw96 <928826483@qq.com> Date: Thu, 16 May 2019 09:09:12 +0800 Subject: [PATCH 2/4] Add different padding supports in SR dnn native scripts for the padding method support in FFmpeg dnn_native --- generate_header_and_model.py | 37 ++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/generate_header_and_model.py b/generate_header_and_model.py index b82a8c3..960ff3c 100644 --- a/generate_header_and_model.py +++ b/generate_header_and_model.py @@ -8,6 +8,11 @@ from models.model_vsrnet import VSRnet from collections import OrderedDict +@enum.unique +class Padding(enum.Enum): + Valid = 0 + Same = 1 + Same_clamp_to_edge = 2 def get_arguments(): parser = argparse.ArgumentParser(description='generate c header with model weights and binary model file') @@ -64,9 +69,9 @@ def dump_to_file(file, values, name): file.write('\n};\n') -def write_conv_layer(kernel, bias, activation, model_file): +def write_conv_layer(kernel, bias, padding, activation, model_file): kernel = np.transpose(kernel, [3, 0, 1, 2]) - np.array([1, activation, kernel.shape[3], kernel.shape[0], kernel.shape[1]], dtype=np.uint32).tofile(model_file) + np.array([1, padding.value, activation, kernel.shape[3], kernel.shape[0], kernel.shape[1]], dtype=np.uint32).tofile(model_file) kernel.tofile(model_file) bias.tofile(model_file) @@ -77,34 +82,34 @@ def write_depth_to_space_layer(block_size, model_file): def prepare_native_mf_srcnn(weights, model_file): np.array([3], dtype=np.uint32).tofile(model_file) - write_conv_layer(weights['srcnn/conv1/kernel:0'], weights['srcnn/conv1/bias:0'], 0, model_file) - write_conv_layer(weights['srcnn/conv2/kernel:0'], weights['srcnn/conv2/bias:0'], 0, model_file) - write_conv_layer(weights['srcnn/conv3/kernel:0'], weights['srcnn/conv3/bias:0'], 0, model_file) + write_conv_layer(weights['srcnn/conv1/kernel:0'], weights['srcnn/conv1/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['srcnn/conv2/kernel:0'], weights['srcnn/conv2/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['srcnn/conv3/kernel:0'], weights['srcnn/conv3/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) def prepare_native_mf_espcn(weights, model_file, scale_factor): np.array([4], dtype=np.uint32).tofile(model_file) - write_conv_layer(weights['espcn/conv1/kernel:0'], weights['espcn/conv1/bias:0'], 1, model_file) - write_conv_layer(weights['espcn/conv2/kernel:0'], weights['espcn/conv2/bias:0'], 1, model_file) - write_conv_layer(weights['espcn/conv3/kernel:0'], weights['espcn/conv3/bias:0'], 2, model_file) + write_conv_layer(weights['espcn/conv1/kernel:0'], weights['espcn/conv1/bias:0'], Padding.Same_clamp_to_edge, 1, model_file) + write_conv_layer(weights['espcn/conv2/kernel:0'], weights['espcn/conv2/bias:0'], Padding.Same_clamp_to_edge, 1, model_file) + write_conv_layer(weights['espcn/conv3/kernel:0'], weights['espcn/conv3/bias:0'], Padding.Same_clamp_to_edge, 2, model_file) write_depth_to_space_layer(scale_factor, model_file) def prepare_native_mf_vespcn(weights, model_file, scale_factor): np.array([6], dtype=np.uint32).tofile(model_file) - write_conv_layer(weights['vespcn/conv1/kernel:0'], weights['vespcn/conv1/bias:0'], 0, model_file) - write_conv_layer(weights['vespcn/conv2/kernel:0'], weights['vespcn/conv2/bias:0'], 0, model_file) - write_conv_layer(weights['vespcn/conv3/kernel:0'], weights['vespcn/conv3/bias:0'], 0, model_file) - write_conv_layer(weights['vespcn/conv4/kernel:0'], weights['vespcn/conv4/bias:0'], 0, model_file) - write_conv_layer(weights['vespcn/conv5/kernel:0'], weights['vespcn/conv5/bias:0'], 0, model_file) + write_conv_layer(weights['vespcn/conv1/kernel:0'], weights['vespcn/conv1/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vespcn/conv2/kernel:0'], weights['vespcn/conv2/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vespcn/conv3/kernel:0'], weights['vespcn/conv3/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vespcn/conv4/kernel:0'], weights['vespcn/conv4/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vespcn/conv5/kernel:0'], weights['vespcn/conv5/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) write_depth_to_space_layer(scale_factor, model_file) def prepare_native_mf_vsrnet(weights, model_file): np.array([3], dtype=np.uint32).tofile(model_file) - write_conv_layer(weights['vsrnet/conv1/kernel:0'], weights['vsrnet/conv1/bias:0'], 0, model_file) - write_conv_layer(weights['vsrnet/conv2/kernel:0'], weights['vsrnet/conv2/bias:0'], 0, model_file) - write_conv_layer(weights['vsrnet/conv3/kernel:0'], weights['vsrnet/conv3/bias:0'], 0, model_file) + write_conv_layer(weights['vsrnet/conv1/kernel:0'], weights['vsrnet/conv1/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vsrnet/conv2/kernel:0'], weights['vsrnet/conv2/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vsrnet/conv3/kernel:0'], weights['vsrnet/conv3/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) def main(): From f1a7a2db49c445a462b525b7e496582d857952e8 Mon Sep 17 00:00:00 2001 From: mxw96 <928826483@qq.com> Date: Mon, 20 May 2019 14:31:06 +0800 Subject: [PATCH 3/4] Add denpendency libaray --- generate_header_and_model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/generate_header_and_model.py b/generate_header_and_model.py index 960ff3c..2ff68da 100644 --- a/generate_header_and_model.py +++ b/generate_header_and_model.py @@ -2,6 +2,7 @@ import numpy as np import argparse import os +import enum from models.model_espcn import ESPCN from models.model_srcnn import SRCNN from models.model_vespcn import VESPCN From 59ba04394885f79b442f007c07e21169cd81a001 Mon Sep 17 00:00:00 2001 From: mxw96 <928826483@qq.com> Date: Wed, 22 May 2019 20:08:06 +0800 Subject: [PATCH 4/4] Add dilation option in generate --- generate_header_and_model.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/generate_header_and_model.py b/generate_header_and_model.py index 2ff68da..0df6a08 100644 --- a/generate_header_and_model.py +++ b/generate_header_and_model.py @@ -70,9 +70,9 @@ def dump_to_file(file, values, name): file.write('\n};\n') -def write_conv_layer(kernel, bias, padding, activation, model_file): +def write_conv_layer(kernel, bias, dilation_rate, padding, activation, model_file): kernel = np.transpose(kernel, [3, 0, 1, 2]) - np.array([1, padding.value, activation, kernel.shape[3], kernel.shape[0], kernel.shape[1]], dtype=np.uint32).tofile(model_file) + np.array([1, dilation_rate, padding.value, activation, kernel.shape[3], kernel.shape[0], kernel.shape[1]], dtype=np.uint32).tofile(model_file) kernel.tofile(model_file) bias.tofile(model_file) @@ -83,34 +83,34 @@ def write_depth_to_space_layer(block_size, model_file): def prepare_native_mf_srcnn(weights, model_file): np.array([3], dtype=np.uint32).tofile(model_file) - write_conv_layer(weights['srcnn/conv1/kernel:0'], weights['srcnn/conv1/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) - write_conv_layer(weights['srcnn/conv2/kernel:0'], weights['srcnn/conv2/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) - write_conv_layer(weights['srcnn/conv3/kernel:0'], weights['srcnn/conv3/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['srcnn/conv1/kernel:0'], weights['srcnn/conv1/bias:0'], 1, Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['srcnn/conv2/kernel:0'], weights['srcnn/conv2/bias:0'], 1, Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['srcnn/conv3/kernel:0'], weights['srcnn/conv3/bias:0'], 1, Padding.Same_clamp_to_edge, 0, model_file) def prepare_native_mf_espcn(weights, model_file, scale_factor): np.array([4], dtype=np.uint32).tofile(model_file) - write_conv_layer(weights['espcn/conv1/kernel:0'], weights['espcn/conv1/bias:0'], Padding.Same_clamp_to_edge, 1, model_file) - write_conv_layer(weights['espcn/conv2/kernel:0'], weights['espcn/conv2/bias:0'], Padding.Same_clamp_to_edge, 1, model_file) - write_conv_layer(weights['espcn/conv3/kernel:0'], weights['espcn/conv3/bias:0'], Padding.Same_clamp_to_edge, 2, model_file) + write_conv_layer(weights['espcn/conv1/kernel:0'], weights['espcn/conv1/bias:0'], 1, Padding.Same_clamp_to_edge, 1, model_file) + write_conv_layer(weights['espcn/conv2/kernel:0'], weights['espcn/conv2/bias:0'], 1, Padding.Same_clamp_to_edge, 1, model_file) + write_conv_layer(weights['espcn/conv3/kernel:0'], weights['espcn/conv3/bias:0'], 1, Padding.Same_clamp_to_edge, 2, model_file) write_depth_to_space_layer(scale_factor, model_file) def prepare_native_mf_vespcn(weights, model_file, scale_factor): np.array([6], dtype=np.uint32).tofile(model_file) - write_conv_layer(weights['vespcn/conv1/kernel:0'], weights['vespcn/conv1/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) - write_conv_layer(weights['vespcn/conv2/kernel:0'], weights['vespcn/conv2/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) - write_conv_layer(weights['vespcn/conv3/kernel:0'], weights['vespcn/conv3/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) - write_conv_layer(weights['vespcn/conv4/kernel:0'], weights['vespcn/conv4/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) - write_conv_layer(weights['vespcn/conv5/kernel:0'], weights['vespcn/conv5/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vespcn/conv1/kernel:0'], weights['vespcn/conv1/bias:0'], 1, Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vespcn/conv2/kernel:0'], weights['vespcn/conv2/bias:0'], 1, Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vespcn/conv3/kernel:0'], weights['vespcn/conv3/bias:0'], 1, Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vespcn/conv4/kernel:0'], weights['vespcn/conv4/bias:0'], 1, Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vespcn/conv5/kernel:0'], weights['vespcn/conv5/bias:0'], 1, Padding.Same_clamp_to_edge, 0, model_file) write_depth_to_space_layer(scale_factor, model_file) def prepare_native_mf_vsrnet(weights, model_file): np.array([3], dtype=np.uint32).tofile(model_file) - write_conv_layer(weights['vsrnet/conv1/kernel:0'], weights['vsrnet/conv1/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) - write_conv_layer(weights['vsrnet/conv2/kernel:0'], weights['vsrnet/conv2/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) - write_conv_layer(weights['vsrnet/conv3/kernel:0'], weights['vsrnet/conv3/bias:0'], Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vsrnet/conv1/kernel:0'], weights['vsrnet/conv1/bias:0'], 1, Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vsrnet/conv2/kernel:0'], weights['vsrnet/conv2/bias:0'], 1, Padding.Same_clamp_to_edge, 0, model_file) + write_conv_layer(weights['vsrnet/conv3/kernel:0'], weights['vsrnet/conv3/bias:0'], 1, Padding.Same_clamp_to_edge, 0, model_file) def main():