From 833a57de6843854ca53b01c8b94febc7bf770627 Mon Sep 17 00:00:00 2001 From: mnijh Date: Mon, 10 May 2021 15:52:32 +0200 Subject: [PATCH 1/8] added exponential, elu and softplus activation function --- KerasWeightsProcessing/convert_weights.py | 21 +++++--- src/lib/mod_activation.F90 | 59 +++++++++++++++++++++++ src/lib/mod_dense_layer.F90 | 9 ++++ 3 files changed, 83 insertions(+), 6 deletions(-) diff --git a/KerasWeightsProcessing/convert_weights.py b/KerasWeightsProcessing/convert_weights.py index 90568cd..a28d2c5 100644 --- a/KerasWeightsProcessing/convert_weights.py +++ b/KerasWeightsProcessing/convert_weights.py @@ -14,7 +14,7 @@ from keras import optimizers INPUT = ['input'] -ACTIVATIONS = ['relu', 'linear', 'leakyrelu', 'sigmoid'] +ACTIVATIONS = ['elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh'] SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT def txt_to_h5(weights_file_name, output_file_name=''): @@ -242,12 +242,21 @@ def h5_to_txt(weights_file_name, output_file_name=''): ) ) # add information about the activation - layer_info.append( - info_str.format( - name = activation, - info = 0 + if (activation == 'elu'): + layer_info.append( + info_str.format( + name = activation, + info = 1 + ) ) - ) + else: + layer_info.append( + info_str.format( + name = activation, + info = 0 + ) + ) + elif class_name == 'batchnormalization': # get beta, gamma, moving_mean, moving_variance from dictionary for key in sorted(model_weights[name][name].keys()): diff --git a/src/lib/mod_activation.F90 b/src/lib/mod_activation.F90 index 04079bb..c4c6334 100644 --- a/src/lib/mod_activation.F90 +++ b/src/lib/mod_activation.F90 @@ -16,6 +16,9 @@ module mod_activation public :: tanhf, tanh_prime public :: linear, linear_prime public :: leaky_relu, leaky_relu_prime + public :: elu, elu_prime + public :: exponential, exponential_prime + public :: softplus, softplus_prime interface pure function activation_function(x, alpha) @@ -28,6 +31,46 @@ end function activation_function contains + pure function elu(x, alpha) result(res) + !! Exponential Linear Unit (ELU) activation function. + real(rk), intent(in) :: x(:) + real(rk), intent(in) :: alpha + real(rk) :: res(size(x)) + where (x > 0) + res = x + elsewhere + res = alpha * (exp(x) - 1) + end where + end function elu + + pure function elu_prime(x, alpha) result(res) + ! First derivative of the Exponential Linear Unit (ELU) activation function. + real(rk), intent(in) :: x(:) + real(rk), intent(in) :: alpha + real(rk) :: res(size(x)) + where (x > 0) + res = 1 + elsewhere + res = alpha * exp(x) + end where + end function elu_prime + + pure function exponential(x, alpha) result(res) + !! Exponential activation function. + real(rk), intent(in) :: x(:) + real(rk), intent(in) :: alpha + real(rk) :: res(size(x)) + res = exp(x) + end function exponential + + pure function exponential_prime(x, alpha) result(res) + !! First derivative of the exponential activation function. + real(rk), intent(in) :: x(:) + real(rk), intent(in) :: alpha + real(rk) :: res(size(x)) + res = exp(x) + end function exponential_prime + pure function gaussian(x, alpha) result(res) ! Gaussian activation function. real(rk), intent(in) :: x(:) @@ -122,6 +165,22 @@ pure function sigmoid_prime(x, alpha) result(res) res = sigmoid(x, tmp_alpha) * (1 - sigmoid(x, tmp_alpha)) end function sigmoid_prime + pure function softplus(x, alpha) result(res) + !! Softplus activation function. + real(rk), intent(in) :: x(:) + real(rk), intent(in) :: alpha + real(rk) :: res(size(x)) + res = log(exp(x) + 1) + end function softplus + + pure function softplus_prime(x, alpha) result(res) + ! First derivative of the Softplus activation function. + real(rk), intent(in) :: x(:) + real(rk), intent(in) :: alpha + real(rk) :: res(size(x)) + res = 1 / (1 + exp(-x)) + end function softplus_prime + pure function step(x, alpha) result(res) ! Step activation function. real(rk), intent(in) :: x(:) diff --git a/src/lib/mod_dense_layer.F90 b/src/lib/mod_dense_layer.F90 index 22ac8ff..7339385 100644 --- a/src/lib/mod_dense_layer.F90 +++ b/src/lib/mod_dense_layer.F90 @@ -59,6 +59,12 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result ! assign activation function select case(trim(activation)) + case('elu') + layer % activation => elu + layer % activation_prime => elu_prime + case('exponential') + layer % activation => exponential + layer % activation_prime => exponential_prime case('gaussian') layer % activation => gaussian layer % activation_prime => gaussian_prime @@ -71,6 +77,9 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result case('sigmoid') layer % activation => sigmoid layer % activation_prime => sigmoid_prime + case('softplus') + layer % activation => softplus + layer % activation_prime => softplus_prime case('step') layer % activation => step layer % activation_prime => step_prime From c50cc0b51d4001a5aa3a9bc87be11863c6ca942e Mon Sep 17 00:00:00 2001 From: Sungduk Yu Date: Tue, 21 Jun 2022 13:35:47 -0700 Subject: [PATCH 2/8] debugged for enabling REAL64, REAL128, and INT64 --- build_steps.sh | 2 +- src/lib/mod_kinds.F90 | 4 ++++ src/lib/mod_layer.F90 | 6 +++--- src/lib/mod_network.F90 | 2 +- src/tests/test_ensembles.F90 | 2 +- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/build_steps.sh b/build_steps.sh index ca60c81..dd24f29 100644 --- a/build_steps.sh +++ b/build_steps.sh @@ -2,7 +2,7 @@ rm -rf build mkdir build cd build -FC=gfortran cmake .. -DSERIAL=1 +FC=ifort cmake .. -DSERIAL=1 -DREAL=64 -DINT=32 # FC='mpif90 -qopenmp' cmake .. -DSERIAL=1 make diff --git a/src/lib/mod_kinds.F90 b/src/lib/mod_kinds.F90 index 2b45118..8be55bf 100644 --- a/src/lib/mod_kinds.F90 +++ b/src/lib/mod_kinds.F90 @@ -11,6 +11,8 @@ module mod_kinds integer,parameter :: rk = real64 #elif REAL128 integer,parameter :: rk = real128 +#elif REAL32 + integer,parameter :: rk = real32 #else integer,parameter :: rk = real32 #endif @@ -19,6 +21,8 @@ module mod_kinds #ifdef INT64 integer, parameter :: ik = int64 +#elif INT32 + integer, parameter :: ik = int32 #else integer, parameter :: ik = int32 #endif diff --git a/src/lib/mod_layer.F90 b/src/lib/mod_layer.F90 index 601d17f..1307cce 100644 --- a/src/lib/mod_layer.F90 +++ b/src/lib/mod_layer.F90 @@ -68,7 +68,7 @@ end subroutine layer_backward pure type(array1d) function array1d_constructor(length) result(a) ! Overloads the default type constructor. - integer, intent(in) :: length + integer(ik), intent(in) :: length allocate(a % array(length)) a % array = 0 end function array1d_constructor @@ -76,7 +76,7 @@ end function array1d_constructor pure type(array2d) function array2d_constructor(dims) result(a) ! Overloads the default type constructor. - integer, intent(in) :: dims(2) + integer(ik), intent(in) :: dims(2) allocate(a % array(dims(1), dims(2))) a % array = 0 end function array2d_constructor @@ -106,7 +106,7 @@ pure subroutine dw_init(dw, dims) do n = 1, nm - 1 dw(n) = array2d(dims(n:n+1)) end do - dw(n) = array2d([dims(n), 1]) + dw(n) = array2d([dims(n), 1_ik]) end subroutine dw_init diff --git a/src/lib/mod_network.F90 b/src/lib/mod_network.F90 index bed055d..cc7eeeb 100644 --- a/src/lib/mod_network.F90 +++ b/src/lib/mod_network.F90 @@ -56,7 +56,7 @@ type(network_type) function net_constructor(layer_names, layer_info) result(net) call net % init(layer_names, layer_info) - call net % sync(1) + call net % sync(1_ik) end function net_constructor diff --git a/src/tests/test_ensembles.F90 b/src/tests/test_ensembles.F90 index f7bbf68..d04675d 100644 --- a/src/tests/test_ensembles.F90 +++ b/src/tests/test_ensembles.F90 @@ -23,7 +23,7 @@ program test_ensembles call get_command_argument(1,args(1)) ! build ensemble from members in specified directory - ensemble = ensemble_type(args(1), 0.0) + ensemble = ensemble_type(args(1), 0.0_rk) input = [1, 2, 3, 4, 5] From fc7baf2796a7efcc8211acd90d39d2a95faf34e7 Mon Sep 17 00:00:00 2001 From: sungdukyu Date: Tue, 21 Jun 2022 15:41:51 -0700 Subject: [PATCH 3/8] revert build_steps.sh to the original version --- build_steps.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_steps.sh b/build_steps.sh index dd24f29..f8b5a42 100644 --- a/build_steps.sh +++ b/build_steps.sh @@ -2,7 +2,8 @@ rm -rf build mkdir build cd build -FC=ifort cmake .. -DSERIAL=1 -DREAL=64 -DINT=32 +FC=gfortran cmake .. -DSERIAL=1 +# FC=gfortran cmake .. -DSERIAL=1 -DREAL=64 # enable double precision real number (note that the default precision for real number is single. See src/lib/mod_kinds.F90) # FC='mpif90 -qopenmp' cmake .. -DSERIAL=1 make From ad8735c2b1a64f61be169b5c25d00ab111797c24 Mon Sep 17 00:00:00 2001 From: Sungduk Yu Date: Thu, 21 Sep 2023 15:05:07 -0700 Subject: [PATCH 4/8] Minor modifications in convert_weights.py --- KerasWeightsProcessing/convert_weights.py | 23 +++++++++++++++++++---- build_steps.sh | 4 ++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/KerasWeightsProcessing/convert_weights.py b/KerasWeightsProcessing/convert_weights.py index a28d2c5..6921899 100644 --- a/KerasWeightsProcessing/convert_weights.py +++ b/KerasWeightsProcessing/convert_weights.py @@ -156,7 +156,10 @@ def h5_to_txt(weights_file_name, output_file_name=''): keras_version = weights_file.attrs['keras_version'] if 'training_config' in weights_file.attrs: - training_config = weights_file.attrs['training_config'].decode('utf-8') + try: + training_config = weights_file.attrs['training_config'].decode('utf-8') + except AttributeError: + training_config = weights_file.attrs['training_config'] training_config = training_config.replace('true','True') training_config = training_config.replace('false','False') training_config = training_config.replace('null','None') @@ -169,7 +172,10 @@ def h5_to_txt(weights_file_name, output_file_name=''): learning_rate = 0.001 # Decode using the utf-8 encoding; change values for eval - model_config = weights_file.attrs['model_config'].decode('utf-8') + try: + model_config = weights_file.attrs['model_config'].decode('utf-8') + except AttributeError: + model_config = weights_file.attrs['model_config'] model_config = model_config.replace('true','True') model_config = model_config.replace('false','False') model_config = model_config.replace('null','None') @@ -197,7 +203,7 @@ def h5_to_txt(weights_file_name, output_file_name=''): input_layers = model_config['config'].get('input_layers',[]) input_names = [layer[0] for layer in input_layers] - else: + else: # 'Sequential' layer_config = model_config['config']['layers'] for idx,layer in enumerate(layer_config): @@ -285,9 +291,18 @@ def h5_to_txt(weights_file_name, output_file_name=''): elif class_name in ACTIVATIONS: # replace previous dense layer with the advanced activation function (LeakyReLU) + + try: + _tmp = layer['config']['alpha'] + except KeyError: + if class_name == 'relu': + _tmp = 0 # alpha is currently not used in mod_activation + else: + raise KeyError("alpha") + layer_info[-1] = info_str.format( name = class_name, - info = layer['config']['alpha'] + info = _tmp ) # if there are multiple outputs, remove what was just added diff --git a/build_steps.sh b/build_steps.sh index f8b5a42..02e01c7 100644 --- a/build_steps.sh +++ b/build_steps.sh @@ -2,6 +2,10 @@ rm -rf build mkdir build cd build +# perlmutter +module load PrgEnv-gnu/8.3.3 +module load gcc/11.2.0 + FC=gfortran cmake .. -DSERIAL=1 # FC=gfortran cmake .. -DSERIAL=1 -DREAL=64 # enable double precision real number (note that the default precision for real number is single. See src/lib/mod_kinds.F90) # FC='mpif90 -qopenmp' cmake .. -DSERIAL=1 From 5ae526634840b17563a9af86a3304e26e49b629b Mon Sep 17 00:00:00 2001 From: Sungduk Yu Date: Thu, 21 Sep 2023 22:34:04 -0700 Subject: [PATCH 5/8] Double precision for ClimSim E3SM implementation. --- build_steps.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_steps.sh b/build_steps.sh index 02e01c7..7939341 100644 --- a/build_steps.sh +++ b/build_steps.sh @@ -6,8 +6,8 @@ cd build module load PrgEnv-gnu/8.3.3 module load gcc/11.2.0 -FC=gfortran cmake .. -DSERIAL=1 -# FC=gfortran cmake .. -DSERIAL=1 -DREAL=64 # enable double precision real number (note that the default precision for real number is single. See src/lib/mod_kinds.F90) +# FC=gfortran cmake .. -DSERIAL=1 +FC=gfortran cmake .. -DSERIAL=1 -DREAL=64 # enable double precision real number (note that the default precision for real number is single. See src/lib/mod_kinds.F90) # FC='mpif90 -qopenmp' cmake .. -DSERIAL=1 make From 68805b62fcc6c11d2731862be31f5664ba78554a Mon Sep 17 00:00:00 2001 From: Sungduk Yu Date: Fri, 9 Feb 2024 14:48:02 -0800 Subject: [PATCH 6/8] Incorporate @singhbalwinder's bug fix on sigmoid function to avoid floating point overflow (esp. when using single precision) --- src/lib/mod_activation.F90 | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/lib/mod_activation.F90 b/src/lib/mod_activation.F90 index c4c6334..8e033c8 100644 --- a/src/lib/mod_activation.F90 +++ b/src/lib/mod_activation.F90 @@ -146,12 +146,19 @@ pure function linear_prime(x, alpha) result(res) res = 1 end function linear_prime + ! Balwinder's version (July 5th, 2022) + ! - addressing floating point overflow pure function sigmoid(x, alpha) result(res) ! Sigmoid activation function. real(rk), intent(in) :: x(:) real(rk), intent(in) :: alpha - real(rk) :: res(size(x)) - res = 1 / (1 + exp(-x)) + real(rk) :: res(size(x)),y(size(x)) + real(rk), parameter :: exp_lim = log(tiny(x)) !precision based limiter + y(:) = x(:) + where(y < exp_lim) + y = exp_lim + end where + res = 1. / (1. + exp(-y)) endfunction sigmoid pure function sigmoid_prime(x, alpha) result(res) From afac13f7c5b563c832d4ca39a4a0e34ab36a534a Mon Sep 17 00:00:00 2001 From: sungdukyu Date: Tue, 20 Feb 2024 17:43:24 -0800 Subject: [PATCH 7/8] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4087ef1..8083d02 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ Check out an example in the [getting started notebook](https://github.com/scient Get the code: ``` -git clone https://github.com/scientific-computing/FKB +git clone https://github.com/sungdukyu/FKB64 ``` Dependencies: From 5944cd0f844b8e2784252fa60e27207a32caa0ad Mon Sep 17 00:00:00 2001 From: Sungduk Yu Date: Thu, 22 Feb 2024 19:00:49 -0800 Subject: [PATCH 8/8] Add swish (silu) activatin function. --- KerasWeightsProcessing/convert_weights.py | 7 ++++--- src/lib/mod_activation.F90 | 21 ++++++++++++++++++++- src/lib/mod_dense_layer.F90 | 6 ++++++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/KerasWeightsProcessing/convert_weights.py b/KerasWeightsProcessing/convert_weights.py index 6921899..dacc8e1 100644 --- a/KerasWeightsProcessing/convert_weights.py +++ b/KerasWeightsProcessing/convert_weights.py @@ -14,7 +14,7 @@ from keras import optimizers INPUT = ['input'] -ACTIVATIONS = ['elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh'] +ACTIVATIONS = ['swish', 'elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh'] SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT def txt_to_h5(weights_file_name, output_file_name=''): @@ -70,8 +70,9 @@ def txt_to_h5(weights_file_name, output_file_name=''): elif layer_type == 'batchnormalization': batchnorm_count += 4 x = BatchNormalization(name='batch_normalization_{}'.format(batchnorm_count // 4))(x) - elif layer_type == 'linear': - x = Activation('linear')(x) + + elif layer_type == 'swish': + x = Activation('swish')(x) elif not layer_type.isalpha(): if lr == False: diff --git a/src/lib/mod_activation.F90 b/src/lib/mod_activation.F90 index 8e033c8..2f3af6a 100644 --- a/src/lib/mod_activation.F90 +++ b/src/lib/mod_activation.F90 @@ -19,6 +19,7 @@ module mod_activation public :: elu, elu_prime public :: exponential, exponential_prime public :: softplus, softplus_prime + public :: swish, swish_prime interface pure function activation_function(x, alpha) @@ -31,6 +32,24 @@ end function activation_function contains + pure function swish(x, alpha) result(res) + !! swish (or silu) activation function. + !! beta is fixed (i.e., beta==1.). + real(rk), intent(in) :: x(:) + real(rk), intent(in) :: alpha + real(rk) :: res(size(x)) + res = x * sigmoid(x, -999._rk) + end function swish + + pure function swish_prime(x, alpha) result(res) + ! First derivative of swish activation function. + ! beta is fixed (i.e., beta==1.). + real(rk), intent(in) :: x(:) + real(rk), intent(in) :: alpha + real(rk) :: res(size(x)) + res = sigmoid(x, -999._rk) + x * sigmoid_prime(x, -999._rk) + end function swish_prime + pure function elu(x, alpha) result(res) !! Exponential Linear Unit (ELU) activation function. real(rk), intent(in) :: x(:) @@ -106,7 +125,7 @@ pure function leaky_relu_prime(x, alpha) result(res) where (0.3 * x > 0) res = 1 elsewhere - res = 0 + res = alpha end where end function leaky_relu_prime diff --git a/src/lib/mod_dense_layer.F90 b/src/lib/mod_dense_layer.F90 index 7339385..50e396f 100644 --- a/src/lib/mod_dense_layer.F90 +++ b/src/lib/mod_dense_layer.F90 @@ -59,6 +59,12 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result ! assign activation function select case(trim(activation)) + case('swish') + layer % activation => swish + layer % activation_prime => swish_prime + case('silu') + layer % activation => swish + layer % activation_prime => swish_prime case('elu') layer % activation => elu layer % activation_prime => elu_prime