From fade0aa28d03c30bf02527c75bca9e89a434b7d1 Mon Sep 17 00:00:00 2001 From: julienChemillier Date: Fri, 26 May 2023 20:46:58 +0200 Subject: [PATCH 1/9] Add 'finetuning' variable to the Network class --- src/cnn/creation.c | 206 +++++++++++++++++++++---------------- src/cnn/free.c | 136 ++++++++++++++---------- src/cnn/include/config.h | 4 +- src/cnn/include/creation.h | 2 +- src/cnn/include/models.h | 10 +- src/cnn/include/struct.h | 1 + src/cnn/models.c | 20 ++-- src/cnn/neuron_io.c | 55 +++++++--- src/cnn/train.c | 4 +- src/cnn/update.c | 29 ++++++ test/cnn_neuron_io.c | 2 +- test/cnn_structure.c | 3 +- test/cnn_utils.c | 4 +- 13 files changed, 298 insertions(+), 178 deletions(-) diff --git a/src/cnn/creation.c b/src/cnn/creation.c index c12e573..c7ef64a 100644 --- a/src/cnn/creation.c +++ b/src/cnn/creation.c @@ -6,10 +6,11 @@ #include "../common/include/utils.h" #include "include/initialisation.h" #include "include/function.h" +#include "include/cnn.h" #include "include/creation.h" -Network* create_network(int max_size, float learning_rate, int dropout, int initialisation, int input_width, int input_depth) { +Network* create_network(int max_size, float learning_rate, int dropout, int initialisation, int input_width, int input_depth, int finetuning) { if (dropout < 0 || dropout > 100) { printf_error("La probabilité de dropout n'est pas respecté, elle doit être comprise entre 0 et 100\n"); } @@ -19,6 +20,7 @@ Network* create_network(int max_size, float learning_rate, int dropout, int init network->dropout = dropout; network->initialisation = initialisation; network->size = 1; + network->finetuning = finetuning; network->input = (float****)nalloc(max_size, sizeof(float***)); network->input_z = (float****)nalloc(max_size, sizeof(float***)); network->kernel = (Kernel**)nalloc(max_size-1, sizeof(Kernel*)); @@ -148,70 +150,87 @@ void add_convolution(Network* network, int kernel_size, int number_of_kernels, i cnn->rows = input_depth; cnn->columns = output_depth; + // Partie toujours initialisée cnn->weights = (float****)nalloc(input_depth, sizeof(float***)); - cnn->d_weights = (float****)nalloc(input_depth, sizeof(float***)); - #ifdef ADAM_CNN_WEIGHTS - cnn->s_d_weights = (float****)nalloc(input_depth, sizeof(float***)); - cnn->v_d_weights = (float****)nalloc(input_depth, sizeof(float***)); - #endif for (int i=0; i < input_depth; i++) { cnn->weights[i] = (float***)nalloc(output_depth, sizeof(float**)); - cnn->d_weights[i] = (float***)nalloc(output_depth, sizeof(float**)); - #ifdef ADAM_CNN_WEIGHTS - cnn->s_d_weights[i] = (float***)nalloc(output_depth, sizeof(float**)); - cnn->v_d_weights[i] = (float***)nalloc(output_depth, sizeof(float**)); - #endif for (int j=0; j < output_depth; j++) { cnn->weights[i][j] = (float**)nalloc(kernel_size, sizeof(float*)); - cnn->d_weights[i][j] = (float**)nalloc(kernel_size, sizeof(float*)); - #ifdef ADAM_CNN_WEIGHTS - cnn->s_d_weights[i][j] = (float**)nalloc(kernel_size, sizeof(float*)); - cnn->v_d_weights[i][j] = (float**)nalloc(kernel_size, sizeof(float*)); - #endif for (int k=0; k < kernel_size; k++) { cnn->weights[i][j][k] = (float*)nalloc(kernel_size, sizeof(float)); - cnn->d_weights[i][j][k] = (float*)nalloc(kernel_size, sizeof(float)); - #ifdef ADAM_CNN_WEIGHTS - cnn->s_d_weights[i][j][k] = (float*)nalloc(kernel_size, sizeof(float)); - cnn->v_d_weights[i][j][k] = (float*)nalloc(kernel_size, sizeof(float)); - #endif - for (int l=0; l < kernel_size; l++) { - cnn->d_weights[i][j][k][l] = 0.; - #ifdef ADAM_CNN_WEIGHTS - cnn->s_d_weights[i][j][k][l] = 0.; - cnn->v_d_weights[i][j][k][l] = 0.; - #endif - } } } } cnn->bias = (float***)nalloc(output_depth, sizeof(float**)); - cnn->d_bias = (float***)nalloc(output_depth, sizeof(float**)); - #ifdef ADAM_CNN_BIAS - cnn->s_d_bias = (float***)nalloc(output_depth, sizeof(float**)); - cnn->v_d_bias = (float***)nalloc(output_depth, sizeof(float**)); - #endif for (int i=0; i < output_depth; i++) { cnn->bias[i] = (float**)nalloc(bias_size, sizeof(float*)); - cnn->d_bias[i] = (float**)nalloc(bias_size, sizeof(float*)); - #ifdef ADAM_CNN_BIAS - cnn->s_d_bias[i] = (float**)nalloc(bias_size, sizeof(float*)); - cnn->v_d_bias[i] = (float**)nalloc(bias_size, sizeof(float*)); - #endif for (int j=0; j < bias_size; j++) { cnn->bias[i][j] = (float*)nalloc(bias_size, sizeof(float)); - cnn->d_bias[i][j] = (float*)nalloc(bias_size, sizeof(float)); - #ifdef ADAM_CNN_BIAS - cnn->s_d_bias[i][j] = (float*)nalloc(bias_size, sizeof(float)); - cnn->v_d_bias[i][j] = (float*)nalloc(bias_size, sizeof(float)); + } + } + + // Partie initialisée que sous certaines conditions + if (network->finetuning == EVERYTHING) { + cnn->d_weights = (float****)nalloc(input_depth, sizeof(float***)); + #ifdef ADAM_CNN_WEIGHTS + cnn->s_d_weights = (float****)nalloc(input_depth, sizeof(float***)); + cnn->v_d_weights = (float****)nalloc(input_depth, sizeof(float***)); + #endif + for (int i=0; i < input_depth; i++) { + cnn->d_weights[i] = (float***)nalloc(output_depth, sizeof(float**)); + #ifdef ADAM_CNN_WEIGHTS + cnn->s_d_weights[i] = (float***)nalloc(output_depth, sizeof(float**)); + cnn->v_d_weights[i] = (float***)nalloc(output_depth, sizeof(float**)); #endif - for (int k=0; k < bias_size; k++) { - cnn->d_bias[i][j][k] = 0.; - #ifdef ADAM_CNN_BIAS - cnn->s_d_bias[i][j][k] = 0.; - cnn->v_d_bias[i][j][k] = 0.; + for (int j=0; j < output_depth; j++) { + cnn->d_weights[i][j] = (float**)nalloc(kernel_size, sizeof(float*)); + #ifdef ADAM_CNN_WEIGHTS + cnn->s_d_weights[i][j] = (float**)nalloc(kernel_size, sizeof(float*)); + cnn->v_d_weights[i][j] = (float**)nalloc(kernel_size, sizeof(float*)); #endif + for (int k=0; k < kernel_size; k++) { + cnn->d_weights[i][j][k] = (float*)nalloc(kernel_size, sizeof(float)); + #ifdef ADAM_CNN_WEIGHTS + cnn->s_d_weights[i][j][k] = (float*)nalloc(kernel_size, sizeof(float)); + cnn->v_d_weights[i][j][k] = (float*)nalloc(kernel_size, sizeof(float)); + #endif + for (int l=0; l < kernel_size; l++) { + cnn->d_weights[i][j][k][l] = 0.; + #ifdef ADAM_CNN_WEIGHTS + cnn->s_d_weights[i][j][k][l] = 0.; + cnn->v_d_weights[i][j][k][l] = 0.; + #endif + } + + } + } + } + + cnn->d_bias = (float***)nalloc(output_depth, sizeof(float**)); + #ifdef ADAM_CNN_BIAS + cnn->s_d_bias = (float***)nalloc(output_depth, sizeof(float**)); + cnn->v_d_bias = (float***)nalloc(output_depth, sizeof(float**)); + #endif + for (int i=0; i < output_depth; i++) { + cnn->d_bias[i] = (float**)nalloc(bias_size, sizeof(float*)); + #ifdef ADAM_CNN_BIAS + cnn->s_d_bias[i] = (float**)nalloc(bias_size, sizeof(float*)); + cnn->v_d_bias[i] = (float**)nalloc(bias_size, sizeof(float*)); + #endif + for (int j=0; j < bias_size; j++) { + cnn->d_bias[i][j] = (float*)nalloc(bias_size, sizeof(float)); + #ifdef ADAM_CNN_BIAS + cnn->s_d_bias[i][j] = (float*)nalloc(bias_size, sizeof(float)); + cnn->v_d_bias[i][j] = (float*)nalloc(bias_size, sizeof(float)); + #endif + for (int k=0; k < bias_size; k++) { + cnn->d_bias[i][j][k] = 0.; + #ifdef ADAM_CNN_BIAS + cnn->s_d_bias[i][j][k] = 0.; + cnn->v_d_bias[i][j][k] = 0.; + #endif + } } } } @@ -245,28 +264,19 @@ void add_dense(Network* network, int size_output, int activation) { nn->size_input = size_input; nn->size_output = size_output; - nn->bias = (float*)nalloc(size_output, sizeof(float)); - nn->d_bias = (float*)nalloc(size_output, sizeof(float)); - #ifdef ADAM_DENSE_BIAS - nn->s_d_bias = (float*)nalloc(size_output, sizeof(float)); - nn->v_d_bias = (float*)nalloc(size_output, sizeof(float)); - #endif - for (int i=0; i < size_output; i++) { - nn->d_bias[i] = 0.; - #ifdef ADAM_DENSE_BIAS - nn->s_d_bias[i] = 0.; - nn->v_d_bias[i] = 0.; - #endif + nn->weights = (float**)nalloc(size_input, sizeof(float*)); + for (int i=0; i < size_input; i++) { + nn->weights[i] = (float*)nalloc(size_output, sizeof(float)); } - nn->weights = (float**)nalloc(size_input, sizeof(float*)); + nn->bias = (float*)nalloc(size_output, sizeof(float)); + nn->d_weights = (float**)nalloc(size_input, sizeof(float*)); #ifdef ADAM_DENSE_WEIGHTS nn->s_d_weights = (float**)nalloc(size_input, sizeof(float*)); nn->v_d_weights = (float**)nalloc(size_input, sizeof(float*)); #endif for (int i=0; i < size_input; i++) { - nn->weights[i] = (float*)nalloc(size_output, sizeof(float)); nn->d_weights[i] = (float*)nalloc(size_output, sizeof(float)); #ifdef ADAM_DENSE_WEIGHTS nn->s_d_weights[i] = (float*)nalloc(size_output, sizeof(float)); @@ -281,6 +291,20 @@ void add_dense(Network* network, int size_output, int activation) { } } + nn->d_bias = (float*)nalloc(size_output, sizeof(float)); + #ifdef ADAM_DENSE_BIAS + nn->s_d_bias = (float*)nalloc(size_output, sizeof(float)); + nn->v_d_bias = (float*)nalloc(size_output, sizeof(float)); + #endif + for (int i=0; i < size_output; i++) { + nn->d_bias[i] = 0.; + #ifdef ADAM_DENSE_BIAS + nn->s_d_bias[i] = 0.; + nn->v_d_bias[i] = 0.; + #endif + } + + initialisation_1d_matrix(network->initialisation, nn->bias, size_output, size_input, size_output); initialisation_2d_matrix(network->initialisation, nn->weights, size_input, size_output, size_input, size_output); create_a_line_input_layer(network, n, size_output); @@ -310,38 +334,46 @@ void add_dense_linearisation(Network* network, int size_output, int activation) nn->size_input = size_input; nn->size_output = size_output; - nn->bias = (float*)nalloc(size_output, sizeof(float)); - nn->d_bias = (float*)nalloc(size_output, sizeof(float)); - #ifdef ADAM_DENSE_BIAS - nn->s_d_bias = (float*)nalloc(size_output, sizeof(float)); - nn->v_d_bias = (float*)nalloc(size_output, sizeof(float)); - #endif - for (int i=0; i < size_output; i++) { - nn->d_bias[i] = 0.; - #ifdef ADAM_DENSE_BIAS - nn->s_d_bias[i] = 0.; - nn->v_d_bias[i] = 0.; - #endif - } - + // Partie toujours initialisée nn->weights = (float**)nalloc(size_input, sizeof(float*)); - nn->d_weights = (float**)nalloc(size_input, sizeof(float*)); - #ifdef ADAM_DENSE_WEIGHTS - nn->s_d_weights = (float**)nalloc(size_input, sizeof(float*)); - nn->v_d_weights = (float**)nalloc(size_input, sizeof(float*)); - #endif for (int i=0; i < size_input; i++) { nn->weights[i] = (float*)nalloc(size_output, sizeof(float)); - nn->d_weights[i] = (float*)nalloc(size_output, sizeof(float)); + } + + nn->bias = (float*)nalloc(size_output, sizeof(float)); + + // Partie initialisée que sous certaines conditions + if (network->finetuning <= NN_AND_LINEARISATION) { + nn->d_weights = (float**)nalloc(size_input, sizeof(float*)); #ifdef ADAM_DENSE_WEIGHTS - nn->s_d_weights[i] = (float*)nalloc(size_output, sizeof(float)); - nn->v_d_weights[i] = (float*)nalloc(size_output, sizeof(float)); + nn->s_d_weights = (float**)nalloc(size_input, sizeof(float*)); + nn->v_d_weights = (float**)nalloc(size_input, sizeof(float*)); #endif - for (int j=0; j < size_output; j++) { - nn->d_weights[i][j] = 0.; + for (int i=0; i < size_input; i++) { + nn->d_weights[i] = (float*)nalloc(size_output, sizeof(float)); #ifdef ADAM_DENSE_WEIGHTS - nn->s_d_weights[i][j] = 0.; - nn->v_d_weights[i][j] = 0.; + nn->s_d_weights[i] = (float*)nalloc(size_output, sizeof(float)); + nn->v_d_weights[i] = (float*)nalloc(size_output, sizeof(float)); + #endif + for (int j=0; j < size_output; j++) { + nn->d_weights[i][j] = 0.; + #ifdef ADAM_DENSE_WEIGHTS + nn->s_d_weights[i][j] = 0.; + nn->v_d_weights[i][j] = 0.; + #endif + } + } + + nn->d_bias = (float*)nalloc(size_output, sizeof(float)); + #ifdef ADAM_DENSE_BIAS + nn->s_d_bias = (float*)nalloc(size_output, sizeof(float)); + nn->v_d_bias = (float*)nalloc(size_output, sizeof(float)); + #endif + for (int i=0; i < size_output; i++) { + nn->d_bias[i] = 0.; + #ifdef ADAM_DENSE_BIAS + nn->s_d_bias[i] = 0.; + nn->v_d_bias[i] = 0.; #endif } } diff --git a/src/cnn/free.c b/src/cnn/free.c index f3faec1..47ee0b2 100644 --- a/src/cnn/free.c +++ b/src/cnn/free.c @@ -3,6 +3,7 @@ #include #include "../common/include/memory_management.h" +#include "include/cnn.h" #include "include/free.h" @@ -42,59 +43,76 @@ void free_convolution(Network* network, int pos) { int r = k_pos->rows; int bias_size = network->width[pos+1]; free_a_cube_input_layer(network, pos+1, network->depth[pos+1], network->width[pos+1]); + + // Partie toujours initialisée (donc à libérer) for (int i=0; i < c; i++) { for (int j=0; j < bias_size; j++) { gree(k_pos->bias[i][j], true); - gree(k_pos->d_bias[i][j], true); - #ifdef ADAM_CNN_BIAS - gree(k_pos->s_d_bias[i][j], true); - gree(k_pos->v_d_bias[i][j], true); - #endif } gree(k_pos->bias[i], true); - gree(k_pos->d_bias[i], true); - #ifdef ADAM_CNN_BIAS - gree(k_pos->s_d_bias[i], true); - gree(k_pos->v_d_bias[i], true); - #endif } gree(k_pos->bias, true); - gree(k_pos->d_bias, true); - #ifdef ADAM_CNN_BIAS - gree(k_pos->s_d_bias, true); - gree(k_pos->v_d_bias, true); - #endif for (int i=0; i < r; i++) { for (int j=0; j < c; j++) { for (int k=0; k < k_size; k++) { gree(k_pos->weights[i][j][k], true); - gree(k_pos->d_weights[i][j][k], true); - #ifdef ADAM_CNN_WEIGHTS - gree(k_pos->s_d_weights[i][j][k], true); - gree(k_pos->v_d_weights[i][j][k], true); - #endif } gree(k_pos->weights[i][j], true); - gree(k_pos->d_weights[i][j], true); - #ifdef ADAM_CNN_WEIGHTS - gree(k_pos->s_d_weights[i][j], true); - gree(k_pos->v_d_weights[i][j], true); - #endif } gree(k_pos->weights[i], true); - gree(k_pos->d_weights[i], true); - #ifdef ADAM_CNN_WEIGHTS - gree(k_pos->s_d_weights[i], true); - gree(k_pos->v_d_weights[i], true); - #endif } gree(k_pos->weights, true); - gree(k_pos->d_weights, true); - #ifdef ADAM_CNN_WEIGHTS - gree(k_pos->s_d_weights, true); - gree(k_pos->v_d_weights, true); - #endif + + // Partie initialisée que sous certaines conditions (donc ne pas toujours libérer) + if (network->finetuning == EVERYTHING) { + for (int i=0; i < c; i++) { + for (int j=0; j < bias_size; j++) { + gree(k_pos->d_bias[i][j], true); + #ifdef ADAM_CNN_BIAS + gree(k_pos->s_d_bias[i][j], true); + gree(k_pos->v_d_bias[i][j], true); + #endif + } + gree(k_pos->d_bias[i], true); + #ifdef ADAM_CNN_BIAS + gree(k_pos->s_d_bias[i], true); + gree(k_pos->v_d_bias[i], true); + #endif + } + gree(k_pos->d_bias, true); + #ifdef ADAM_CNN_BIAS + gree(k_pos->s_d_bias, true); + gree(k_pos->v_d_bias, true); + #endif + + for (int i=0; i < r; i++) { + for (int j=0; j < c; j++) { + for (int k=0; k < k_size; k++) { + gree(k_pos->d_weights[i][j][k], true); + #ifdef ADAM_CNN_WEIGHTS + gree(k_pos->s_d_weights[i][j][k], true); + gree(k_pos->v_d_weights[i][j][k], true); + #endif + } + gree(k_pos->d_weights[i][j], true); + #ifdef ADAM_CNN_WEIGHTS + gree(k_pos->s_d_weights[i][j], true); + gree(k_pos->v_d_weights[i][j], true); + #endif + } + gree(k_pos->d_weights[i], true); + #ifdef ADAM_CNN_WEIGHTS + gree(k_pos->s_d_weights[i], true); + gree(k_pos->v_d_weights[i], true); + #endif + } + gree(k_pos->d_weights, true); + #ifdef ADAM_CNN_WEIGHTS + gree(k_pos->s_d_weights, true); + gree(k_pos->v_d_weights, true); + #endif + } gree(k_pos, true); } @@ -103,22 +121,26 @@ void free_dense(Network* network, int pos) { free_a_line_input_layer(network, pos+1); Kernel_nn* k_pos = network->kernel[pos]->nn; int dim = k_pos->size_input; + for (int i=0; i < dim; i++) { gree(k_pos->weights[i], true); + } + gree(k_pos->weights, true); + gree(k_pos->bias, true); + + for (int i=0; i < dim; i++) { gree(k_pos->d_weights[i], true); #ifdef ADAM_DENSE_WEIGHTS gree(k_pos->s_d_weights[i], true); gree(k_pos->v_d_weights[i], true); #endif } - gree(k_pos->weights, true); gree(k_pos->d_weights, true); #ifdef ADAM_DENSE_WEIGHTS gree(k_pos->s_d_weights, true); gree(k_pos->v_d_weights, true); #endif - gree(k_pos->bias, true); gree(k_pos->d_bias, true); #ifdef ADAM_DENSE_BIAS gree(k_pos->s_d_bias, true); @@ -132,27 +154,35 @@ void free_dense_linearisation(Network* network, int pos) { free_a_line_input_layer(network, pos+1); Kernel_nn* k_pos = network->kernel[pos]->nn; int dim = k_pos->size_input; + + // Partie toujours initialisée (donc à libérer) for (int i=0; i < dim; i++) { gree(k_pos->weights[i], true); - gree(k_pos->d_weights[i], true); - #ifdef ADAM_DENSE_WEIGHTS - gree(k_pos->s_d_weights[i], true); - gree(k_pos->v_d_weights[i], true); - #endif } gree(k_pos->weights, true); - gree(k_pos->d_weights, true); - #ifdef ADAM_DENSE_WEIGHTS - gree(k_pos->s_d_weights, true); - gree(k_pos->v_d_weights, true); - #endif - gree(k_pos->bias, true); - gree(k_pos->d_bias, true); - #ifdef ADAM_DENSE_BIAS - gree(k_pos->s_d_bias, true); - gree(k_pos->v_d_bias, true); - #endif + + // Partie initialisée que sous certaines conditions (donc ne pas toujours libérer) + if (network->finetuning <= NN_AND_LINEARISATION) { + for (int i=0; i < dim; i++) { + gree(k_pos->d_weights[i], true); + #ifdef ADAM_DENSE_WEIGHTS + gree(k_pos->s_d_weights[i], true); + gree(k_pos->v_d_weights[i], true); + #endif + } + gree(k_pos->d_weights, true); + #ifdef ADAM_DENSE_WEIGHTS + gree(k_pos->s_d_weights, true); + gree(k_pos->v_d_weights, true); + #endif + + gree(k_pos->d_bias, true); + #ifdef ADAM_DENSE_BIAS + gree(k_pos->s_d_bias, true); + gree(k_pos->v_d_bias, true); + #endif + } gree(k_pos, true); } diff --git a/src/cnn/include/config.h b/src/cnn/include/config.h index df9d89e..79d0076 100644 --- a/src/cnn/include/config.h +++ b/src/cnn/include/config.h @@ -18,10 +18,10 @@ //* Options d'ADAM optimizer //* Activer ou désactiver Adam sur les couches dense -//#define ADAM_DENSE_WEIGHTS +#define ADAM_DENSE_WEIGHTS //#define ADAM_DENSE_BIAS //* Activer ou désactiver Adam sur les couches convolutives -//#define ADAM_CNN_WEIGHTS +#define ADAM_CNN_WEIGHTS //#define ADAM_CNN_BIAS diff --git a/src/cnn/include/creation.h b/src/cnn/include/creation.h index 07ce339..0a5468e 100644 --- a/src/cnn/include/creation.h +++ b/src/cnn/include/creation.h @@ -7,7 +7,7 @@ /* * Créé un réseau qui peut contenir max_size couche (dont celle d'input et d'output) */ -Network* create_network(int max_size, float learning_rate, int dropout, int initialisation, int input_width, int input_depth); +Network* create_network(int max_size, float learning_rate, int dropout, int initialisation, int input_width, int input_depth, int finetuning); /* * Créé et alloue de la mémoire à une couche de type input cube diff --git a/src/cnn/include/models.h b/src/cnn/include/models.h index 748f20e..40b14c7 100644 --- a/src/cnn/include/models.h +++ b/src/cnn/include/models.h @@ -8,29 +8,29 @@ /* * Renvoie un réseau suivant l'architecture LeNet5 */ -Network* create_network_lenet5(float learning_rate, int dropout, int activation, int initialisation, int input_width, int input_depth); +Network* create_network_lenet5(float learning_rate, int dropout, int activation, int initialisation, int input_width, int input_depth, int finetuning); /* * Renvoie un réseau suivant l'architecture AlexNet * C'est à dire en entrée 3x227x227 et une sortie de taille 'size_output' */ -Network* create_network_alexnet(float learning_rate, int dropout, int activation, int initialisation, int size_output); +Network* create_network_alexnet(float learning_rate, int dropout, int activation, int initialisation, int size_output, int finetuning); /* * Renvoie un réseau suivant l'architecture VGG16 modifiée pour prendre en entrée 3x256x256 * et une sortie de taille 'size_output' */ -Network* create_network_VGG16(float learning_rate, int dropout, int activation, int initialisation, int size_output); +Network* create_network_VGG16(float learning_rate, int dropout, int activation, int initialisation, int size_output, int finetuning); /* * Renvoie un réseau suivant l'architecture VGG16 originel pour prendre en entrée 3x227x227 * et une sortie de taille 1 000 */ -Network* create_network_VGG16_227(float learning_rate, int dropout, int activation, int initialisation); +Network* create_network_VGG16_227(float learning_rate, int dropout, int activation, int initialisation, int finetuning); /* * Renvoie un réseau sans convolution, similaire à celui utilisé dans src/dense */ -Network* create_simple_one(float learning_rate, int dropout, int activation, int initialisation, int input_width, int input_depth); +Network* create_simple_one(float learning_rate, int dropout, int activation, int initialisation, int input_width, int input_depth, int finetuning); #endif \ No newline at end of file diff --git a/src/cnn/include/struct.h b/src/cnn/include/struct.h index 6d489fa..fdb0f75 100644 --- a/src/cnn/include/struct.h +++ b/src/cnn/include/struct.h @@ -67,6 +67,7 @@ typedef struct Network{ int dropout; // Probabilité d'abandon d'un neurone dans [0, 100] (entiers) float learning_rate; // Taux d'apprentissage du réseau int initialisation; // Id du type d'initialisation + int finetuning; // backpropagation: 0 sur tout; 1 sur dense et linéarisation; 2 sur dense int max_size; // Taille du tableau contenant le réseau int size; // Taille actuelle du réseau (size ≤ max_size) diff --git a/src/cnn/models.c b/src/cnn/models.c index 7f3130a..8116ebf 100644 --- a/src/cnn/models.c +++ b/src/cnn/models.c @@ -7,8 +7,8 @@ #include "include/models.h" -Network* create_network_lenet5(float learning_rate, int dropout, int activation, int initialisation, int input_width, int input_depth) { - Network* network = create_network(8, learning_rate, dropout, initialisation, input_width, input_depth); +Network* create_network_lenet5(float learning_rate, int dropout, int activation, int initialisation, int input_width, int input_depth, int finetuning) { + Network* network = create_network(8, learning_rate, dropout, initialisation, input_width, input_depth, finetuning); add_convolution(network, 5, 6, 1, 0, activation); add_average_pooling(network, 2, 2, 0); add_convolution(network, 5, 16, 1, 0, activation); @@ -19,8 +19,8 @@ Network* create_network_lenet5(float learning_rate, int dropout, int activation, return network; } -Network* create_network_alexnet(float learning_rate, int dropout, int activation, int initialisation, int size_output) { - Network* network = create_network(12, learning_rate, dropout, initialisation, 227, 3); +Network* create_network_alexnet(float learning_rate, int dropout, int activation, int initialisation, int size_output, int finetuning) { + Network* network = create_network(12, learning_rate, dropout, initialisation, 227, 3, finetuning); add_convolution(network, 11, 96, 4, 0, activation); add_average_pooling(network, 3, 2, 0); add_convolution(network, 5, 256, 1, 2, activation); @@ -35,8 +35,8 @@ Network* create_network_alexnet(float learning_rate, int dropout, int activation return network; } -Network* create_network_VGG16(float learning_rate, int dropout, int activation, int initialisation, int size_output) { - Network* network = create_network(22, learning_rate, dropout, initialisation, 256, 3); +Network* create_network_VGG16(float learning_rate, int dropout, int activation, int initialisation, int size_output, int finetuning) { + Network* network = create_network(22, learning_rate, dropout, initialisation, 256, 3, finetuning); add_convolution(network, 3, 64, 1, 1, activation); // Conv3-64 add_convolution(network, 3, 64, 1, 1, activation); // Conv3-64 add_average_pooling(network, 2, 2, 0); // Max Pool @@ -66,8 +66,8 @@ Network* create_network_VGG16(float learning_rate, int dropout, int activation, return network; } -Network* create_network_VGG16_227(float learning_rate, int dropout, int activation, int initialisation) { - Network* network = create_network(22, learning_rate, dropout, initialisation, 227, 3); +Network* create_network_VGG16_227(float learning_rate, int dropout, int activation, int initialisation, int finetuning) { + Network* network = create_network(22, learning_rate, dropout, initialisation, 227, 3, finetuning); add_convolution(network, 3, 64, 1, 1, activation); // Conv3-64 add_convolution(network, 3, 64, 1, 1, activation); // Conv3-64 add_average_pooling(network, 2, 2, 0); // Max Pool @@ -97,8 +97,8 @@ Network* create_network_VGG16_227(float learning_rate, int dropout, int activati return network; } -Network* create_simple_one(float learning_rate, int dropout, int activation, int initialisation, int input_width, int input_depth) { - Network* network = create_network(3, learning_rate, dropout, initialisation, input_width, input_depth); +Network* create_simple_one(float learning_rate, int dropout, int activation, int initialisation, int input_width, int input_depth, int finetuning) { + Network* network = create_network(3, learning_rate, dropout, initialisation, input_width, input_depth, finetuning); add_dense_linearisation(network, 80, activation); add_dense(network, 10, SOFTMAX); return network; diff --git a/src/cnn/neuron_io.c b/src/cnn/neuron_io.c index bd5f30c..b240299 100644 --- a/src/cnn/neuron_io.c +++ b/src/cnn/neuron_io.c @@ -187,6 +187,7 @@ Network* read_network(char* filename) { network->initialisation = initialisation; (void) !fread(&dropout, sizeof(uint32_t), 1, ptr); network->dropout = dropout; + network->finetuning = 0; // Lecture de la taille de l'entrée des différentes matrices network->width = (int*)nalloc(size, sizeof(int)); @@ -268,28 +269,35 @@ Kernel* read_kernel(int type_couche, int output_width, FILE* ptr) { float tmp; cnn->bias = (float***)nalloc(cnn->columns, sizeof(float**)); + for (int i=0; i < cnn->columns; i++) { + cnn->bias[i] = (float**)nalloc(output_width, sizeof(float*)); + for (int j=0; j < output_width; j++) { + cnn->bias[i][j] = (float*)nalloc(output_width, sizeof(float)); + for (int k=0; k < output_width; k++) { + (void) !fread(&tmp, sizeof(tmp), 1, ptr); + cnn->bias[i][j][k] = tmp; + } + } + } + cnn->d_bias = (float***)nalloc(cnn->columns, sizeof(float**)); #ifdef ADAM_CNN_BIAS cnn->s_d_bias = (float***)nalloc(cnn->columns, sizeof(float**)); cnn->v_d_bias = (float***)nalloc(cnn->columns, sizeof(float**)); #endif for (int i=0; i < cnn->columns; i++) { - cnn->bias[i] = (float**)nalloc(output_width, sizeof(float*)); cnn->d_bias[i] = (float**)nalloc(output_width, sizeof(float*)); #ifdef ADAM_CNN_BIAS cnn->s_d_bias[i] = (float**)nalloc(output_width, sizeof(float*)); cnn->v_d_bias[i] = (float**)nalloc(output_width, sizeof(float*)); #endif for (int j=0; j < output_width; j++) { - cnn->bias[i][j] = (float*)nalloc(output_width, sizeof(float)); cnn->d_bias[i][j] = (float*)nalloc(output_width, sizeof(float)); #ifdef ADAM_CNN_BIAS cnn->s_d_bias[i][j] = (float*)nalloc(output_width, sizeof(float)); cnn->v_d_bias[i][j] = (float*)nalloc(output_width, sizeof(float)); #endif for (int k=0; k < output_width; k++) { - (void) !fread(&tmp, sizeof(tmp), 1, ptr); - cnn->bias[i][j][k] = tmp; cnn->d_bias[i][j][k] = 0.; #ifdef ADAM_CNN_BIAS cnn->s_d_bias[i][j][k] = 0.; @@ -299,36 +307,46 @@ Kernel* read_kernel(int type_couche, int output_width, FILE* ptr) { } } + cnn->weights = (float****)nalloc(cnn->rows, sizeof(float***)); + for (int i=0; i < cnn->rows; i++) { + cnn->weights[i] = (float***)nalloc(cnn->columns, sizeof(float**)); + for (int j=0; j < cnn->columns; j++) { + cnn->weights[i][j] = (float**)nalloc(cnn->k_size, sizeof(float*)); + for (int k=0; k < cnn->k_size; k++) { + cnn->weights[i][j][k] = (float*)nalloc(cnn->k_size, sizeof(float)); + for (int l=0; l < cnn->k_size; l++) { + (void) !fread(&tmp, sizeof(tmp), 1, ptr); + cnn->weights[i][j][k][l] = tmp; + } + } + } + } + cnn->d_weights = (float****)nalloc(cnn->rows, sizeof(float***)); #ifdef ADAM_CNN_WEIGHTS cnn->s_d_weights = (float****)nalloc(cnn->rows, sizeof(float***)); cnn->v_d_weights = (float****)nalloc(cnn->rows, sizeof(float***)); #endif for (int i=0; i < cnn->rows; i++) { - cnn->weights[i] = (float***)nalloc(cnn->columns, sizeof(float**)); cnn->d_weights[i] = (float***)nalloc(cnn->columns, sizeof(float**)); #ifdef ADAM_CNN_WEIGHTS cnn->s_d_weights[i] = (float***)nalloc(cnn->columns, sizeof(float**)); cnn->v_d_weights[i] = (float***)nalloc(cnn->columns, sizeof(float**)); #endif for (int j=0; j < cnn->columns; j++) { - cnn->weights[i][j] = (float**)nalloc(cnn->k_size, sizeof(float*)); cnn->d_weights[i][j] = (float**)nalloc(cnn->k_size, sizeof(float*)); #ifdef ADAM_CNN_WEIGHTS cnn->s_d_weights[i][j] = (float**)nalloc(cnn->k_size, sizeof(float*)); cnn->v_d_weights[i][j] = (float**)nalloc(cnn->k_size, sizeof(float*)); #endif for (int k=0; k < cnn->k_size; k++) { - cnn->weights[i][j][k] = (float*)nalloc(cnn->k_size, sizeof(float)); cnn->d_weights[i][j][k] = (float*)nalloc(cnn->k_size, sizeof(float)); #ifdef ADAM_CNN_WEIGHTS cnn->s_d_weights[i][j][k] = (float*)nalloc(cnn->k_size, sizeof(float)); cnn->v_d_weights[i][j][k] = (float*)nalloc(cnn->k_size, sizeof(float)); #endif for (int l=0; l < cnn->k_size; l++) { - (void) !fread(&tmp, sizeof(tmp), 1, ptr); - cnn->weights[i][j][k][l] = tmp; cnn->d_weights[i][j][k][l] = 0.; #ifdef ADAM_CNN_WEIGHTS cnn->s_d_weights[i][j][k][l] = 0.; @@ -357,14 +375,17 @@ Kernel* read_kernel(int type_couche, int output_width, FILE* ptr) { float tmp; nn->bias = (float*)nalloc(nn->size_output, sizeof(float)); + for (int i=0; i < nn->size_output; i++) { + (void) !fread(&tmp, sizeof(tmp), 1, ptr); + nn->bias[i] = tmp; + } + nn->d_bias = (float*)nalloc(nn->size_output, sizeof(float)); #ifdef ADAM_DENSE_BIAS nn->s_d_bias = (float*)nalloc(nn->size_output, sizeof(float)); nn->v_d_bias = (float*)nalloc(nn->size_output, sizeof(float)); #endif for (int i=0; i < nn->size_output; i++) { - (void) !fread(&tmp, sizeof(tmp), 1, ptr); - nn->bias[i] = tmp; nn->d_bias[i] = 0.; #ifdef ADAM_DENSE_BIAS nn->s_d_bias[i] = 0.; @@ -373,21 +394,26 @@ Kernel* read_kernel(int type_couche, int output_width, FILE* ptr) { } nn->weights = (float**)nalloc(nn->size_input, sizeof(float*)); + for (int i=0; i < nn->size_input; i++) { + nn->weights[i] = (float*)nalloc(nn->size_output, sizeof(float)); + for (int j=0; j < nn->size_output; j++) { + (void) !fread(&tmp, sizeof(tmp), 1, ptr); + nn->weights[i][j] = tmp; + } + } + nn->d_weights = (float**)nalloc(nn->size_input, sizeof(float*)); #ifdef ADAM_DENSE_WEIGHTS nn->s_d_weights = (float**)nalloc(nn->size_input, sizeof(float*)); nn->v_d_weights = (float**)nalloc(nn->size_input, sizeof(float*)); #endif for (int i=0; i < nn->size_input; i++) { - nn->weights[i] = (float*)nalloc(nn->size_output, sizeof(float)); nn->d_weights[i] = (float*)nalloc(nn->size_output, sizeof(float)); #ifdef ADAM_DENSE_WEIGHTS nn->s_d_weights[i] = (float*)nalloc(nn->size_output, sizeof(float)); nn->v_d_weights[i] = (float*)nalloc(nn->size_output, sizeof(float)); #endif for (int j=0; j < nn->size_output; j++) { - (void) !fread(&tmp, sizeof(tmp), 1, ptr); - nn->weights[i][j] = tmp; nn->d_weights[i][j] = 0.; #ifdef ADAM_DENSE_WEIGHTS nn->s_d_weights[i][j] = 0.; @@ -395,6 +421,7 @@ Kernel* read_kernel(int type_couche, int output_width, FILE* ptr) { #endif } } + } else if (type_couche == POOLING) { // Cas du Pooling Layer uint32_t pooling, linearisation, stride, padding; (void) !fread(&linearisation, sizeof(linearisation), 1, ptr); diff --git a/src/cnn/train.c b/src/cnn/train.c index 5840585..e2c91bb 100644 --- a/src/cnn/train.c +++ b/src/cnn/train.c @@ -232,10 +232,10 @@ void train(int dataset_type, char* images_file, char* labels_file, char* data_di Network* network; if (!recover) { if (dataset_type == 0) { - network = create_network_lenet5(LEARNING_RATE, 0, LEAKY_RELU, HE, input_width, input_depth); + network = create_network_lenet5(LEARNING_RATE, 0, LEAKY_RELU, HE, input_width, input_depth, finetuning); //network = create_simple_one(LEARNING_RATE, 0, RELU, GLOROT, input_width, input_depth); } else { - network = create_network_VGG16(LEARNING_RATE, 0, RELU, HE, dataset->numCategories); + network = create_network_VGG16(LEARNING_RATE, 0, RELU, HE, dataset->numCategories, finetuning); #ifdef USE_MULTITHREADING printf_warning("Utilisation de VGG16 avec multithreading. La quantité de RAM utilisée peut devenir excessive\n"); diff --git a/src/cnn/update.c b/src/cnn/update.c index f8fa448..a23e476 100644 --- a/src/cnn/update.c +++ b/src/cnn/update.c @@ -4,6 +4,7 @@ #include "include/update.h" #include "include/struct.h" +#include "include/cnn.h" #include "include/config.h" @@ -31,6 +32,9 @@ void update_weights(Network* network, Network* d_network) { int output_width = network->width[i+1]; if (k_i->cnn) { // Convolution + if (network->finetuning != EVERYTHING) { + return; // Alors on a finit de backpropager + } Kernel_cnn* cnn = k_i->cnn; Kernel_cnn* d_cnn = dk_i->cnn; int k_size = cnn->k_size; @@ -70,6 +74,9 @@ void update_weights(Network* network, Network* d_network) { } } } else { // Matrice -> vecteur + if (network->finetuning == NN_ONLY) { + return; // Alors on a finit de backpropager + } Kernel_nn* nn = k_i->nn; Kernel_nn* d_nn = dk_i->nn; @@ -105,6 +112,9 @@ void update_bias(Network* network, Network* d_network) { int output_depth = network->depth[i+1]; if (k_i->cnn) { // Convolution + if (network->finetuning != EVERYTHING) { + return; // Alors on a finit de backpropager + } Kernel_cnn* cnn = k_i->cnn; Kernel_cnn* d_cnn = dk_i->cnn; @@ -124,6 +134,11 @@ void update_bias(Network* network, Network* d_network) { } } } else if (k_i->nn) { // Full connection + if (k_i->linearisation == DO_LINEARISE) {// Matrice -> vecteur + if (network->finetuning == NN_ONLY) { + return; // Alors on a finit de backpropager + } + } Kernel_nn* nn = k_i->nn; Kernel_nn* d_nn = dk_i->nn; @@ -157,6 +172,9 @@ void reset_d_weights(Network* network) { int output_width = network->width[i+1]; if (k_i->cnn) { // Convolution + if (network->finetuning != EVERYTHING) { + continue; // On n'a pas initialisé donc on n'a pas besoin de reset + } Kernel_cnn* cnn = k_i_1->cnn; int k_size = cnn->k_size; @@ -180,6 +198,9 @@ void reset_d_weights(Network* network) { } } } else { // Matrice -> vecteur + if (network->finetuning == NN_ONLY) { + continue; // On n'a pas initialisé donc on n'a pas besoin de reset + } Kernel_nn* nn = k_i_1->nn; int size_input = input_width*input_width*input_depth; @@ -206,6 +227,9 @@ void reset_d_bias(Network* network) { int output_depth = network->depth[i+1]; if (k_i->cnn) { // Convolution + if (network->finetuning != EVERYTHING) { + continue; // On n'a pas initialisé donc on n'a pas besoin de reset + } Kernel_cnn* cnn = k_i_1->cnn; for (int a=0; a < output_depth; a++) { @@ -216,6 +240,11 @@ void reset_d_bias(Network* network) { } } } else if (k_i->nn) { // Full connection + if (k_i->linearisation == DO_LINEARISE) { + if (network->finetuning == NN_ONLY) { + continue; // On n'a pas initialisé donc on n'a pas besoin de reset + } + } Kernel_nn* nn = k_i_1->nn; for (int a=0; a < output_width; a++) { diff --git a/test/cnn_neuron_io.c b/test/cnn_neuron_io.c index 987c448..f19f332 100644 --- a/test/cnn_neuron_io.c +++ b/test/cnn_neuron_io.c @@ -13,7 +13,7 @@ int main() { printf("Création du réseau\n"); - Network* network = create_network_lenet5(0, 0, 3, GLOROT, 32, 1); + Network* network = create_network_lenet5(0, 0, 3, GLOROT, 32, 1, 2); // Pas besoin d'initialiser toute la backprop printf(GREEN "OK\n" RESET); printf("Écriture du réseau\n"); diff --git a/test/cnn_structure.c b/test/cnn_structure.c index 1f8d594..705d395 100644 --- a/test/cnn_structure.c +++ b/test/cnn_structure.c @@ -8,12 +8,13 @@ #include "../src/cnn/include/models.h" #include "../src/cnn/include/utils.h" #include "../src/cnn/include/free.h" +#include "../src/cnn/include/cnn.h" int main() { Kernel* kernel; printf("Création du réseau\n"); - Network* network = create_network_lenet5(0, 0, 3, 2, 32, 1); + Network* network = create_network_lenet5(0, 0, 3, 2, 32, 1, NN_ONLY); // Pas besoin d'initialiser toute la backprop printf(GREEN "OK\n" RESET); printf("Architecture LeNet5:\n"); diff --git a/test/cnn_utils.c b/test/cnn_utils.c index 22d212a..72cda04 100644 --- a/test/cnn_utils.c +++ b/test/cnn_utils.c @@ -9,8 +9,8 @@ int main() { printf("Création du réseau\n"); - Network* network = create_network_lenet5(0, 0, 3, 2, 32, 1); - Network* network2 = create_network_lenet5(0, 0, 3, 2, 32, 1); + Network* network = create_network_lenet5(0, 0, 3, 2, 32, 1, 0); + Network* network2 = create_network_lenet5(0, 0, 3, 2, 32, 1, 0); printf(GREEN "OK\n" RESET); printf("Copie du réseau via copy_network\n"); From 208b121c734f7f58eb5fa2dfbeb41de660fe62c1 Mon Sep 17 00:00:00 2001 From: julienChemillier Date: Fri, 26 May 2023 22:16:26 +0200 Subject: [PATCH 2/9] Add a new classe: 'D_Network' --- src/cnn/include/struct.h | 55 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/cnn/include/struct.h b/src/cnn/include/struct.h index fdb0f75..3a0cbf1 100644 --- a/src/cnn/include/struct.h +++ b/src/cnn/include/struct.h @@ -10,6 +10,9 @@ #define DOESNT_LINEARISE 0 #define DO_LINEARISE 1 + +//-------------------------- Réseau classique -------------------------- + typedef struct Kernel_cnn { // Noyau ayant une couche matricielle en sortie int k_size; // k_size = 2*padding + input_width + stride - output_width*stride @@ -63,6 +66,7 @@ typedef struct Kernel { } Kernel; + typedef struct Network{ int dropout; // Probabilité d'abandon d'un neurone dans [0, 100] (entiers) float learning_rate; // Taux d'apprentissage du réseau @@ -80,4 +84,55 @@ typedef struct Network{ float**** input; // input[i] = f(input_z[i]) où f est la fonction d'activation de la couche i } Network; + + +//------------------- Réseau pour la backpropagation ------------------- + +/* +* On définit ici la classe D_Network associé à la classe Network +* Elle permet la backpropagation des réseaux auxquels elle est associée +*/ + + +typedef struct D_Kernel_cnn { + // Noyau ayant une couche matricielle en sortie + + float*** d_bias; // d_bias[columns][output_width][output_width] + #ifdef ADAM_CNN_BIAS + float*** s_d_bias; // s_d_bias[columns][output_width][output_width] + float*** v_d_bias; // v_d_bias[columns][output_width][output_width] + #endif + + float**** d_weights; // d_weights[rows][columns][k_size][k_size] + #ifdef ADAM_CNN_WEIGHTS + float**** s_d_weights; // s_d_weights[rows][columns][k_size][k_size] + float**** v_d_weights; // v_d_weights[rows][columns][k_size][k_size] + #endif +} D_Kernel_cnn; + +typedef struct D_Kernel_nn { + // Noyau ayant une couche vectorielle en sortie + + float* d_bias; // d_bias[size_output] + #ifdef ADAM_DENSE_BIAS + float* s_d_bias; // s_d_bias[size_output] + float* v_d_bias; // v_d_bias[size_output] + #endif + + float** d_weights; // d_weights[size_input][size_output] + #ifdef ADAM_DENSE_WEIGHTS + float** s_d_weights; // s_d_weights[size_input][size_output] + float** v_d_weights; // v_d_weights[size_input][size_output] + #endif +} D_Kernel_nn; + +typedef struct D_Kernel { + D_Kernel_cnn* cnn; // NULL si ce n'est pas un cnn + D_Kernel_nn* nn; // NULL si ce n'est pas un nn +} D_Kernel; + +typedef struct D_Network{ + D_Kernel** kernel; // kernel[size], contient tous les kernels +} D_Network; + #endif \ No newline at end of file From 84e552105a0beb562aa82d3ec219d1a725331ac0 Mon Sep 17 00:00:00 2001 From: julienChemillier Date: Sat, 27 May 2023 20:22:29 +0200 Subject: [PATCH 3/9] Modification in the structures --- src/cnn/backpropagation.c | 42 ++--- src/cnn/backpropagation.cu | 44 ++--- src/cnn/cnn.c | 8 +- src/cnn/creation.c | 272 +++++++++++++-------------- src/cnn/free.c | 223 +++++++++++++--------- src/cnn/include/backpropagation.h | 6 +- src/cnn/include/creation.h | 7 + src/cnn/include/free.h | 20 ++ src/cnn/include/struct.h | 135 ++++++------- src/cnn/include/update.h | 4 +- src/cnn/neuron_io.c | 96 ---------- src/cnn/train.c | 13 +- src/cnn/update.c | 49 ++--- src/cnn/utils.c | 71 +------ src/scripts/convolution_benchmark.cu | 37 +--- test/cnn_convolution.cu | 30 --- 16 files changed, 446 insertions(+), 611 deletions(-) diff --git a/src/cnn/backpropagation.c b/src/cnn/backpropagation.c index 8b12822..df499de 100644 --- a/src/cnn/backpropagation.c +++ b/src/cnn/backpropagation.c @@ -316,12 +316,12 @@ __global__ void backward_dense_kernel_2(float** weights, float* input, float* in input[idx] = tmp*( (*d_f)(input_z[idx]) ); } -void backward_dense_device(Kernel_nn* ker, float* input, float* input_z, float* output, int size_input, int size_output, int activation, int is_first) { +void backward_dense_device(Kernel_nn* ker, D_Kernel_nn* d_ker, float* input, float* input_z, float* output, int size_input, int size_output, int activation, int is_first) { // Make computation dim3 gridSize1(i_div_up(size_input, BLOCKSIZE_x), i_div_up(size_output, BLOCKSIZE_y)); dim3 blockSize1(BLOCKSIZE_x, BLOCKSIZE_y); - backward_dense_kernel_1<<>>(ker->d_weights, ker->d_bias, input, output, size_input, size_output); + backward_dense_kernel_1<<>>(d_ker->d_weights, d_ker->d_bias, input, output, size_input, size_output); gpuErrchk( cudaPeekAtLastError() ); gpuErrchk( cudaDeviceSynchronize() ); @@ -341,18 +341,18 @@ void backward_dense_device(Kernel_nn* ker, float* input, float* input_z, float* } #endif -void backward_dense_cpu(Kernel_nn* ker, float* input, float* input_z, float* output, int size_input, int size_output, int activation, int is_first) { +void backward_dense_cpu(Kernel_nn* ker, D_Kernel_nn* d_ker, float* input, float* input_z, float* output, int size_input, int size_output, int activation, int is_first) { funcPtr d_function = get_activation_function(activation); // Bias for (int j=0; j < size_output; j++) { - ker->d_bias[j] += output[j]; + d_ker->d_bias[j] += output[j]; } // Weights for (int i=0; i < size_input; i++) { for (int j=0; j < size_output; j++) { - ker->d_weights[i][j] += input[i]*output[j]; + d_ker->d_weights[i][j] += input[i]*output[j]; } } @@ -373,11 +373,11 @@ void backward_dense_cpu(Kernel_nn* ker, float* input, float* input_z, float* out #ifdef __CUDACC__ extern "C" #endif -void backward_dense(Kernel_nn* ker, float* input, float* input_z, float* output, int size_input, int size_output, int activation, int is_first) { +void backward_dense(Kernel_nn* ker, D_Kernel_nn* d_ker, float* input, float* input_z, float* output, int size_input, int size_output, int activation, int is_first) { #ifndef __CUDACC__ - backward_dense_cpu(ker, input, input_z, output, size_input, size_output, activation, is_first); + backward_dense_cpu(ker, d_ker, input, input_z, output, size_input, size_output, activation, is_first); #else - backward_dense_device(ker, input, input_z, output, size_input, size_output, activation, is_first); + backward_dense_device(ker, d_ker, input, input_z, output, size_input, size_output, activation, is_first); #endif } @@ -425,12 +425,12 @@ __global__ void backward_linearisation_kernel_2(float** weights, float*** input, input[idx][idy][idz] = tmp*( (*d_f)(input_z[idx][idy][idz]) ); } -void backward_linearisation_device(Kernel_nn* ker, float*** input, float*** input_z, float* output, int input_depth, int input_width, int size_output, int activation) { +void backward_linearisation_device(Kernel_nn* ker, D_Kernel_nn* d_ker, float*** input, float*** input_z, float* output, int input_depth, int input_width, int size_output, int activation) { // Make computation dim3 gridSize(i_div_up(input_depth, BLOCKSIZE_x), i_div_up(input_width, BLOCKSIZE_y), i_div_up(input_width, BLOCKSIZE_y)); dim3 blockSize(BLOCKSIZE_x, BLOCKSIZE_y, BLOCKSIZE_z); - backward_linearisation_kernel_1<<>>(ker->d_weights, ker->d_bias, input, output, input_depth, input_width, size_output); + backward_linearisation_kernel_1<<>>(d_ker->d_weights, d_ker->d_bias, input, output, input_depth, input_width, size_output); gpuErrchk( cudaPeekAtLastError() ); gpuErrchk( cudaDeviceSynchronize() ); @@ -445,13 +445,13 @@ void backward_linearisation_device(Kernel_nn* ker, float*** input, float*** inpu } #endif -void backward_linearisation_cpu(Kernel_nn* ker, float*** input, float*** input_z, float* output, int input_depth, int input_width, int size_output, int activation) { +void backward_linearisation_cpu(Kernel_nn* ker, D_Kernel_nn* d_ker, float*** input, float*** input_z, float* output, int input_depth, int input_width, int size_output, int activation) { funcPtr d_function = get_activation_function(activation); // Bias for (int j=0; j < size_output; j++) { - ker->d_bias[j] += output[j]; + d_ker->d_bias[j] += output[j]; } // Weights @@ -460,7 +460,7 @@ void backward_linearisation_cpu(Kernel_nn* ker, float*** input, float*** input_z for (int k=0; k < input_width; k++) { for (int l=0; l < input_width; l++) { for (int j=0; j < size_output; j++) { - ker->d_weights[cpt][j] += input[i][k][l]*output[j]; + d_ker->d_weights[cpt][j] += input[i][k][l]*output[j]; } cpt++; } @@ -486,9 +486,9 @@ void backward_linearisation_cpu(Kernel_nn* ker, float*** input, float*** input_z #ifdef __CUDACC__ extern "C" #endif -void backward_linearisation(Kernel_nn* ker, float*** input, float*** input_z, float* output, int input_depth, int input_width, int size_output, int activation) { +void backward_linearisation(Kernel_nn* ker, D_Kernel_nn* d_ker, float*** input, float*** input_z, float* output, int input_depth, int input_width, int size_output, int activation) { #ifndef __CUDACC__ - backward_linearisation_cpu(ker, input, input_z, output, input_depth, input_width, size_output, activation); + backward_linearisation_cpu(ker, d_ker, input, input_z, output, input_depth, input_width, size_output, activation); #else backward_linearisation_device(ker, input, input_z, output, input_depth, input_width, size_output, activation); #endif @@ -610,7 +610,7 @@ void backward_convolution_device(Kernel_cnn* kernel, float*** input, float*** in #endif -void backward_convolution_cpu(Kernel_cnn* ker, float*** input, float*** input_z, float*** output, int input_depth, int input_width, int output_depth, int output_width, int activation, int is_first, int kernel_size, int padding, int stride) { +void backward_convolution_cpu(Kernel_cnn* ker, D_Kernel_cnn* d_ker, float*** input, float*** input_z, float*** output, int input_depth, int input_width, int output_depth, int output_width, int activation, int is_first, int kernel_size, int padding, int stride) { funcPtr d_function = get_activation_function(activation); int max_move = kernel_size - padding; @@ -619,7 +619,7 @@ void backward_convolution_cpu(Kernel_cnn* ker, float*** input, float*** input_z, for (int i=0; i < output_depth; i++) { for (int j=0; j < output_width; j++) { for (int k=0; k < output_width; k++) { - ker->d_bias[i][j][k] += output[i][j][k]; + d_ker->d_bias[i][j][k] += output[i][j][k]; } } } @@ -637,7 +637,7 @@ void backward_convolution_cpu(Kernel_cnn* ker, float*** input, float*** input_z, } } } - ker->d_weights[h][i][j+padding][k+padding] += tmp; + d_ker->d_weights[h][i][j+padding][k+padding] += tmp; } } } @@ -680,10 +680,10 @@ void backward_convolution_cpu(Kernel_cnn* ker, float*** input, float*** input_z, #ifdef __CUDACC__ extern "C" #endif -void backward_convolution(Kernel_cnn* ker, float*** input, float*** input_z, float*** output, int input_depth, int input_width, int output_depth, int output_width, int activation, int is_first, int kernel_size, int padding, int stride) { +void backward_convolution(Kernel_cnn* ker, D_Kernel_cnn* d_ker, float*** input, float*** input_z, float*** output, int input_depth, int input_width, int output_depth, int output_width, int activation, int is_first, int kernel_size, int padding, int stride) { #ifndef __CUDACC__ - backward_convolution_cpu(ker, input, input_z, output, input_depth, input_width, output_depth, output_width, activation, is_first, kernel_size, padding, stride); + backward_convolution_cpu(ker, d_ker, input, input_z, output, input_depth, input_width, output_depth, output_width, activation, is_first, kernel_size, padding, stride); #else - backward_convolution_device(ker, input, input_z, output, input_depth, input_width, output_depth, output_width, activation, is_first, kernel_size, padding, stride); + backward_convolution_device(ker, d_ker, input, input_z, output, input_depth, input_width, output_depth, output_width, activation, is_first, kernel_size, padding, stride); #endif } \ No newline at end of file diff --git a/src/cnn/backpropagation.cu b/src/cnn/backpropagation.cu index 1d23a2b..df499de 100644 --- a/src/cnn/backpropagation.cu +++ b/src/cnn/backpropagation.cu @@ -316,12 +316,12 @@ __global__ void backward_dense_kernel_2(float** weights, float* input, float* in input[idx] = tmp*( (*d_f)(input_z[idx]) ); } -void backward_dense_device(Kernel_nn* ker, float* input, float* input_z, float* output, int size_input, int size_output, int activation, int is_first) { +void backward_dense_device(Kernel_nn* ker, D_Kernel_nn* d_ker, float* input, float* input_z, float* output, int size_input, int size_output, int activation, int is_first) { // Make computation dim3 gridSize1(i_div_up(size_input, BLOCKSIZE_x), i_div_up(size_output, BLOCKSIZE_y)); dim3 blockSize1(BLOCKSIZE_x, BLOCKSIZE_y); - backward_dense_kernel_1<<>>(ker->d_weights, ker->d_bias, input, output, size_input, size_output); + backward_dense_kernel_1<<>>(d_ker->d_weights, d_ker->d_bias, input, output, size_input, size_output); gpuErrchk( cudaPeekAtLastError() ); gpuErrchk( cudaDeviceSynchronize() ); @@ -341,18 +341,18 @@ void backward_dense_device(Kernel_nn* ker, float* input, float* input_z, float* } #endif -void backward_dense_cpu(Kernel_nn* ker, float* input, float* input_z, float* output, int size_input, int size_output, int activation, int is_first) { +void backward_dense_cpu(Kernel_nn* ker, D_Kernel_nn* d_ker, float* input, float* input_z, float* output, int size_input, int size_output, int activation, int is_first) { funcPtr d_function = get_activation_function(activation); // Bias for (int j=0; j < size_output; j++) { - ker->d_bias[j] += output[j]; + d_ker->d_bias[j] += output[j]; } // Weights for (int i=0; i < size_input; i++) { for (int j=0; j < size_output; j++) { - ker->d_weights[i][j] += input[i]*output[j]; + d_ker->d_weights[i][j] += input[i]*output[j]; } } @@ -373,11 +373,11 @@ void backward_dense_cpu(Kernel_nn* ker, float* input, float* input_z, float* out #ifdef __CUDACC__ extern "C" #endif -void backward_dense(Kernel_nn* ker, float* input, float* input_z, float* output, int size_input, int size_output, int activation, int is_first) { +void backward_dense(Kernel_nn* ker, D_Kernel_nn* d_ker, float* input, float* input_z, float* output, int size_input, int size_output, int activation, int is_first) { #ifndef __CUDACC__ - backward_dense_cpu(ker, input, input_z, output, size_input, size_output, activation, is_first); + backward_dense_cpu(ker, d_ker, input, input_z, output, size_input, size_output, activation, is_first); #else - backward_dense_device(ker, input, input_z, output, size_input, size_output, activation, is_first); + backward_dense_device(ker, d_ker, input, input_z, output, size_input, size_output, activation, is_first); #endif } @@ -425,12 +425,12 @@ __global__ void backward_linearisation_kernel_2(float** weights, float*** input, input[idx][idy][idz] = tmp*( (*d_f)(input_z[idx][idy][idz]) ); } -void backward_linearisation_device(Kernel_nn* ker, float*** input, float*** input_z, float* output, int input_depth, int input_width, int size_output, int activation) { +void backward_linearisation_device(Kernel_nn* ker, D_Kernel_nn* d_ker, float*** input, float*** input_z, float* output, int input_depth, int input_width, int size_output, int activation) { // Make computation dim3 gridSize(i_div_up(input_depth, BLOCKSIZE_x), i_div_up(input_width, BLOCKSIZE_y), i_div_up(input_width, BLOCKSIZE_y)); dim3 blockSize(BLOCKSIZE_x, BLOCKSIZE_y, BLOCKSIZE_z); - backward_linearisation_kernel_1<<>>(ker->d_weights, ker->d_bias, input, output, input_depth, input_width, size_output); + backward_linearisation_kernel_1<<>>(d_ker->d_weights, d_ker->d_bias, input, output, input_depth, input_width, size_output); gpuErrchk( cudaPeekAtLastError() ); gpuErrchk( cudaDeviceSynchronize() ); @@ -445,13 +445,13 @@ void backward_linearisation_device(Kernel_nn* ker, float*** input, float*** inpu } #endif -void backward_linearisation_cpu(Kernel_nn* ker, float*** input, float*** input_z, float* output, int input_depth, int input_width, int size_output, int activation) { +void backward_linearisation_cpu(Kernel_nn* ker, D_Kernel_nn* d_ker, float*** input, float*** input_z, float* output, int input_depth, int input_width, int size_output, int activation) { funcPtr d_function = get_activation_function(activation); // Bias for (int j=0; j < size_output; j++) { - ker->d_bias[j] += output[j]; + d_ker->d_bias[j] += output[j]; } // Weights @@ -460,7 +460,7 @@ void backward_linearisation_cpu(Kernel_nn* ker, float*** input, float*** input_z for (int k=0; k < input_width; k++) { for (int l=0; l < input_width; l++) { for (int j=0; j < size_output; j++) { - ker->d_weights[cpt][j] += input[i][k][l]*output[j]; + d_ker->d_weights[cpt][j] += input[i][k][l]*output[j]; } cpt++; } @@ -486,9 +486,9 @@ void backward_linearisation_cpu(Kernel_nn* ker, float*** input, float*** input_z #ifdef __CUDACC__ extern "C" #endif -void backward_linearisation(Kernel_nn* ker, float*** input, float*** input_z, float* output, int input_depth, int input_width, int size_output, int activation) { +void backward_linearisation(Kernel_nn* ker, D_Kernel_nn* d_ker, float*** input, float*** input_z, float* output, int input_depth, int input_width, int size_output, int activation) { #ifndef __CUDACC__ - backward_linearisation_cpu(ker, input, input_z, output, input_depth, input_width, size_output, activation); + backward_linearisation_cpu(ker, d_ker, input, input_z, output, input_depth, input_width, size_output, activation); #else backward_linearisation_device(ker, input, input_z, output, input_depth, input_width, size_output, activation); #endif @@ -610,7 +610,7 @@ void backward_convolution_device(Kernel_cnn* kernel, float*** input, float*** in #endif -void backward_convolution_cpu(Kernel_cnn* ker, float*** input, float*** input_z, float*** output, int input_depth, int input_width, int output_depth, int output_width, int activation, int is_first, int kernel_size, int padding, int stride) { +void backward_convolution_cpu(Kernel_cnn* ker, D_Kernel_cnn* d_ker, float*** input, float*** input_z, float*** output, int input_depth, int input_width, int output_depth, int output_width, int activation, int is_first, int kernel_size, int padding, int stride) { funcPtr d_function = get_activation_function(activation); int max_move = kernel_size - padding; @@ -619,7 +619,7 @@ void backward_convolution_cpu(Kernel_cnn* ker, float*** input, float*** input_z, for (int i=0; i < output_depth; i++) { for (int j=0; j < output_width; j++) { for (int k=0; k < output_width; k++) { - ker->d_bias[i][j][k] += output[i][j][k]; + d_ker->d_bias[i][j][k] += output[i][j][k]; } } } @@ -637,13 +637,13 @@ void backward_convolution_cpu(Kernel_cnn* ker, float*** input, float*** input_z, } } } - ker->d_weights[h][i][j+padding][k+padding] += tmp; + d_ker->d_weights[h][i][j+padding][k+padding] += tmp; } } } } - // Input TODO + // Input if (is_first==1) // Pas besoin de backpropager dans l'input return; for (int i=0; i < input_depth; i++) { @@ -680,10 +680,10 @@ void backward_convolution_cpu(Kernel_cnn* ker, float*** input, float*** input_z, #ifdef __CUDACC__ extern "C" #endif -void backward_convolution(Kernel_cnn* ker, float*** input, float*** input_z, float*** output, int input_depth, int input_width, int output_depth, int output_width, int activation, int is_first, int kernel_size, int padding, int stride) { +void backward_convolution(Kernel_cnn* ker, D_Kernel_cnn* d_ker, float*** input, float*** input_z, float*** output, int input_depth, int input_width, int output_depth, int output_width, int activation, int is_first, int kernel_size, int padding, int stride) { #ifndef __CUDACC__ - backward_convolution_cpu(ker, input, input_z, output, input_depth, input_width, output_depth, output_width, activation, is_first, kernel_size, padding, stride); + backward_convolution_cpu(ker, d_ker, input, input_z, output, input_depth, input_width, output_depth, output_width, activation, is_first, kernel_size, padding, stride); #else - backward_convolution_device(ker, input, input_z, output, input_depth, input_width, output_depth, output_width, activation, is_first, kernel_size, padding, stride); + backward_convolution_device(ker, d_ker, input, input_z, output, input_depth, input_width, output_depth, output_width, activation, is_first, kernel_size, padding, stride); #endif } \ No newline at end of file diff --git a/src/cnn/cnn.c b/src/cnn/cnn.c index 639f232..28e0a71 100644 --- a/src/cnn/cnn.c +++ b/src/cnn/cnn.c @@ -253,6 +253,7 @@ void forward_propagation(Network* network) { void backward_propagation(Network* network, int wanted_number, int finetuning) { int n = network->size; // Nombre de couches du réseau + D_Network* d_network = network->d_network; // Backward sur la dernière couche qui utilise toujours SOFTMAX float* wanted_output = generate_wanted_output(wanted_number, network->width[network->size -1]); // Sortie désirée, permet d'initialiser une erreur @@ -269,6 +270,7 @@ void backward_propagation(Network* network, int wanted_number, int finetuning) { for (int i=n-2; i >= 0; i--) { // Modifie 'k_i' à partir d'une comparaison d'informations entre 'input' et 'output' Kernel* k_i = network->kernel[i]; + D_Kernel* d_k_i = d_network->kernel[i]; float*** input = network->input[i]; float*** input_z = network->input_z[i]; @@ -290,15 +292,15 @@ void backward_propagation(Network* network, int wanted_number, int finetuning) { return; // On arrête la backpropagation } int kernel_size = k_i->cnn->k_size; - backward_convolution(k_i->cnn, input, input_z, output, input_depth, input_width, output_depth, output_width, -activation, is_last_layer, kernel_size, padding, stride); + backward_convolution(k_i->cnn, d_k_i->cnn, input, input_z, output, input_depth, input_width, output_depth, output_width, -activation, is_last_layer, kernel_size, padding, stride); } else if (k_i->nn) { // Full connection if (k_i->linearisation == DOESNT_LINEARISE) { // Vecteur -> Vecteur - backward_dense(k_i->nn, input[0][0], input_z[0][0], output[0][0], input_width, output_width, -activation, is_last_layer); + backward_dense(k_i->nn, d_k_i->nn, input[0][0], input_z[0][0], output[0][0], input_width, output_width, -activation, is_last_layer); } else { // Matrice -> vecteur if (finetuning == NN_ONLY) { return; // On arrête la backpropagation } - backward_linearisation(k_i->nn, input, input_z, output[0][0], input_depth, input_width, output_width, -activation); + backward_linearisation(k_i->nn, d_k_i->nn, input, input_z, output[0][0], input_depth, input_width, output_width, -activation); } } else { // Pooling int kernel_size = 2*padding + input_width + stride - output_width*stride; diff --git a/src/cnn/creation.c b/src/cnn/creation.c index c7ef64a..d175870 100644 --- a/src/cnn/creation.c +++ b/src/cnn/creation.c @@ -15,6 +15,7 @@ Network* create_network(int max_size, float learning_rate, int dropout, int init printf_error("La probabilité de dropout n'est pas respecté, elle doit être comprise entre 0 et 100\n"); } Network* network = (Network*)nalloc(1, sizeof(Network)); + network->d_network = NULL; network->learning_rate = learning_rate; network->max_size = max_size; network->dropout = dropout; @@ -36,6 +37,141 @@ Network* create_network(int max_size, float learning_rate, int dropout, int init return network; } +D_Network* create_d_network(Network* network) { + + // On initialise le réseau + int max_size = network->max_size; + D_Network* d_network = (D_Network*)nalloc(1, sizeof(D_Network)); + if (pthread_mutex_init(&(d_network->lock), NULL) != 0) + { + printf_error("Le mutex ne s'est pas initialisé correctement \n"); + } + d_network->kernel = (D_Kernel**)nalloc(max_size-1, sizeof(D_Kernel*)); + for (int i=0; i < max_size-1; i++) { + d_network->kernel[i] = (D_Kernel*)nalloc(1, sizeof(D_Kernel)); + } + + // Puis toutes ses couches + int n = network->size; + for (int i=0; i<(n-1); i++) { + Kernel* k_i = network->kernel[i]; + D_Kernel* d_k_i = d_network->kernel[i]; + + if (k_i->cnn) { // Convolution + int k_size = k_i->cnn->k_size; + int rows = k_i->cnn->rows; + int columns = k_i->cnn->columns; + int output_width = network->width[i+1]; + d_k_i->cnn = (D_Kernel_cnn*)nalloc(1, sizeof(D_Kernel_cnn)); + D_Kernel_cnn* cnn = d_k_i->cnn; + // Weights + cnn->d_weights = (float****)nalloc(rows, sizeof(float***)); + #ifdef ADAM_CNN_WEIGHTS + cnn->s_d_weights = (float****)nalloc(rows, sizeof(float***)); + cnn->v_d_weights = (float****)nalloc(rows, sizeof(float***)); + #endif + for (int i=0; i < rows; i++) { + cnn->d_weights[i] = (float***)nalloc(columns, sizeof(float**)); + #ifdef ADAM_CNN_WEIGHTS + cnn->s_d_weights[i] = (float***)nalloc(columns, sizeof(float**)); + cnn->v_d_weights[i] = (float***)nalloc(columns, sizeof(float**)); + #endif + for (int j=0; j < columns; j++) { + cnn->d_weights[i][j] = (float**)nalloc(k_size, sizeof(float*)); + #ifdef ADAM_CNN_WEIGHTS + cnn->s_d_weights[i][j] = (float**)nalloc(k_size, sizeof(float*)); + cnn->v_d_weights[i][j] = (float**)nalloc(k_size, sizeof(float*)); + #endif + for (int k=0; k < k_size; k++) { + cnn->d_weights[i][j][k] = (float*)nalloc(k_size, sizeof(float)); + #ifdef ADAM_CNN_WEIGHTS + cnn->s_d_weights[i][j][k] = (float*)nalloc(k_size, sizeof(float)); + cnn->v_d_weights[i][j][k] = (float*)nalloc(k_size, sizeof(float)); + #endif + for (int l=0; l < k_size; l++) { + cnn->d_weights[i][j][k][l] = 0.; + #ifdef ADAM_CNN_WEIGHTS + cnn->s_d_weights[i][j][k][l] = 0.; + cnn->v_d_weights[i][j][k][l] = 0.; + #endif + } + + } + } + } + //Bias + cnn->d_bias = (float***)nalloc(columns, sizeof(float**)); + #ifdef ADAM_CNN_BIAS + cnn->s_d_bias = (float***)nalloc(columns, sizeof(float**)); + cnn->v_d_bias = (float***)nalloc(columns, sizeof(float**)); + #endif + for (int i=0; i < columns; i++) { + cnn->d_bias[i] = (float**)nalloc(output_width, sizeof(float*)); + #ifdef ADAM_CNN_BIAS + cnn->s_d_bias[i] = (float**)nalloc(output_width, sizeof(float*)); + cnn->v_d_bias[i] = (float**)nalloc(output_width, sizeof(float*)); + #endif + for (int j=0; j < output_width; j++) { + cnn->d_bias[i][j] = (float*)nalloc(output_width, sizeof(float)); + #ifdef ADAM_CNN_BIAS + cnn->s_d_bias[i][j] = (float*)nalloc(output_width, sizeof(float)); + cnn->v_d_bias[i][j] = (float*)nalloc(output_width, sizeof(float)); + #endif + for (int k=0; k < output_width; k++) { + cnn->d_bias[i][j][k] = 0.; + #ifdef ADAM_CNN_BIAS + cnn->s_d_bias[i][j][k] = 0.; + cnn->v_d_bias[i][j][k] = 0.; + #endif + } + } + } + } else if (k_i->nn) { + d_k_i->nn = (D_Kernel_nn*)nalloc(1, sizeof(D_Kernel_nn)); + D_Kernel_nn* nn = d_k_i->nn; + int size_input = k_i->nn->size_input; + int size_output = k_i->nn->size_output; + // Weights + nn->d_weights = (float**)nalloc(size_input, sizeof(float*)); + #ifdef ADAM_DENSE_WEIGHTS + nn->s_d_weights = (float**)nalloc(size_input, sizeof(float*)); + nn->v_d_weights = (float**)nalloc(size_input, sizeof(float*)); + #endif + for (int i=0; i < size_input; i++) { + nn->d_weights[i] = (float*)nalloc(size_output, sizeof(float)); + #ifdef ADAM_DENSE_WEIGHTS + nn->s_d_weights[i] = (float*)nalloc(size_output, sizeof(float)); + nn->v_d_weights[i] = (float*)nalloc(size_output, sizeof(float)); + #endif + for (int j=0; j < size_output; j++) { + nn->d_weights[i][j] = 0.; + #ifdef ADAM_DENSE_WEIGHTS + nn->s_d_weights[i][j] = 0.; + nn->v_d_weights[i][j] = 0.; + #endif + } + } + // Bias + nn->d_bias = (float*)nalloc(size_output, sizeof(float)); + #ifdef ADAM_DENSE_BIAS + nn->s_d_bias = (float*)nalloc(size_output, sizeof(float)); + nn->v_d_bias = (float*)nalloc(size_output, sizeof(float)); + #endif + for (int i=0; i < size_output; i++) { + nn->d_bias[i] = 0.; + #ifdef ADAM_DENSE_BIAS + nn->s_d_bias[i] = 0.; + nn->v_d_bias[i] = 0.; + #endif + } + } + // Sinon c'est un pooling donc on ne fait rien + + } + + return d_network; +} + void create_a_cube_input_layer(Network* network, int pos, int depth, int dim) { network->input[pos] = (float***)nalloc(depth, sizeof(float**)); for (int i=0; i < depth; i++) { @@ -150,7 +286,6 @@ void add_convolution(Network* network, int kernel_size, int number_of_kernels, i cnn->rows = input_depth; cnn->columns = output_depth; - // Partie toujours initialisée cnn->weights = (float****)nalloc(input_depth, sizeof(float***)); for (int i=0; i < input_depth; i++) { cnn->weights[i] = (float***)nalloc(output_depth, sizeof(float**)); @@ -170,71 +305,6 @@ void add_convolution(Network* network, int kernel_size, int number_of_kernels, i } } - // Partie initialisée que sous certaines conditions - if (network->finetuning == EVERYTHING) { - cnn->d_weights = (float****)nalloc(input_depth, sizeof(float***)); - #ifdef ADAM_CNN_WEIGHTS - cnn->s_d_weights = (float****)nalloc(input_depth, sizeof(float***)); - cnn->v_d_weights = (float****)nalloc(input_depth, sizeof(float***)); - #endif - for (int i=0; i < input_depth; i++) { - cnn->d_weights[i] = (float***)nalloc(output_depth, sizeof(float**)); - #ifdef ADAM_CNN_WEIGHTS - cnn->s_d_weights[i] = (float***)nalloc(output_depth, sizeof(float**)); - cnn->v_d_weights[i] = (float***)nalloc(output_depth, sizeof(float**)); - #endif - for (int j=0; j < output_depth; j++) { - cnn->d_weights[i][j] = (float**)nalloc(kernel_size, sizeof(float*)); - #ifdef ADAM_CNN_WEIGHTS - cnn->s_d_weights[i][j] = (float**)nalloc(kernel_size, sizeof(float*)); - cnn->v_d_weights[i][j] = (float**)nalloc(kernel_size, sizeof(float*)); - #endif - for (int k=0; k < kernel_size; k++) { - cnn->d_weights[i][j][k] = (float*)nalloc(kernel_size, sizeof(float)); - #ifdef ADAM_CNN_WEIGHTS - cnn->s_d_weights[i][j][k] = (float*)nalloc(kernel_size, sizeof(float)); - cnn->v_d_weights[i][j][k] = (float*)nalloc(kernel_size, sizeof(float)); - #endif - for (int l=0; l < kernel_size; l++) { - cnn->d_weights[i][j][k][l] = 0.; - #ifdef ADAM_CNN_WEIGHTS - cnn->s_d_weights[i][j][k][l] = 0.; - cnn->v_d_weights[i][j][k][l] = 0.; - #endif - } - - } - } - } - - cnn->d_bias = (float***)nalloc(output_depth, sizeof(float**)); - #ifdef ADAM_CNN_BIAS - cnn->s_d_bias = (float***)nalloc(output_depth, sizeof(float**)); - cnn->v_d_bias = (float***)nalloc(output_depth, sizeof(float**)); - #endif - for (int i=0; i < output_depth; i++) { - cnn->d_bias[i] = (float**)nalloc(bias_size, sizeof(float*)); - #ifdef ADAM_CNN_BIAS - cnn->s_d_bias[i] = (float**)nalloc(bias_size, sizeof(float*)); - cnn->v_d_bias[i] = (float**)nalloc(bias_size, sizeof(float*)); - #endif - for (int j=0; j < bias_size; j++) { - cnn->d_bias[i][j] = (float*)nalloc(bias_size, sizeof(float)); - #ifdef ADAM_CNN_BIAS - cnn->s_d_bias[i][j] = (float*)nalloc(bias_size, sizeof(float)); - cnn->v_d_bias[i][j] = (float*)nalloc(bias_size, sizeof(float)); - #endif - for (int k=0; k < bias_size; k++) { - cnn->d_bias[i][j][k] = 0.; - #ifdef ADAM_CNN_BIAS - cnn->s_d_bias[i][j][k] = 0.; - cnn->v_d_bias[i][j][k] = 0.; - #endif - } - } - } - } - int n_in = network->width[n-1]*network->width[n-1]*network->depth[n-1]; int n_out = network->width[n]*network->width[n]*network->depth[n]; initialisation_3d_matrix(network->initialisation, cnn->bias, output_depth, output_width, output_width, n_in, n_out); @@ -271,39 +341,6 @@ void add_dense(Network* network, int size_output, int activation) { nn->bias = (float*)nalloc(size_output, sizeof(float)); - nn->d_weights = (float**)nalloc(size_input, sizeof(float*)); - #ifdef ADAM_DENSE_WEIGHTS - nn->s_d_weights = (float**)nalloc(size_input, sizeof(float*)); - nn->v_d_weights = (float**)nalloc(size_input, sizeof(float*)); - #endif - for (int i=0; i < size_input; i++) { - nn->d_weights[i] = (float*)nalloc(size_output, sizeof(float)); - #ifdef ADAM_DENSE_WEIGHTS - nn->s_d_weights[i] = (float*)nalloc(size_output, sizeof(float)); - nn->v_d_weights[i] = (float*)nalloc(size_output, sizeof(float)); - #endif - for (int j=0; j < size_output; j++) { - nn->d_weights[i][j] = 0.; - #ifdef ADAM_DENSE_WEIGHTS - nn->s_d_weights[i][j] = 0.; - nn->v_d_weights[i][j] = 0.; - #endif - } - } - - nn->d_bias = (float*)nalloc(size_output, sizeof(float)); - #ifdef ADAM_DENSE_BIAS - nn->s_d_bias = (float*)nalloc(size_output, sizeof(float)); - nn->v_d_bias = (float*)nalloc(size_output, sizeof(float)); - #endif - for (int i=0; i < size_output; i++) { - nn->d_bias[i] = 0.; - #ifdef ADAM_DENSE_BIAS - nn->s_d_bias[i] = 0.; - nn->v_d_bias[i] = 0.; - #endif - } - initialisation_1d_matrix(network->initialisation, nn->bias, size_output, size_input, size_output); initialisation_2d_matrix(network->initialisation, nn->weights, size_input, size_output, size_input, size_output); @@ -334,7 +371,6 @@ void add_dense_linearisation(Network* network, int size_output, int activation) nn->size_input = size_input; nn->size_output = size_output; - // Partie toujours initialisée nn->weights = (float**)nalloc(size_input, sizeof(float*)); for (int i=0; i < size_input; i++) { nn->weights[i] = (float*)nalloc(size_output, sizeof(float)); @@ -342,42 +378,6 @@ void add_dense_linearisation(Network* network, int size_output, int activation) nn->bias = (float*)nalloc(size_output, sizeof(float)); - // Partie initialisée que sous certaines conditions - if (network->finetuning <= NN_AND_LINEARISATION) { - nn->d_weights = (float**)nalloc(size_input, sizeof(float*)); - #ifdef ADAM_DENSE_WEIGHTS - nn->s_d_weights = (float**)nalloc(size_input, sizeof(float*)); - nn->v_d_weights = (float**)nalloc(size_input, sizeof(float*)); - #endif - for (int i=0; i < size_input; i++) { - nn->d_weights[i] = (float*)nalloc(size_output, sizeof(float)); - #ifdef ADAM_DENSE_WEIGHTS - nn->s_d_weights[i] = (float*)nalloc(size_output, sizeof(float)); - nn->v_d_weights[i] = (float*)nalloc(size_output, sizeof(float)); - #endif - for (int j=0; j < size_output; j++) { - nn->d_weights[i][j] = 0.; - #ifdef ADAM_DENSE_WEIGHTS - nn->s_d_weights[i][j] = 0.; - nn->v_d_weights[i][j] = 0.; - #endif - } - } - - nn->d_bias = (float*)nalloc(size_output, sizeof(float)); - #ifdef ADAM_DENSE_BIAS - nn->s_d_bias = (float*)nalloc(size_output, sizeof(float)); - nn->v_d_bias = (float*)nalloc(size_output, sizeof(float)); - #endif - for (int i=0; i < size_output; i++) { - nn->d_bias[i] = 0.; - #ifdef ADAM_DENSE_BIAS - nn->s_d_bias[i] = 0.; - nn->v_d_bias[i] = 0.; - #endif - } - } - initialisation_1d_matrix(network->initialisation, nn->bias, size_output, size_input, size_output); initialisation_2d_matrix(network->initialisation, nn->weights, size_input, size_output, size_input, size_output); create_a_line_input_layer(network, n, size_output); diff --git a/src/cnn/free.c b/src/cnn/free.c index 47ee0b2..125634d 100644 --- a/src/cnn/free.c +++ b/src/cnn/free.c @@ -64,56 +64,6 @@ void free_convolution(Network* network, int pos) { } gree(k_pos->weights, true); - // Partie initialisée que sous certaines conditions (donc ne pas toujours libérer) - if (network->finetuning == EVERYTHING) { - for (int i=0; i < c; i++) { - for (int j=0; j < bias_size; j++) { - gree(k_pos->d_bias[i][j], true); - #ifdef ADAM_CNN_BIAS - gree(k_pos->s_d_bias[i][j], true); - gree(k_pos->v_d_bias[i][j], true); - #endif - } - gree(k_pos->d_bias[i], true); - #ifdef ADAM_CNN_BIAS - gree(k_pos->s_d_bias[i], true); - gree(k_pos->v_d_bias[i], true); - #endif - } - gree(k_pos->d_bias, true); - #ifdef ADAM_CNN_BIAS - gree(k_pos->s_d_bias, true); - gree(k_pos->v_d_bias, true); - #endif - - for (int i=0; i < r; i++) { - for (int j=0; j < c; j++) { - for (int k=0; k < k_size; k++) { - gree(k_pos->d_weights[i][j][k], true); - #ifdef ADAM_CNN_WEIGHTS - gree(k_pos->s_d_weights[i][j][k], true); - gree(k_pos->v_d_weights[i][j][k], true); - #endif - } - gree(k_pos->d_weights[i][j], true); - #ifdef ADAM_CNN_WEIGHTS - gree(k_pos->s_d_weights[i][j], true); - gree(k_pos->v_d_weights[i][j], true); - #endif - } - gree(k_pos->d_weights[i], true); - #ifdef ADAM_CNN_WEIGHTS - gree(k_pos->s_d_weights[i], true); - gree(k_pos->v_d_weights[i], true); - #endif - } - gree(k_pos->d_weights, true); - #ifdef ADAM_CNN_WEIGHTS - gree(k_pos->s_d_weights, true); - gree(k_pos->v_d_weights, true); - #endif - } - gree(k_pos, true); } @@ -128,25 +78,6 @@ void free_dense(Network* network, int pos) { gree(k_pos->weights, true); gree(k_pos->bias, true); - for (int i=0; i < dim; i++) { - gree(k_pos->d_weights[i], true); - #ifdef ADAM_DENSE_WEIGHTS - gree(k_pos->s_d_weights[i], true); - gree(k_pos->v_d_weights[i], true); - #endif - } - gree(k_pos->d_weights, true); - #ifdef ADAM_DENSE_WEIGHTS - gree(k_pos->s_d_weights, true); - gree(k_pos->v_d_weights, true); - #endif - - gree(k_pos->d_bias, true); - #ifdef ADAM_DENSE_BIAS - gree(k_pos->s_d_bias, true); - gree(k_pos->v_d_bias, true); - #endif - gree(k_pos, true); } @@ -155,35 +86,12 @@ void free_dense_linearisation(Network* network, int pos) { Kernel_nn* k_pos = network->kernel[pos]->nn; int dim = k_pos->size_input; - // Partie toujours initialisée (donc à libérer) for (int i=0; i < dim; i++) { gree(k_pos->weights[i], true); } gree(k_pos->weights, true); gree(k_pos->bias, true); - // Partie initialisée que sous certaines conditions (donc ne pas toujours libérer) - if (network->finetuning <= NN_AND_LINEARISATION) { - for (int i=0; i < dim; i++) { - gree(k_pos->d_weights[i], true); - #ifdef ADAM_DENSE_WEIGHTS - gree(k_pos->s_d_weights[i], true); - gree(k_pos->v_d_weights[i], true); - #endif - } - gree(k_pos->d_weights, true); - #ifdef ADAM_DENSE_WEIGHTS - gree(k_pos->s_d_weights, true); - gree(k_pos->v_d_weights, true); - #endif - - gree(k_pos->d_bias, true); - #ifdef ADAM_DENSE_BIAS - gree(k_pos->s_d_bias, true); - gree(k_pos->v_d_bias, true); - #endif - } - gree(k_pos, true); } @@ -203,6 +111,7 @@ void free_network_creation(Network* network) { gree(network, true); } + void free_network(Network* network) { #if (defined(USE_CUDA) || defined(TEST_MEMORY_MANAGEMENT)) && defined(FREE_ALL_OPT) // Supprimer toute la mémoire allouée avec nalloc directement @@ -236,3 +145,133 @@ void free_network(Network* network) { free_network_creation(network); #endif } + +// ----------------------- Pour le d_network ----------------------- + +void free_d_convolution(Network* network, D_Network* d_network, int pos) { + Kernel_cnn* k_pos = network->kernel[pos]->cnn; + D_Kernel_cnn* d_k_pos = d_network->kernel[pos]->cnn; + int c = k_pos->columns; + int k_size = k_pos->k_size; + int r = k_pos->rows; + int bias_size = network->width[pos+1]; + + if (network->finetuning == EVERYTHING) { + for (int i=0; i < c; i++) { + for (int j=0; j < bias_size; j++) { + gree(d_k_pos->d_bias[i][j], true); + #ifdef ADAM_CNN_BIAS + gree(d_k_pos->s_d_bias[i][j], true); + gree(d_k_pos->v_d_bias[i][j], true); + #endif + } + gree(d_k_pos->d_bias[i], true); + #ifdef ADAM_CNN_BIAS + gree(d_k_pos->s_d_bias[i], true); + gree(d_k_pos->v_d_bias[i], true); + #endif + } + gree(d_k_pos->d_bias, true); + #ifdef ADAM_CNN_BIAS + gree(d_k_pos->s_d_bias, true); + gree(d_k_pos->v_d_bias, true); + #endif + + for (int i=0; i < r; i++) { + for (int j=0; j < c; j++) { + for (int k=0; k < k_size; k++) { + gree(d_k_pos->d_weights[i][j][k], true); + #ifdef ADAM_CNN_WEIGHTS + gree(d_k_pos->s_d_weights[i][j][k], true); + gree(d_k_pos->v_d_weights[i][j][k], true); + #endif + } + gree(d_k_pos->d_weights[i][j], true); + #ifdef ADAM_CNN_WEIGHTS + gree(d_k_pos->s_d_weights[i][j], true); + gree(d_k_pos->v_d_weights[i][j], true); + #endif + } + gree(d_k_pos->d_weights[i], true); + #ifdef ADAM_CNN_WEIGHTS + gree(d_k_pos->s_d_weights[i], true); + gree(d_k_pos->v_d_weights[i], true); + #endif + } + gree(d_k_pos->d_weights, true); + #ifdef ADAM_CNN_WEIGHTS + gree(d_k_pos->s_d_weights, true); + gree(d_k_pos->v_d_weights, true); + #endif + } +} + +void free_d_dense(Network* network, D_Network* d_network, int pos) { + D_Kernel_nn* d_k_pos = d_network->kernel[pos]->nn; + int dim = network->kernel[pos]->nn->size_input; + for (int i=0; i < dim; i++) { + gree(d_k_pos->d_weights[i], true); + #ifdef ADAM_DENSE_WEIGHTS + gree(d_k_pos->s_d_weights[i], true); + gree(d_k_pos->v_d_weights[i], true); + #endif + } + gree(d_k_pos->d_weights, true); + #ifdef ADAM_DENSE_WEIGHTS + gree(d_k_pos->s_d_weights, true); + gree(d_k_pos->v_d_weights, true); + #endif + + gree(d_k_pos->d_bias, true); + #ifdef ADAM_DENSE_BIAS + gree(d_k_pos->s_d_bias, true); + gree(d_k_pos->v_d_bias, true); + #endif +} + +void free_d_dense_linearisation(Network* network, D_Network* d_network, int pos) { + D_Kernel_nn* d_k_pos = d_network->kernel[pos]->nn; + int dim = network->kernel[pos]->nn->size_input; + + if (network->finetuning <= NN_AND_LINEARISATION) { + for (int i=0; i < dim; i++) { + gree(d_k_pos->d_weights[i], true); + #ifdef ADAM_DENSE_WEIGHTS + gree(d_k_pos->s_d_weights[i], true); + gree(d_k_pos->v_d_weights[i], true); + #endif + } + gree(d_k_pos->d_weights, true); + #ifdef ADAM_DENSE_WEIGHTS + gree(d_k_pos->s_d_weights, true); + gree(d_k_pos->v_d_weights, true); + #endif + + gree(d_k_pos->d_bias, true); + #ifdef ADAM_DENSE_BIAS + gree(d_k_pos->s_d_bias, true); + gree(d_k_pos->v_d_bias, true); + #endif + } + + gree(d_k_pos, true); +} + +void free_d_network_creation(Network* network, D_Network* d_network) { + for (int i=0; i < network->max_size-1; i++) { + D_Kernel* d_k_i = d_network->kernel[i]; + if (d_k_i->cnn) { // Convolution + free_d_convolution(network, d_network, i); + } else if (d_k_i->nn) { // Dense + if (network->kernel[i]->linearisation == DOESNT_LINEARISE) { // Vecteur -> Vecteur + free_d_dense(network, d_network, i); + } else { // Matrice -> Vecteur + free_d_dense_linearisation(network, d_network, i); + } + } + gree(network->kernel[i], true); + } + gree(network->kernel, true); + pthread_mutex_destroy(&(d_network->lock)); + gree(network, true); +} \ No newline at end of file diff --git a/src/cnn/include/backpropagation.h b/src/cnn/include/backpropagation.h index 049cc9e..832bdcf 100644 --- a/src/cnn/include/backpropagation.h +++ b/src/cnn/include/backpropagation.h @@ -50,7 +50,7 @@ extern "C" /* * Transfert les informations d'erreur à travers une couche fully connected */ -void backward_dense(Kernel_nn* ker, float* input, float* input_z, float* output, int size_input, int size_output, int activation, int is_first); +void backward_dense(Kernel_nn* ker, D_Kernel_nn* d_ker, float* input, float* input_z, float* output, int size_input, int size_output, int activation, int is_first); #ifdef __CUDACC__ @@ -59,7 +59,7 @@ extern "C" /* * Transfert les informations d'erreur à travers une couche de linéarisation */ -void backward_linearisation(Kernel_nn* ker, float*** input, float*** input_z, float* output, int input_depth, int input_width, int size_output, int activation); +void backward_linearisation(Kernel_nn* ker, D_Kernel_nn* d_ker, float*** input, float*** input_z, float* output, int input_depth, int input_width, int size_output, int activation); #ifdef __CUDACC__ @@ -68,6 +68,6 @@ extern "C" /* * Transfert les informations d'erreur à travers un couche de convolution */ -void backward_convolution(Kernel_cnn* ker, float*** input, float*** input_z, float*** output, int input_depth, int input_width, int output_depth, int output_width, int activation, int is_first, int kernel_size, int padding, int stride); +void backward_convolution(Kernel_cnn* ker, D_Kernel_cnn* d_ker, float*** input, float*** input_z, float*** output, int input_depth, int input_width, int output_depth, int output_width, int activation, int is_first, int kernel_size, int padding, int stride); #endif diff --git a/src/cnn/include/creation.h b/src/cnn/include/creation.h index 0a5468e..266a60d 100644 --- a/src/cnn/include/creation.h +++ b/src/cnn/include/creation.h @@ -9,6 +9,13 @@ */ Network* create_network(int max_size, float learning_rate, int dropout, int initialisation, int input_width, int input_depth, int finetuning); +/* +* Créer un réseau associé à 'network' pour la backpropagation en suivant la même +* architecture que 'network' +* Pour cela, la fonction alloue le réseau et ses couches +*/ +D_Network* create_d_network(Network* network); + /* * Créé et alloue de la mémoire à une couche de type input cube */ diff --git a/src/cnn/include/free.h b/src/cnn/include/free.h index 1dac665..9eefe2f 100644 --- a/src/cnn/include/free.h +++ b/src/cnn/include/free.h @@ -47,4 +47,24 @@ void free_network_creation(Network* network); */ void free_network(Network* network); +/* +* Libère l'espace mémoire alloué pour une d_convolution +*/ +void free_d_convolution(Network* network, D_Network* d_network, int pos); + +/* +* Libère l'espace mémoire alloué pour une d_dense +*/ +void free_d_dense(Network* network, D_Network* d_network, int pos); + +/* +* Libère l'espace mémoire alloué pour une d_dense_linearisation +*/ +void free_d_dense_linearisation(Network* network, D_Network* d_network, int pos); + +/* +* Libère entièrement l'espace mémoire alloué dans 'create_d_network' (creation.c) +*/ +void free_d_network_creation(Network* network, D_Network* d_network); + #endif \ No newline at end of file diff --git a/src/cnn/include/struct.h b/src/cnn/include/struct.h index 3a0cbf1..bde1960 100644 --- a/src/cnn/include/struct.h +++ b/src/cnn/include/struct.h @@ -1,8 +1,11 @@ #ifndef DEF_STRUCT_H #define DEF_STRUCT_H +#include + #include "config.h" + #define NO_POOLING 0 #define AVG_POOLING 1 #define MAX_POOLING 2 @@ -11,81 +14,6 @@ #define DO_LINEARISE 1 -//-------------------------- Réseau classique -------------------------- - -typedef struct Kernel_cnn { - // Noyau ayant une couche matricielle en sortie - int k_size; // k_size = 2*padding + input_width + stride - output_width*stride - int rows; // Depth de l'input - int columns; // Depth de l'output - - float*** bias; // bias[columns][output_width][output_width] <=> bias[output depth][output width][output width] - float*** d_bias; // d_bias[columns][output_width][output_width] - #ifdef ADAM_CNN_BIAS - float*** s_d_bias; // s_d_bias[columns][output_width][output_width] - float*** v_d_bias; // v_d_bias[columns][output_width][output_width] - #endif - - float**** weights; // weights[rows][columns][k_size][k_size] <=> weights[input depth][output depth][kernel size][kernel size] - float**** d_weights; // d_weights[rows][columns][k_size][k_size] - #ifdef ADAM_CNN_WEIGHTS - float**** s_d_weights; // s_d_weights[rows][columns][k_size][k_size] - float**** v_d_weights; // v_d_weights[rows][columns][k_size][k_size] - #endif -} Kernel_cnn; - -typedef struct Kernel_nn { - // Noyau ayant une couche vectorielle en sortie - int size_input; // Nombre d'éléments en entrée - int size_output; // Nombre d'éléments en sortie - - float* bias; // bias[size_output] - float* d_bias; // d_bias[size_output] - #ifdef ADAM_DENSE_BIAS - float* s_d_bias; // s_d_bias[size_output] - float* v_d_bias; // v_d_bias[size_output] - #endif - - float** weights; // weight[size_input][size_output] - float** d_weights; // d_weights[size_input][size_output] - #ifdef ADAM_DENSE_WEIGHTS - float** s_d_weights; // s_d_weights[size_input][size_output] - float** v_d_weights; // v_d_weights[size_input][size_output] - #endif -} Kernel_nn; - -typedef struct Kernel { - Kernel_cnn* cnn; // NULL si ce n'est pas un cnn - Kernel_nn* nn; // NULL si ce n'est pas un nn - - int activation; // Id de la fonction d'activation et -Id de sa dérivée - int linearisation; // 1 si c'est la linéarisation d'une couche, 0 sinon - int pooling; // 0 si pas pooling, 1 si average_pooling, 2 si max_pooling - int stride; // Valable uniquement une pooling et un cnn - int padding; // Valable uniquement une pooling et un cnn -} Kernel; - - - -typedef struct Network{ - int dropout; // Probabilité d'abandon d'un neurone dans [0, 100] (entiers) - float learning_rate; // Taux d'apprentissage du réseau - int initialisation; // Id du type d'initialisation - int finetuning; // backpropagation: 0 sur tout; 1 sur dense et linéarisation; 2 sur dense - - int max_size; // Taille du tableau contenant le réseau - int size; // Taille actuelle du réseau (size ≤ max_size) - - int* width; // width[size] - int* depth; // depth[size] - - Kernel** kernel; // kernel[size], contient tous les kernels - float**** input_z; // Tableau de toutes les couches du réseau input_z[size][couche->depth][couche->width][couche->width] - float**** input; // input[i] = f(input_z[i]) où f est la fonction d'activation de la couche i -} Network; - - - //------------------- Réseau pour la backpropagation ------------------- /* @@ -129,10 +57,67 @@ typedef struct D_Kernel_nn { typedef struct D_Kernel { D_Kernel_cnn* cnn; // NULL si ce n'est pas un cnn D_Kernel_nn* nn; // NULL si ce n'est pas un nn + // Ajouter un mutex } D_Kernel; typedef struct D_Network{ D_Kernel** kernel; // kernel[size], contient tous les kernels + pthread_mutex_t lock; // Restreint les modifications de d_network à un seul réseau à la fois } D_Network; + +//-------------------------- Réseau classique -------------------------- + +typedef struct Kernel_cnn { + // Noyau ayant une couche matricielle en sortie + int k_size; // k_size = 2*padding + input_width + stride - output_width*stride + int rows; // Depth de l'input + int columns; // Depth de l'output + + float*** bias; // bias[columns][output_width][output_width] <=> bias[output depth][output width][output width] + float**** weights; // weights[rows][columns][k_size][k_size] <=> weights[input depth][output depth][kernel size][kernel size] +} Kernel_cnn; + +typedef struct Kernel_nn { + // Noyau ayant une couche vectorielle en sortie + int size_input; // Nombre d'éléments en entrée + int size_output; // Nombre d'éléments en sortie + + float* bias; // bias[size_output] + float** weights; // weight[size_input][size_output] +} Kernel_nn; + +typedef struct Kernel { + Kernel_cnn* cnn; // NULL si ce n'est pas un cnn + Kernel_nn* nn; // NULL si ce n'est pas un nn + + int activation; // Id de la fonction d'activation et -Id de sa dérivée + int linearisation; // 1 si c'est la linéarisation d'une couche, 0 sinon + int pooling; // 0 si pas pooling, 1 si average_pooling, 2 si max_pooling + int stride; // Valable uniquement une pooling et un cnn + int padding; // Valable uniquement une pooling et un cnn +} Kernel; + + + +typedef struct Network{ + int dropout; // Probabilité d'abandon d'un neurone dans [0, 100] (entiers) + float learning_rate; // Taux d'apprentissage du réseau + int initialisation; // Id du type d'initialisation + int finetuning; // backpropagation: 0 sur tout; 1 sur dense et linéarisation; 2 sur dense + + int max_size; // Taille du tableau contenant le réseau + int size; // Taille actuelle du réseau (size ≤ max_size) + + int* width; // width[size] + int* depth; // depth[size] + + Kernel** kernel; // kernel[size], contient tous les kernels + float**** input_z; // Tableau de toutes les couches du réseau input_z[size][couche->depth][couche->width][couche->width] + float**** input; // input[i] = f(input_z[i]) où f est la fonction d'activation de la couche i + D_Network* d_network; // Réseau utilisé pour la backpropagation + // Ce dernier peut être commun à plusieurs réseau 'Network' +} Network; + + #endif \ No newline at end of file diff --git a/src/cnn/include/update.h b/src/cnn/include/update.h index 21f17ae..8f5d4d0 100644 --- a/src/cnn/include/update.h +++ b/src/cnn/include/update.h @@ -15,13 +15,13 @@ float clip(float a); * Met à jours les poids à partir de données obtenus après plusieurs backpropagations * Puis met à 0 tous les d_weights */ -void update_weights(Network* network, Network* d_network); +void update_weights(Network* network); /* * Met à jours les biais à partir de données obtenus après plusieurs backpropagations * Puis met à 0 tous les d_bias */ -void update_bias(Network* network, Network* d_network); +void update_bias(Network* network); /* * Met à 0 toutes les données de backpropagation de poids diff --git a/src/cnn/neuron_io.c b/src/cnn/neuron_io.c index b240299..3694ecd 100644 --- a/src/cnn/neuron_io.c +++ b/src/cnn/neuron_io.c @@ -280,34 +280,6 @@ Kernel* read_kernel(int type_couche, int output_width, FILE* ptr) { } } - cnn->d_bias = (float***)nalloc(cnn->columns, sizeof(float**)); - #ifdef ADAM_CNN_BIAS - cnn->s_d_bias = (float***)nalloc(cnn->columns, sizeof(float**)); - cnn->v_d_bias = (float***)nalloc(cnn->columns, sizeof(float**)); - #endif - for (int i=0; i < cnn->columns; i++) { - cnn->d_bias[i] = (float**)nalloc(output_width, sizeof(float*)); - #ifdef ADAM_CNN_BIAS - cnn->s_d_bias[i] = (float**)nalloc(output_width, sizeof(float*)); - cnn->v_d_bias[i] = (float**)nalloc(output_width, sizeof(float*)); - #endif - for (int j=0; j < output_width; j++) { - cnn->d_bias[i][j] = (float*)nalloc(output_width, sizeof(float)); - #ifdef ADAM_CNN_BIAS - cnn->s_d_bias[i][j] = (float*)nalloc(output_width, sizeof(float)); - cnn->v_d_bias[i][j] = (float*)nalloc(output_width, sizeof(float)); - #endif - for (int k=0; k < output_width; k++) { - cnn->d_bias[i][j][k] = 0.; - #ifdef ADAM_CNN_BIAS - cnn->s_d_bias[i][j][k] = 0.; - cnn->v_d_bias[i][j][k] = 0.; - #endif - } - } - } - - cnn->weights = (float****)nalloc(cnn->rows, sizeof(float***)); for (int i=0; i < cnn->rows; i++) { cnn->weights[i] = (float***)nalloc(cnn->columns, sizeof(float**)); @@ -322,40 +294,6 @@ Kernel* read_kernel(int type_couche, int output_width, FILE* ptr) { } } } - - cnn->d_weights = (float****)nalloc(cnn->rows, sizeof(float***)); - #ifdef ADAM_CNN_WEIGHTS - cnn->s_d_weights = (float****)nalloc(cnn->rows, sizeof(float***)); - cnn->v_d_weights = (float****)nalloc(cnn->rows, sizeof(float***)); - #endif - for (int i=0; i < cnn->rows; i++) { - cnn->d_weights[i] = (float***)nalloc(cnn->columns, sizeof(float**)); - #ifdef ADAM_CNN_WEIGHTS - cnn->s_d_weights[i] = (float***)nalloc(cnn->columns, sizeof(float**)); - cnn->v_d_weights[i] = (float***)nalloc(cnn->columns, sizeof(float**)); - #endif - for (int j=0; j < cnn->columns; j++) { - cnn->d_weights[i][j] = (float**)nalloc(cnn->k_size, sizeof(float*)); - #ifdef ADAM_CNN_WEIGHTS - cnn->s_d_weights[i][j] = (float**)nalloc(cnn->k_size, sizeof(float*)); - cnn->v_d_weights[i][j] = (float**)nalloc(cnn->k_size, sizeof(float*)); - #endif - for (int k=0; k < cnn->k_size; k++) { - cnn->d_weights[i][j][k] = (float*)nalloc(cnn->k_size, sizeof(float)); - #ifdef ADAM_CNN_WEIGHTS - cnn->s_d_weights[i][j][k] = (float*)nalloc(cnn->k_size, sizeof(float)); - cnn->v_d_weights[i][j][k] = (float*)nalloc(cnn->k_size, sizeof(float)); - #endif - for (int l=0; l < cnn->k_size; l++) { - cnn->d_weights[i][j][k][l] = 0.; - #ifdef ADAM_CNN_WEIGHTS - cnn->s_d_weights[i][j][k][l] = 0.; - cnn->v_d_weights[i][j][k][l] = 0.; - #endif - } - } - } - } } else if (type_couche == NN) { // Cas du NN // Lecture du "Pré-corps" kernel->nn = (Kernel_nn*)nalloc(1, sizeof(Kernel_nn)); @@ -380,19 +318,6 @@ Kernel* read_kernel(int type_couche, int output_width, FILE* ptr) { nn->bias[i] = tmp; } - nn->d_bias = (float*)nalloc(nn->size_output, sizeof(float)); - #ifdef ADAM_DENSE_BIAS - nn->s_d_bias = (float*)nalloc(nn->size_output, sizeof(float)); - nn->v_d_bias = (float*)nalloc(nn->size_output, sizeof(float)); - #endif - for (int i=0; i < nn->size_output; i++) { - nn->d_bias[i] = 0.; - #ifdef ADAM_DENSE_BIAS - nn->s_d_bias[i] = 0.; - nn->v_d_bias[i] = 0.; - #endif - } - nn->weights = (float**)nalloc(nn->size_input, sizeof(float*)); for (int i=0; i < nn->size_input; i++) { nn->weights[i] = (float*)nalloc(nn->size_output, sizeof(float)); @@ -401,27 +326,6 @@ Kernel* read_kernel(int type_couche, int output_width, FILE* ptr) { nn->weights[i][j] = tmp; } } - - nn->d_weights = (float**)nalloc(nn->size_input, sizeof(float*)); - #ifdef ADAM_DENSE_WEIGHTS - nn->s_d_weights = (float**)nalloc(nn->size_input, sizeof(float*)); - nn->v_d_weights = (float**)nalloc(nn->size_input, sizeof(float*)); - #endif - for (int i=0; i < nn->size_input; i++) { - nn->d_weights[i] = (float*)nalloc(nn->size_output, sizeof(float)); - #ifdef ADAM_DENSE_WEIGHTS - nn->s_d_weights[i] = (float*)nalloc(nn->size_output, sizeof(float)); - nn->v_d_weights[i] = (float*)nalloc(nn->size_output, sizeof(float)); - #endif - for (int j=0; j < nn->size_output; j++) { - nn->d_weights[i][j] = 0.; - #ifdef ADAM_DENSE_WEIGHTS - nn->s_d_weights[i][j] = 0.; - nn->v_d_weights[i][j] = 0.; - #endif - } - } - } else if (type_couche == POOLING) { // Cas du Pooling Layer uint32_t pooling, linearisation, stride, padding; (void) !fread(&linearisation, sizeof(linearisation), 1, ptr); diff --git a/src/cnn/train.c b/src/cnn/train.c index e2c91bb..4447895 100644 --- a/src/cnn/train.c +++ b/src/cnn/train.c @@ -14,6 +14,7 @@ #include "include/initialisation.h" #include "include/test_network.h" #include "include/neuron_io.h" +#include "include/creation.h" #include "include/function.h" #include "include/update.h" #include "include/models.h" @@ -246,6 +247,10 @@ void train(int dataset_type, char* images_file, char* labels_file, char* data_di network->learning_rate = LEARNING_RATE; } + // On ajoute le réseau de backpropagation au réseau précédemment initialisé + D_Network* d_network = create_d_network(network); + network->d_network = d_network; + /* shuffle_index[i] contient le nouvel index de l'élément à l'emplacement i avant mélange Cela permet de réordonner le jeu d'apprentissage pour éviter certains biais @@ -392,8 +397,8 @@ void train(int dataset_type, char* images_file, char* labels_file, char* data_di // On attend que tous les fils aient fini avant d'appliquer des modifications au réseau principal for (int k=0; k < nb_threads; k++) { if (train_parameters[k]->network) { // Si le fil a été utilisé - update_weights(network, train_parameters[k]->network); - update_bias(network, train_parameters[k]->network); + update_weights(network); // , train_parameters[k]->network + update_bias(network); // , train_parameters[k]->network } } current_accuracy = accuracy * nb_images_total/((j+1)*BATCHES); @@ -416,8 +421,8 @@ void train(int dataset_type, char* images_file, char* labels_file, char* data_di loss += train_params->loss/nb_images_total; batch_loss += train_params->loss/BATCHES; - update_weights(network, network); - update_bias(network, network); + update_weights(network); + update_bias(network); printf("\rÉpoque [%d/%d]\tImage [%d/%d]\tAccuracy: " YELLOW "%0.4f%%" RESET "\tBatch Accuracy: " YELLOW "%0.2f%%" RESET, i, epochs, BATCHES*(j+1), nb_images_total, current_accuracy*100, batch_accuracy*100); #endif diff --git a/src/cnn/update.c b/src/cnn/update.c index a23e476..46e13ad 100644 --- a/src/cnn/update.c +++ b/src/cnn/update.c @@ -18,12 +18,14 @@ float clip(float a) { return a; } -void update_weights(Network* network, Network* d_network) { +void update_weights(Network* network) { int n = network->size; + D_Network* d_network = network->d_network; + pthread_mutex_lock(&(d_network->lock)); for (int i=0; i < (n-1); i++) { Kernel* k_i = network->kernel[i]; - Kernel* dk_i = d_network->kernel[i]; + D_Kernel* d_k_i = d_network->kernel[i]; int input_depth = network->depth[i]; int input_width = network->width[i]; @@ -36,7 +38,7 @@ void update_weights(Network* network, Network* d_network) { return; // Alors on a finit de backpropager } Kernel_cnn* cnn = k_i->cnn; - Kernel_cnn* d_cnn = dk_i->cnn; + D_Kernel_cnn* d_cnn = d_k_i->cnn; int k_size = cnn->k_size; for (int a=0; a < input_depth; a++) { for (int b=0; b < output_depth; b++) { @@ -59,7 +61,7 @@ void update_weights(Network* network, Network* d_network) { } else if (k_i->nn) { // Full connection if (k_i->linearisation == DOESNT_LINEARISE) { // Vecteur -> Vecteur Kernel_nn* nn = k_i->nn; - Kernel_nn* d_nn = dk_i->nn; + D_Kernel_nn* d_nn = d_k_i->nn; for (int a=0; a < input_width; a++) { for (int b=0; b < output_width; b++) { @@ -78,7 +80,7 @@ void update_weights(Network* network, Network* d_network) { return; // Alors on a finit de backpropager } Kernel_nn* nn = k_i->nn; - Kernel_nn* d_nn = dk_i->nn; + D_Kernel_nn* d_nn = d_k_i->nn; int size_input = input_width*input_width*input_depth; @@ -100,14 +102,16 @@ void update_weights(Network* network, Network* d_network) { } // Une couche de pooling ne nécessite pas de traitement } + pthread_mutex_unlock(&(d_network->lock)); } -void update_bias(Network* network, Network* d_network) { +void update_bias(Network* network) { int n = network->size; + D_Network* d_network = network->d_network; for (int i=0; i < (n-1); i++) { Kernel* k_i = network->kernel[i]; - Kernel* dk_i = d_network->kernel[i]; + D_Kernel* d_k_i = d_network->kernel[i]; int output_width = network->width[i+1]; int output_depth = network->depth[i+1]; @@ -116,7 +120,7 @@ void update_bias(Network* network, Network* d_network) { return; // Alors on a finit de backpropager } Kernel_cnn* cnn = k_i->cnn; - Kernel_cnn* d_cnn = dk_i->cnn; + D_Kernel_cnn* d_cnn = d_k_i->cnn; for (int a=0; a < output_depth; a++) { for (int b=0; b < output_width; b++) { @@ -140,7 +144,7 @@ void update_bias(Network* network, Network* d_network) { } } Kernel_nn* nn = k_i->nn; - Kernel_nn* d_nn = dk_i->nn; + D_Kernel_nn* d_nn = d_k_i->nn; for (int a=0; a < output_width; a++) { #ifdef ADAM_DENSE_BIAS @@ -160,10 +164,12 @@ void update_bias(Network* network, Network* d_network) { void reset_d_weights(Network* network) { int n = network->size; + D_Network* d_network = network->d_network; for (int i=0; i < (n-1); i++) { Kernel* k_i = network->kernel[i]; Kernel* k_i_1 = network->kernel[i+1]; + D_Kernel* d_k_i_1 = d_network->kernel[i+1]; int input_depth = network->depth[i]; int input_width = network->width[i]; @@ -175,39 +181,39 @@ void reset_d_weights(Network* network) { if (network->finetuning != EVERYTHING) { continue; // On n'a pas initialisé donc on n'a pas besoin de reset } - Kernel_cnn* cnn = k_i_1->cnn; + D_Kernel_cnn* d_cnn = d_k_i_1->cnn; - int k_size = cnn->k_size; + int k_size = k_i_1->cnn->k_size; for (int a=0; a < input_depth; a++) { for (int b=0; b < output_depth; b++) { for (int c=0; c < k_size; c++) { for (int d=0; d < k_size; d++) { - cnn->d_weights[a][b][c][d] = 0; + d_cnn->d_weights[a][b][c][d] = 0; } } } } } else if (k_i->nn) { // Full connection if (k_i->linearisation == DOESNT_LINEARISE) { // Vecteur -> Vecteur - Kernel_nn* nn = k_i_1->nn; + D_Kernel_nn* d_nn = d_k_i_1->nn; for (int a=0; a < input_width; a++) { for (int b=0; b < output_width; b++) { - nn->d_weights[a][b] = 0; + d_nn->d_weights[a][b] = 0; } } } else { // Matrice -> vecteur if (network->finetuning == NN_ONLY) { continue; // On n'a pas initialisé donc on n'a pas besoin de reset } - Kernel_nn* nn = k_i_1->nn; + D_Kernel_nn* d_nn = d_k_i_1->nn; int size_input = input_width*input_width*input_depth; for (int a=0; a < size_input; a++) { for (int b=0; b < output_width; b++) { - nn->d_weights[a][b] = 0; + d_nn->d_weights[a][b] = 0; } } } @@ -218,10 +224,11 @@ void reset_d_weights(Network* network) { void reset_d_bias(Network* network) { int n = network->size; + D_Network* d_network = network->d_network; for (int i=0; i < (n-1); i++) { Kernel* k_i = network->kernel[i]; - Kernel* k_i_1 = network->kernel[i+1]; + D_Kernel* d_k_i_1 = d_network->kernel[i+1]; int output_width = network->width[i+1]; int output_depth = network->depth[i+1]; @@ -230,12 +237,12 @@ void reset_d_bias(Network* network) { if (network->finetuning != EVERYTHING) { continue; // On n'a pas initialisé donc on n'a pas besoin de reset } - Kernel_cnn* cnn = k_i_1->cnn; + D_Kernel_cnn* d_cnn = d_k_i_1->cnn; for (int a=0; a < output_depth; a++) { for (int b=0; b < output_width; b++) { for (int c=0; c < output_width; c++) { - cnn->d_bias[a][b][c] = 0; + d_cnn->d_bias[a][b][c] = 0; } } } @@ -245,10 +252,10 @@ void reset_d_bias(Network* network) { continue; // On n'a pas initialisé donc on n'a pas besoin de reset } } - Kernel_nn* nn = k_i_1->nn; + D_Kernel_nn* d_nn = d_k_i_1->nn; for (int a=0; a < output_width; a++) { - nn->d_bias[a] = 0; + d_nn->d_bias[a] = 0; } } // Une couche de pooling ne nécessite pas de traitement diff --git a/src/cnn/utils.c b/src/cnn/utils.c index 6b65c20..025ca8d 100644 --- a/src/cnn/utils.c +++ b/src/cnn/utils.c @@ -115,6 +115,7 @@ Network* copy_network(Network* network) { copyVar(initialisation); copyVar(max_size); copyVar(size); + copyVar(d_network); // Les deux réseaux partagent ainsi le même réseau pour la backpropagation network_cp->width = (int*)nalloc(size, sizeof(int)); network_cp->depth = (int*)nalloc(size, sizeof(int)); @@ -153,40 +154,15 @@ Network* copy_network(Network* network) { copyVar(kernel[i]->nn->size_output); network_cp->kernel[i]->nn->bias = (float*)nalloc(size_output, sizeof(float)); - network_cp->kernel[i]->nn->d_bias = (float*)nalloc(size_output, sizeof(float)); - #ifdef ADAM_DENSE_BIAS - network_cp->kernel[i]->nn->s_d_bias = (float*)nalloc(size_output, sizeof(float)); - network_cp->kernel[i]->nn->v_d_bias = (float*)nalloc(size_output, sizeof(float)); - #endif for (int j=0; j < size_output; j++) { copyVar(kernel[i]->nn->bias[j]); - network_cp->kernel[i]->nn->d_bias[j] = 0.; - #ifdef ADAM_DENSE_BIAS - network_cp->kernel[i]->nn->s_d_bias[j] = 0.; - network_cp->kernel[i]->nn->v_d_bias[j] = 0.; - #endif } network_cp->kernel[i]->nn->weights = (float**)nalloc(size_input, sizeof(float*)); - network_cp->kernel[i]->nn->d_weights = (float**)nalloc(size_input, sizeof(float*)); - #ifdef ADAM_DENSE_WEIGHTS - network_cp->kernel[i]->nn->s_d_weights = (float**)nalloc(size_input, sizeof(float*)); - network_cp->kernel[i]->nn->v_d_weights = (float**)nalloc(size_input, sizeof(float*)); - #endif for (int j=0; j < size_input; j++) { network_cp->kernel[i]->nn->weights[j] = (float*)nalloc(size_output, sizeof(float)); - network_cp->kernel[i]->nn->d_weights[j] = (float*)nalloc(size_output, sizeof(float)); - #ifdef ADAM_DENSE_WEIGHTS - network_cp->kernel[i]->nn->s_d_weights[j] = (float*)nalloc(size_output, sizeof(float)); - network_cp->kernel[i]->nn->v_d_weights[j] = (float*)nalloc(size_output, sizeof(float)); - #endif for (int k=0; k < size_output; k++) { copyVar(kernel[i]->nn->weights[j][k]); - network_cp->kernel[i]->nn->d_weights[j][k] = 0.; - #ifdef ADAM_DENSE_WEIGHTS - network_cp->kernel[i]->nn->s_d_weights[j][k] = 0.; - network_cp->kernel[i]->nn->v_d_weights[j][k] = 0.; - #endif } } } @@ -211,70 +187,25 @@ Network* copy_network(Network* network) { copyVar(kernel[i]->cnn->columns); network_cp->kernel[i]->cnn->bias = (float***)nalloc(columns, sizeof(float**)); - network_cp->kernel[i]->cnn->d_bias = (float***)nalloc(columns, sizeof(float**)); - #ifdef ADAM_CNN_BIAS - network_cp->kernel[i]->cnn->s_d_bias = (float***)nalloc(columns, sizeof(float**)); - network_cp->kernel[i]->cnn->v_d_bias = (float***)nalloc(columns, sizeof(float**)); - #endif for (int j=0; j < columns; j++) { network_cp->kernel[i]->cnn->bias[j] = (float**)nalloc(output_width, sizeof(float*)); - network_cp->kernel[i]->cnn->d_bias[j] = (float**)nalloc(output_width, sizeof(float*)); - #ifdef ADAM_CNN_BIAS - network_cp->kernel[i]->cnn->s_d_bias[j] = (float**)nalloc(output_width, sizeof(float*)); - network_cp->kernel[i]->cnn->v_d_bias[j] = (float**)nalloc(output_width, sizeof(float*)); - #endif for (int k=0; k < output_width; k++) { network_cp->kernel[i]->cnn->bias[j][k] = (float*)nalloc(output_width, sizeof(float)); - network_cp->kernel[i]->cnn->d_bias[j][k] = (float*)nalloc(output_width, sizeof(float)); - #ifdef ADAM_CNN_BIAS - network_cp->kernel[i]->cnn->s_d_bias[j][k] = (float*)nalloc(output_width, sizeof(float)); - network_cp->kernel[i]->cnn->v_d_bias[j][k] = (float*)nalloc(output_width, sizeof(float)); - #endif for (int l=0; l < output_width; l++) { copyVar(kernel[i]->cnn->bias[j][k][l]); - network_cp->kernel[i]->cnn->d_bias[j][k][l] = 0.; - #ifdef ADAM_CNN_BIAS - network_cp->kernel[i]->cnn->s_d_bias[j][k][l] = 0.; - network_cp->kernel[i]->cnn->v_d_bias[j][k][l] = 0.; - #endif } } } network_cp->kernel[i]->cnn->weights = (float****)nalloc(rows, sizeof(float***)); - network_cp->kernel[i]->cnn->d_weights = (float****)nalloc(rows, sizeof(float***)); - #ifdef ADAM_CNN_WEIGHTS - network_cp->kernel[i]->cnn->s_d_weights = (float****)nalloc(rows, sizeof(float***)); - network_cp->kernel[i]->cnn->v_d_weights = (float****)nalloc(rows, sizeof(float***)); - #endif for (int j=0; j < rows; j++) { network_cp->kernel[i]->cnn->weights[j] = (float***)nalloc(columns, sizeof(float**)); - network_cp->kernel[i]->cnn->d_weights[j] = (float***)nalloc(columns, sizeof(float**)); - #ifdef ADAM_CNN_WEIGHTS - network_cp->kernel[i]->cnn->s_d_weights[j] = (float***)nalloc(columns, sizeof(float**)); - network_cp->kernel[i]->cnn->v_d_weights[j] = (float***)nalloc(columns, sizeof(float**)); - #endif for (int k=0; k < columns; k++) { network_cp->kernel[i]->cnn->weights[j][k] = (float**)nalloc(k_size, sizeof(float*)); - network_cp->kernel[i]->cnn->d_weights[j][k] = (float**)nalloc(k_size, sizeof(float*)); - #ifdef ADAM_CNN_WEIGHTS - network_cp->kernel[i]->cnn->s_d_weights[j][k] = (float**)nalloc(k_size, sizeof(float*)); - network_cp->kernel[i]->cnn->v_d_weights[j][k] = (float**)nalloc(k_size, sizeof(float*)); - #endif for (int l=0; l < k_size; l++) { network_cp->kernel[i]->cnn->weights[j][k][l] = (float*)nalloc(k_size, sizeof(float)); - network_cp->kernel[i]->cnn->d_weights[j][k][l] = (float*)nalloc(k_size, sizeof(float)); - #ifdef ADAM_CNN_WEIGHTS - network_cp->kernel[i]->cnn->s_d_weights[j][k][l] = (float*)nalloc(k_size, sizeof(float)); - network_cp->kernel[i]->cnn->v_d_weights[j][k][l] = (float*)nalloc(k_size, sizeof(float)); - #endif for (int m=0; m < k_size; m++) { copyVar(kernel[i]->cnn->weights[j][k][l][m]); - network_cp->kernel[i]->cnn->d_weights[j][k][l][m] = 0.; - #ifdef ADAM_CNN_WEIGHTS - network_cp->kernel[i]->cnn->s_d_weights[j][k][l][m] = 0.; - network_cp->kernel[i]->cnn->v_d_weights[j][k][l][m] = 0.; - #endif } } } diff --git a/src/scripts/convolution_benchmark.cu b/src/scripts/convolution_benchmark.cu index 92accc4..5425dfa 100644 --- a/src/scripts/convolution_benchmark.cu +++ b/src/scripts/convolution_benchmark.cu @@ -115,34 +115,14 @@ void run_convolution_test(int input_width, int output_width, int rows, int colum // bias[kernel->columns] kernel->bias = (float*)malloc(kernel->columns, sizeof(float)); - kernel->d_bias = (float*)malloc(kernel->columns, sizeof(float)); - #ifdef ADAM_CNN_BIAS - kernel->s_d_bias = (float*)malloc(kernel->columns, sizeof(float)); - kernel->v_d_bias = (float*)malloc(kernel->columns, sizeof(float)); - #endif for (int i=0; icolumns; i++) { kernel->bias[i] = random_float(0.0f, 15.0f); - kernel->d_bias[i] = random_float(0.0f, 1.5f); - #ifdef ADAM_CNN_BIAS - kernel->s_d_bias[i] = random_float(0.0f, 1.5f); - kernel->v_d_bias[i] = random_float(0.0f, 1.5f); - #endif } // weights[rows][columns][k_size][k_size] kernel->weights = (float****)malloc(sizeof(float***)*kernel->rows); - kernel->d_weights = (float****)malloc(sizeof(float***)*kernel->rows); - #ifdef ADAM_CNN_WEIGHTS - kernel->s_d_weights = (float****)malloc(sizeof(float***)*kernel->rows); - kernel->v_d_weights = (float****)malloc(sizeof(float***)*kernel->rows); - #endif for (int i=0; i < kernel->rows; i++) { kernel->weights[i] = create_matrix(kernel->columns, kernel->k_size, kernel->k_size, 15.0f); - kernel->d_weights[i] = create_matrix(kernel->columns, kernel->k_size, kernel->k_size, 1.5f); - #ifdef ADAM_CNN_WEIGHTS - kernel->s_d_weights[i] = create_matrix(kernel->columns, kernel->k_size, kernel->k_size, 1.5f); - kernel->v_d_weights[i] = create_matrix(kernel->columns, kernel->k_size, kernel->k_size, 1.5f); - #endif } float*** input = create_matrix(kernel->rows, input_width, input_width, 5.0f); @@ -165,7 +145,7 @@ void run_convolution_test(int input_width, int output_width, int rows, int colum start = clock(); - make_convolution_cpu(kernel, input, output_cpu, output_width, 1); + make_convolution_cpu(kernel, input, output_cpu, output_width, 1, 0); end = clock(); cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; @@ -179,26 +159,11 @@ void run_convolution_test(int input_width, int output_width, int rows, int colum //printf(GREEN "OK\n" RESET); free(kernel->bias); - free(kernel->d_bias); - #ifdef ADAM_CNN_BIAS - free(kernel->s_d_bias); - free(kernel->v_d_bias); - #endif for (int i=0; i < kernel->rows; i++) { free_matrix(kernel->weights[i], kernel->columns, kernel->k_size); - free_matrix(kernel->d_weights[i], kernel->columns, kernel->k_size); - #ifdef ADAM_CNN_WEIGHTS - free_matrix(kernel->s_d_weights[i], kernel->columns, kernel->k_size); - free_matrix(kernel->v_d_weights[i], kernel->columns, kernel->k_size); - #endif } free(kernel->weights); - free(kernel->d_weights); - #ifdef ADAM_CNN_WEIGHTS - free(kernel->s_d_weights); - free(kernel->v_d_weights); - #endif free_matrix(input, kernel->rows, input_width); free_matrix(output_cpu, kernel->columns, output_width); diff --git a/test/cnn_convolution.cu b/test/cnn_convolution.cu index 5dcba85..84ecccf 100644 --- a/test/cnn_convolution.cu +++ b/test/cnn_convolution.cu @@ -106,26 +106,11 @@ void run_convolution_test(int input_width, int output_width, int rows, int colum // bias[kernel->columns][output_width][output_width] kernel->bias = create_matrix(kernel->columns, output_width, output_width, 15.0f); - kernel->d_bias = create_matrix(kernel->columns, output_width, output_width, 1.5f); - #ifdef ADAM_CNN_BIAS - kernel->s_d_bias = create_matrix(kernel->columns, output_width, output_width, 1.5f); - kernel->v_d_bias = create_matrix(kernel->columns, output_width, output_width, 1.5f); - #endif // weights[rows][columns][k_size][k_size] kernel->weights = (float****)nalloc(kernel->rows, sizeof(float***)); - kernel->d_weights = (float****)nalloc(kernel->rows, sizeof(float***)); - #ifdef ADAM_CNN_WEIGHTS - kernel->s_d_weights = (float****)nalloc(kernel->rows, sizeof(float***)); - kernel->v_d_weights = (float****)nalloc(kernel->rows, sizeof(float***)); - #endif for (int i=0; i < kernel->rows; i++) { kernel->weights[i] = create_matrix(kernel->columns, kernel->k_size, kernel->k_size, 15.0f); - kernel->d_weights[i] = create_matrix(kernel->columns, kernel->k_size, kernel->k_size, 1.5f); - #ifdef ADAM_CNN_WEIGHTS - kernel->s_d_weights[i] = create_matrix(kernel->columns, kernel->k_size, kernel->k_size, 1.5f); - kernel->v_d_weights[i] = create_matrix(kernel->columns, kernel->k_size, kernel->k_size, 1.5f); - #endif } float*** input = create_matrix(kernel->rows, input_width, input_width, 5.0f); @@ -163,26 +148,11 @@ void run_convolution_test(int input_width, int output_width, int rows, int colum printf(GREEN "OK\n" RESET); free_matrix(kernel->bias, kernel->columns, output_width); - free_matrix(kernel->d_bias, kernel->columns, output_width); - #ifdef ADAM_CNN_BIAS - free_matrix(kernel->s_d_bias, kernel->columns, output_width); - free_matrix(kernel->v_d_bias, kernel->columns, output_width); - #endif for (int i=0; i < kernel->rows; i++) { free_matrix(kernel->weights[i], kernel->columns, kernel->k_size); - free_matrix(kernel->d_weights[i], kernel->columns, kernel->k_size); - #ifdef ADAM_CNN_WEIGHTS - free_matrix(kernel->s_d_weights[i], kernel->columns, kernel->k_size); - free_matrix(kernel->v_d_weights[i], kernel->columns, kernel->k_size); - #endif } gree(kernel->weights, false); - gree(kernel->d_weights, false); - #ifdef ADAM_CNN_WEIGHTS - gree(kernel->s_d_weights, false); - gree(kernel->v_d_weights, false); - #endif free_matrix(input, kernel->rows, input_width); free_matrix(output_cpu, kernel->columns, output_width); From 4ad116511e04d39cd4a99dace5a87240ff204e11 Mon Sep 17 00:00:00 2001 From: julienChemillier Date: Sat, 27 May 2023 21:09:04 +0200 Subject: [PATCH 4/9] Fix an argument error in 'backpropagation' file --- src/cnn/backpropagation.c | 8 ++++---- src/cnn/backpropagation.cu | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/cnn/backpropagation.c b/src/cnn/backpropagation.c index df499de..01f9a7c 100644 --- a/src/cnn/backpropagation.c +++ b/src/cnn/backpropagation.c @@ -490,7 +490,7 @@ void backward_linearisation(Kernel_nn* ker, D_Kernel_nn* d_ker, float*** input, #ifndef __CUDACC__ backward_linearisation_cpu(ker, d_ker, input, input_z, output, input_depth, input_width, size_output, activation); #else - backward_linearisation_device(ker, input, input_z, output, input_depth, input_width, size_output, activation); + backward_linearisation_device(ker, d_ker, input, input_z, output, input_depth, input_width, size_output, activation); #endif } @@ -569,12 +569,12 @@ __global__ void backward_convolution_apply_propagate_kernel(float*** input, floa } } -void backward_convolution_device(Kernel_cnn* kernel, float*** input, float*** input_z, float*** output, int input_depth, int input_width, int output_depth, int output_width, int activation, int is_first, int kernel_size, int padding, int stride) { +void backward_convolution_device(Kernel_cnn* kernel, D_Kernel_cnn* d_kernel, float*** input, float*** input_z, float*** output, int input_depth, int input_width, int output_depth, int output_width, int activation, int is_first, int kernel_size, int padding, int stride) { // Bias Kernel dim3 gridSize1(i_div_up(output_depth, BLOCKSIZE_x), i_div_up(output_width, BLOCKSIZE_y), i_div_up(output_width, BLOCKSIZE_y)); dim3 blockSize1(BLOCKSIZE_x, BLOCKSIZE_y, BLOCKSIZE_z); - backward_convolution_dbias_kernel<<>>(kernel->d_bias, output, output_depth, output_width); + backward_convolution_dbias_kernel<<>>(d_kernel->d_bias, output, output_depth, output_width); gpuErrchk( cudaPeekAtLastError() ); gpuErrchk( cudaDeviceSynchronize() ); @@ -582,7 +582,7 @@ void backward_convolution_device(Kernel_cnn* kernel, float*** input, float*** in dim3 gridSize2(i_div_up(output_width, BLOCKSIZE_x), i_div_up(output_width, BLOCKSIZE_y), i_div_up(output_depth, BLOCKSIZE_y)); dim3 blockSize2(BLOCKSIZE_x, BLOCKSIZE_y, BLOCKSIZE_z); - backward_convolution_dweight_kernel<<>>(kernel->d_weights, input, output, input_depth, output_depth, input_width, output_width, kernel_size, stride, padding); + backward_convolution_dweight_kernel<<>>(d_kernel->d_weights, input, output, input_depth, output_depth, input_width, output_width, kernel_size, stride, padding); gpuErrchk( cudaPeekAtLastError() ); gpuErrchk( cudaDeviceSynchronize() ); diff --git a/src/cnn/backpropagation.cu b/src/cnn/backpropagation.cu index df499de..01f9a7c 100644 --- a/src/cnn/backpropagation.cu +++ b/src/cnn/backpropagation.cu @@ -490,7 +490,7 @@ void backward_linearisation(Kernel_nn* ker, D_Kernel_nn* d_ker, float*** input, #ifndef __CUDACC__ backward_linearisation_cpu(ker, d_ker, input, input_z, output, input_depth, input_width, size_output, activation); #else - backward_linearisation_device(ker, input, input_z, output, input_depth, input_width, size_output, activation); + backward_linearisation_device(ker, d_ker, input, input_z, output, input_depth, input_width, size_output, activation); #endif } @@ -569,12 +569,12 @@ __global__ void backward_convolution_apply_propagate_kernel(float*** input, floa } } -void backward_convolution_device(Kernel_cnn* kernel, float*** input, float*** input_z, float*** output, int input_depth, int input_width, int output_depth, int output_width, int activation, int is_first, int kernel_size, int padding, int stride) { +void backward_convolution_device(Kernel_cnn* kernel, D_Kernel_cnn* d_kernel, float*** input, float*** input_z, float*** output, int input_depth, int input_width, int output_depth, int output_width, int activation, int is_first, int kernel_size, int padding, int stride) { // Bias Kernel dim3 gridSize1(i_div_up(output_depth, BLOCKSIZE_x), i_div_up(output_width, BLOCKSIZE_y), i_div_up(output_width, BLOCKSIZE_y)); dim3 blockSize1(BLOCKSIZE_x, BLOCKSIZE_y, BLOCKSIZE_z); - backward_convolution_dbias_kernel<<>>(kernel->d_bias, output, output_depth, output_width); + backward_convolution_dbias_kernel<<>>(d_kernel->d_bias, output, output_depth, output_width); gpuErrchk( cudaPeekAtLastError() ); gpuErrchk( cudaDeviceSynchronize() ); @@ -582,7 +582,7 @@ void backward_convolution_device(Kernel_cnn* kernel, float*** input, float*** in dim3 gridSize2(i_div_up(output_width, BLOCKSIZE_x), i_div_up(output_width, BLOCKSIZE_y), i_div_up(output_depth, BLOCKSIZE_y)); dim3 blockSize2(BLOCKSIZE_x, BLOCKSIZE_y, BLOCKSIZE_z); - backward_convolution_dweight_kernel<<>>(kernel->d_weights, input, output, input_depth, output_depth, input_width, output_width, kernel_size, stride, padding); + backward_convolution_dweight_kernel<<>>(d_kernel->d_weights, input, output, input_depth, output_depth, input_width, output_width, kernel_size, stride, padding); gpuErrchk( cudaPeekAtLastError() ); gpuErrchk( cudaDeviceSynchronize() ); From d7d5a7ae6e76bfb946173d1cd944724bc8a02b11 Mon Sep 17 00:00:00 2001 From: julienChemillier Date: Sun, 28 May 2023 09:07:30 +0200 Subject: [PATCH 5/9] Reduce arguments in functions in 'free.c' --- src/cnn/free.c | 15 +++++++++------ src/cnn/include/free.h | 6 +++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/cnn/free.c b/src/cnn/free.c index 125634d..701f52f 100644 --- a/src/cnn/free.c +++ b/src/cnn/free.c @@ -148,8 +148,9 @@ void free_network(Network* network) { // ----------------------- Pour le d_network ----------------------- -void free_d_convolution(Network* network, D_Network* d_network, int pos) { +void free_d_convolution(Network* network, int pos) { Kernel_cnn* k_pos = network->kernel[pos]->cnn; + D_Network* d_network = network->d_network; D_Kernel_cnn* d_k_pos = d_network->kernel[pos]->cnn; int c = k_pos->columns; int k_size = k_pos->k_size; @@ -206,8 +207,9 @@ void free_d_convolution(Network* network, D_Network* d_network, int pos) { } } -void free_d_dense(Network* network, D_Network* d_network, int pos) { +void free_d_dense(Network* network, int pos) { D_Kernel_nn* d_k_pos = d_network->kernel[pos]->nn; + D_Network* d_network = network->d_network; int dim = network->kernel[pos]->nn->size_input; for (int i=0; i < dim; i++) { gree(d_k_pos->d_weights[i], true); @@ -229,8 +231,9 @@ void free_d_dense(Network* network, D_Network* d_network, int pos) { #endif } -void free_d_dense_linearisation(Network* network, D_Network* d_network, int pos) { +void free_d_dense_linearisation(Network* network, int pos) { D_Kernel_nn* d_k_pos = d_network->kernel[pos]->nn; + D_Network* d_network = network->d_network; int dim = network->kernel[pos]->nn->size_input; if (network->finetuning <= NN_AND_LINEARISATION) { @@ -261,12 +264,12 @@ void free_d_network_creation(Network* network, D_Network* d_network) { for (int i=0; i < network->max_size-1; i++) { D_Kernel* d_k_i = d_network->kernel[i]; if (d_k_i->cnn) { // Convolution - free_d_convolution(network, d_network, i); + free_d_convolution(network, i); } else if (d_k_i->nn) { // Dense if (network->kernel[i]->linearisation == DOESNT_LINEARISE) { // Vecteur -> Vecteur - free_d_dense(network, d_network, i); + free_d_dense(network, i); } else { // Matrice -> Vecteur - free_d_dense_linearisation(network, d_network, i); + free_d_dense_linearisation(network, i); } } gree(network->kernel[i], true); diff --git a/src/cnn/include/free.h b/src/cnn/include/free.h index 9eefe2f..c705223 100644 --- a/src/cnn/include/free.h +++ b/src/cnn/include/free.h @@ -50,17 +50,17 @@ void free_network(Network* network); /* * Libère l'espace mémoire alloué pour une d_convolution */ -void free_d_convolution(Network* network, D_Network* d_network, int pos); +void free_d_convolution(Network* network, int pos); /* * Libère l'espace mémoire alloué pour une d_dense */ -void free_d_dense(Network* network, D_Network* d_network, int pos); +void free_d_dense(Network* network, int pos); /* * Libère l'espace mémoire alloué pour une d_dense_linearisation */ -void free_d_dense_linearisation(Network* network, D_Network* d_network, int pos); +void free_d_dense_linearisation(Network* network, int pos); /* * Libère entièrement l'espace mémoire alloué dans 'create_d_network' (creation.c) From d916c6c86bce218a6ebf11bbe01497c03c3c18f3 Mon Sep 17 00:00:00 2001 From: julienChemillier Date: Sun, 28 May 2023 09:12:52 +0200 Subject: [PATCH 6/9] Rectification of a d_network leak --- src/cnn/free.c | 7 ++++--- src/cnn/include/free.h | 2 +- src/cnn/train.c | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/cnn/free.c b/src/cnn/free.c index 701f52f..f586c45 100644 --- a/src/cnn/free.c +++ b/src/cnn/free.c @@ -208,8 +208,8 @@ void free_d_convolution(Network* network, int pos) { } void free_d_dense(Network* network, int pos) { - D_Kernel_nn* d_k_pos = d_network->kernel[pos]->nn; D_Network* d_network = network->d_network; + D_Kernel_nn* d_k_pos = d_network->kernel[pos]->nn; int dim = network->kernel[pos]->nn->size_input; for (int i=0; i < dim; i++) { gree(d_k_pos->d_weights[i], true); @@ -232,8 +232,8 @@ void free_d_dense(Network* network, int pos) { } void free_d_dense_linearisation(Network* network, int pos) { - D_Kernel_nn* d_k_pos = d_network->kernel[pos]->nn; D_Network* d_network = network->d_network; + D_Kernel_nn* d_k_pos = d_network->kernel[pos]->nn; int dim = network->kernel[pos]->nn->size_input; if (network->finetuning <= NN_AND_LINEARISATION) { @@ -260,7 +260,8 @@ void free_d_dense_linearisation(Network* network, int pos) { gree(d_k_pos, true); } -void free_d_network_creation(Network* network, D_Network* d_network) { +void free_d_network(Network* network) { + D_Network* d_network = network->d_network; for (int i=0; i < network->max_size-1; i++) { D_Kernel* d_k_i = d_network->kernel[i]; if (d_k_i->cnn) { // Convolution diff --git a/src/cnn/include/free.h b/src/cnn/include/free.h index c705223..be0a064 100644 --- a/src/cnn/include/free.h +++ b/src/cnn/include/free.h @@ -65,6 +65,6 @@ void free_d_dense_linearisation(Network* network, int pos); /* * Libère entièrement l'espace mémoire alloué dans 'create_d_network' (creation.c) */ -void free_d_network_creation(Network* network, D_Network* d_network); +void free_d_network(Network* network); #endif \ No newline at end of file diff --git a/src/cnn/train.c b/src/cnn/train.c index 4447895..35bd118 100644 --- a/src/cnn/train.c +++ b/src/cnn/train.c @@ -463,6 +463,7 @@ void train(int dataset_type, char* images_file, char* labels_file, char* data_di write_network(out, network); } free(shuffle_index); + free_d_network(network); free_network(network); #ifdef USE_MULTITHREADING From ba05923c782b0a87439f4e06f216a5886991ded0 Mon Sep 17 00:00:00 2001 From: julienChemillier Date: Tue, 30 May 2023 13:52:13 +0200 Subject: [PATCH 7/9] Remove an useless allocation of memory --- src/cnn/creation.c | 1 - src/cnn/free.c | 12 +++++++++++- src/cnn/include/free.h | 7 +++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/cnn/creation.c b/src/cnn/creation.c index d175870..a7858f0 100644 --- a/src/cnn/creation.c +++ b/src/cnn/creation.c @@ -33,7 +33,6 @@ Network* create_network(int max_size, float learning_rate, int dropout, int init network->width[0] = input_width; network->depth[0] = input_depth; create_a_cube_input_layer(network, 0, input_depth, input_width); - create_a_cube_input_z_layer(network, 0, input_depth, input_width); return network; } diff --git a/src/cnn/free.c b/src/cnn/free.c index f586c45..c66fca5 100644 --- a/src/cnn/free.c +++ b/src/cnn/free.c @@ -20,6 +20,16 @@ void free_a_cube_input_layer(Network* network, int pos, int depth, int dim) { gree(network->input_z[pos], true); } +void free_a_cube_input_layer_without_z(Network* network, int pos, int depth, int dim) { + for (int i=0; i < depth; i++) { + for (int j=0; j < dim; j++) { + gree(network->input[pos][i][j], true); + } + gree(network->input[pos][i], true); + } + gree(network->input[pos], true); +} + void free_a_line_input_layer(Network* network, int pos) { // Libère l'espace mémoire de network->input[pos] et network->input_z[pos] // lorsque ces couches sont denses (donc sont des matrice de dimension 1) @@ -97,7 +107,7 @@ void free_dense_linearisation(Network* network, int pos) { void free_network_creation(Network* network) { // On libère l'input correspondant à l'image: input[0] (car elle n'appartient à aucune couche) - free_a_cube_input_layer(network, 0, network->depth[0], network->width[0]); + free_a_cube_input_layer_without_z(network, 0, network->depth[0], network->width[0]); for (int i=0; i < network->max_size-1; i++) { gree(network->kernel[i], true); diff --git a/src/cnn/include/free.h b/src/cnn/include/free.h index be0a064..59a2bd5 100644 --- a/src/cnn/include/free.h +++ b/src/cnn/include/free.h @@ -10,6 +10,13 @@ */ void free_a_cube_input_layer(Network* network, int pos, int depth, int dim); +/* +* Libère l'espace mémoire de network->input[pos] +* lorsque cette couches est non denses (donc est une matrice de dimension 3) +* Libère donc l'espace mémoire alloué dans 'create_a_cube_input_layer' (creation.c) +*/ +void free_a_cube_input_layer_without_z(Network* network, int pos, int depth, int dim); + /* * Libère l'espace mémoire de network->input[pos] et network->input_z[pos] * lorsque ces couches sont denses (donc sont des matrice de dimension 1) From 7e4f9167fa384f51220e6cdc7af746c3ffb9db28 Mon Sep 17 00:00:00 2001 From: julienChemillier Date: Wed, 31 May 2023 08:52:14 +0200 Subject: [PATCH 8/9] Fix free issue --- src/cnn/free.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cnn/free.c b/src/cnn/free.c index c66fca5..13f7dcb 100644 --- a/src/cnn/free.c +++ b/src/cnn/free.c @@ -283,9 +283,9 @@ void free_d_network(Network* network) { free_d_dense_linearisation(network, i); } } - gree(network->kernel[i], true); + gree(d_network->kernel[i], true); } - gree(network->kernel, true); + gree(d_network->kernel, true); pthread_mutex_destroy(&(d_network->lock)); - gree(network, true); + gree(d_network, true); } \ No newline at end of file From 6f0b360dc9670bede42ae6e226361a3a36888e9d Mon Sep 17 00:00:00 2001 From: julienChemillier Date: Wed, 31 May 2023 11:15:46 +0200 Subject: [PATCH 9/9] Remove Adam optimize options by default --- src/cnn/include/config.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cnn/include/config.h b/src/cnn/include/config.h index 79d0076..df9d89e 100644 --- a/src/cnn/include/config.h +++ b/src/cnn/include/config.h @@ -18,10 +18,10 @@ //* Options d'ADAM optimizer //* Activer ou désactiver Adam sur les couches dense -#define ADAM_DENSE_WEIGHTS +//#define ADAM_DENSE_WEIGHTS //#define ADAM_DENSE_BIAS //* Activer ou désactiver Adam sur les couches convolutives -#define ADAM_CNN_WEIGHTS +//#define ADAM_CNN_WEIGHTS //#define ADAM_CNN_BIAS