From 2a88621c34fb9825c60b0aa836be54a7c9bcc33e Mon Sep 17 00:00:00 2001 From: julienChemillier Date: Sun, 12 Mar 2023 11:13:26 +0100 Subject: [PATCH] Change bias implementation to fix keras's one --- src/cnn/backpropagation.c | 3 ++- src/cnn/convolution.c | 4 ++-- src/cnn/convolution.cu | 4 ++-- src/cnn/creation.c | 20 ++++++----------- src/cnn/free.c | 9 -------- src/cnn/include/cnn.h | 3 ++- src/cnn/include/convolution.h | 2 +- src/cnn/include/struct.h | 4 ++-- src/cnn/neuron_io.c | 14 +++++------- src/cnn/print.c | 2 +- src/cnn/update.c | 17 ++++----------- src/cnn/utils.c | 32 +++++++--------------------- src/scripts/convolution_benchmark.cu | 14 +++++++----- test/cnn_convolution.cu | 14 +++++++----- 14 files changed, 53 insertions(+), 89 deletions(-) diff --git a/src/cnn/backpropagation.c b/src/cnn/backpropagation.c index 3b48556..b35bf94 100644 --- a/src/cnn/backpropagation.c +++ b/src/cnn/backpropagation.c @@ -146,10 +146,11 @@ void backward_linearisation(Kernel_nn* ker, float*** input, float*** input_z, fl void backward_convolution(Kernel_cnn* ker, float*** input, float*** input_z, float*** output, int depth_input, int dim_input, int depth_output, int dim_output, ptr d_function, int is_first) { // Bias + int n = dim_output*dim_output; for (int i=0; i < depth_output; i++) { for (int j=0; j < dim_output; j++) { for (int k=0; k < dim_output; k++) { - ker->d_bias[i][j][k] += output[i][j][k]; + ker->d_bias[i] += output[i][j][k]/n; } } } diff --git a/src/cnn/convolution.c b/src/cnn/convolution.c index 1f17c30..6c656f1 100644 --- a/src/cnn/convolution.c +++ b/src/cnn/convolution.c @@ -20,7 +20,7 @@ void make_convolution_cpu(Kernel_cnn* kernel, float*** input, float*** output, i for (int i=0; i < kernel->columns; i++) { // filtre for (int j=0; j < output_dim; j++) { // ligne de sortie for (int k=0; k < output_dim; k++) { // colonne de sortie - f = kernel->bias[i][j][k]; + f = kernel->bias[i]; for (int a=0; a < kernel->rows; a++) { // Canal de couleur for (int b=0; b < kernel->k_size; b++) { // ligne du filtre for (int c=0; c < kernel->k_size; c++) { // colonne du filtre @@ -46,7 +46,7 @@ __global__ void make_convolution_kernel(Kernel_cnn* kernel, float*** input, floa return; } - float f = kernel->bias[idx][idy][idz]; + float f = kernel->bias[idx]; for (int a=0; a < kernel->rows; a++) { for (int b=0; b < kernel->k_size; b++) { diff --git a/src/cnn/convolution.cu b/src/cnn/convolution.cu index 22994fc..6aab479 100644 --- a/src/cnn/convolution.cu +++ b/src/cnn/convolution.cu @@ -20,7 +20,7 @@ void make_convolution_cpu(Kernel_cnn* kernel, float*** input, float*** output, i for (int i=0; i < kernel->columns; i++) { // filtre for (int j=0; j < output_dim; j++) { // ligne de sortie for (int k=0; k < output_dim; k++) { // colonne de sortie - f = kernel->bias[i][j][k]; + f = kernel->bias[i]; for (int a=0; a < kernel->rows; a++) { // Canal de couleur for (int b=0; b < kernel->k_size; b++) { // ligne du filtre for (int c=0; c < kernel->k_size; c++) { // colonne du filtre @@ -46,7 +46,7 @@ __global__ void make_convolution_kernel(Kernel_cnn* kernel, float*** input, floa return; } - float f = kernel->bias[idx][idy][idz]; + float f = kernel->bias[idx]; for (int a=0; a < kernel->rows; a++) { for (int b=0; b < kernel->k_size; b++) { diff --git a/src/cnn/creation.c b/src/cnn/creation.c index 12f32f4..4177b3b 100644 --- a/src/cnn/creation.c +++ b/src/cnn/creation.c @@ -181,22 +181,14 @@ void add_convolution(Network* network, int depth_output, int dim_output, int act } } } - cnn->bias = (float***)nalloc(depth_output, sizeof(float**)); - cnn->d_bias = (float***)nalloc(depth_output, sizeof(float**)); + cnn->bias = (float*)nalloc(depth_output, sizeof(float)); + cnn->d_bias = (float*)nalloc(depth_output, sizeof(float)); for (int i=0; i < depth_output; i++) { - cnn->bias[i] = (float**)nalloc(bias_size, sizeof(float*)); - cnn->d_bias[i] = (float**)nalloc(bias_size, sizeof(float*)); - for (int j=0; j < bias_size; j++) { - cnn->bias[i][j] = (float*)nalloc(bias_size, sizeof(float)); - cnn->d_bias[i][j] = (float*)nalloc(bias_size, sizeof(float)); - for (int k=0; k < bias_size; k++) { - cnn->d_bias[i][j][k] = 0.; - } - } + cnn->d_bias[i] = 0; } - int n_in = network->width[n-1]*network->width[n-1]*network->depth[n-1]; + int n_in = kernel_size*kernel_size; int n_out = network->width[n]*network->width[n]*network->depth[n]; - initialisation_3d_matrix(network->initialisation, cnn->bias, depth_output, dim_output, dim_output, n_in, n_out); + initialisation_1d_matrix(network->initialisation, cnn->bias, depth_output, n_in, n_out); initialisation_4d_matrix(network->initialisation, cnn->weights, depth_input, depth_output, kernel_size, kernel_size, n_in, n_out); create_a_cube_input_layer(network, n, depth_output, bias_size); create_a_cube_input_z_layer(network, n, depth_output, bias_size); @@ -234,7 +226,7 @@ void add_dense(Network* network, int size_output, int activation) { nn->d_weights[i][j] = 0.; } } - + initialisation_1d_matrix(network->initialisation, nn->bias, size_output, size_input, size_output); initialisation_2d_matrix(network->initialisation, nn->weights, size_input, size_output, size_input, size_output); create_a_line_input_layer(network, n, size_output); diff --git a/src/cnn/free.c b/src/cnn/free.c index b9a4943..c3b927b 100644 --- a/src/cnn/free.c +++ b/src/cnn/free.c @@ -36,16 +36,7 @@ void free_convolution(Network* network, int pos) { int c = k_pos->columns; int k_size = k_pos->k_size; int r = k_pos->rows; - int bias_size = network->width[pos+1]; // Not sure of the value free_a_cube_input_layer(network, pos+1, network->depth[pos+1], network->width[pos+1]); - for (int i=0; i < c; i++) { - for (int j=0; j < bias_size; j++) { - gree(k_pos->bias[i][j]); - gree(k_pos->d_bias[i][j]); - } - gree(k_pos->bias[i]); - gree(k_pos->d_bias[i]); - } gree(k_pos->bias); gree(k_pos->d_bias); diff --git a/src/cnn/include/cnn.h b/src/cnn/include/cnn.h index bd8d664..5f117d0 100644 --- a/src/cnn/include/cnn.h +++ b/src/cnn/include/cnn.h @@ -25,7 +25,8 @@ void write_image_in_network_32(int** image, int height, int width, float** input void write_image_in_network_260(unsigned char* image, int height, int width, float*** input); /* -* Propage en avant le cnn +* Propage en avant le cnn. Le dropout est actif que si le réseau est en phase d'apprentissage. +* */ void forward_propagation(Network* network); diff --git a/src/cnn/include/convolution.h b/src/cnn/include/convolution.h index e39a6ea..2f599b1 100644 --- a/src/cnn/include/convolution.h +++ b/src/cnn/include/convolution.h @@ -9,7 +9,7 @@ void make_convolution_cpu(Kernel_cnn* kernel, float*** input, float*** output, i /* * Kernel de la convolution sur carte graphique */ -__global__ void make_convolution_kernel(int k_size, int columns, int rows, float*** bias, size_t pitch_bias, float**** weights, size_t pitch_weights, float*** input, size_t pitch_input, float*** output, size_t pitch_output, int output_dim); +__global__ void make_convolution_kernel(int k_size, int columns, int rows, float* bias, size_t pitch_bias, float**** weights, size_t pitch_weights, float*** input, size_t pitch_input, float*** output, size_t pitch_output, int output_dim); /* * Effectue la convolution naïvement sur la carte graphique diff --git a/src/cnn/include/struct.h b/src/cnn/include/struct.h index 90df7eb..c3a62f6 100644 --- a/src/cnn/include/struct.h +++ b/src/cnn/include/struct.h @@ -13,8 +13,8 @@ typedef struct Kernel_cnn { int k_size; // k_size = dim_input - dim_output + 1 int rows; // Depth de l'input int columns; // Depth de l'output - float*** bias; // bias[columns][dim_output][dim_output] - float*** d_bias; // d_bias[columns][dim_output][dim_output] + float* bias; // bias[columns] + float* d_bias; // d_bias[columns] float**** weights; // weights[rows][columns][k_size][k_size] float**** d_weights; // d_weights[rows][columns][k_size][k_size] } Kernel_cnn; diff --git a/src/cnn/neuron_io.c b/src/cnn/neuron_io.c index c29fdc1..6f43812 100644 --- a/src/cnn/neuron_io.c +++ b/src/cnn/neuron_io.c @@ -91,7 +91,7 @@ void write_couche(Network* network, int indice_couche, int type_couche, FILE* pt float buffer[output_dim*output_dim]; for (int j=0; j < output_dim; j++) { for (int k=0; k < output_dim; k++) { - bufferAdd(cnn->bias[i][j][k]); + bufferAdd(cnn->bias[i]); } } fwrite(buffer, sizeof(buffer), 1, ptr); @@ -247,18 +247,14 @@ Kernel* read_kernel(int type_couche, int output_dim, FILE* ptr) { Kernel_cnn* cnn = kernel->cnn; float tmp; - cnn->bias = (float***)nalloc(cnn->columns, sizeof(float**)); - cnn->d_bias = (float***)nalloc(cnn->columns, sizeof(float**)); + cnn->bias = (float*)nalloc(cnn->columns, sizeof(float)); + cnn->d_bias = (float*)nalloc(cnn->columns, sizeof(float)); for (int i=0; i < cnn->columns; i++) { - cnn->bias[i] = (float**)nalloc(output_dim, sizeof(float*)); - cnn->d_bias[i] = (float**)nalloc(output_dim, sizeof(float*)); for (int j=0; j < output_dim; j++) { - cnn->bias[i][j] = (float*)nalloc(output_dim, sizeof(float)); - cnn->d_bias[i][j] = (float*)nalloc(output_dim, sizeof(float)); for (int k=0; k < output_dim; k++) { (void) !fread(&tmp, sizeof(tmp), 1, ptr); - cnn->bias[i][j][k] = tmp; - cnn->d_bias[i][j][k] = 0.; + cnn->bias[i] = tmp; + cnn->d_bias[i] = 0.; } } } diff --git a/src/cnn/print.c b/src/cnn/print.c index 4005218..72ba091 100644 --- a/src/cnn/print.c +++ b/src/cnn/print.c @@ -18,7 +18,7 @@ void print_kernel_cnn(Kernel_cnn* ker, int depth_input, int dim_input, int depth for (int i=0; ibias[i][j][k]); + printf("%.2f", ker->bias[i]); } print_space; } diff --git a/src/cnn/update.c b/src/cnn/update.c index 8ec3807..e0b3b30 100644 --- a/src/cnn/update.c +++ b/src/cnn/update.c @@ -87,14 +87,9 @@ void update_bias(Network* network, Network* d_network) { Kernel_cnn* d_cnn = dk_i->cnn; for (int a=0; a < output_depth; a++) { - for (int b=0; b < output_width; b++) { - for (int c=0; c < output_width; c++) { - cnn->bias[a][b][c] -= network->learning_rate * d_cnn->d_bias[a][b][c]; - d_cnn->d_bias[a][b][c] = 0; - - cnn->bias[a][b][c] = clip(cnn->bias[a][b][c]); - } - } + cnn->bias[a] -= network->learning_rate * d_cnn->d_bias[a]; + d_cnn->d_bias[a] = 0; + cnn->bias[a] = clip(cnn->bias[a]); } } else if (k_i->nn) { // Full connection Kernel_nn* nn = k_i->nn; @@ -177,11 +172,7 @@ void reset_d_bias(Network* network) { Kernel_cnn* cnn = k_i_1->cnn; for (int a=0; a < output_depth; a++) { - for (int b=0; b < output_width; b++) { - for (int c=0; c < output_width; c++) { - cnn->d_bias[a][b][c] = 0; - } - } + cnn->d_bias[a] = 0; } } else if (k_i->nn) { // Full connection Kernel_nn* nn = k_i_1->nn; diff --git a/src/cnn/utils.c b/src/cnn/utils.c index 1a211ba..fdb119d 100644 --- a/src/cnn/utils.c +++ b/src/cnn/utils.c @@ -75,7 +75,7 @@ bool equals_networks(Network* network1, Network* network2) { for (int j=0; j < network1->kernel[i]->cnn->columns; j++) { for (int k=0; k < output_dim; k++) { for (int l=0; l < output_dim; l++) { - checkEquals(kernel[i]->cnn->bias[j][k][l], "kernel[i]->cnn->bias[j][k][l]", l); + checkEquals(kernel[i]->cnn->bias[j], "kernel[i]->cnn->bias[j][k][l]", j); } } } @@ -83,7 +83,7 @@ bool equals_networks(Network* network1, Network* network2) { for (int k=0; k < network1->kernel[i]->cnn->columns; k++) { for (int l=0; l < network1->kernel[i]->cnn->k_size; l++) { for (int m=0; m < network1->kernel[i]->cnn->k_size; m++) { - checkEquals(kernel[i]->cnn->weights[j][k][l][m], "kernel[i]->cnn->bias[j][k][l][m]", m); + checkEquals(kernel[i]->cnn->weights[j][k][l][m], "kernel[i]->cnn->weights[j][k][l][m]", m); } } } @@ -182,19 +182,11 @@ Network* copy_network(Network* network) { copyVar(kernel[i]->cnn->k_size); copyVar(kernel[i]->cnn->columns); - network_cp->kernel[i]->cnn->bias = (float***)nalloc(columns, sizeof(float**)); - network_cp->kernel[i]->cnn->d_bias = (float***)nalloc(columns, sizeof(float**)); + network_cp->kernel[i]->cnn->bias = (float*)nalloc(columns, sizeof(float)); + network_cp->kernel[i]->cnn->d_bias = (float*)nalloc(columns, sizeof(float)); for (int j=0; j < columns; j++) { - network_cp->kernel[i]->cnn->bias[j] = (float**)nalloc(output_dim, sizeof(float*)); - network_cp->kernel[i]->cnn->d_bias[j] = (float**)nalloc(output_dim, sizeof(float*)); - for (int k=0; k < output_dim; k++) { - network_cp->kernel[i]->cnn->bias[j][k] = (float*)nalloc(output_dim, sizeof(float)); - network_cp->kernel[i]->cnn->d_bias[j][k] = (float*)nalloc(output_dim, sizeof(float)); - for (int l=0; l < output_dim; l++) { - copyVar(kernel[i]->cnn->bias[j][k][l]); - network_cp->kernel[i]->cnn->d_bias[j][k][l] = 0.; - } - } + copyVar(kernel[i]->cnn->bias[j]); + network_cp->kernel[i]->cnn->d_bias[j] = 0.; } network_cp->kernel[i]->cnn->weights = (float****)nalloc(rows, sizeof(float***)); @@ -287,11 +279,7 @@ void copy_network_parameters(Network* network_src, Network* network_dest) { output_dim = network_src->width[i+1]; for (int j=0; j < columns; j++) { - for (int k=0; k < output_dim; k++) { - for (int l=0; l < output_dim; l++) { - copyVarParams(kernel[i]->cnn->bias[j][k][l]); - } - } + copyVarParams(kernel[i]->cnn->bias[j]); } for (int j=0; j < rows; j++) { for (int k=0; k < columns; k++) { @@ -346,11 +334,7 @@ int count_null_weights(Network* network) { output_dim = network->width[i+1]; for (int j=0; j < columns; j++) { - for (int k=0; k < output_dim; k++) { - for (int l=0; l < output_dim; l++) { - null_bias += fabs(network->kernel[i]->cnn->bias[j][k][l]) <= epsilon; - } - } + null_bias += fabs(network->kernel[i]->cnn->bias[j]) <= epsilon; } for (int j=0; j < rows; j++) { for (int k=0; k < columns; k++) { diff --git a/src/scripts/convolution_benchmark.cu b/src/scripts/convolution_benchmark.cu index 605b72d..11b5b03 100644 --- a/src/scripts/convolution_benchmark.cu +++ b/src/scripts/convolution_benchmark.cu @@ -111,9 +111,13 @@ void run_convolution_test(int input_dim, int output_dim, int rows, int columns) kernel->rows = rows; kernel->columns = columns; - // bias[kernel->columns][dim_output][dim_output] - kernel->bias = create_matrix(kernel->columns, output_dim, output_dim, 15.0f); - kernel->d_bias = create_matrix(kernel->columns, output_dim, output_dim, 1.5f); + // bias[kernel->columns] + kernel->bias = (float*)malloc(kernel->columns, sizeof(float)); + kernel->d_bias = (float*)malloc(kernel->columns, sizeof(float)); + for (int i=0; icolumns; i++) { + kernel->bias[i] = random_float(0.0f, 15.0f); + kernel->d_bias[i] = random_float(0.0f, 1.5f); + } // weights[rows][columns][k_size][k_size] kernel->weights = (float****)malloc(sizeof(float***)*kernel->rows); @@ -156,8 +160,8 @@ void run_convolution_test(int input_dim, int output_dim, int rows, int columns) } //printf(GREEN "OK\n" RESET); - free_matrix(kernel->bias, kernel->columns, output_dim); - free_matrix(kernel->d_bias, kernel->columns, output_dim); + free(kernel->bias); + free(kernel->d_bias); for (int i=0; i < kernel->rows; i++) { free_matrix(kernel->weights[i], kernel->columns, kernel->k_size); diff --git a/test/cnn_convolution.cu b/test/cnn_convolution.cu index 049657b..7d2cd8e 100644 --- a/test/cnn_convolution.cu +++ b/test/cnn_convolution.cu @@ -104,9 +104,13 @@ void run_convolution_test(int input_dim, int output_dim, int rows, int columns) kernel->rows = rows; kernel->columns = columns; - // bias[kernel->columns][dim_output][dim_output] - kernel->bias = create_matrix(kernel->columns, output_dim, output_dim, 15.0f); - kernel->d_bias = create_matrix(kernel->columns, output_dim, output_dim, 1.5f); + // bias[kernel->columns] + kernel->bias = (float*)nalloc(kernel->columns, sizeof(float)); + kernel->d_bias = (float*)nalloc(kernel->columns, sizeof(float)); + for (int i=0; icolumns; i++) { + kernel->bias[i] = random_float(0.0f, 15.0f); + kernel->d_bias[i] = random_float(0.0f, 1.5f); + } // weights[rows][columns][k_size][k_size] kernel->weights = (float****)nalloc(kernel->rows, sizeof(float***)); @@ -150,8 +154,8 @@ void run_convolution_test(int input_dim, int output_dim, int rows, int columns) } printf(GREEN "OK\n" RESET); - free_matrix(kernel->bias, kernel->columns, output_dim); - free_matrix(kernel->d_bias, kernel->columns, output_dim); + gree(kernel->bias); + gree(kernel->d_bias); for (int i=0; i < kernel->rows; i++) { free_matrix(kernel->weights[i], kernel->columns, kernel->k_size);