Change bias implementation to fix keras's one

src/cnn/backpropagation.c

@@ -146,10 +146,11 @@ void backward_linearisation(Kernel_nn* ker, float*** input, float*** input_z, fl
 
 void backward_convolution(Kernel_cnn* ker, float*** input, float*** input_z, float*** output, int depth_input, int dim_input, int depth_output, int dim_output, ptr d_function, int is_first) {
     // Bias
+    int n = dim_output*dim_output;
     for (int i=0; i < depth_output; i++) {
         for (int j=0; j < dim_output; j++) {
             for (int k=0; k < dim_output; k++) {
-                ker->d_bias[i][j][k] += output[i][j][k];
+                ker->d_bias[i] += output[i][j][k]/n;
             }
         }
     }

src/cnn/convolution.c

@@ -20,7 +20,7 @@ void make_convolution_cpu(Kernel_cnn* kernel, float*** input, float*** output, i
     for (int i=0; i < kernel->columns; i++) { // filtre
         for (int j=0; j < output_dim; j++) { // ligne de sortie
             for (int k=0; k < output_dim; k++) { // colonne de sortie
-                f = kernel->bias[i][j][k];
+                f = kernel->bias[i];
                 for (int a=0; a < kernel->rows; a++) { // Canal de couleur
                     for (int b=0; b < kernel->k_size; b++) { // ligne du filtre
                         for (int c=0; c < kernel->k_size; c++) { // colonne du filtre
@@ -46,7 +46,7 @@ __global__ void make_convolution_kernel(Kernel_cnn* kernel, float*** input, floa
         return;
     }
 
-    float f = kernel->bias[idx][idy][idz];
+    float f = kernel->bias[idx];
 
     for (int a=0; a < kernel->rows; a++) {
         for (int b=0; b < kernel->k_size; b++) {

src/cnn/convolution.cu

@@ -20,7 +20,7 @@ void make_convolution_cpu(Kernel_cnn* kernel, float*** input, float*** output, i
     for (int i=0; i < kernel->columns; i++) { // filtre
         for (int j=0; j < output_dim; j++) { // ligne de sortie
             for (int k=0; k < output_dim; k++) { // colonne de sortie
-                f = kernel->bias[i][j][k];
+                f = kernel->bias[i];
                 for (int a=0; a < kernel->rows; a++) { // Canal de couleur
                     for (int b=0; b < kernel->k_size; b++) { // ligne du filtre
                         for (int c=0; c < kernel->k_size; c++) { // colonne du filtre
@@ -46,7 +46,7 @@ __global__ void make_convolution_kernel(Kernel_cnn* kernel, float*** input, floa
         return;
     }
 
-    float f = kernel->bias[idx][idy][idz];
+    float f = kernel->bias[idx];
 
     for (int a=0; a < kernel->rows; a++) {
         for (int b=0; b < kernel->k_size; b++) {

src/cnn/creation.c

@@ -181,22 +181,14 @@ void add_convolution(Network* network, int depth_output, int dim_output, int act
             }
         }
     }
-    cnn->bias = (float***)nalloc(depth_output, sizeof(float**));
-    cnn->d_bias = (float***)nalloc(depth_output, sizeof(float**));
+    cnn->bias = (float*)nalloc(depth_output, sizeof(float));
+    cnn->d_bias = (float*)nalloc(depth_output, sizeof(float));
     for (int i=0; i < depth_output; i++) {
-        cnn->bias[i] = (float**)nalloc(bias_size, sizeof(float*));
-        cnn->d_bias[i] = (float**)nalloc(bias_size, sizeof(float*));
-        for (int j=0; j < bias_size; j++) {
-            cnn->bias[i][j] = (float*)nalloc(bias_size, sizeof(float));
-            cnn->d_bias[i][j] = (float*)nalloc(bias_size, sizeof(float));
-            for (int k=0; k < bias_size; k++) {
-                cnn->d_bias[i][j][k] = 0.;
-            }
-        }
+        cnn->d_bias[i] = 0;
     }
-    int n_in = network->width[n-1]*network->width[n-1]*network->depth[n-1];
+    int n_in = kernel_size*kernel_size;
     int n_out = network->width[n]*network->width[n]*network->depth[n];
-    initialisation_3d_matrix(network->initialisation, cnn->bias, depth_output, dim_output, dim_output, n_in, n_out);
+    initialisation_1d_matrix(network->initialisation, cnn->bias, depth_output, n_in, n_out);
     initialisation_4d_matrix(network->initialisation, cnn->weights, depth_input, depth_output, kernel_size, kernel_size, n_in, n_out);
     create_a_cube_input_layer(network, n, depth_output, bias_size);
     create_a_cube_input_z_layer(network, n, depth_output, bias_size);
@@ -234,7 +226,7 @@ void add_dense(Network* network, int size_output, int activation) {
             nn->d_weights[i][j] = 0.;
         }
     }
-
+    
     initialisation_1d_matrix(network->initialisation, nn->bias, size_output, size_input, size_output);
     initialisation_2d_matrix(network->initialisation, nn->weights, size_input, size_output, size_input, size_output);
     create_a_line_input_layer(network, n, size_output);

src/cnn/free.c

@@ -36,16 +36,7 @@ void free_convolution(Network* network, int pos) {
     int c = k_pos->columns;
     int k_size = k_pos->k_size;
     int r = k_pos->rows;
-    int bias_size = network->width[pos+1]; // Not sure of the value
     free_a_cube_input_layer(network, pos+1, network->depth[pos+1], network->width[pos+1]);
-    for (int i=0; i < c; i++) {
-        for (int j=0; j < bias_size; j++) {
-            gree(k_pos->bias[i][j]);
-            gree(k_pos->d_bias[i][j]);
-        }
-        gree(k_pos->bias[i]);
-        gree(k_pos->d_bias[i]);
-    }
     gree(k_pos->bias);
     gree(k_pos->d_bias);
 

src/cnn/include/cnn.h

@@ -25,7 +25,8 @@ void write_image_in_network_32(int** image, int height, int width, float** input
 void write_image_in_network_260(unsigned char* image, int height, int width, float*** input);
 
 /*
-* Propage en avant le cnn
+* Propage en avant le cnn. Le dropout est actif que si le réseau est en phase d'apprentissage.
+* 
 */
 void forward_propagation(Network* network);
 

src/cnn/include/convolution.h

@@ -9,7 +9,7 @@ void make_convolution_cpu(Kernel_cnn* kernel, float*** input, float*** output, i
 /*
 * Kernel de la convolution sur carte graphique
 */
-__global__ void make_convolution_kernel(int k_size, int columns, int rows, float*** bias, size_t pitch_bias, float**** weights, size_t pitch_weights, float*** input, size_t pitch_input, float*** output, size_t pitch_output, int output_dim);
+__global__ void make_convolution_kernel(int k_size, int columns, int rows, float* bias, size_t pitch_bias, float**** weights, size_t pitch_weights, float*** input, size_t pitch_input, float*** output, size_t pitch_output, int output_dim);
 
 /*
 * Effectue la convolution naïvement sur la carte graphique

src/cnn/include/struct.h

@@ -13,8 +13,8 @@ typedef struct Kernel_cnn {
     int k_size; // k_size = dim_input - dim_output + 1
     int rows; // Depth de l'input
     int columns; // Depth de l'output
-    float*** bias; // bias[columns][dim_output][dim_output]
-    float*** d_bias; // d_bias[columns][dim_output][dim_output]
+    float* bias; // bias[columns]
+    float* d_bias; // d_bias[columns]
     float**** weights; // weights[rows][columns][k_size][k_size]
     float**** d_weights; // d_weights[rows][columns][k_size][k_size]
 } Kernel_cnn;

src/cnn/neuron_io.c

@@ -91,7 +91,7 @@ void write_couche(Network* network, int indice_couche, int type_couche, FILE* pt
             float buffer[output_dim*output_dim];
             for (int j=0; j < output_dim; j++) {
                 for (int k=0; k < output_dim; k++) {
-                    bufferAdd(cnn->bias[i][j][k]);
+                    bufferAdd(cnn->bias[i]);
                 }
             }
             fwrite(buffer, sizeof(buffer), 1, ptr);
@@ -247,18 +247,14 @@ Kernel* read_kernel(int type_couche, int output_dim, FILE* ptr) {
         Kernel_cnn* cnn = kernel->cnn;
         float tmp;
 
-        cnn->bias = (float***)nalloc(cnn->columns, sizeof(float**));
-        cnn->d_bias = (float***)nalloc(cnn->columns, sizeof(float**));
+        cnn->bias = (float*)nalloc(cnn->columns, sizeof(float));
+        cnn->d_bias = (float*)nalloc(cnn->columns, sizeof(float));
         for (int i=0; i < cnn->columns; i++) {
-            cnn->bias[i] = (float**)nalloc(output_dim, sizeof(float*));
-            cnn->d_bias[i] = (float**)nalloc(output_dim, sizeof(float*));
             for (int j=0; j < output_dim; j++) {
-                cnn->bias[i][j] = (float*)nalloc(output_dim, sizeof(float));
-                cnn->d_bias[i][j] = (float*)nalloc(output_dim, sizeof(float));
                 for (int k=0; k < output_dim; k++) {
                     (void) !fread(&tmp, sizeof(tmp), 1, ptr);
-                    cnn->bias[i][j][k] = tmp;
-                    cnn->d_bias[i][j][k] = 0.;
+                    cnn->bias[i] = tmp;
+                    cnn->d_bias[i] = 0.;
                 }
             }
         }

src/cnn/print.c

@@ -18,7 +18,7 @@ void print_kernel_cnn(Kernel_cnn* ker, int depth_input, int dim_input, int depth
     for (int i=0; i<depth_output; i++) {
         for (int j=0; j<dim_output; j++) {
             for (int k=0; k<dim_output; k++) {
-                printf("%.2f", ker->bias[i][j][k]);
+                printf("%.2f", ker->bias[i]);
             }
             print_space;
         }

src/cnn/update.c

@@ -87,14 +87,9 @@ void update_bias(Network* network, Network* d_network) {
             Kernel_cnn* d_cnn = dk_i->cnn;
 
             for (int a=0; a < output_depth; a++) {
-                for (int b=0; b < output_width; b++) {
-                    for (int c=0; c < output_width; c++) {
-                        cnn->bias[a][b][c] -= network->learning_rate * d_cnn->d_bias[a][b][c];
-                        d_cnn->d_bias[a][b][c] = 0;
-
-                        cnn->bias[a][b][c] = clip(cnn->bias[a][b][c]);
-                    }
-                }
+                cnn->bias[a] -= network->learning_rate * d_cnn->d_bias[a];
+                d_cnn->d_bias[a] = 0;
+                cnn->bias[a] = clip(cnn->bias[a]);
             }
         } else if (k_i->nn) { // Full connection
             Kernel_nn* nn = k_i->nn;
@@ -177,11 +172,7 @@ void reset_d_bias(Network* network) {
             Kernel_cnn* cnn = k_i_1->cnn;
 
             for (int a=0; a < output_depth; a++) {
-                for (int b=0; b < output_width; b++) {
-                    for (int c=0; c < output_width; c++) {
-                        cnn->d_bias[a][b][c] = 0;
-                    }
-                }
+                cnn->d_bias[a] = 0;
             }
         } else if (k_i->nn) { // Full connection
             Kernel_nn* nn = k_i_1->nn;

src/cnn/utils.c

@@ -75,7 +75,7 @@ bool equals_networks(Network* network1, Network* network2) {
             for (int j=0; j < network1->kernel[i]->cnn->columns; j++) {
                 for (int k=0; k < output_dim; k++) {
                     for (int l=0; l < output_dim; l++) {
-                        checkEquals(kernel[i]->cnn->bias[j][k][l], "kernel[i]->cnn->bias[j][k][l]", l);
+                        checkEquals(kernel[i]->cnn->bias[j], "kernel[i]->cnn->bias[j][k][l]", j);
                     }
                 }
             }
@@ -83,7 +83,7 @@ bool equals_networks(Network* network1, Network* network2) {
                 for (int k=0; k < network1->kernel[i]->cnn->columns; k++) {
                     for (int l=0; l < network1->kernel[i]->cnn->k_size; l++) {
                         for (int m=0; m < network1->kernel[i]->cnn->k_size; m++) {
-                            checkEquals(kernel[i]->cnn->weights[j][k][l][m], "kernel[i]->cnn->bias[j][k][l][m]", m);
+                            checkEquals(kernel[i]->cnn->weights[j][k][l][m], "kernel[i]->cnn->weights[j][k][l][m]", m);
                         }
                     }
                 }
@@ -182,19 +182,11 @@ Network* copy_network(Network* network) {
             copyVar(kernel[i]->cnn->k_size);
             copyVar(kernel[i]->cnn->columns);
 
-            network_cp->kernel[i]->cnn->bias = (float***)nalloc(columns, sizeof(float**));
-            network_cp->kernel[i]->cnn->d_bias = (float***)nalloc(columns, sizeof(float**));
+            network_cp->kernel[i]->cnn->bias = (float*)nalloc(columns, sizeof(float));
+            network_cp->kernel[i]->cnn->d_bias = (float*)nalloc(columns, sizeof(float));
             for (int j=0; j < columns; j++) {
-                network_cp->kernel[i]->cnn->bias[j] = (float**)nalloc(output_dim, sizeof(float*));
-                network_cp->kernel[i]->cnn->d_bias[j] = (float**)nalloc(output_dim, sizeof(float*));
-                for (int k=0; k < output_dim; k++) {
-                    network_cp->kernel[i]->cnn->bias[j][k] = (float*)nalloc(output_dim, sizeof(float));
-                    network_cp->kernel[i]->cnn->d_bias[j][k] = (float*)nalloc(output_dim, sizeof(float));
-                    for (int l=0; l < output_dim; l++) {
-                        copyVar(kernel[i]->cnn->bias[j][k][l]);
-                        network_cp->kernel[i]->cnn->d_bias[j][k][l] = 0.;
-                    }
-                }
+                copyVar(kernel[i]->cnn->bias[j]);
+                network_cp->kernel[i]->cnn->d_bias[j] = 0.;
             }
 
             network_cp->kernel[i]->cnn->weights = (float****)nalloc(rows, sizeof(float***));
@@ -287,11 +279,7 @@ void copy_network_parameters(Network* network_src, Network* network_dest) {
             output_dim = network_src->width[i+1];
 
             for (int j=0; j < columns; j++) {
-                for (int k=0; k < output_dim; k++) {
-                    for (int l=0; l < output_dim; l++) {
-                        copyVarParams(kernel[i]->cnn->bias[j][k][l]);
-                    }
-                }
+                copyVarParams(kernel[i]->cnn->bias[j]);
             }
             for (int j=0; j < rows; j++) {
                 for (int k=0; k < columns; k++) {
@@ -346,11 +334,7 @@ int count_null_weights(Network* network) {
             output_dim = network->width[i+1];
 
             for (int j=0; j < columns; j++) {
-                for (int k=0; k < output_dim; k++) {
-                    for (int l=0; l < output_dim; l++) {
-                        null_bias += fabs(network->kernel[i]->cnn->bias[j][k][l]) <= epsilon;
-                    }
-                }
+                null_bias += fabs(network->kernel[i]->cnn->bias[j]) <= epsilon;
             }
             for (int j=0; j < rows; j++) {
                 for (int k=0; k < columns; k++) {

src/scripts/convolution_benchmark.cu

@@ -111,9 +111,13 @@ void run_convolution_test(int input_dim, int output_dim, int rows, int columns)
     kernel->rows = rows;
     kernel->columns = columns;
 
-    // bias[kernel->columns][dim_output][dim_output]
-    kernel->bias = create_matrix(kernel->columns, output_dim, output_dim, 15.0f);
-    kernel->d_bias = create_matrix(kernel->columns, output_dim, output_dim, 1.5f);
+    // bias[kernel->columns]
+    kernel->bias = (float*)malloc(kernel->columns, sizeof(float));
+    kernel->d_bias = (float*)malloc(kernel->columns, sizeof(float));
+    for (int i=0; i<kernel->columns; i++) {
+        kernel->bias[i] = random_float(0.0f, 15.0f);
+        kernel->d_bias[i] = random_float(0.0f, 1.5f);
+    }
 
     // weights[rows][columns][k_size][k_size]
     kernel->weights = (float****)malloc(sizeof(float***)*kernel->rows);
@@ -156,8 +160,8 @@ void run_convolution_test(int input_dim, int output_dim, int rows, int columns)
     }
     //printf(GREEN "OK\n" RESET);
 
-    free_matrix(kernel->bias, kernel->columns, output_dim);
-    free_matrix(kernel->d_bias, kernel->columns, output_dim);
+    free(kernel->bias);
+    free(kernel->d_bias);
 
     for (int i=0; i < kernel->rows; i++) {
         free_matrix(kernel->weights[i], kernel->columns, kernel->k_size);

test/cnn_convolution.cu

@@ -104,9 +104,13 @@ void run_convolution_test(int input_dim, int output_dim, int rows, int columns)
     kernel->rows = rows;
     kernel->columns = columns;
 
-    // bias[kernel->columns][dim_output][dim_output]
-    kernel->bias = create_matrix(kernel->columns, output_dim, output_dim, 15.0f);
-    kernel->d_bias = create_matrix(kernel->columns, output_dim, output_dim, 1.5f);
+    // bias[kernel->columns]
+    kernel->bias = (float*)nalloc(kernel->columns, sizeof(float));
+    kernel->d_bias = (float*)nalloc(kernel->columns, sizeof(float));
+    for (int i=0; i<kernel->columns; i++) {
+        kernel->bias[i] = random_float(0.0f, 15.0f);
+        kernel->d_bias[i] = random_float(0.0f, 1.5f);
+    }
 
     // weights[rows][columns][k_size][k_size]
     kernel->weights = (float****)nalloc(kernel->rows, sizeof(float***));
@@ -150,8 +154,8 @@ void run_convolution_test(int input_dim, int output_dim, int rows, int columns)
     }
     printf(GREEN "OK\n" RESET);
 
-    free_matrix(kernel->bias, kernel->columns, output_dim);
-    free_matrix(kernel->d_bias, kernel->columns, output_dim);
+    gree(kernel->bias);
+    gree(kernel->d_bias);
 
     for (int i=0; i < kernel->rows; i++) {
         free_matrix(kernel->weights[i], kernel->columns, kernel->k_size);