augustin64/tipe
1
1
Fork 0
mirror of https://github.com/augustin64/projet-tipe synced 2025-01-23 15:16:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: augustin64:677de892e7403571385ebf737727e6c6641051f3
Branches Tags
augustin64:main
..
compare: augustin64:57954a27c09130fdb621bf0ffdbfab28096e6379
Branches Tags
augustin64:main

2 Commits
677de892e7 ... 57954a27c0

Author SHA1 Message Date
augustin64
57954a27c0 Update tests 2023-03-27 18:30:19 +02:00
julienChemillier
5f684d9f91 Implementation of Adam Optimizer 2023-03-27 18:17:50 +02:00
11 changed files with 377 additions and 1 deletions
Show Stats Expand all files Collapse all files

76
src/cnn/creation.c
View File

@ -165,31 +165,67 @@ void add_convolution(Network* network, int depth_output, int dim_output, int act
cnn->columns = depth_output;
cnn->weights = (float****)nalloc(depth_input, sizeof(float***));
cnn->d_weights = (float****)nalloc(depth_input, sizeof(float***));
#ifdef ADAM_CNN_WEIGHTS
cnn->s_d_weights = (float****)nalloc(depth_input, sizeof(float***));
cnn->v_d_weights = (float****)nalloc(depth_input, sizeof(float***));
#endif
for (int i=0; i < depth_input; i++) {
cnn->weights[i] = (float***)nalloc(depth_output, sizeof(float**));
cnn->d_weights[i] = (float***)nalloc(depth_output, sizeof(float**));
#ifdef ADAM_CNN_WEIGHTS
cnn->s_d_weights[i] = (float***)nalloc(depth_output, sizeof(float**));
cnn->v_d_weights[i] = (float***)nalloc(depth_output, sizeof(float**));
#endif
for (int j=0; j < depth_output; j++) {
cnn->weights[i][j] = (float**)nalloc(kernel_size, sizeof(float*));
cnn->d_weights[i][j] = (float**)nalloc(kernel_size, sizeof(float*));
#ifdef ADAM_CNN_WEIGHTS
cnn->s_d_weights[i][j] = (float**)nalloc(kernel_size, sizeof(float*));
cnn->v_d_weights[i][j] = (float**)nalloc(kernel_size, sizeof(float*));
#endif
for (int k=0; k < kernel_size; k++) {
cnn->weights[i][j][k] = (float*)nalloc(kernel_size, sizeof(float));
cnn->d_weights[i][j][k] = (float*)nalloc(kernel_size, sizeof(float));
#ifdef ADAM_CNN_WEIGHTS
cnn->s_d_weights[i][j][k] = (float*)nalloc(kernel_size, sizeof(float));
cnn->v_d_weights[i][j][k] = (float*)nalloc(kernel_size, sizeof(float));
#endif
for (int l=0; l < kernel_size; l++) {
cnn->d_weights[i][j][k][l] = 0.;
#ifdef ADAM_CNN_WEIGHTS
cnn->s_d_weights[i][j][k][l] = 0.;
cnn->v_d_weights[i][j][k][l] = 0.;
#endif
}
}
}
}
cnn->bias = (float***)nalloc(depth_output, sizeof(float**));
cnn->d_bias = (float***)nalloc(depth_output, sizeof(float**));
#ifdef ADAM_CNN_BIAS
cnn->s_d_bias = (float***)nalloc(depth_output, sizeof(float**));
cnn->v_d_bias = (float***)nalloc(depth_output, sizeof(float**));
#endif
for (int i=0; i < depth_output; i++) {
cnn->bias[i] = (float**)nalloc(bias_size, sizeof(float*));
cnn->d_bias[i] = (float**)nalloc(bias_size, sizeof(float*));
#ifdef ADAM_CNN_BIAS
cnn->s_d_bias[i] = (float**)nalloc(bias_size, sizeof(float*));
cnn->v_d_bias[i] = (float**)nalloc(bias_size, sizeof(float*));
#endif
for (int j=0; j < bias_size; j++) {
cnn->bias[i][j] = (float*)nalloc(bias_size, sizeof(float));
cnn->d_bias[i][j] = (float*)nalloc(bias_size, sizeof(float));
#ifdef ADAM_CNN_BIAS
cnn->s_d_bias[i][j] = (float*)nalloc(bias_size, sizeof(float));
cnn->v_d_bias[i][j] = (float*)nalloc(bias_size, sizeof(float));
#endif
for (int k=0; k < bias_size; k++) {
cnn->d_bias[i][j][k] = 0.;
#ifdef ADAM_CNN_BIAS
cnn->s_d_bias[i][j][k] = 0.;
cnn->v_d_bias[i][j][k] = 0.;
#endif
}
}
}
@ -220,17 +256,37 @@ void add_dense(Network* network, int size_output, int activation) {
nn->size_output = size_output;
nn->bias = (float*)nalloc(size_output, sizeof(float));
nn->d_bias = (float*)nalloc(size_output, sizeof(float));
#ifdef ADAM_DENSE_BIAS
nn->s_d_bias = (float*)nalloc(size_output, sizeof(float));
nn->v_d_bias = (float*)nalloc(size_output, sizeof(float));
#endif
for (int i=0; i < size_output; i++) {
nn->d_bias[i] = 0.;
#ifdef ADAM_DENSE_BIAS
nn->s_d_bias[i] = 0.;
nn->v_d_bias[i] = 0.;
#endif
}
nn->weights = (float**)nalloc(size_input, sizeof(float*));
nn->d_weights = (float**)nalloc(size_input, sizeof(float*));
#ifdef ADAM_DENSE_WEIGHTS
nn->s_d_weights = (float**)nalloc(size_input, sizeof(float*));
nn->v_d_weights = (float**)nalloc(size_input, sizeof(float*));
#endif
for (int i=0; i < size_input; i++) {
nn->weights[i] = (float*)nalloc(size_output, sizeof(float));
nn->d_weights[i] = (float*)nalloc(size_output, sizeof(float));
#ifdef ADAM_DENSE_WEIGHTS
nn->s_d_weights[i] = (float*)nalloc(size_output, sizeof(float));
nn->v_d_weights[i] = (float*)nalloc(size_output, sizeof(float));
#endif
for (int j=0; j < size_output; j++) {
nn->d_weights[i][j] = 0.;
#ifdef ADAM_DENSE_WEIGHTS
nn->s_d_weights[i][j] = 0.;
nn->v_d_weights[i][j] = 0.;
#endif
}
}
@ -262,16 +318,36 @@ void add_dense_linearisation(Network* network, int size_output, int activation)
nn->bias = (float*)nalloc(size_output, sizeof(float));
nn->d_bias = (float*)nalloc(size_output, sizeof(float));
#ifdef ADAM_DENSE_BIAS
nn->s_d_bias = (float*)nalloc(size_output, sizeof(float));
nn->v_d_bias = (float*)nalloc(size_output, sizeof(float));
#endif
for (int i=0; i < size_output; i++) {
nn->d_bias[i] = 0.;
#ifdef ADAM_DENSE_BIAS
nn->s_d_bias[i] = 0.;
nn->v_d_bias[i] = 0.;
#endif
}
nn->weights = (float**)nalloc(size_input, sizeof(float*));
nn->d_weights = (float**)nalloc(size_input, sizeof(float*));
#ifdef ADAM_DENSE_WEIGHTS
nn->s_d_weights = (float**)nalloc(size_input, sizeof(float*));
nn->v_d_weights = (float**)nalloc(size_input, sizeof(float*));
#endif
for (int i=0; i < size_input; i++) {
nn->weights[i] = (float*)nalloc(size_output, sizeof(float));
nn->d_weights[i] = (float*)nalloc(size_output, sizeof(float));
#ifdef ADAM_DENSE_WEIGHTS
nn->s_d_weights[i] = (float*)nalloc(size_output, sizeof(float));
nn->v_d_weights[i] = (float*)nalloc(size_output, sizeof(float));
#endif
for (int j=0; j < size_output; j++) {
nn->d_weights[i][j] = 0.;
#ifdef ADAM_DENSE_WEIGHTS
nn->s_d_weights[i][j] = 0.;
nn->v_d_weights[i][j] = 0.;
#endif
}
}
initialisation_1d_matrix(network->initialisation, nn->bias, size_output, size_input, size_output);

52
src/cnn/free.c
View File

@ -42,27 +42,55 @@ void free_convolution(Network* network, int pos) {
for (int j=0; j < bias_size; j++) {
gree(k_pos->bias[i][j]);
gree(k_pos->d_bias[i][j]);
#ifdef ADAM_CNN_BIAS
gree(k_pos->s_d_bias[i][j]);
gree(k_pos->v_d_bias[i][j]);
#endif
}
gree(k_pos->bias[i]);
gree(k_pos->d_bias[i]);
#ifdef ADAM_CNN_BIAS
gree(k_pos->s_d_bias[i]);
gree(k_pos->v_d_bias[i]);
#endif
}
gree(k_pos->bias);
gree(k_pos->d_bias);
#ifdef ADAM_CNN_BIAS
gree(k_pos->s_d_bias);
gree(k_pos->v_d_bias);
#endif
for (int i=0; i < r; i++) {
for (int j=0; j < c; j++) {
for (int k=0; k < k_size; k++) {
gree(k_pos->weights[i][j][k]);
gree(k_pos->d_weights[i][j][k]);
#ifdef ADAM_CNN_WEIGHTS
gree(k_pos->s_d_weights[i][j][k]);
gree(k_pos->v_d_weights[i][j][k]);
#endif
}
gree(k_pos->weights[i][j]);
gree(k_pos->d_weights[i][j]);
#ifdef ADAM_CNN_WEIGHTS
gree(k_pos->s_d_weights[i][j]);
gree(k_pos->v_d_weights[i][j]);
#endif
}
gree(k_pos->weights[i]);
gree(k_pos->d_weights[i]);
#ifdef ADAM_CNN_WEIGHTS
gree(k_pos->s_d_weights[i]);
gree(k_pos->v_d_weights[i]);
#endif
}
gree(k_pos->weights);
gree(k_pos->d_weights);
#ifdef ADAM_CNN_WEIGHTS
gree(k_pos->s_d_weights);
gree(k_pos->v_d_weights);
#endif
gree(k_pos);
}
@ -74,12 +102,24 @@ void free_dense(Network* network, int pos) {
for (int i=0; i < dim; i++) {
gree(k_pos->weights[i]);
gree(k_pos->d_weights[i]);
#ifdef ADAM_DENSE_WEIGHTS
gree(k_pos->s_d_weights[i]);
gree(k_pos->v_d_weights[i]);
#endif
}
gree(k_pos->weights);
gree(k_pos->d_weights);
#ifdef ADAM_DENSE_WEIGHTS
gree(k_pos->s_d_weights);
gree(k_pos->v_d_weights);
#endif
gree(k_pos->bias);
gree(k_pos->d_bias);
#ifdef ADAM_DENSE_BIAS
gree(k_pos->s_d_bias);
gree(k_pos->v_d_bias);
#endif
gree(k_pos);
}
@ -91,12 +131,24 @@ void free_dense_linearisation(Network* network, int pos) {
for (int i=0; i < dim; i++) {
gree(k_pos->weights[i]);
gree(k_pos->d_weights[i]);
#ifdef ADAM_DENSE_WEIGHTS
gree(k_pos->s_d_weights[i]);
gree(k_pos->v_d_weights[i]);
#endif
}
gree(k_pos->weights);
gree(k_pos->d_weights);
#ifdef ADAM_DENSE_WEIGHTS
gree(k_pos->s_d_weights);
gree(k_pos->v_d_weights);
#endif
gree(k_pos->bias);
gree(k_pos->d_bias);
#ifdef ADAM_DENSE_BIAS
gree(k_pos->s_d_bias);
gree(k_pos->v_d_bias);
#endif
gree(k_pos);
}

15
src/cnn/include/config.h
View File

@ -1,12 +1,27 @@
#ifndef DEF_CONFIG_H
#define DEF_CONFIG_H
//* Paramètres d'entraînement
#define EPOCHS 10 // Nombre d'époques par défaut (itérations sur toutes les images)
#define BATCHES 32 // Nombre d'images à voir avant de mettre le réseau à jour
#define LEARNING_RATE 3e-4 // Taux d'apprentissage
#define USE_MULTITHREADING // Commenter pour utiliser un seul coeur durant l'apprentissage (meilleur pour des tailles de batchs traités rapidement)
//* Paramètres d'ADAM optimizer
#define ALPHA 3e-4
#define BETA_1 0.9
#define BETA_2 0.999
#define Epsilon 1e-7
//* Options d'ADAM optimizer
//* Activer ou désactiver Adam sur les couches dense
//#define ADAM_DENSE_WEIGHTS
//#define ADAM_DENSE_BIAS
//* Activer ou désactiver Adam sur les couches convolutives
//#define ADAM_CNN_WEIGHTS
//#define ADAM_CNN_BIAS
//* Paramètre d'optimisation pour un dataset Jpeg
// keep images in ram e.g re-read and decompress each time

18
src/cnn/include/struct.h
View File

@ -1,6 +1,8 @@
#ifndef DEF_STRUCT_H
#define DEF_STRUCT_H
#include "config.h"
#define NO_POOLING 0
#define AVG_POOLING 1
#define MAX_POOLING 2
@ -15,8 +17,16 @@ typedef struct Kernel_cnn {
int columns; // Depth de l'output
float*** bias; // bias[columns][dim_output][dim_output]
float*** d_bias; // d_bias[columns][dim_output][dim_output]
#ifdef ADAM_CNN_BIAS
float*** s_d_bias; // s_d_bias[columns][dim_output][dim_output]
float*** v_d_bias; // v_d_bias[columns][dim_output][dim_output]
#endif
float**** weights; // weights[rows][columns][k_size][k_size]
float**** d_weights; // d_weights[rows][columns][k_size][k_size]
#ifdef ADAM_CNN_WEIGHTS
float**** s_d_weights; // s_d_weights[rows][columns][k_size][k_size]
float**** v_d_weights; // v_d_weights[rows][columns][k_size][k_size]
#endif
} Kernel_cnn;
typedef struct Kernel_nn {
@ -25,8 +35,16 @@ typedef struct Kernel_nn {
int size_output; // Nombre d'éléments en sortie
float* bias; // bias[size_output]
float* d_bias; // d_bias[size_output]
#ifdef ADAM_DENSE_BIAS
float* s_d_bias; // s_d_bias[size_output]
float* v_d_bias; // v_d_bias[size_output]
#endif
float** weights; // weight[size_input][size_output]
float** d_weights; // d_weights[size_input][size_output]
#ifdef ADAM_DENSE_WEIGHTS
float** s_d_weights; // s_d_weights[size_input][size_output]
float** v_d_weights; // v_d_weights[size_input][size_output]
#endif
} Kernel_nn;
typedef struct Kernel {

56
src/cnn/neuron_io.c
View File

@ -249,35 +249,71 @@ Kernel* read_kernel(int type_couche, int output_dim, FILE* ptr) {
cnn->bias = (float***)nalloc(cnn->columns, sizeof(float**));
cnn->d_bias = (float***)nalloc(cnn->columns, sizeof(float**));
#ifdef ADAM_CNN_BIAS
cnn->s_d_bias = (float***)nalloc(cnn->columns, sizeof(float**));
cnn->v_d_bias = (float***)nalloc(cnn->columns, sizeof(float**));
#endif
for (int i=0; i < cnn->columns; i++) {
cnn->bias[i] = (float**)nalloc(output_dim, sizeof(float*));
cnn->d_bias[i] = (float**)nalloc(output_dim, sizeof(float*));
#ifdef ADAM_CNN_BIAS
cnn->s_d_bias[i] = (float**)nalloc(output_dim, sizeof(float*));
cnn->v_d_bias[i] = (float**)nalloc(output_dim, sizeof(float*));
#endif
for (int j=0; j < output_dim; j++) {
cnn->bias[i][j] = (float*)nalloc(output_dim, sizeof(float));
cnn->d_bias[i][j] = (float*)nalloc(output_dim, sizeof(float));
#ifdef ADAM_CNN_BIAS
cnn->s_d_bias[i][j] = (float*)nalloc(output_dim, sizeof(float));
cnn->v_d_bias[i][j] = (float*)nalloc(output_dim, sizeof(float));
#endif
for (int k=0; k < output_dim; k++) {
(void) !fread(&tmp, sizeof(tmp), 1, ptr);
cnn->bias[i][j][k] = tmp;
cnn->d_bias[i][j][k] = 0.;
#ifdef ADAM_CNN_BIAS
cnn->s_d_bias[i][j][k] = 0.;
cnn->v_d_bias[i][j][k] = 0.;
#endif
}
}
}
cnn->weights = (float****)nalloc(cnn->rows, sizeof(float***));
cnn->d_weights = (float****)nalloc(cnn->rows, sizeof(float***));
#ifdef ADAM_CNN_WEIGHTS
cnn->s_d_weights = (float****)nalloc(cnn->rows, sizeof(float***));
cnn->v_d_weights = (float****)nalloc(cnn->rows, sizeof(float***));
#endif
for (int i=0; i < cnn->rows; i++) {
cnn->weights[i] = (float***)nalloc(cnn->columns, sizeof(float**));
cnn->d_weights[i] = (float***)nalloc(cnn->columns, sizeof(float**));
#ifdef ADAM_CNN_WEIGHTS
cnn->s_d_weights[i] = (float***)nalloc(cnn->columns, sizeof(float**));
cnn->v_d_weights[i] = (float***)nalloc(cnn->columns, sizeof(float**));
#endif
for (int j=0; j < cnn->columns; j++) {
cnn->weights[i][j] = (float**)nalloc(cnn->k_size, sizeof(float*));
cnn->d_weights[i][j] = (float**)nalloc(cnn->k_size, sizeof(float*));
#ifdef ADAM_CNN_WEIGHTS
cnn->s_d_weights[i][j] = (float**)nalloc(cnn->k_size, sizeof(float*));
cnn->v_d_weights[i][j] = (float**)nalloc(cnn->k_size, sizeof(float*));
#endif
for (int k=0; k < cnn->k_size; k++) {
cnn->weights[i][j][k] = (float*)nalloc(cnn->k_size, sizeof(float));
cnn->d_weights[i][j][k] = (float*)nalloc(cnn->k_size, sizeof(float));
#ifdef ADAM_CNN_WEIGHTS
cnn->s_d_weights[i][j][k] = (float*)nalloc(cnn->k_size, sizeof(float));
cnn->v_d_weights[i][j][k] = (float*)nalloc(cnn->k_size, sizeof(float));
#endif
for (int l=0; l < cnn->k_size; l++) {
(void) !fread(&tmp, sizeof(tmp), 1, ptr);
cnn->weights[i][j][k][l] = tmp;
cnn->d_weights[i][j][k][l] = 0.;
#ifdef ADAM_CNN_WEIGHTS
cnn->s_d_weights[i][j][k][l] = 0.;
cnn->v_d_weights[i][j][k][l] = 0.;
#endif
}
}
}
@ -300,21 +336,41 @@ Kernel* read_kernel(int type_couche, int output_dim, FILE* ptr) {
nn->bias = (float*)nalloc(nn->size_output, sizeof(float));
nn->d_bias = (float*)nalloc(nn->size_output, sizeof(float));
#ifdef ADAM_DENSE_BIAS
nn->s_d_bias = (float*)nalloc(nn->size_output, sizeof(float));
nn->v_d_bias = (float*)nalloc(nn->size_output, sizeof(float));
#endif
for (int i=0; i < nn->size_output; i++) {
(void) !fread(&tmp, sizeof(tmp), 1, ptr);
nn->bias[i] = tmp;
nn->d_bias[i] = 0.;
#ifdef ADAM_DENSE_BIAS
nn->s_d_bias[i] = 0.;
nn->v_d_bias[i] = 0.;
#endif
}
nn->weights = (float**)nalloc(nn->size_input, sizeof(float*));
nn->d_weights = (float**)nalloc(nn->size_input, sizeof(float*));
#ifdef ADAM_DENSE_WEIGHTS
nn->s_d_weights = (float**)nalloc(nn->size_input, sizeof(float*));
nn->v_d_weights = (float**)nalloc(nn->size_input, sizeof(float*));
#endif
for (int i=0; i < nn->size_input; i++) {
nn->weights[i] = (float*)nalloc(nn->size_output, sizeof(float));
nn->d_weights[i] = (float*)nalloc(nn->size_output, sizeof(float));
#ifdef ADAM_DENSE_WEIGHTS
nn->s_d_weights[i] = (float*)nalloc(nn->size_output, sizeof(float));
nn->v_d_weights[i] = (float*)nalloc(nn->size_output, sizeof(float));
#endif
for (int j=0; j < nn->size_output; j++) {
(void) !fread(&tmp, sizeof(tmp), 1, ptr);
nn->weights[i][j] = tmp;
nn->d_weights[i][j] = 0.;
#ifdef ADAM_DENSE_WEIGHTS
nn->s_d_weights[i][j] = 0.;
nn->v_d_weights[i][j] = 0.;
#endif
}
}
} else if (type_couche == POOLING) { // Cas du Pooling Layer

32
src/cnn/update.c
View File

@ -1,8 +1,12 @@
#include <stdio.h>
#include <math.h>
#include <float.h>
#include "include/update.h"
#include "include/struct.h"
#include "include/config.h"
float clip(float a) {
if (a > NETWORK_CLIP_VALUE) {
return NETWORK_CLIP_VALUE;
@ -34,7 +38,13 @@ void update_weights(Network* network, Network* d_network) {
for (int b=0; b < output_depth; b++) {
for (int c=0; c < k_size; c++) {
for (int d=0; d < k_size; d++) {
#ifdef ADAM_CNN_WEIGHTS
d_cnn->v_d_weights[a][b][c][d] = BETA_1*d_cnn->v_d_weights[a][b][c][d] + (1-BETA_1)*d_cnn->d_weights[a][b][c][d];
d_cnn->s_d_weights[a][b][c][d] = BETA_2*d_cnn->s_d_weights[a][b][c][d] + (1-BETA_2)*d_cnn->d_weights[a][b][c][d]*d_cnn->d_weights[a][b][c][d];
cnn->weights[a][b][c][d] -= ALPHA*(d_cnn->v_d_weights[a][b][c][d]/sqrt(d_cnn->s_d_weights[a][b][c][d]+Epsilon));
#else
cnn->weights[a][b][c][d] -= network->learning_rate * d_cnn->d_weights[a][b][c][d];
#endif
d_cnn->d_weights[a][b][c][d] = 0;
cnn->weights[a][b][c][d] = clip(cnn->weights[a][b][c][d]);
@ -49,7 +59,13 @@ void update_weights(Network* network, Network* d_network) {
for (int a=0; a < input_width; a++) {
for (int b=0; b < output_width; b++) {
#ifdef ADAM_DENSE_WEIGHTS
d_nn->v_d_weights[a][b] = BETA_1*d_nn->v_d_weights[a][b] + (1-BETA_1)*d_nn->d_weights[a][b];
d_nn->s_d_weights[a][b] = BETA_2*d_nn->s_d_weights[a][b] + (1-BETA_2)*d_nn->d_weights[a][b]*d_nn->d_weights[a][b];
nn->weights[a][b] -= ALPHA*(d_nn->v_d_weights[a][b]/sqrt(d_nn->s_d_weights[a][b]+Epsilon));
#else
nn->weights[a][b] -= network->learning_rate * d_nn->d_weights[a][b];
#endif
d_nn->d_weights[a][b] = 0;
}
}
@ -61,7 +77,13 @@ void update_weights(Network* network, Network* d_network) {
for (int a=0; a < size_input; a++) {
for (int b=0; b < output_width; b++) {
#ifdef ADAM_DENSE_WEIGHTS
d_nn->v_d_weights[a][b] = BETA_1*d_nn->v_d_weights[a][b] + (1-BETA_1)*d_nn->d_weights[a][b];
d_nn->s_d_weights[a][b] = BETA_2*d_nn->s_d_weights[a][b] + (1-BETA_2)*d_nn->d_weights[a][b]*d_nn->d_weights[a][b];
nn->weights[a][b] -= ALPHA*(d_nn->d_weights[a][b]/sqrt(d_nn->s_d_weights[a][b]+Epsilon));
#else
nn->weights[a][b] -= network->learning_rate * d_nn->d_weights[a][b];
#endif
d_nn->d_weights[a][b] = 0;
nn->weights[a][b] = clip(nn->weights[a][b]);
@ -89,7 +111,12 @@ void update_bias(Network* network, Network* d_network) {
for (int a=0; a < output_depth; a++) {
for (int b=0; b < output_width; b++) {
for (int c=0; c < output_width; c++) {
#ifdef ADAM_CNN_BIAS
d_cnn->s_d_bias[a][b][c] = BETA_2*d_cnn->s_d_bias[a][b][c] + (1-BETA_2)*d_cnn->d_bias[a][b][c]*d_cnn->d_bias[a][b][c];
cnn->bias[a][b][c] -= ALPHA*(d_cnn->d_bias[a][b][c]/sqrt(d_cnn->s_d_bias[a][b][c]+Epsilon));
#else
cnn->bias[a][b][c] -= network->learning_rate * d_cnn->d_bias[a][b][c];
#endif
d_cnn->d_bias[a][b][c] = 0;
cnn->bias[a][b][c] = clip(cnn->bias[a][b][c]);
@ -101,7 +128,12 @@ void update_bias(Network* network, Network* d_network) {
Kernel_nn* d_nn = dk_i->nn;
for (int a=0; a < output_width; a++) {
#ifdef ADAM_DENSE_BIAS
d_nn->s_d_bias[a] = BETA_2*d_nn->s_d_bias[a] + (1-BETA_2)*d_nn->d_bias[a]*d_nn->d_bias[a];
nn->bias[a] -= ALPHA*(d_nn->d_bias[a]/sqrt(d_nn->s_d_bias[a]+Epsilon));
#else
nn->bias[a] -= network->learning_rate * d_nn->d_bias[a];
#endif
d_nn->d_bias[a] = 0;
nn->bias[a] = clip(nn->bias[a]);

56
src/cnn/utils.c
View File

@ -148,19 +148,39 @@ Network* copy_network(Network* network) {
network_cp->kernel[i]->nn->bias = (float*)nalloc(size_output, sizeof(float));
network_cp->kernel[i]->nn->d_bias = (float*)nalloc(size_output, sizeof(float));
#ifdef ADAM_DENSE_BIAS
network_cp->kernel[i]->nn->s_d_bias = (float*)nalloc(size_output, sizeof(float));
network_cp->kernel[i]->nn->v_d_bias = (float*)nalloc(size_output, sizeof(float));
#endif
for (int j=0; j < size_output; j++) {
copyVar(kernel[i]->nn->bias[j]);
network_cp->kernel[i]->nn->d_bias[j] = 0.;
#ifdef ADAM_DENSE_BIAS
network_cp->kernel[i]->nn->s_d_bias[j] = 0.;
network_cp->kernel[i]->nn->v_d_bias[j] = 0.;
#endif
}
network_cp->kernel[i]->nn->weights = (float**)nalloc(size_input, sizeof(float*));
network_cp->kernel[i]->nn->d_weights = (float**)nalloc(size_input, sizeof(float*));
#ifdef ADAM_DENSE_WEIGHTS
network_cp->kernel[i]->nn->s_d_weights = (float**)nalloc(size_input, sizeof(float*));
network_cp->kernel[i]->nn->v_d_weights = (float**)nalloc(size_input, sizeof(float*));
#endif
for (int j=0; j < size_input; j++) {
network_cp->kernel[i]->nn->weights[j] = (float*)nalloc(size_output, sizeof(float));
network_cp->kernel[i]->nn->d_weights[j] = (float*)nalloc(size_output, sizeof(float));
#ifdef ADAM_DENSE_WEIGHTS
network_cp->kernel[i]->nn->s_d_weights[j] = (float*)nalloc(size_output, sizeof(float));
network_cp->kernel[i]->nn->v_d_weights[j] = (float*)nalloc(size_output, sizeof(float));
#endif
for (int k=0; k < size_output; k++) {
copyVar(kernel[i]->nn->weights[j][k]);
network_cp->kernel[i]->nn->d_weights[j][k] = 0.;
#ifdef ADAM_DENSE_WEIGHTS
network_cp->kernel[i]->nn->s_d_weights[j][k] = 0.;
network_cp->kernel[i]->nn->v_d_weights[j][k] = 0.;
#endif
}
}
}
@ -184,33 +204,69 @@ Network* copy_network(Network* network) {
network_cp->kernel[i]->cnn->bias = (float***)nalloc(columns, sizeof(float**));
network_cp->kernel[i]->cnn->d_bias = (float***)nalloc(columns, sizeof(float**));
#ifdef ADAM_CNN_BIAS
network_cp->kernel[i]->cnn->s_d_bias = (float***)nalloc(columns, sizeof(float**));
network_cp->kernel[i]->cnn->v_d_bias = (float***)nalloc(columns, sizeof(float**));
#endif
for (int j=0; j < columns; j++) {
network_cp->kernel[i]->cnn->bias[j] = (float**)nalloc(output_dim, sizeof(float*));
network_cp->kernel[i]->cnn->d_bias[j] = (float**)nalloc(output_dim, sizeof(float*));
#ifdef ADAM_CNN_BIAS
network_cp->kernel[i]->cnn->s_d_bias[j] = (float**)nalloc(output_dim, sizeof(float*));
network_cp->kernel[i]->cnn->v_d_bias[j] = (float**)nalloc(output_dim, sizeof(float*));
#endif
for (int k=0; k < output_dim; k++) {
network_cp->kernel[i]->cnn->bias[j][k] = (float*)nalloc(output_dim, sizeof(float));
network_cp->kernel[i]->cnn->d_bias[j][k] = (float*)nalloc(output_dim, sizeof(float));
#ifdef ADAM_CNN_BIAS
network_cp->kernel[i]->cnn->s_d_bias[j][k] = (float*)nalloc(output_dim, sizeof(float));
network_cp->kernel[i]->cnn->v_d_bias[j][k] = (float*)nalloc(output_dim, sizeof(float));
#endif
for (int l=0; l < output_dim; l++) {
copyVar(kernel[i]->cnn->bias[j][k][l]);
network_cp->kernel[i]->cnn->d_bias[j][k][l] = 0.;
#ifdef ADAM_CNN_BIAS
network_cp->kernel[i]->cnn->s_d_bias[j][k][l] = 0.;
network_cp->kernel[i]->cnn->v_d_bias[j][k][l] = 0.;
#endif
}
}
}
network_cp->kernel[i]->cnn->weights = (float****)nalloc(rows, sizeof(float***));
network_cp->kernel[i]->cnn->d_weights = (float****)nalloc(rows, sizeof(float***));
#ifdef ADAM_CNN_WEIGHTS
network_cp->kernel[i]->cnn->s_d_weights = (float****)nalloc(rows, sizeof(float***));
network_cp->kernel[i]->cnn->v_d_weights = (float****)nalloc(rows, sizeof(float***));
#endif
for (int j=0; j < rows; j++) {
network_cp->kernel[i]->cnn->weights[j] = (float***)nalloc(columns, sizeof(float**));
network_cp->kernel[i]->cnn->d_weights[j] = (float***)nalloc(columns, sizeof(float**));
#ifdef ADAM_CNN_WEIGHTS
network_cp->kernel[i]->cnn->s_d_weights[j] = (float***)nalloc(columns, sizeof(float**));
network_cp->kernel[i]->cnn->v_d_weights[j] = (float***)nalloc(columns, sizeof(float**));
#endif
for (int k=0; k < columns; k++) {
network_cp->kernel[i]->cnn->weights[j][k] = (float**)nalloc(k_size, sizeof(float*));
network_cp->kernel[i]->cnn->d_weights[j][k] = (float**)nalloc(k_size, sizeof(float*));
#ifdef ADAM_CNN_WEIGHTS
network_cp->kernel[i]->cnn->s_d_weights[j][k] = (float**)nalloc(k_size, sizeof(float*));
network_cp->kernel[i]->cnn->v_d_weights[j][k] = (float**)nalloc(k_size, sizeof(float*));
#endif
for (int l=0; l < k_size; l++) {
network_cp->kernel[i]->cnn->weights[j][k][l] = (float*)nalloc(k_size, sizeof(float));
network_cp->kernel[i]->cnn->d_weights[j][k][l] = (float*)nalloc(k_size, sizeof(float));
#ifdef ADAM_CNN_WEIGHTS
network_cp->kernel[i]->cnn->s_d_weights[j][k][l] = (float*)nalloc(k_size, sizeof(float));
network_cp->kernel[i]->cnn->v_d_weights[j][k][l] = (float*)nalloc(k_size, sizeof(float));
#endif
for (int m=0; m < k_size; m++) {
copyVar(kernel[i]->cnn->weights[j][k][l][m]);
network_cp->kernel[i]->cnn->d_weights[j][k][l][m] = 0.;
#ifdef ADAM_CNN_WEIGHTS
network_cp->kernel[i]->cnn->s_d_weights[j][k][l][m] = 0.;
network_cp->kernel[i]->cnn->v_d_weights[j][k][l][m] = 0.;
#endif
}
}
}

30
src/scripts/convolution_benchmark.cu
View File

@ -1,3 +1,5 @@
//! This file uses an old implementation of convolution which uses linearised matrices
//! It is therefore not compatible nor compilable now.
#include <stdlib.h>
#include <stdio.h>
#include <stdbool.h>
@ -114,17 +116,33 @@ void run_convolution_test(int input_dim, int output_dim, int rows, int columns)
// bias[kernel->columns]
kernel->bias = (float*)malloc(kernel->columns, sizeof(float));
kernel->d_bias = (float*)malloc(kernel->columns, sizeof(float));
#ifdef ADAM_CNN_BIAS
kernel->s_d_bias = (float*)malloc(kernel->columns, sizeof(float));
kernel->v_d_bias = (float*)malloc(kernel->columns, sizeof(float));
#endif
for (int i=0; i<kernel->columns; i++) {
kernel->bias[i] = random_float(0.0f, 15.0f);
kernel->d_bias[i] = random_float(0.0f, 1.5f);
#ifdef ADAM_CNN_BIAS
kernel->s_d_bias[i] = random_float(0.0f, 1.5f);
kernel->v_d_bias[i] = random_float(0.0f, 1.5f);
#endif
}
// weights[rows][columns][k_size][k_size]
kernel->weights = (float****)malloc(sizeof(float***)*kernel->rows);
kernel->d_weights = (float****)malloc(sizeof(float***)*kernel->rows);
#ifdef ADAM_CNN_WEIGHTS
kernel->s_d_weights = (float****)malloc(sizeof(float***)*kernel->rows);
kernel->v_d_weights = (float****)malloc(sizeof(float***)*kernel->rows);
#endif
for (int i=0; i < kernel->rows; i++) {
kernel->weights[i] = create_matrix(kernel->columns, kernel->k_size, kernel->k_size, 15.0f);
kernel->d_weights[i] = create_matrix(kernel->columns, kernel->k_size, kernel->k_size, 1.5f);
#ifdef ADAM_CNN_WEIGHTS
kernel->s_d_weights[i] = create_matrix(kernel->columns, kernel->k_size, kernel->k_size, 1.5f);
kernel->v_d_weights[i] = create_matrix(kernel->columns, kernel->k_size, kernel->k_size, 1.5f);
#endif
}
float*** input = create_matrix(kernel->rows, input_dim, input_dim, 5.0f);
@ -162,13 +180,25 @@ void run_convolution_test(int input_dim, int output_dim, int rows, int columns)
free(kernel->bias);
free(kernel->d_bias);
#ifdef ADAM_CNN_BIAS
free(kernel->s_d_bias);
free(kernel->v_d_bias);
#endif
for (int i=0; i < kernel->rows; i++) {
free_matrix(kernel->weights[i], kernel->columns, kernel->k_size);
free_matrix(kernel->d_weights[i], kernel->columns, kernel->k_size);
#ifdef ADAM_CNN_WEIGHTS
free_matrix(kernel->s_d_weights[i], kernel->columns, kernel->k_size);
free_matrix(kernel->v_d_weights[i], kernel->columns, kernel->k_size);
#endif
}
free(kernel->weights);
free(kernel->d_weights);
#ifdef ADAM_CNN_WEIGHTS
free(kernel->s_d_weights);
free(kernel->v_d_weights);
#endif
free_matrix(input, kernel->rows, input_dim);
free_matrix(output_cpu, kernel->columns, output_dim);

24
test/cnn_convolution.cu
View File

@ -107,13 +107,25 @@ void run_convolution_test(int input_dim, int output_dim, int rows, int columns)
// bias[kernel->columns][dim_output][dim_output]
kernel->bias = create_matrix(kernel->columns, output_dim, output_dim, 15.0f);
kernel->d_bias = create_matrix(kernel->columns, output_dim, output_dim, 1.5f);
#ifdef ADAM_CNN_BIAS
kernel->s_d_bias = create_matrix(kernel->columns, output_dim, output_dim, 1.5f);
kernel->v_d_bias = create_matrix(kernel->columns, output_dim, output_dim, 1.5f);
#endif
// weights[rows][columns][k_size][k_size]
kernel->weights = (float****)nalloc(kernel->rows, sizeof(float***));
kernel->d_weights = (float****)nalloc(kernel->rows, sizeof(float***));
#ifdef ADAM_CNN_WEIGHTS
kernel->s_d_weights = (float****)nalloc(kernel->rows, sizeof(float***));
kernel->v_d_weights = (float****)nalloc(kernel->rows, sizeof(float***));
#endif
for (int i=0; i < kernel->rows; i++) {
kernel->weights[i] = create_matrix(kernel->columns, kernel->k_size, kernel->k_size, 15.0f);
kernel->d_weights[i] = create_matrix(kernel->columns, kernel->k_size, kernel->k_size, 1.5f);
#ifdef ADAM_CNN_WEIGHTS
kernel->s_d_weights[i] = create_matrix(kernel->columns, kernel->k_size, kernel->k_size, 1.5f);
kernel->v_d_weights[i] = create_matrix(kernel->columns, kernel->k_size, kernel->k_size, 1.5f);
#endif
}
float*** input = create_matrix(kernel->rows, input_dim, input_dim, 5.0f);
@ -152,13 +164,25 @@ void run_convolution_test(int input_dim, int output_dim, int rows, int columns)
free_matrix(kernel->bias, kernel->columns, output_dim);
free_matrix(kernel->d_bias, kernel->columns, output_dim);
#ifdef ADAM_CNN_BIAS
free_matrix(kernel->s_d_bias, kernel->columns, output_dim);
free_matrix(kernel->v_d_bias, kernel->columns, output_dim);
#endif
for (int i=0; i < kernel->rows; i++) {
free_matrix(kernel->weights[i], kernel->columns, kernel->k_size);
free_matrix(kernel->d_weights[i], kernel->columns, kernel->k_size);
#ifdef ADAM_CNN_WEIGHTS
free_matrix(kernel->s_d_weights[i], kernel->columns, kernel->k_size);
free_matrix(kernel->v_d_weights[i], kernel->columns, kernel->k_size);
#endif
}
gree(kernel->weights);
gree(kernel->d_weights);
#ifdef ADAM_CNN_WEIGHTS
gree(kernel->s_d_weights);
gree(kernel->v_d_weights);
#endif
free_matrix(input, kernel->rows, input_dim);
free_matrix(output_cpu, kernel->columns, output_dim);

11
test/cnn_function.cu
View File

@ -5,10 +5,19 @@
#include "../src/include/memory_management.h"
#include "../src/cnn/include/function.h"
#include "../src/include/colors.h"
#include "../src/include/utils.h"
int main() {
printf("Initialisation\n");
printf("Checking CUDA compatibility.\n");
bool cuda_compatible = check_cuda_compatibility();
if (!cuda_compatible) {
printf(RED "CUDA not compatible, skipping tests.\n" RESET);
return 0;
}
printf(GREEN "OK\n" RESET);
printf("Initialisation OK\n");
// Initialise values
int depth = 10;
int rows = 10;

8
test/memory_management.cu
View File

@ -17,6 +17,14 @@ __global__ void check_access(int* array, int range) {
int main() {
printf("Checking CUDA compatibility.\n");
bool cuda_compatible = check_cuda_compatibility();
if (!cuda_compatible) {
printf(RED "CUDA not compatible, skipping tests.\n" RESET);
return 0;
}
printf(GREEN "OK\n" RESET);
printf("Pollution de la mémoire\n");
int mem_used;
int blocks_used;