From b12a03c1baa8e8505066fa07ae2f20882a24854b Mon Sep 17 00:00:00 2001 From: augustin64 Date: Thu, 29 Sep 2022 22:21:33 +0200 Subject: [PATCH] Rename mnist out files: out/* -> out/mnist_* --- .github/workflows/tests.yml | 2 +- .vscode/tasks.json | 31 ++------ make.sh | 70 ++++++++++++------- src/parallel/app.py | 6 +- src/parallel/client.py | 6 +- src/parallel/structures.py | 12 ++-- src/webserver/app.py | 2 +- test/{mnist.c => mnist_data_io.c} | 0 ...eural_network.c => mnist_neural_network.c} | 9 +-- test/mnist_neuron_io.c | 4 ++ test/mnist_utils.sh | 22 ++++++ test/test_utils.sh | 22 ------ 12 files changed, 92 insertions(+), 94 deletions(-) rename test/{mnist.c => mnist_data_io.c} (100%) rename test/{neural_network.c => mnist_neural_network.c} (80%) create mode 100755 test/mnist_utils.sh delete mode 100755 test/test_utils.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 923bc6c..880ebc8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,7 +18,7 @@ jobs: - name: run tests run: ./make.sh test run - name: mnist main test - run: out/main train + run: out/mnist_main train --epochs 1 --images data/mnist/t10k-images-idx3-ubyte --labels data/mnist/t10k-labels-idx1-ubyte diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 35d9ce9..3509bee 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -29,32 +29,13 @@ "detail": "Tâche générée par le débogueur." }, { - "type": "build", - "label": "C: gcc générer le fichier actif", - "command": "gcc", + "label": "build", + "command": "${workspaceFolder}/make.sh", "args": [ - "-g", - "${file}", - "-o", - "${workspaceFolder}/out/${fileBasenameNoExtension}", - "-lm", - "-lpthread", - "-std=c99", - "-Wall", - "-Wextra" - ], - "options": { - "cwd": "${fileDirname}" - }, - "problemMatcher": [ - "$gcc" - ], - "group": { - "kind": "build", - "isDefault": true - }, - "detail": "Tâche générée par le débogueur." - } + "build", + "all" + ] + }, ], "version": "2.0.0" } \ No newline at end of file diff --git a/make.sh b/make.sh index 7b3303a..0e02eff 100755 --- a/make.sh +++ b/make.sh @@ -15,19 +15,19 @@ compile_cuda () { build () { mkdir -p "$OUT" - [[ $1 ]] || set "main" - if [[ $1 == "main" ]]; then + [[ $1 ]] || set "mnist-main" + if [[ $1 == "mnist-main" ]]; then echo "Compilation de src/mnist/main.c" - $CC src/mnist/main.c -o "$OUT/main" $FLAGS + $CC src/mnist/main.c -o "$OUT/mnist_main" $FLAGS echo "Fait." return 0 - elif [[ $1 == "preview" ]]; then + elif [[ $1 == "mnist-preview" ]]; then echo "Compilation de src/mnist/preview.c" - $CC src/mnist/preview.c -o "$OUT/preview_mnist" $FLAGS + $CC src/mnist/preview.c -o "$OUT/mnist_preview" $FLAGS echo "Fait." return 0 elif [[ $1 == "test" ]]; then - [[ -f "$OUT/test_"* ]] && rm "$OUT/test_"* + rm "$OUT/test_"* || true for i in "test/"*".c"; do echo "Compilation de $i" $CC "$i" -o "$OUT/test_$(echo $i | awk -F. '{print $1}' | awk -F/ '{print $NF}')" $FLAGS @@ -45,16 +45,30 @@ build () { done fi return 0 - elif [[ $1 == "utils" ]]; then + elif [[ $1 == "mnist-utils" ]]; then echo "Compilation de src/mnist/utils.c" - $CC "src/mnist/utils.c" -o "$OUT/utils" $FLAGS + $CC "src/mnist/utils.c" -o "$OUT/mnist_utils" $FLAGS echo "Fait." return 0 + elif [[ $1 == "mnist" ]]; then + build mnist-main + build mnist-preview + build mnist-utils + elif [[ $1 == "cnn-main" ]]; then + echo "Compilation de src/cnn/main.c" + $CC "src/cnn/main.c" -o "$OUT/cnn_main" $FLAGS + echo "Fait." + elif [[ $1 == "cnn" ]]; then + build cnn-main else - build main - build preview + echo -e "\033[1m\033[34m### Building mnist ###\033[0m" + build mnist-main + build mnist-preview + build mnist-utils + echo -e "\033[1m\033[34m### Building cnn ###\033[0m" + build cnn + echo -e "\033[1m\033[34m### Building tests ###\033[0m" build test - build utils return 0 fi } @@ -64,12 +78,12 @@ preview () { build preview return 0 elif [[ $1 == "train" ]]; then - [[ -f "$OUT/preview_mnist" ]] || $0 build preview - "$OUT/preview_mnist" data/mnist/train-images-idx3-ubyte data/mnist/train-labels-idx1-ubyte + [[ -f "$OUT/mnist_preview" ]] || $0 build preview + "$OUT/mnist_preview" data/mnist/train-images-idx3-ubyte data/mnist/train-labels-idx1-ubyte return 0 elif [[ $1 == "t10k" ]]; then - [[ -f "$OUT/preview_mnist" ]] || $0 build preview - "$OUT/preview_mnist" data/mnist/t10k-images-idx3-ubyte data/mnist/t10k-labels-idx1-ubyte + [[ -f "$OUT/mnist_preview" ]] || $0 build preview + "$OUT/mnist_preview" data/mnist/t10k-images-idx3-ubyte data/mnist/t10k-labels-idx1-ubyte return 0 fi } @@ -95,11 +109,11 @@ test () { } train () { - [[ -f "$OUT/main" ]] || build main + [[ -f "$OUT/mnist_main" ]] || build mnist-main [[ $1 ]] || set -- "train" [[ $2 == "-r" || $2 == "--recover" ]] && RECOVER="-r .cache/reseau.bin" mkdir -p .cache - "$OUT/main" train \ + "$OUT/mnist_main" train \ --images "data/mnist/$1-images-idx3-ubyte" \ --labels "data/mnist/$1-labels-idx1-ubyte" \ --out ".cache/reseau.bin" \ @@ -108,10 +122,10 @@ train () { } test_reseau () { - [[ -f "$OUT/main" ]] || build main + [[ -f "$OUT/mnist_main" ]] || build mnist-main [[ $1 ]] || set -- "train" [[ -f ".cache/reseau.bin" ]] || train train - "$OUT/main" test \ + "$OUT/mnist_main" test \ --images "data/mnist/$1-images-idx3-ubyte" \ --labels "data/mnist/$1-labels-idx1-ubyte" \ --modele ".cache/reseau.bin" @@ -121,9 +135,9 @@ test_reseau () { recognize () { if [[ $1 ]]; then [[ $2 ]] || set -- "$2" "text" - [[ -f "$OUT/main" ]] || build main + [[ -f "$OUT/mnist_main" ]] || build mnist-main [[ -f ".cache/reseau.bin" ]] || train train - "$OUT/main" recognize \ + "$OUT/mnist_main" recognize \ --modele ".cache/reseau.bin" \ --in "$1" \ --out "$2" @@ -135,13 +149,13 @@ recognize () { } utils () { - [[ -f "$OUT/utils" ]] || build utils - "$OUT/utils" ${*:1} + [[ -f "$OUT/mnist_utils" ]] || build mnist-utils + "$OUT/mnist_utils" ${*:1} return 0 } webserver () { - [[ -f "$OUT/main" ]] || build main + [[ -f "$OUT/mnist_main" ]] || build mnist-main [[ -f ".cache/reseau.bin" ]] || train train FLASK_APP="src/webserver/app.py" flask run return 0 @@ -149,7 +163,13 @@ webserver () { usage () { echo "Usage:" - echo -e "\t$0 build ( main | preview | train | utils | all )\n" + echo -e "\t$0 build ( test | all | ... )" + echo -e "\t\t\tmnist: mnist" + echo -e "\t\t\t\tmnist-main" + echo -e "\t\t\t\tmnist-preview" + echo -e "\t\t\t\tmnist-utils" + echo -e "\t\t\tcnn: cnn" + echo -e "\t\t\t\tcnn-main\n" echo -e "\t$0 train ( train | t10k ) ( -r | --recover )" echo -e "\t$0 preview ( train | t10k )" echo -e "\t$0 test_reseau ( train | t10k )\n" diff --git a/src/parallel/app.py b/src/parallel/app.py index 0bf2242..2a7c287 100644 --- a/src/parallel/app.py +++ b/src/parallel/app.py @@ -27,11 +27,11 @@ training = Training(BATCHS, DATASET, TEST_SET, CACHE) os.makedirs(CACHE, exist_ok=True) # On crée un réseau aléatoire si il n'existe pas encore if not os.path.isfile(RESEAU): - if not os.path.isfile("out/main"): - subprocess.call(["./make.sh", "build", "main"]) + if not os.path.isfile("out/mnist_main"): + subprocess.call(["./make.sh", "build", "mnist-main"]) subprocess.call( [ - "out/main", "train", + "out/mnist_main", "train", "--epochs", "0", "--images", "data/mnist/train-images-idx3-ubyte", "--labels", "data/mnist/train-labels-idx1-ubyte", diff --git a/src/parallel/client.py b/src/parallel/client.py index 431486a..f5bb375 100755 --- a/src/parallel/client.py +++ b/src/parallel/client.py @@ -118,13 +118,13 @@ def train_shared(dataset, start, nb_elem, epochs=1, out=DELTA): raise NotImplementedError # On compile out/main si il n'existe pas encore - if not os.path.isfile("out/main"): - subprocess.call(["./make.sh", "build", "main"]) + if not os.path.isfile("out/mnist_main"): + subprocess.call(["./make.sh", "build", "mnist-main"]) # Entraînement du réseau subprocess.call( [ - "out/main", "train", + "out/mnist-main", "train", "--epochs", str(epochs), "--images", images, "--labels", labels, diff --git a/src/parallel/structures.py b/src/parallel/structures.py index 1e186b1..5e97beb 100644 --- a/src/parallel/structures.py +++ b/src/parallel/structures.py @@ -95,12 +95,12 @@ class Training: return self.lock_test = True - if not os.path.isfile("out/main"): - subprocess.call(["./make.sh", "build", "main"]) + if not os.path.isfile("out/mnist_main"): + subprocess.call(["./make.sh", "build", "mnist-main"]) subprocess.call( [ - "out/main", "test", + "out/mnist_main", "test", "--images", self.test_images, "--labels", self.test_labels, "--modele", self.reseau @@ -127,11 +127,11 @@ class Training: with open(self.reseau + ".lock", "w", encoding="utf8") as file: file.write("") - if not os.path.isfile("out/utils"): - subprocess.call(["./make.sh", "build", "utils"]) + if not os.path.isfile("out/mnist_utils"): + subprocess.call(["./make.sh", "build", "mnist-utils"]) subprocess.call( [ - "out/utils", "patch-network", + "out/mnist_utils", "patch-network", "--network", self.reseau, "--delta", self.delta, ]) diff --git a/src/webserver/app.py b/src/webserver/app.py index a682535..8af9be0 100644 --- a/src/webserver/app.py +++ b/src/webserver/app.py @@ -38,7 +38,7 @@ def recognize_mnist(image): try: output = subprocess.check_output([ - 'out/main', + 'out/mnist_main', 'recognize', '--modele', '.cache/reseau.bin', '--in', '.cache/image-idx3-ubyte', diff --git a/test/mnist.c b/test/mnist_data_io.c similarity index 100% rename from test/mnist.c rename to test/mnist_data_io.c diff --git a/test/neural_network.c b/test/mnist_neural_network.c similarity index 80% rename from test/neural_network.c rename to test/mnist_neural_network.c index ae7583a..c0691fd 100644 --- a/test/neural_network.c +++ b/test/mnist_neural_network.c @@ -8,22 +8,15 @@ int main() { printf("Création du réseau\n"); - Network* network = (Network*)malloc(sizeof(Network)); int tab[5] = {30, 25, 20, 15, 10}; network_creation(network, tab, 5); - printf("OK\n"); + printf("Initialisation du réseau\n"); - network_initialisation(network); - printf("OK\n"); - printf("Enregistrement du réseau\n"); - write_network(".test-cache/random_network.bin", network); - - printf("OK\n"); deletion_of_network(network); return 0; } \ No newline at end of file diff --git a/test/mnist_neuron_io.c b/test/mnist_neuron_io.c index ca46fff..5a86dcb 100644 --- a/test/mnist_neuron_io.c +++ b/test/mnist_neuron_io.c @@ -71,7 +71,11 @@ int main() { printf("Vérification de l'accès en lecture\n"); Network* network2 = read_network(".test-cache/neuron_io.bin"); printf("OK\n"); + + printf("Suppression des réseaux\n"); deletion_of_network(network); deletion_of_network(network2); + printf("OK\n"); + return 0; } \ No newline at end of file diff --git a/test/mnist_utils.sh b/test/mnist_utils.sh new file mode 100755 index 0000000..3c699b2 --- /dev/null +++ b/test/mnist_utils.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +OUT="$1" +[[ -f "$OUT/mnist_utils" ]] || "$2" build mnist-utils + +echo "Compte des labels" +"$OUT/mnist_utils" count-labels -l data/mnist/t10k-labels-idx1-ubyte > /dev/null +echo "OK" + +echo "Création du réseau" +"$OUT/mnist_utils" creer-reseau -n 3 -o .test-cache/reseau.bin > /dev/null +echo "OK" + +echo "Affichage poids" +"$OUT/mnist_utils" print-poids -r .test-cache/reseau.bin > /dev/null +echo "OK" + +echo "Affichage biais" +"$OUT/mnist_utils" print-biais -r .test-cache/reseau.bin > /dev/null +echo "OK" \ No newline at end of file diff --git a/test/test_utils.sh b/test/test_utils.sh deleted file mode 100755 index 66d7029..0000000 --- a/test/test_utils.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -set -e - -OUT="$1" -[[ -f "$OUT/utils" ]] || "$2" build utils - -echo "Compte des labels" -"$OUT/utils" count-labels -l data/mnist/t10k-labels-idx1-ubyte > /dev/null -echo "OK" - -echo "Création du réseau" -"$OUT/utils" creer-reseau -n 3 -o .test-cache/reseau.bin > /dev/null -echo "OK" - -echo "Affichage poids" -"$OUT/utils" print-poids -r .test-cache/reseau.bin > /dev/null -echo "OK" - -echo "Affichage biais" -"$OUT/utils" print-biais -r .test-cache/reseau.bin > /dev/null -echo "OK" \ No newline at end of file