search: Multithreaded the downloads

search: Thumbnails are now created on-demand

partitioncloud/__init__.py

@@ -4,12 +4,12 @@ Main file
 """
 import os
 
-from flask import Flask, g, redirect, render_template, request, send_file, flash, session
+from flask import Flask, g, redirect, render_template, request, send_file, flash, session, abort
 from werkzeug.security import generate_password_hash
 
 from .modules.utils import User, Album, get_all_albums
 from .modules import albums, auth, partition, admin
-from .modules.auth import admin_required
+from .modules.auth import admin_required, login_required
 from .modules.db import get_db
 
 app = Flask(__name__)
@@ -82,6 +82,32 @@ def add_user():
     return render_template("auth/register.html", albums=get_all_albums(), user=current_user)
 
 
+@app.route("/static/search-thumbnails/<uuid>.jpg")
+@login_required
+def search_thumbnail(uuid):
+    db = get_db()
+    partition = db.execute(
+        """
+        SELECT uuid, url FROM search_results
+        WHERE uuid = ?
+        """,
+        (uuid,)
+    ).fetchone()
+
+    if partition is None:
+        abort(404)
+    if not os.path.exists(os.path.join(app.static_folder, "search-thumbnails", f"{uuid}.jpg")):
+        os.system(
+            f'/usr/bin/convert -thumbnail\
+            "178^>" -background white -alpha \
+            remove -crop 178x178+0+0 \
+            partitioncloud/search-partitions/{uuid}.pdf[0] \
+            partitioncloud/static/search-thumbnails/{uuid}.jpg'
+        )
+
+    return send_file(os.path.join(app.static_folder, "search-thumbnails", f"{uuid}.jpg"))
+
+
 @app.after_request
 def after_request(response):
     """Automatically close db after each request"""

partitioncloud/modules/search.py

@@ -4,6 +4,7 @@ Module implémentant la recherche de partitions par mots-clés
 """
 from uuid import uuid4
 import urllib.request
+import threading
 import os
 
 import googlesearch
@@ -40,6 +41,18 @@ def local_search(query, partitions):
     return sorted_partitions[:min(5,len(sorted_partitions))]
 
 
+def download_search_result(element):
+    uuid = element["uuid"]
+    url = element["url"]
+
+    try:
+        urllib.request.urlretrieve(url, f"partitioncloud/search-partitions/{uuid}.pdf")
+
+    except (urllib.error.HTTPError, urllib.error.URLError) as e:
+        with open(f"partitioncloud/search-partitions/{uuid}.pdf",'a') as f: 
+            pass # Create empty file
+
+
 def online_search(query, num_queries):
     """
     Renvoie les 3 résultats les plus pertinents depuis google
@@ -65,26 +78,32 @@ def online_search(query, num_queries):
                     (uuid, element,)
                 )
                 db.commit()
-                urllib.request.urlretrieve(element, f"partitioncloud/search-partitions/{uuid}.pdf")
 
-                os.system(
-                    f'/usr/bin/convert -thumbnail\
-                    "178^>" -background white -alpha \
-                    remove -crop 178x178+0+0 \
-                    partitioncloud/search-partitions/{uuid}.pdf[0] \
-                    partitioncloud/static/search-thumbnails/{uuid}.jpg'
-                )
                 partitions.append(
                     {
                         "name": element.split("://")[1].split("/")[0],
-                        "uuid": uuid
+                        "uuid": uuid,
+                        "url": element
                     }
                 )
                 break
             except db.IntegrityError:
                 pass
-            except (urllib.error.HTTPError, urllib.error.URLError) as e:
+
-                print(e, element)
+    threads = [threading.Thread(target=download_search_result, args=(elem,)) for elem in partitions]
+
+    for thread in threads:
+        thread.start()
+
+    for thread in threads:
+        thread.join()
+
+    for element in partitions:
+        pass
+        uuid = element["uuid"]
+        url = element["url"]
+        if os.stat(f"partitioncloud/search-partitions/{uuid}.pdf").st_size == 0:
+            print("An error occured", url)
             db.execute(
                 """
                 DELETE FROM search_results
@@ -93,7 +112,11 @@ def online_search(query, num_queries):
                 (uuid,)
             )
             db.commit()
-                break
+
+            os.remove(f"partitioncloud/search-partitions/{uuid}.pdf")
+
+            partitions.remove(element)
+
     return partitions