augustin64/partitioncloud-server
1
1
Fork 0
mirror of https://github.com/partitioncloud/partitioncloud-server.git synced 2025-01-23 17:26:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

search: Multithreaded the downloads

Browse Source 
search: Thumbnails are now created on-demand
This commit is contained in:
augustin64 2023-08-28 14:14:57 +02:00
parent e4777cf7a6
commit 08908ffdf7
2 changed files with 71 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
partitioncloud/__init__.py
View File

@ -4,12 +4,12 @@ Main file
"""
import os
from flask import Flask, g, redirect, render_template, request, send_file, flash, session
from flask import Flask, g, redirect, render_template, request, send_file, flash, session, abort
from werkzeug.security import generate_password_hash
from .modules.utils import User, Album, get_all_albums
from .modules import albums, auth, partition, admin
from .modules.auth import admin_required
from .modules.auth import admin_required, login_required
from .modules.db import get_db
app = Flask(__name__)
@ -82,6 +82,32 @@ def add_user():
return render_template("auth/register.html", albums=get_all_albums(), user=current_user)
@app.route("/static/search-thumbnails/<uuid>.jpg")
@login_required
def search_thumbnail(uuid):
db = get_db()
partition = db.execute(
"""
SELECT uuid, url FROM search_results
WHERE uuid = ?
""",
(uuid,)
).fetchone()
if partition is None:
abort(404)
if not os.path.exists(os.path.join(app.static_folder, "search-thumbnails", f"{uuid}.jpg")):
os.system(
f'/usr/bin/convert -thumbnail\
"178^>" -background white -alpha \
remove -crop 178x178+0+0 \
partitioncloud/search-partitions/{uuid}.pdf[0] \
partitioncloud/static/search-thumbnails/{uuid}.jpg'
)
return send_file(os.path.join(app.static_folder, "search-thumbnails", f"{uuid}.jpg"))
@app.after_request
def after_request(response):
"""Automatically close db after each request"""

47
partitioncloud/modules/search.py
View File

@ -4,6 +4,7 @@ Module implémentant la recherche de partitions par mots-clés
"""
from uuid import uuid4
import urllib.request
import threading
import os
import googlesearch
@ -40,6 +41,18 @@ def local_search(query, partitions):
return sorted_partitions[:min(5,len(sorted_partitions))]
def download_search_result(element):
uuid = element["uuid"]
url = element["url"]
try:
urllib.request.urlretrieve(url, f"partitioncloud/search-partitions/{uuid}.pdf")
except (urllib.error.HTTPError, urllib.error.URLError) as e:
with open(f"partitioncloud/search-partitions/{uuid}.pdf",'a') as f:
pass # Create empty file
def online_search(query, num_queries):
"""
Renvoie les 3 résultats les plus pertinents depuis google
@ -65,26 +78,32 @@ def online_search(query, num_queries):
(uuid, element,)
)
db.commit()
urllib.request.urlretrieve(element, f"partitioncloud/search-partitions/{uuid}.pdf")
os.system(
f'/usr/bin/convert -thumbnail\
"178^>" -background white -alpha \
remove -crop 178x178+0+0 \
partitioncloud/search-partitions/{uuid}.pdf[0] \
partitioncloud/static/search-thumbnails/{uuid}.jpg'
)
partitions.append(
{
"name": element.split("://")[1].split("/")[0],
"uuid": uuid
"uuid": uuid,
"url": element
}
)
break
except db.IntegrityError:
pass
except (urllib.error.HTTPError, urllib.error.URLError) as e:
print(e, element)
threads = [threading.Thread(target=download_search_result, args=(elem,)) for elem in partitions]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
for element in partitions:
pass
uuid = element["uuid"]
url = element["url"]
if os.stat(f"partitioncloud/search-partitions/{uuid}.pdf").st_size == 0:
print("An error occured", url)
db.execute(
"""
DELETE FROM search_results
@ -93,7 +112,11 @@ def online_search(query, num_queries):
(uuid,)
)
db.commit()
break
os.remove(f"partitioncloud/search-partitions/{uuid}.pdf")
partitions.remove(element)
return partitions