From 870a5f25f3db734288e102ee94bd8b09c2ceef09 Mon Sep 17 00:00:00 2001 From: Youen Date: Mon, 15 Aug 2022 22:38:08 +0200 Subject: [PATCH] Improved log viewing so that it updates in real time --- src/api/api_document.py | 17 +++---- src/api/api_task.py | 20 ++++++++ src/app.py | 3 ++ src/data/document.py | 47 +++++++++-------- src/static/app.js | 11 +++- src/static/log-stream.js | 34 +++++++++++++ src/static/style.css | 23 +++++++++ src/templates/admin/command_output.html | 11 ++-- src/templates/admin/document/manage.html | 2 +- src/templates/admin/index.html | 4 +- src/web/admin/admin_document.py | 11 +++- src/web_utils/task.py | 64 ++++++++++++++++++++++++ 12 files changed, 205 insertions(+), 42 deletions(-) create mode 100644 src/api/api_task.py create mode 100644 src/static/log-stream.js create mode 100644 src/web_utils/task.py diff --git a/src/api/api_document.py b/src/api/api_document.py index 81d8fc3..54dac1c 100644 --- a/src/api/api_document.py +++ b/src/api/api_document.py @@ -1,7 +1,5 @@ from flask import Blueprint, request -from markupsafe import escape from web_utils.get_arg import get_arg -from web_utils.run import run import os from data.document import Document @@ -18,14 +16,10 @@ def build(): if doc.get_api_key() != apikey: raise Exception("Invalid API key") - output = "" - output += "\n# Pulling source...\n" - output += doc.pull() + build_task = doc.build() + build_task.join() - output += "\n# Compiling...\n" - output += doc.build() - - return output.replace('\n', '
') + return build_task.get_output_str().replace('\n', '
') @bp.route('/clone') def clone(): @@ -34,6 +28,7 @@ def clone(): branch = get_arg('branch', 'master') source_dir = get_arg('source', 'source') - output = Document.clone(repo, branch, doc_name, source_dir) + clone_task = Document.clone(repo, branch, doc_name, source_dir) + clone_task.join() - return output.replace('\n', '
') + return clone_task.get_output_str().replace('\n', '
') diff --git a/src/api/api_task.py b/src/api/api_task.py new file mode 100644 index 0000000..816f072 --- /dev/null +++ b/src/api/api_task.py @@ -0,0 +1,20 @@ +import time +from flask import Blueprint, request +from web_utils.get_arg import get_arg +from web_utils.task import Task + +bp = Blueprint('api_task', __name__, url_prefix='/api/task') + +@bp.route('/log') +def log(): + task_id = get_arg('task_id') + from_char = int(get_arg('from', '0')) + + task = Task.get(task_id) + if task == None: + return { "task_finished": True, "new_output": "" } + + while task.is_alive() and len(task.get_output_str()) <= from_char: + time.sleep(100) + + return { "task_finished": not task.is_alive(), "new_output": task.get_output_str()[from_char:-1] } diff --git a/src/app.py b/src/app.py index 5570603..be23dc9 100644 --- a/src/app.py +++ b/src/app.py @@ -36,6 +36,9 @@ def create_app(): from api import api_document app.register_blueprint(api_document.bp) + from api import api_task + app.register_blueprint(api_task.bp) + from web import web_document app.register_blueprint(web_document.bp) diff --git a/src/data/document.py b/src/data/document.py index b844ea9..87aa456 100644 --- a/src/data/document.py +++ b/src/data/document.py @@ -1,7 +1,7 @@ import os import uuid from flask import current_app -from web_utils.run import run +from web_utils.task import ProcessTask import shutil from unicodedata import normalize import string @@ -45,11 +45,15 @@ class Document: def build(self): #venv_path = os.getenv('VIRTUAL_ENV') - cmd = "sphinx-build -M html \""+self.doc_path + "/repo/source\" \""+self.doc_path+"/build\"" - return run(cmd) - - def pull(self): - return run("cd \"" + self.doc_path + "/repo\" && git pull") + + cmd = [] + cmd.append(['git', 'pull']) + cmd.append(['sphinx-build', '-M', 'html', self.doc_path + "/repo/source", self.doc_path + "/build"]) + + task = ProcessTask(cmd, cwd = self.doc_path + "/repo") + task.start() + + return task def delete(self): shutil.rmtree(self.doc_path) @@ -90,24 +94,23 @@ class Document: apikey = str(uuid.uuid4()) target_dir = doc_path + "/repo" + os.makedirs(target_dir, exist_ok = True) + with open(doc_path + "/apikey", "wb") as apikey_file: + apikey_file:write(apikey) + + cmd = [] + cmd.append(['git', 'init', '--initial-branch=' + branch]) + cmd.append(['git', 'remote', 'add', '-f', 'origin', repo]) + cmd.append(['git', 'sparse-checkout', 'init']) + cmd.append(['git', 'sparse-checkout', 'set', source_dir]) + cmd.append(['git', 'pull', 'origin', branch]) + cmd.append(['git', 'branch', '--set-upstream-to=origin/' + branch, branch]) - cmd = "" - cmd += "mkdir -p \"" + target_dir + "\"\n" - cmd += "echo \""+apikey+"\" > \"" + doc_path + "/apikey\"\n" - cmd += "cd \"" + target_dir + "\"\n" - cmd += "git init \"--initial-branch=" + branch + "\"\n" - cmd += "git remote add -f origin \"" + repo + "\"\n" - cmd += "git sparse-checkout init\n" - cmd += "git sparse-checkout set \"" + source_dir + "\"\n" - cmd += "git pull origin \"" + branch + "\"\n" - cmd += "git branch \"--set-upstream-to=origin/" + branch + "\" \"" +branch + "\"" + task = ProcessTask(cmd, cwd = target_dir) + task.on_fail(lambda : shutil.rmtree(doc_path, ignore_errors = True)) + task.start() - try: - return run(cmd) - except Exception as e: - # cloning failed, clean up and raise the same exception again - shutil.rmtree(doc_path) - raise e + return task @staticmethod def list(): diff --git a/src/static/app.js b/src/static/app.js index d6aa92c..49a6744 100644 --- a/src/static/app.js +++ b/src/static/app.js @@ -1,4 +1,4 @@ -var confirm_elements = document.querySelectorAll('a[data-confirm]'); +var confirm_elements = document.querySelectorAll('[data-confirm]'); for (let elt of confirm_elements) { elt.addEventListener('click', (e) => { @@ -6,3 +6,12 @@ for (let elt of confirm_elements) e.preventDefault(); }, false); } + +var buttons = document.querySelectorAll('.button'); +for (let elt of buttons) +{ + elt.addEventListener('click', (e) => { + if(elt.classList.contains('disabled')) + e.preventDefault(); + }, false); +} diff --git a/src/static/log-stream.js b/src/static/log-stream.js new file mode 100644 index 0000000..50516fe --- /dev/null +++ b/src/static/log-stream.js @@ -0,0 +1,34 @@ +function log_stream(element, logUrl) { + element.innerText = ""; + + function fetchMore() { + fetch(logUrl + "&from=" + element.innerText.length) + .then(response => response.text()) + .then(jsonStr => { + let json = JSON.parse(jsonStr); + + element.innerText += json.new_output; + + const scrollOffset = 60; + const bodyTop = document.body.getBoundingClientRect().top; + const viewportHeight = window.innerHeight; + const elementBottom = element.getBoundingClientRect().bottom; + + window.scrollTo({ + top: elementBottom - viewportHeight - bodyTop + scrollOffset + }); + + if(json.task_finished) { + let validationButtonId = element.getAttribute('data-validation-button-id'); + if(validationButtonId) { + document.getElementById(validationButtonId).classList.remove('disabled', false); + } + } + else { + setTimeout(fetchMore, 100); + } + }); + } + + fetchMore(); +} diff --git a/src/static/style.css b/src/static/style.css index e69de29..2d1acf8 100644 --- a/src/static/style.css +++ b/src/static/style.css @@ -0,0 +1,23 @@ +.button { + display: inline-block; + border: 1px solid gray; + border-radius: 4px; + padding: 3px 10px; + margin: 2px; + color: black; +} + +a.button { + text-decoration: none; +} + +.button.danger { + color: darkred; + border-color: darkred; +} + +.button.disabled { + color: lightgray; + border-color: lightgray; + cursor: default; +} diff --git a/src/templates/admin/command_output.html b/src/templates/admin/command_output.html index 7f7d776..3848e27 100644 --- a/src/templates/admin/command_output.html +++ b/src/templates/admin/command_output.html @@ -1,8 +1,13 @@ -{% extends 'base.html' %} +{% extends 'base.html' %} {% block title %}Exécution...{% endblock %} {% block content %} - 
{{ output }}
- OK + 
Chargement...
+ OK + + + {% endblock %} diff --git a/src/templates/admin/document/manage.html b/src/templates/admin/document/manage.html index 483caf1..7323fc5 100644 --- a/src/templates/admin/document/manage.html +++ b/src/templates/admin/document/manage.html @@ -6,7 +6,7 @@

Gestion de {{ doc.doc_name }} / {{ doc.branch }}

URL permettant de déclencher la compilation : {{ url_for('api_document.build', doc = doc.doc_name, branch = doc.branch, apikey = doc.get_api_key(), _external = True) }}

Consulter
- Compiler
+ Compiler

Supprimer {% endblock %} diff --git a/src/templates/admin/index.html b/src/templates/admin/index.html index 585a591..40e8b63 100644 --- a/src/templates/admin/index.html +++ b/src/templates/admin/index.html @@ -4,13 +4,13 @@ {% block content %}

Administration des documents

- Nouveau document... + Nouveau document...

Liste des documents

{% if documents|length > 0 %}