Improved log viewing so that it updates in real time

src/api/api_document.py

@@ -1,7 +1,5 @@
 from flask import Blueprint, request
-from markupsafe import escape
 from web_utils.get_arg import get_arg
-from web_utils.run import run
 import os
 
 from data.document import Document
@@ -18,14 +16,10 @@ def build():
 	if doc.get_api_key() != apikey:
 		raise Exception("Invalid API key")
 	
-	output = ""
+	build_task = doc.build()
-	output += "\n# Pulling source...\n"
+	build_task.join()
-	output += doc.pull()
 	
-	output += "\n# Compiling...\n"
+	return build_task.get_output_str().replace('\n', '<br/>')
-	output += doc.build()
-	
-	return output.replace('\n', '<br/>')
 
 @bp.route('/clone')
 def clone():
@@ -34,6 +28,7 @@ def clone():
 	branch = get_arg('branch', 'master')
 	source_dir = get_arg('source', 'source')
 	
-	output = Document.clone(repo, branch, doc_name, source_dir)
+	clone_task = Document.clone(repo, branch, doc_name, source_dir)
+	clone_task.join()
 	
-	return output.replace('\n', '<br/>')
+	return clone_task.get_output_str().replace('\n', '<br/>')

src/api/api_task.py

@@ -0,0 +1,20 @@
+import time
+from flask import Blueprint, request
+from web_utils.get_arg import get_arg
+from web_utils.task import Task
+
+bp = Blueprint('api_task', __name__, url_prefix='/api/task')
+
+@bp.route('/log')
+def log():
+	task_id = get_arg('task_id')
+	from_char = int(get_arg('from', '0'))
+	
+	task = Task.get(task_id)
+	if task == None:
+		return { "task_finished": True, "new_output": "" }
+	
+	while task.is_alive() and len(task.get_output_str()) <= from_char:
+		time.sleep(100)
+	
+	return  { "task_finished": not task.is_alive(), "new_output": task.get_output_str()[from_char:-1] }

src/app.py

@@ -36,6 +36,9 @@ def create_app():
 	from api import api_document
 	app.register_blueprint(api_document.bp)
 	
+	from api import api_task
+	app.register_blueprint(api_task.bp)
+	
 	from web import web_document
 	app.register_blueprint(web_document.bp)
 	

src/data/document.py

@@ -1,7 +1,7 @@
 import os
 import uuid
 from flask import current_app
-from web_utils.run import run
+from web_utils.task import ProcessTask
 import shutil
 from unicodedata import normalize
 import string
@@ -45,11 +45,15 @@ class Document:
 	
 	def build(self):
 		#venv_path = os.getenv('VIRTUAL_ENV')
-		cmd = "sphinx-build -M html \""+self.doc_path + "/repo/source\" \""+self.doc_path+"/build\""
+		
-		return run(cmd)
+		cmd = []
-	
+		cmd.append(['git', 'pull'])
-	def pull(self):
+		cmd.append(['sphinx-build', '-M', 'html', self.doc_path + "/repo/source", self.doc_path + "/build"])
-		return run("cd \"" + self.doc_path + "/repo\" && git pull")
+		
+		task = ProcessTask(cmd, cwd = self.doc_path + "/repo")
+		task.start()
+		
+		return task
 	
 	def delete(self):
 		shutil.rmtree(self.doc_path)
@@ -90,24 +94,23 @@ class Document:
 		apikey = str(uuid.uuid4())
 		
 		target_dir = doc_path + "/repo"
+		os.makedirs(target_dir, exist_ok = True)
+		with open(doc_path + "/apikey", "wb") as apikey_file:
+			apikey_file:write(apikey)
+		
+		cmd = []
+		cmd.append(['git', 'init', '--initial-branch=' + branch])
+		cmd.append(['git', 'remote', 'add', '-f', 'origin',  repo])
+		cmd.append(['git', 'sparse-checkout', 'init'])
+		cmd.append(['git', 'sparse-checkout', 'set', source_dir])
+		cmd.append(['git', 'pull', 'origin', branch])
+		cmd.append(['git', 'branch', '--set-upstream-to=origin/' + branch, branch])
 		
-		cmd = ""
+		task = ProcessTask(cmd, cwd = target_dir)
-		cmd += "mkdir -p \"" + target_dir + "\"\n"
+		task.on_fail(lambda : shutil.rmtree(doc_path, ignore_errors = True))
-		cmd += "echo \""+apikey+"\" > \"" + doc_path + "/apikey\"\n"
+		task.start()
-		cmd += "cd \"" + target_dir + "\"\n"
-		cmd += "git init \"--initial-branch=" + branch + "\"\n"
-		cmd += "git remote add -f origin \"" + repo + "\"\n"
-		cmd += "git sparse-checkout init\n"
-		cmd += "git sparse-checkout set \"" + source_dir + "\"\n"
-		cmd += "git pull origin \"" + branch + "\"\n"
-		cmd += "git branch \"--set-upstream-to=origin/" + branch + "\" \"" +branch + "\""
 		
-		try:
+		return task
-			return run(cmd)
-		except Exception as e:
-			# cloning failed, clean up and raise the same exception again
-			shutil.rmtree(doc_path)
-			raise e
 	
 	@staticmethod
 	def list():

src/static/app.js

@@ -1,4 +1,4 @@
-var confirm_elements = document.querySelectorAll('a[data-confirm]');
+var confirm_elements = document.querySelectorAll('[data-confirm]');
 for (let elt of confirm_elements)
 {
     elt.addEventListener('click', (e) => {
@@ -6,3 +6,12 @@ for (let elt of confirm_elements)
 			e.preventDefault();
 	}, false);
 }
+
+var buttons = document.querySelectorAll('.button');
+for (let elt of buttons)
+{
+    elt.addEventListener('click', (e) => {
+		if(elt.classList.contains('disabled'))
+			e.preventDefault();
+	}, false);
+}

src/static/log-stream.js

@@ -0,0 +1,34 @@
+function log_stream(element, logUrl) {
+	element.innerText = "";
+	
+	function fetchMore() {
+		fetch(logUrl + "&from=" + element.innerText.length)
+			.then(response => response.text())
+			.then(jsonStr => {
+				let json = JSON.parse(jsonStr);
+			
+				element.innerText += json.new_output;
+			
+				const scrollOffset = 60;
+				const bodyTop = document.body.getBoundingClientRect().top;
+				const viewportHeight = window.innerHeight;
+				const elementBottom = element.getBoundingClientRect().bottom;
+
+				window.scrollTo({
+				  top: elementBottom - viewportHeight - bodyTop + scrollOffset
+				});
+			
+				if(json.task_finished) {
+					let validationButtonId = element.getAttribute('data-validation-button-id');
+					if(validationButtonId) {
+						document.getElementById(validationButtonId).classList.remove('disabled', false);
+					}
+				}
+				else {
+					setTimeout(fetchMore, 100);
+				}
+			});
+	}
+	
+	fetchMore();
+}

src/static/style.css

@@ -0,0 +1,23 @@
+.button {
+	display: inline-block;
+	border: 1px solid gray;
+	border-radius: 4px;
+	padding: 3px 10px;
+	margin: 2px;
+	color: black;
+}
+
+a.button {
+	text-decoration: none;
+}
+
+.button.danger {
+	color: darkred;
+	border-color: darkred;
+}
+
+.button.disabled {
+	color: lightgray;
+	border-color: lightgray;
+	cursor: default;
+}

src/templates/admin/command_output.html

@@ -1,8 +1,13 @@
-{% extends 'base.html' %}
+{% extends 'base.html' %}
 
 {% block title %}Exécution...{% endblock %}
 
 {% block content %}
-	<pre>{{ output }}</pre>
+	<pre id="log" data-validation-button-id="next_button">Chargement...</pre>
-	<a href="{{ next }}" class="button">OK</a>
+	<a id="next_button" href="{{ next }}" class="button disabled">OK</a>
+
+	<script type="text/javascript" src="{{ url_for('static', filename='log-stream.js') }}"></script>
+	<script type="text/javascript">
+		log_stream(document.getElementById('log'), '{{ url_for('api_task.log', task_id=task.task_id) }}');
+	</script>
 {% endblock %}

src/templates/admin/document/manage.html

@@ -6,7 +6,7 @@
 	<h1>Gestion de {{ doc.doc_name }} / {{ doc.branch }}</h1>
 	<p>URL permettant de déclencher la compilation : {{ url_for('api_document.build', doc = doc.doc_name, branch = doc.branch, apikey = doc.get_api_key(), _external = True) }}</p>
 	<a href="{{doc.get_url()}}" class="button">Consulter</a><br/>
-	<a href="{{ url_for('api_document.build', doc = doc.doc_name, branch = doc.branch, apikey = doc.get_api_key()) }}" class="button">Compiler</a><br/>
+	<a href="{{ url_for('admin_document.build', doc_name = doc.doc_name, branch = doc.branch) }}" class="button">Compiler</a><br/>
 	<br/>
 	<a href="{{ url_for('admin_document.delete', doc_name = doc.doc_name, branch = doc.branch) }}" class="button danger" data-confirm="Êtes-vous sûr de vouloir supprimer le document {{ doc.doc_name }} / {{ doc.branch }} ?">Supprimer</a>
 {% endblock %}

src/templates/admin/index.html

@@ -4,13 +4,13 @@
 
 {% block content %}
 	<h1>Administration des documents</h1>
-	<a href="{{ url_for('admin_document.new') }}">Nouveau document...</a>
+	<a href="{{ url_for('admin_document.new') }}" class="button">Nouveau document...</a>
 	<h2>Liste des documents</h2>
 	{% if documents|length > 0 %}
 		<ul>
 			{% for doc in documents %}
 				{% if doc.valid %}
-					<li>{{ doc.doc_name }} / {{ doc.branch }} <a href="{{ doc.get_url() }}">Consulter</a> <a href="{{ url_for('admin_document.manage', doc_name = doc.doc_name, branch = doc.branch) }}" class="button">Gérer</a></li>
+					<li>{{ doc.doc_name }} / {{ doc.branch }} <a href="{{ doc.get_url() }}" class="button">Consulter</a> <a href="{{ url_for('admin_document.manage', doc_name = doc.doc_name, branch = doc.branch) }}" class="button">Gérer</a></li>
 				{% else %}
 					<li>{{ doc.doc_name }} / {{ doc.branch }} (document invalide) <a href="{{ url_for('admin_document.delete_invalid', doc_name = doc.doc_name, branch = doc.branch) }}" class="confirm danger">Supprimer le dossier</a></li>
 				{% endif %}

src/web/admin/admin_document.py

@@ -18,9 +18,9 @@ def new():
 		if doc_name == "":
 			doc_name = os.path.splitext(os.path.basename(repo))[0]
 		
-		output = Document.clone(repo, branch, doc_name, source_dir)
+		clone_task = Document.clone(repo, branch, doc_name, source_dir)
 		
-		return render_template("admin/command_output.html", output = output, next = url_for('admin_document.manage', doc_name = doc_name, branch = branch))
+		return render_template("admin/command_output.html", task = clone_task, next = url_for('admin_document.manage', doc_name = doc_name, branch = branch))
 	else:
 		return render_template("admin/document/new.html")
 
@@ -28,6 +28,13 @@ def new():
 def manage(doc_name, branch):
 	return render_template("admin/document/manage.html", doc=Document(doc_name, branch))
 
+@bp.route('/build/<doc_name>/<branch>')
+def build(doc_name, branch):
+	doc = Document(doc_name, branch)
+	build_task = doc.build()
+	
+	return render_template("admin/command_output.html", task = build_task, next = url_for('admin_document.manage', doc_name = doc_name, branch = branch))
+
 @bp.route('/delete/<doc_name>/<branch>')
 def delete(doc_name, branch):
 	doc = Document(doc_name, branch)

src/web_utils/task.py

@@ -0,0 +1,64 @@
+from threading import Thread, Lock
+from io import StringIO
+from subprocess import Popen, PIPE, STDOUT, DEVNULL
+import uuid
+
+tasks = {}
+
+class Task(Thread):
+	def __init__(self):
+		Thread.__init__(self)
+		self.task_id = uuid.uuid4()
+		self.__mutex = Lock()
+		self.__output = StringIO()
+		tasks[str(self.task_id)] = self
+		
+	def print(self, str):
+		self.__mutex.acquire()
+		self.__output.write(str)
+		self.__mutex.release()
+		
+	def get_output_str(self):
+		self.__mutex.acquire()
+		result = self.__output.getvalue()
+		self.__mutex.release()
+		return result
+	
+	@staticmethod
+	def get(task_id):
+		return tasks[task_id]
+	
+class ProcessTask(Task):
+	def __init__(self, commands, cwd = None, hide_passwords = []):
+		Task.__init__(self)
+		self.__commands = commands
+		self.__cwd = cwd
+		self.__hide_passwords = hide_passwords
+		
+	def on_fail(self, callback):
+		self.__fail_callback = callback
+	
+	def run(self):
+		is_first_command = True
+		for command in self.__commands:
+			command_display = ' '.join(command)
+			for password in self.__hide_passwords:
+				command_display = command_display.replace(password, '******')
+				
+			if not is_first_command:
+				self.print('\n')
+			is_first_command = False
+			self.print('> ' + command_display + '\n')
+			
+			# We use setsid to make a non-interactive session, otherwise any command that expects an input (confirmation, password, etc.) would hang forever
+			self.__process = Popen(['setsid'] + command, cwd = self.__cwd, stdin=DEVNULL, stdout = PIPE, stderr = STDOUT, shell = False)
+
+			for line in self.__process.stdout:
+				self.print(line.decode())
+
+			self.__process.wait()
+
+			if self.__process.returncode != 0:
+				if self.__fail_callback:
+					self.__fail_callback()
+				raise Exception("Command failed ("+str(self.__process.returncode)+")\n"+self.get_output_str())