Commit dea70812 authored by Daniele Venzano's avatar Daniele Venzano

Implement a different way of retrieving service logs, with better performance and reliability

parent f6a64ca4
......@@ -23,7 +23,7 @@ Because of this in Zoe we decided to leave the maximum freedom to administrators
In this case the logs command line, API and web interface will not be operational.
Docker engine integrated log management
---------------------------------------
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
When using the Docker Engine back-end Zoe can configure the containers to produce the output in UDP GELF format and send them to a configured destination, via the ``gelf-address`` option. Each messages is enriched with labels to help matching each log line to the ZApp and service that produced it.
......@@ -36,3 +36,8 @@ Additionally the Zoe master can itself be configured to act as a log collector.
In this case the logs command line, API and web interface will work normally.
Please note that the GELF listener implemented in the Zoe Master process is not built to manage high loads of incoming log messages. If the incoming rate is too high, UDP packets (and hence log lines) may be dropped and lost.
There are two ways to show the logs on the web interface:
1. An implementation of log streaming via web sockets
2. the directory tree created above must be exported via HTTP by an external web server that supports the Range header. This choice is due to performance and reliability. The option ``log-url`` must contain the base url where the directory tree is exposed.
......@@ -156,6 +156,8 @@ class ServiceLogsWeb(ZoeWebRequestHandler):
template_vars = {
"service": service,
"websocket_base": get_conf().websocket_base + get_conf().reverse_proxy_path
"log_path": "{}/{}/{}/{}.txt".format(get_conf().log_url, get_conf().deployment_name, service.execution_id, service.name),
"websocket_base": get_conf().websocket_base + get_conf().reverse_proxy_path,
'use_websockets': get_conf().log_use_websockets
}
self.render('service_logs.jinja2', **template_vars)
/* Copyright (c) 2012: Daniel Richman. License: GNU GPL 3 */
/* Additional features: Priyesh Patel */
function logtail(url) {
var dataelem = "#logoutput";
var pausetoggle = "#pause";
var scrollelems = ["#logoutput"];
var fix_rn = true;
var load = 30 * 1024; /* 30KB */
var poll = 1000; /* 1s */
var kill = false;
var loading = false;
var pause = false;
var reverse = false;
var log_data = "";
var log_file_size = 0;
/* :-( https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/parseInt */
function parseInt2(value) {
if(!(/^[0-9]+$/.test(value))) throw "Invalid integer " + value;
var v = Number(value);
if (isNaN(v)) throw "Invalid integer " + value;
return v;
}
function get_log() {
if (kill || loading) return;
loading = true;
var range;
var first_load;
var must_get_206;
if (log_file_size === 0) {
/* Get the last 'load' bytes */
range = "-" + load.toString();
first_load = true;
must_get_206 = false;
} else {
/* Get the (log_file_size - 1)th byte, onwards. */
range = (log_file_size - 1).toString() + "-";
first_load = false;
must_get_206 = log_file_size > 1;
}
/* The "log_file_size - 1" deliberately reloads the last byte, which we already
* have. This is to prevent a 416 "Range unsatisfiable" error: a response
* of length 1 tells us that the file hasn't changed yet. A 416 shows that
* the file has been truncated */
$.ajax(url, {
dataType: "text",
cache: false,
headers: {Range: "bytes=" + range},
success: function (data, s, xhr) {
loading = false;
var content_size;
if (xhr.status === 206) {
var c_r = xhr.getResponseHeader("Content-Range");
if (!c_r)
throw "Server did not respond with a Content-Range";
log_file_size = parseInt2(c_r.split("/")[1]);
content_size = parseInt2(xhr.getResponseHeader("Content-Length"));
} else if (xhr.status === 200) {
if (must_get_206)
throw "Expected 206 Partial Content";
content_size = log_file_size =
parseInt2(xhr.getResponseHeader("Content-Length"));
} else {
throw "Unexpected status " + xhr.status;
}
if (first_load && data.length > load)
throw "Server's response was too long";
var added = false;
if (first_load) {
/* Clip leading part-line if not the whole file */
if (content_size < log_file_size) {
var start = data.indexOf("\n");
log_data = data.substring(start + 1);
} else {
log_data = data;
}
added = true;
} else {
/* Drop the first byte (see above) */
log_data += data.substring(1);
if (log_data.length > load) {
var start = log_data.indexOf("\n", log_data.length - load);
log_data = log_data.substring(start + 1);
}
if (data.length > 1)
added = true;
}
if (added)
show_log(added);
setTimeout(get_log, poll);
},
error: function (xhr, s, t) {
loading = false;
if (xhr.status === 416 || xhr.status == 404) {
/* 416: Requested range not satisfiable: log was truncated. */
/* 404: Retry soon, I guess */
log_file_size = 0;
log_data = "";
show_log();
setTimeout(get_log, poll);
} else {
throw "Unknown AJAX Error (status " + xhr.status + ")";
}
}
});
}
function scroll(where) {
for (var i = 0; i < scrollelems.length; i++) {
var s = $(scrollelems[i]);
if (where === -1)
s.scrollTop(s[0].scrollHeight);
else
s.scrollTop(where);
}
}
function show_log() {
if (pause) return;
var t = log_data;
if (reverse) {
var t_a = t.split(/\n/g);
t_a.reverse();
if (t_a[0] == "")
t_a.shift();
t = t_a.join("\n");
}
if (fix_rn)
t = t.replace(/\n/g, "\r\n");
$(dataelem).text(t);
if (!reverse)
scroll(-1);
}
function error(what) {
kill = true;
$(dataelem).text("An error occured :-(.\r\n" +
"Reloading may help; no promises.\r\n" +
what);
scroll(0);
return false;
}
$(document).ready(function () {
window.onerror = error;
/* If URL is /logtail/?noreverse display in chronological order */
var hash = location.search.replace(/^\?/, "");
if (hash == "noreverse")
reverse = false;
/* Add pause toggle */
$(pausetoggle).click(function (e) {
pause = !pause;
$(pausetoggle).text(pause ? "Unpause" : "Pause");
show_log();
e.preventDefault();
});
get_log();
});
}
{% extends "base_user.jinja2" %}
{% block title %}Service {{ service.name }} logs{% endblock %}
{% block custom_head %}
<script type="text/javascript" src="{{ static_url("logtail.js") }}"></script>
{% endblock %}
{% block content %}
<h1>Zoe - Analytics on demand</h1>
......@@ -9,10 +12,16 @@
<textarea class="logoutput" id="logoutput" readonly>
</textarea>
<a id="pause" href='#'>Pause</a>
{% if not use_websockets %}
<p>Please note: only the last 30kB of the log are shown when the page is loaded.</p>
{% endif %}
<p><a href="{{ reverse_url("execution_inspect", service.execution_id) }}">Back to execution details</a></p>
<script type="application/javascript">
{% if use_websockets %}
var ws = new WebSocket('{{ websocket_base }}/websocket');
ws.onopen = function (e) {
ws.send(JSON.stringify({
......@@ -20,13 +29,14 @@ ws.onopen = function (e) {
service_id: {{ service.id }}
}));
};
var log_element = $('#logoutput');
ws.onmessage = function (evt) {
log_element.append(evt.data);
log_element.scrollTop(log_element[0].scrollHeight);
};
{% else %}
logtail("{{ log_path }}");
{% endif %}
</script>
{% endblock %}
......@@ -74,6 +74,8 @@ def load_configuration(test_conf=None):
argparser.add_argument('--gelf-address', help='Enable Docker GELF log output to this destination (ex. udp://1.2.3.4:7896)', default='')
argparser.add_argument('--gelf-listener', type=int, help='Enable the internal GELF log listener on this port, set to 0 to disable', default='7896')
argparser.add_argument('--service-logs-base-path', help='Path where service logs coming from the GELF listener will be stored', default='/var/lib/zoe/service-logs')
argparser.add_argument('--log-url', help='URL where log files are available via HTTP as /deployment-name/execution-id/service-name.txt', default='https://cloud-platform.eurecom.fr/zoe-logs/')
argparser.add_argument('--log-use-websockets', help='Use websockets or standard ajax with an external web server for serving service logs', action="store_true")
# API options
argparser.add_argument('--listen-address', help='Address to listen to for incoming connections', default="0.0.0.0")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment