Commit 97f2dfd0 authored by Daniele Venzano's avatar Daniele Venzano

Several fixes

Use the termiantion_notice flag to prevent sending multiple emails
Automatically create directories inside the history path
Do not show the Open link in the home page if there is nothing to open
parent 03363c15
......@@ -7,6 +7,18 @@ from common.configuration import zoeconf
log = logging.getLogger(__name__)
def init_history_paths() -> bool:
if not os.path.exists(zoeconf.history_path):
try:
os.makedirs(zoeconf.history_path)
except OSError:
log.error("Cannot create history directory in {}".format(zoeconf.history_path))
return False
os.makedirs(os.path.join(zoeconf.history_path, 'apps'))
os.makedirs(os.path.join(zoeconf.history_path, 'logs'))
return True
def application_data_upload(application: Application, data: bytes) -> bool:
fpath = os.path.join(zoeconf.history_path, 'apps', 'app-{}.zip'.format(application.id))
open(fpath, "wb").write(data)
......
......@@ -8,6 +8,7 @@ from rpyc.utils.server import ThreadedServer
from zoe_scheduler.rpyc_service import ZoeSchedulerRPCService
from zoe_scheduler.scheduler import zoe_sched
from zoe_scheduler.periodic_tasks import PeriodicTaskManager
from common.object_storage import init_history_paths
log = logging.getLogger('zoe')
loop = None
......@@ -45,9 +46,11 @@ def main():
rpyc_logger = logging.getLogger('rpyc')
rpyc_logger.setLevel(logging.WARNING)
if not init_history_paths():
return
tm = PeriodicTaskManager()
# rpyc_server = RPyCAsyncIOServer(ZoeSchedulerRPCService, '0.0.0.0', port=4000, auto_register=True)
rpyc_server = ThreadedServer(ZoeSchedulerRPCService, '0.0.0.0', port=4000,
auto_register=not args.rpyc_no_auto_register,
protocol_config={"allow_public_attrs": True},
......
......@@ -214,9 +214,10 @@ class PlatformManager:
self.execution_terminate(state, e)
notify_notebook_termination(e)
if datetime.now() - pr.last_access > timedelta(hours=zoeconf.notebook_max_age_no_activity) - timedelta(hours=zoeconf.notebook_warning_age_no_activity):
log.info("Spark notebook {} is on notice for inactivity".format(e.id))
e.termination_notice = True
notify_notebook_notice(e)
if not e.termination_notice:
log.info("Spark notebook {} is on notice for inactivity".format(e.id))
e.termination_notice = True
notify_notebook_notice(e)
state.commit()
......
......@@ -64,11 +64,12 @@
<td>{{ e[1]['scheduled_at']|format_timestamp }}</td>
{% if e[1]['started_at'] == 'None' %}
<td>not yet</td>
<td></td>
{% else %}
<td>{{ e[1]['started_at']|format_timestamp }}</td>
<td><a href="{{ e[2] }}">Open</a></td>
{% endif %}
<td>{{ e[1]['cluster']|count }}</td>
<td><a href="{{ e[2] }}">Open</a></td>
<td><a href="{{ url_for('web.execution_terminate', exec_id=e[1]['id']) }}">Terminate</a></td>
</tr>
{% endfor %}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment