Commit 2e5d80aa authored by Daniele Venzano's avatar Daniele Venzano

Rationalize state classes

Refactor the state classes to differentiate between State classes, coming from SqlAlchemy, "normal" classes that are moved areoung and passed to client applications and statistic classes that are generated when required, on the fly.
parent e165b498
import os import os
import logging import logging
from common.state import Application, Execution from common.state import ApplicationState, ExecutionState
from common.configuration import zoeconf from common.configuration import zoeconf
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
...@@ -19,18 +19,18 @@ def init_history_paths() -> bool: ...@@ -19,18 +19,18 @@ def init_history_paths() -> bool:
return True return True
def application_data_upload(application: Application, data: bytes) -> bool: def application_data_upload(application: ApplicationState, data: bytes) -> bool:
fpath = os.path.join(zoeconf.history_path, 'apps', 'app-{}.zip'.format(application.id)) fpath = os.path.join(zoeconf.history_path, 'apps', 'app-{}.zip'.format(application.id))
open(fpath, "wb").write(data) open(fpath, "wb").write(data)
def application_data_download(application: Application) -> bytes: def application_data_download(application: ApplicationState) -> bytes:
fpath = os.path.join(zoeconf.history_path, 'apps', 'app-{}.zip'.format(application.id)) fpath = os.path.join(zoeconf.history_path, 'apps', 'app-{}.zip'.format(application.id))
data = open(fpath, "rb").read() data = open(fpath, "rb").read()
return data return data
def application_data_delete(application: Application): def application_data_delete(application: ApplicationState):
fpath = os.path.join(zoeconf.history_path, 'apps', 'app-{}.zip'.format(application.id)) fpath = os.path.join(zoeconf.history_path, 'apps', 'app-{}.zip'.format(application.id))
try: try:
os.unlink(fpath) os.unlink(fpath)
...@@ -38,18 +38,18 @@ def application_data_delete(application: Application): ...@@ -38,18 +38,18 @@ def application_data_delete(application: Application):
log.warning("Binary data for application {} not found, cannot delete".format(application.id)) log.warning("Binary data for application {} not found, cannot delete".format(application.id))
def logs_archive_upload(execution: Execution, data: bytes) -> bool: def logs_archive_upload(execution: ExecutionState, data: bytes) -> bool:
fpath = os.path.join(zoeconf.history_path, 'logs', 'log-{}.zip'.format(execution.id)) fpath = os.path.join(zoeconf.history_path, 'logs', 'log-{}.zip'.format(execution.id))
open(fpath, "wb").write(data) open(fpath, "wb").write(data)
def logs_archive_download(execution: Execution) -> bytes: def logs_archive_download(execution: ExecutionState) -> bytes:
fpath = os.path.join(zoeconf.history_path, 'logs', 'log-{}.zip'.format(execution.id)) fpath = os.path.join(zoeconf.history_path, 'logs', 'log-{}.zip'.format(execution.id))
data = open(fpath, "rb").read() data = open(fpath, "rb").read()
return data return data
def logs_archive_delete(execution: Execution): def logs_archive_delete(execution: ExecutionState):
fpath = os.path.join(zoeconf.history_path, 'logs', 'log-{}.zip'.format(execution.id)) fpath = os.path.join(zoeconf.history_path, 'logs', 'log-{}.zip'.format(execution.id))
try: try:
os.unlink(fpath) os.unlink(fpath)
......
...@@ -9,12 +9,12 @@ Base = declarative_base() ...@@ -9,12 +9,12 @@ Base = declarative_base()
_engine = create_engine(zoeconf.db_url, echo=False) _engine = create_engine(zoeconf.db_url, echo=False)
AlchemySession = sessionmaker(bind=_engine) AlchemySession = sessionmaker(bind=_engine)
from common.state.container import Container from common.state.container import ContainerState
from common.state.cluster import Cluster from common.state.cluster import ClusterState
from common.state.application import Application, SparkApplication, SparkNotebookApplication, SparkSubmitApplication from common.state.application import ApplicationState, SparkApplicationState, SparkNotebookApplicationState, SparkSubmitApplicationState
from common.state.user import User from common.state.user import UserState
from common.state.proxy import Proxy from common.state.proxy import ProxyState
from common.state.execution import Execution, SparkSubmitExecution from common.state.execution import ExecutionState, SparkSubmitExecutionState
def create_tables(): def create_tables():
......
...@@ -4,7 +4,7 @@ from sqlalchemy.orm import relationship ...@@ -4,7 +4,7 @@ from sqlalchemy.orm import relationship
from common.state import Base from common.state import Base
class Application(Base): class ApplicationState(Base):
__tablename__ = 'applications' __tablename__ = 'applications'
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
...@@ -12,7 +12,7 @@ class Application(Base): ...@@ -12,7 +12,7 @@ class Application(Base):
required_resources = Column(PickleType()) # JSON resource description required_resources = Column(PickleType()) # JSON resource description
user_id = Column(Integer, ForeignKey('users.id')) user_id = Column(Integer, ForeignKey('users.id'))
executions = relationship("Execution", order_by="Execution.id", backref="application") executions = relationship("ExecutionState", order_by="ExecutionState.id", backref="application")
type = Column(String(20)) # Needed by sqlalchemy to manage class inheritance type = Column(String(20)) # Needed by sqlalchemy to manage class inheritance
...@@ -21,19 +21,6 @@ class Application(Base): ...@@ -21,19 +21,6 @@ class Application(Base):
'polymorphic_identity': 'application' 'polymorphic_identity': 'application'
} }
def to_dict(self) -> dict:
ret = {
'id': self.id,
'name': self.name,
'required_resources': self.required_resources.__dict__.copy(),
'user_id': self.user_id
}
return ret
def __repr__(self):
return "<Application(name='%s', id='%s', required_resourced='%s')>" % (
self.name, self.id, self.required_resources)
def executions_running(self): def executions_running(self):
ret = [] ret = []
for e in self.executions: for e in self.executions:
...@@ -42,17 +29,10 @@ class Application(Base): ...@@ -42,17 +29,10 @@ class Application(Base):
return ret return ret
def extract(self): def extract(self):
ret = PlainApplication() return Application(self)
ret.id = self.id
ret.name = self.name
ret.required_resources = self.required_resources
ret.user_id = self.user_id
ret.executions = [x.id for x in self.executions]
ret.type = self.type
return ret
class SparkApplication(Application): class SparkApplicationState(ApplicationState):
master_image = Column(String(256)) master_image = Column(String(256))
worker_image = Column(String(256)) worker_image = Column(String(256))
...@@ -60,63 +40,67 @@ class SparkApplication(Application): ...@@ -60,63 +40,67 @@ class SparkApplication(Application):
'polymorphic_identity': 'spark-application' 'polymorphic_identity': 'spark-application'
} }
def to_dict(self) -> dict:
ret = super().to_dict()
ret["master_image"] = self.master_image
ret["worker_image"] = self.worker_image
return ret
def extract(self): def extract(self):
ret = super().extract() return Application(self)
ret.master_image = self.master_image
ret.worker_image = self.worker_image
return ret
class SparkNotebookApplication(SparkApplication): class SparkNotebookApplicationState(SparkApplicationState):
notebook_image = Column(String(256)) notebook_image = Column(String(256))
__mapper_args__ = { __mapper_args__ = {
'polymorphic_identity': 'spark-notebook' 'polymorphic_identity': 'spark-notebook'
} }
def to_dict(self) -> dict:
ret = super().to_dict()
ret["notebook_image"] = self.notebook_image
return ret
def extract(self): def extract(self):
ret = super().extract() return Application(self)
ret.notebook_image = self.notebook_image
return ret
class SparkSubmitApplication(SparkApplication): class SparkSubmitApplicationState(SparkApplicationState):
submit_image = Column(String(256)) submit_image = Column(String(256))
__mapper_args__ = { __mapper_args__ = {
'polymorphic_identity': 'spark-submit' 'polymorphic_identity': 'spark-submit'
} }
def to_dict(self) -> dict:
ret = super().to_dict()
ret["submit_image"] = self.submit_image
return ret
def extract(self): def extract(self):
ret = super().extract() return Application(self)
ret.submit_image = self.submit_image
return ret
class Application:
"""
class PlainApplication: :type id: int
id = None :type name: str
name = None :type required_resources: ApplicationResources
required_resources = None :type user_id: int
user_id = None :type type: str
executions = None :type master_image: str
type = None :type worker_image: str
master_image = None :type notebook_image: str
worker_image = None :type submit_image: str
notebook_image = None :type executions: list[Execution]
submit_image = None """
def __init__(self, application: ApplicationState) -> None:
self.id = application.id
self.name = application.name
self.required_resources = application.required_resources
self.user_id = application.user_id
self.type = application.type
if isinstance(application, SparkApplicationState):
self.master_image = application.master_image
self.worker_image = application.worker_image
if isinstance(application, SparkNotebookApplicationState):
self.notebook_image = application.notebook_image
if isinstance(application, SparkSubmitApplicationState):
self.submit_image = application.submit_image
self.executions = []
for e in application.executions:
self.executions.append(e.extract())
def __str__(self):
s = "Application"
s += " - Name: {}".format(self.name)
s += " - Type: {}".format(self.type)
# FIXME add missing fields
return s
...@@ -4,15 +4,11 @@ from sqlalchemy.orm import relationship ...@@ -4,15 +4,11 @@ from sqlalchemy.orm import relationship
from common.state import Base from common.state import Base
class Cluster(Base): class ClusterState(Base):
__tablename__ = 'clusters' __tablename__ = 'clusters'
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
execution_id = Column(Integer, ForeignKey('executions.id')) execution_id = Column(Integer, ForeignKey('executions.id'))
containers = relationship("Container", order_by="Container.id", backref="cluster") containers = relationship("ContainerState", order_by="ContainerState.id", backref="cluster")
proxies = relationship("Proxy", order_by="Proxy.id", backref="cluster") proxies = relationship("ProxyState", order_by="ProxyState.id", backref="cluster")
def __repr__(self):
return "<Cluster(id='%s', execution_id='%s')>" % (
self.id, self.execution_id)
...@@ -4,7 +4,7 @@ from sqlalchemy.orm import relationship ...@@ -4,7 +4,7 @@ from sqlalchemy.orm import relationship
from common.state import Base from common.state import Base
class Container(Base): class ContainerState(Base):
__tablename__ = 'containers' __tablename__ = 'containers'
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
...@@ -13,8 +13,25 @@ class Container(Base): ...@@ -13,8 +13,25 @@ class Container(Base):
ip_address = Column(String(16)) ip_address = Column(String(16))
readable_name = Column(String(32)) readable_name = Column(String(32))
proxies = relationship("Proxy", order_by="Proxy.id", backref="container") proxies = relationship("ProxyState", order_by="ProxyState.id", backref="container")
def __repr__(self): def extract(self):
return "<Container(name='%s', id='%s', docker_id='%s', cluster_id='%s', ip_address='%s')>" % ( """
self.readable_name, self.id, self.docker_id, self.cluster_id, self.ip_address) Generates a normal object not attached to SQLAlchemy
:rtype : Container
"""
return Container(self)
class Container:
def __init__(self, container: ContainerState):
self.id = container.id
self.docker_id = container.docker_id
self.cluster_id = container.cluster_id
self.ip_address = container.ip_address
self.readable_name = container.readable_name
self.proxies = []
for p in container.proxies:
self.proxies.append(p.extract())
...@@ -6,7 +6,7 @@ from sqlalchemy.orm import relationship ...@@ -6,7 +6,7 @@ from sqlalchemy.orm import relationship
from common.state import Base from common.state import Base
class Execution(Base): class ExecutionState(Base):
__tablename__ = 'executions' __tablename__ = 'executions'
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
...@@ -19,7 +19,7 @@ class Execution(Base): ...@@ -19,7 +19,7 @@ class Execution(Base):
status = Column(String(32)) status = Column(String(32))
termination_notice = Column(Boolean, default=False) termination_notice = Column(Boolean, default=False)
cluster = relationship("Cluster", uselist=False, backref="execution") cluster = relationship("ClusterState", uselist=False, backref="execution")
type = Column(String(32)) # Needed by sqlalchemy to manage class inheritance type = Column(String(32)) # Needed by sqlalchemy to manage class inheritance
...@@ -49,28 +49,11 @@ class Execution(Base): ...@@ -49,28 +49,11 @@ class Execution(Base):
if c.readable_name == name: if c.readable_name == name:
return c return c
def __repr__(self):
return "<Execution(name='%s', id='%s', assigned_resourced='%s', application_id='%s', )>" % (
self.name, self.id, self.assigned_resources, self.application_id)
def extract(self): def extract(self):
ret = PlainExecution() return Execution(self)
ret.id = self.id
ret.name = self.name
ret.assigned_resources = self.assigned_resources class SparkSubmitExecutionState(ExecutionState):
ret.application_id = self.application_id
ret.time_started = self.time_started
ret.time_scheduled = self.time_scheduled
ret.time_finished = self.time_finished
ret.status = self.status
ret.termination_notice = self.termination_notice
if self.cluster is not None:
ret.cluster_id = self.cluster.id
ret.type = self.type
return ret
class SparkSubmitExecution(Execution):
commandline = Column(String(1024)) commandline = Column(String(1024))
spark_opts = Column(String(1024)) spark_opts = Column(String(1024))
...@@ -79,23 +62,32 @@ class SparkSubmitExecution(Execution): ...@@ -79,23 +62,32 @@ class SparkSubmitExecution(Execution):
} }
def extract(self): def extract(self):
ret = super().extract() return Execution(self)
ret.commandline = self.commandline
ret.spark_opts = self.spark_opts
return ret class Execution:
def __init__(self, execution: ExecutionState):
self.id = execution.id
class PlainExecution: self.name = execution.name
id = None self.assigned_resources = execution.assigned_resources
name = None self.application_id = execution.application_id
assigned_resources = None self.time_started = execution.time_started
application_id = None self.time_scheduled = execution.time_scheduled
time_started = None self.time_finished = execution.time_finished
time_scheduled = None self.status = execution.status
time_finished = None self.termination_notice = execution.termination_notice
status = None if execution.cluster is not None:
termination_notice = None self.cluster_id = execution.cluster.id
cluster_id = None else:
type = None self.cluster_id = None
commandline = None self.type = execution.type
spark_opts = None
if isinstance(execution, SparkSubmitExecutionState):
self.commandline = execution.commandline
self.spark_opts = execution.spark_opts
self.containers = []
if execution.cluster is not None:
for c in execution.cluster.containers:
self.containers.append(c.extract())
...@@ -3,7 +3,7 @@ from sqlalchemy import Column, Integer, String, ForeignKey, DateTime, func ...@@ -3,7 +3,7 @@ from sqlalchemy import Column, Integer, String, ForeignKey, DateTime, func
from common.state import Base from common.state import Base
class Proxy(Base): class ProxyState(Base):
__tablename__ = 'proxies' __tablename__ = 'proxies'
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
...@@ -13,6 +13,15 @@ class Proxy(Base): ...@@ -13,6 +13,15 @@ class Proxy(Base):
service_name = Column(String(32)) service_name = Column(String(32))
last_access = Column(DateTime, default=func.now()) last_access = Column(DateTime, default=func.now())
def __repr__(self): def extract(self):
return "<Proxy(service_name='%s', id='%s', internal_url='%s', cluster_id='%s', container_id='%s', last_access='%s')>" % ( return Proxy(self)
self.service_name, self.id, self.internal_url, self.cluster_id, self.container_id, self.last_access)
class Proxy:
def __init__(self, proxy: ProxyState):
self.id = proxy.id
self.internal_url = proxy.internal_url
self.cluster_id = proxy.cluster_id
self.container_id = proxy.container_id
self.service_name = proxy.service_name
self.last_access = proxy.last_access
...@@ -4,26 +4,19 @@ from sqlalchemy.orm import relationship ...@@ -4,26 +4,19 @@ from sqlalchemy.orm import relationship
from common.state import Base from common.state import Base
class User(Base): class UserState(Base):
__tablename__ = 'users' __tablename__ = 'users'
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
email = Column(String(128)) email = Column(String(128))
applications = relationship("Application", order_by="Application.id", backref="user") applications = relationship("ApplicationState", order_by="ApplicationState.id", backref="user")
def __repr__(self):
return "<User(id='%s', email='%s')>" % (
self.id, self.app_id)
def extract(self): def extract(self):
ret = PlainUser() return User(self)
ret.id = self.id
ret.email = self.email
return ret
class PlainUser:
id = None
email = None
class User:
def __init__(self, user: UserState):
self.id = user.id
self.email = user.email
import time
from common.state import ApplicationState, ExecutionState, ContainerState, ProxyState
class Stats:
def __init__(self):
self.timestamp = None
class SwarmNodeStats(Stats):
def __init__(self, name):
super().__init__()
self.name = name
self.docker_endpoint = None
self.container_count = 0
self.cores_total = 0
self.cores_reserved = 0
self.memory_total = 0
self.memory_reserved = 0
self.labels = {}
def __str__(self):
s = " -- Node {}\n".format(self.name)
s += " -- Docker endpoint: {}\n".format(self.docker_endpoint)
s += " -- Container count: {}\n".format(self.container_count)
s += " -- Memory total: {}\n".format(self.memory_total)
s += " -- Memory reserved: {}\n".format(self.memory_reserved)
s += " -- Cores total: {}\n".format(self.cores_total)
s += " -- Cores reserved: {}\n".format(self.cores_reserved)
s += " -- Labels: {}\n".format(self.labels)
return s
class SwarmStats(Stats):
def __init__(self):
super().__init__()
self.container_count = 0
self.image_count = 0
self.memory_total = 0
self.cores_total = 0
self.placement_strategy = ''
self.active_filters = []
self.nodes = []
def __str__(self):
s = " - Container count: {}\n".format(self.container_count)
s += " - Image count: {}\n".format(self.image_count)
s += " - Memory total: {}\n".format(self.memory_total)
s += " - Cores total: {}\n".format(self.cores_total)
s += " - Placement strategy: {}\n".format(self.placement_strategy)
s += " - Active filters: {}\n".format(self.active_filters)
for node in self.nodes:
s += str(node)
return s
class SchedulerStats(Stats):
def __init__(self):
super().__init__()
self.count_running = 0
self.count_waiting = 0
def __str__(self):
return " - Apps running: {}\n - Apps waiting: {}\n".format(self.count_running, self.count_waiting)
class PlatformStats(Stats):
def __init__(self):
super().__init__()
self.swarm = SwarmStats()
self.scheduler = SchedulerStats()
def __str__(self):
return "Swarm:\n{}\nScheduler:\n{}\n".format(self.swarm, self.scheduler)
from pprint import pformat
from zoe_scheduler.swarm_status import SwarmStatus
from common.state import Application, Execution, SparkSubmitExecution
class Report:
def __init__(self):
self.report = {}
def __str__(self):
return pformat(self.report)
class PlatformStatusReport(Report):
def include_swarm_status(self, sw_status: SwarmStatus):
self.report["swarm"] = sw_status.to_dict()
class ApplicationStatusReport(Report):
def __init__(self, application: Application):
super().__init__()
self.report["executions"] = []
self._app_to_dict(application)
def _app_to_dict(self, application: Application):
self.report.update(application.to_dict())
def add_execution(self, execution: Execution):
exrep = {
'id': execution.id,
'name': execution.name,
'status': execution.status,
'type': execution.type
}
if execution.time_scheduled is None:
exrep['scheduled_at'] = None
else:
exrep['scheduled_at'] = execution.time_scheduled.timestamp()
if execution.time_started is None:
exrep['started_at'] = None
else:
exrep['started_at'] = execution.time_started.timestamp()
if execution.time_finished is None:
exrep['finished_at'] = None
else:
exrep['finished_at'] = execution.time_finished.timestamp()
if isinstance(execution, SparkSubmitExecution):
exrep["commandline"] = execution.commandline
exrep["spark_opts"] = execution.spark_opts
exrep["cluster"] = []
if execution.cluster is None:
self.report['executions'].append(exrep)
return
for c in execution.cluster.containers:
cd = {
'id': c.id,
'docker_id': c.docker_id,
'ip_address': c.ip_address,
'name': c.readable_name,
'proxies': []
}
for p in c.proxies: