Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Simone Rossi
main
Commits
434956e4
Commit
434956e4
authored
Sep 01, 2015
by
Daniele Venzano
Browse files
The web interface now starts correctly
Implement missing status information parsing from Docker
parent
b4c70be0
Changes
13
Hide whitespace changes
Inline
Side-by-side
common/status.py
View file @
434956e4
...
...
@@ -6,9 +6,7 @@ from common.state import Application, Execution, SparkSubmitExecution
class
Report
:
def
__init__
(
self
):
self
.
report
=
{
'executions'
:
[]
}
self
.
report
=
{}
def
__str__
(
self
):
return
pformat
(
self
.
report
)
...
...
@@ -16,14 +14,13 @@ class Report:
class
PlatformStatusReport
(
Report
):
def
include_swarm_status
(
self
,
sw_status
:
SwarmStatus
):
self
.
report
[
"swarm"
]
=
{
'total cores'
:
sw_status
.
cores_total
}
self
.
report
[
"swarm"
]
=
sw_status
.
to_dict
()
class
ApplicationStatusReport
(
Report
):
def
__init__
(
self
,
application
:
Application
):
super
().
__init__
()
self
.
report
[
"executions"
]
=
[]
self
.
_app_to_dict
(
application
)
def
_app_to_dict
(
self
,
application
:
Application
):
...
...
zoe_scheduler/platform.py
View file @
434956e4
...
...
@@ -17,6 +17,7 @@ from common.object_storage import logs_archive_upload
class
PlatformManager
:
def
__init__
(
self
):
self
.
swarm
=
SwarmClient
()
pm
.
update_proxy
()
def
start_execution
(
self
,
execution_id
:
int
,
resources
:
ApplicationResources
)
->
bool
:
state
=
AlchemySession
()
...
...
zoe_scheduler/rpyc_server.py
View file @
434956e4
...
...
@@ -43,7 +43,9 @@ class RPyCAsyncIOServer:
self
.
auto_register
=
bool
(
registrar
)
else
:
self
.
auto_register
=
auto_register
self
.
protocol_config
=
protocol_config
self
.
protocol_config
=
{
"allow_public_attrs"
:
True
}
if
protocol_config
is
not
None
:
self
.
protocol_config
.
update
(
protocol_config
)
self
.
hostname
=
hostname
self
.
port
=
port
...
...
zoe_scheduler/swarm_client.py
View file @
434956e4
...
...
@@ -8,7 +8,7 @@ import docker.errors
from
common.configuration
import
conf
from
zoe_scheduler.swarm_status
import
SwarmStatus
from
zoe_scheduler.swarm_status
import
SwarmStatus
,
SwarmNodeStatus
class
SwarmClient
:
...
...
@@ -30,8 +30,23 @@ class SwarmClient:
pl_status
.
placement_strategy
=
info
[
"DriverStatus"
][
idx
][
1
]
idx
=
2
assert
'Filters'
in
info
[
"DriverStatus"
][
idx
][
0
]
pl_status
.
active_filters
=
info
[
"DriverStatus"
][
idx
][
1
].
split
(
", "
)
pl_status
.
active_filters
=
[
x
.
strip
()
for
x
in
info
[
"DriverStatus"
][
idx
][
1
].
split
(
", "
)]
idx
=
3
assert
'Nodes'
in
info
[
"DriverStatus"
][
idx
][
0
]
node_count
=
int
(
info
[
"DriverStatus"
][
idx
][
1
])
idx
=
4
for
node
in
range
(
node_count
):
ns
=
SwarmNodeStatus
(
info
[
"DriverStatus"
][
idx
+
node
][
0
])
ns
.
docker_endpoint
=
info
[
"DriverStatus"
][
idx
+
node
][
1
]
ns
.
container_count
=
int
(
info
[
"DriverStatus"
][
idx
+
node
+
1
][
1
])
ns
.
cores_reserved
=
int
(
info
[
"DriverStatus"
][
idx
+
node
+
2
][
1
].
split
(
' / '
)[
0
])
ns
.
cores_total
=
int
(
info
[
"DriverStatus"
][
idx
+
node
+
2
][
1
].
split
(
' / '
)[
1
])
ns
.
memory_reserved
=
info
[
"DriverStatus"
][
idx
+
node
+
3
][
1
].
split
(
' / '
)[
0
]
ns
.
memory_total
=
info
[
"DriverStatus"
][
idx
+
node
+
3
][
1
].
split
(
' / '
)[
1
]
ns
.
labels
=
ns
.
cores_reserved
=
info
[
"DriverStatus"
][
idx
+
node
+
4
][
1
:]
pl_status
.
nodes
.
append
(
ns
)
idx
+=
4
pl_status
.
timestamp
=
time
.
time
()
return
pl_status
...
...
zoe_scheduler/swarm_status.py
View file @
434956e4
...
...
@@ -12,6 +12,18 @@ class SwarmNodeStatus:
self
.
memory_reserved
=
0
self
.
labels
=
{}
def
to_dict
(
self
):
return
{
'name'
:
self
.
name
,
'docker_endpoint'
:
self
.
docker_endpoint
,
'container_count'
:
self
.
container_count
,
'cores_total'
:
self
.
cores_total
,
'cores_reserved'
:
self
.
cores_reserved
,
'memory_total'
:
self
.
memory_total
,
'memory_reserved'
:
self
.
memory_reserved
,
'labels'
:
self
.
labels
.
copy
()
}
class
SwarmStatus
:
def
__init__
(
self
):
...
...
@@ -23,3 +35,16 @@ class SwarmStatus:
self
.
active_filters
=
[]
self
.
nodes
=
[]
self
.
timestamp
=
time
.
time
()
def
to_dict
(
self
):
ret
=
{
'container_count'
:
self
.
container_count
,
'image_count'
:
self
.
image_count
,
'memory_total'
:
self
.
memory_total
,
'cores_total'
:
self
.
cores_total
,
'placement_strategy'
:
self
.
placement_strategy
,
'active_filters'
:
self
.
active_filters
.
copy
(),
'timestamp'
:
self
.
timestamp
,
'nodes'
:
[
x
.
to_dict
()
for
x
in
self
.
nodes
]
}
return
ret
zoe_web/__init__.py
View file @
434956e4
from
flask
import
Flask
from
flask
import
Flask
,
url_for
,
abort
from
zoe_web.api
import
api_bp
from
zoe_web.web
import
web_bp
app
=
Flask
(
__name__
)
app
.
register_blueprint
(
web_bp
,
url_prefix
=
'
/web
'
)
app
.
register_blueprint
(
api_bp
,
url_prefix
=
'/
web/
api'
)
app
=
Flask
(
__name__
,
static_url_path
=
'/does-not-exist'
)
app
.
register_blueprint
(
web_bp
,
url_prefix
=
''
)
app
.
register_blueprint
(
api_bp
,
url_prefix
=
'/api'
)
app
.
secret_key
=
b
"
\xc3\xb0\xa7\xff\x8f
H'
\xf7
m
\x1c\xa2\x92
F
\x1d\xdc
z
\x05\xe6
CJN5
\x83
!"
def
debug_list_routes
():
output
=
[]
for
rule
in
app
.
url_map
.
iter_rules
():
options
=
{}
for
arg
in
rule
.
arguments
:
options
[
arg
]
=
"[{0}]"
.
format
(
arg
)
methods
=
','
.
join
(
rule
.
methods
)
url
=
url_for
(
rule
.
endpoint
,
**
options
)
line
=
"{:50s} {:20s} {}"
.
format
(
rule
.
endpoint
,
methods
,
url
)
output
.
append
(
line
)
for
line
in
sorted
(
output
):
print
(
line
)
@
app
.
errorhandler
(
404
)
def
page_not_found
(
_
):
debug_list_routes
()
zoe_web/api/__init__.py
View file @
434956e4
from
flask
import
Blueprint
,
abort
from
flask
import
Blueprint
,
abort
,
jsonify
api_bp
=
Blueprint
(
'api'
,
__name__
)
from
zoe_client
import
get_zoe_client
@
api_bp
.
route
(
'/status/basic'
)
def
basic_status
():
abort
(
404
)
def
status_basic
():
client
=
get_zoe_client
()
platform_report
=
client
.
platform_status
()
ret
=
{
'num_nodes'
:
len
(
platform_report
.
report
[
"swarm"
][
"nodes"
]),
'num_containers'
:
platform_report
.
report
[
"swarm"
][
"container_count"
]
}
return
jsonify
(
**
ret
)
zoe_web/
web/
utils.py
→
zoe_web/utils.py
View file @
434956e4
from
flask
import
session
,
redirect
,
url_for
from
zoe_client
import
ZoeClient
from
zoe_web.web
import
web_bp
from
common.configuration
import
conf
def
get_zoe_client
():
if
conf
[
'client_rpyc_autodiscovery'
]:
return
ZoeClient
()
else
:
return
ZoeClient
(
conf
[
'client_rpyc_server'
],
conf
[
'client_rpyc_port'
])
def
check_user
(
zoeclient
:
ZoeClient
):
if
'user_id'
not
in
session
:
return
redirect
(
url_for
(
web_bp
.
index
))
return
redirect
(
url_for
(
'
web_bp.index
'
))
if
not
zoeclient
.
user_check
(
session
[
'user_id'
]):
return
redirect
(
url_for
(
'index'
))
return
redirect
(
url_for
(
'
web_bp.
index'
))
zoe_web/web/__init__.py
View file @
434956e4
from
flask
import
Blueprint
web_bp
=
Blueprint
(
'web'
,
__name__
,
template_folder
=
'templates'
,
static_folder
=
'static'
)
import
zoe_web.web.start
import
zoe_web.web.status
zoe_web/web/start.py
View file @
434956e4
...
...
@@ -2,7 +2,7 @@ from flask import render_template
from
zoe_client
import
get_zoe_client
from
zoe_web.web
import
web_bp
import
zoe_web.
web.
utils
as
web_utils
import
zoe_web.utils
as
web_utils
@
web_bp
.
route
(
'/'
)
...
...
zoe_web/web/status.py
0 → 100644
View file @
434956e4
from
flask
import
render_template
from
zoe_client
import
get_zoe_client
from
zoe_web.web
import
web_bp
import
zoe_web.utils
as
web_utils
@
web_bp
.
route
(
'/status/platform'
)
def
status_platform
():
client
=
get_zoe_client
()
user
=
web_utils
.
check_user
(
client
)
template_vars
=
{
"user_id"
:
user
.
id
,
"email"
:
user
.
email
}
return
render_template
(
'status.html'
,
**
template_vars
)
zoe_web/web/templates/base.html
View file @
434956e4
...
...
@@ -11,7 +11,7 @@
<body>
<script>
function
update_status
()
{
$
.
getJSON
(
"
{{ url_for(
"
api
_
status
"
) }}
"
)
$
.
getJSON
(
"
{{ url_for(
"
api
.
status
_basic
"
) }}
"
)
.
done
(
function
(
data
)
{
$
(
"
#num_nodes
"
).
text
(
data
.
num_nodes
);
$
(
"
#num_containers
"
).
text
(
data
.
num_containers
);
...
...
@@ -29,7 +29,7 @@
{% block footer %}
<div
class=
"status_line"
>
<p>
<span
id=
"app_name"
><a
href=
"{{ url_for("
web
_
status
")
}}"
>
Zoe
</a></span>
<span
id=
"app_name"
><a
href=
"{{ url_for("
web
.
status
_platform
")
}}"
>
Zoe
</a></span>
<span
id=
"status"
>
[
<span
id=
"num_nodes"
>
N/A
</span>
swarm nodes,
<span
id=
"num_containers"
>
N/A
</span>
active containers]
</span>
...
...
zoectl.py
View file @
434956e4
...
...
@@ -30,7 +30,7 @@ def user_new_cmd(args):
def
user_get_cmd
(
args
):
client
=
get_zoe_client
()
user
=
client
.
user_get
(
args
.
email
)
print
(
"User ID: {}"
.
format
(
user
.
email
))
print
(
"User ID: {}"
.
format
(
user
.
id
))
def
spark_cluster_new_cmd
(
args
):
...
...
@@ -216,7 +216,4 @@ def main():
args
.
func
(
args
)
if
__name__
==
"__main__"
:
try
:
main
()
except
AttributeError
:
argparser
.
print_help
()
main
()
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment