Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Simone Rossi
main
Commits
dd9226c8
Commit
dd9226c8
authored
Aug 12, 2015
by
Daniele Venzano
Browse files
Cleanups and fixes
parent
c177a3fc
Changes
10
Expand all
Hide whitespace changes
Inline
Side-by-side
.idea/workspace.xml
View file @
dd9226c8
This diff is collapsed.
Click to expand it.
caaas/api.py
View file @
dd9226c8
...
...
@@ -18,30 +18,6 @@ def api_status():
return
jsonify
(
**
data
)
@
app
.
route
(
"/api/full-status"
)
def
api_full_status
():
db
=
CAaaState
()
data
=
{}
user_list
=
db
.
get_all_users
()
for
user_id
,
username
in
user_list
:
data
[
username
]
=
{}
data
[
username
][
"num_clusters"
]
=
db
.
count_clusters
(
user_id
)
data
[
username
][
"num_containers"
]
=
db
.
count_containers
(
user_id
)
data
[
username
][
"has_notebook"
]
=
db
.
has_notebook
(
user_id
)
return
jsonify
(
**
data
)
@
app
.
route
(
"/api/<username>/status"
)
def
api_user_status
(
username
):
db
=
CAaaState
()
user_id
=
db
.
get_user_id
(
username
)
cluster_list
=
db
.
get_clusters
(
user_id
)
for
clid
in
cluster_list
:
cluster_list
[
clid
][
"is_notebook"
]
=
cluster_list
[
clid
][
"name"
]
==
"notebook"
cluster_list
[
clid
][
"num_containers"
]
=
db
.
count_containers
(
user_id
,
clid
)
return
jsonify
(
**
cluster_list
)
@
app
.
route
(
"/api/<username>/cluster/<cluster_id>/terminate"
)
def
api_terminate_cluster
(
username
,
cluster_id
):
db
=
CAaaState
()
...
...
caaas/sql.py
View file @
dd9226c8
...
...
@@ -352,6 +352,20 @@ class CAaaState:
cursor
.
execute
(
q
,
(
app_id
,))
self
.
_close_cursor
(
cursor
)
def
get_applications
(
self
,
user_id
=
None
)
->
dict
:
cursor
=
self
.
_get_cursor
(
dictionary
=
True
)
if
user_id
is
None
:
q
=
"SELECT * FROM applications"
cursor
.
execute
(
q
)
else
:
q
=
"SELECT * FROM applications WHERE user_id=%s"
cursor
.
execute
(
q
,
(
user_id
,))
res
=
[]
for
row
in
cursor
:
res
.
append
((
dict
(
row
)))
self
.
_close_cursor
(
cursor
)
return
res
def
get_application
(
self
,
app_id
:
int
)
->
dict
:
cursor
=
self
.
_get_cursor
(
dictionary
=
True
)
q
=
"SELECT * FROM applications WHERE id=%s"
...
...
caaas/swarm_manager.py
View file @
dd9226c8
...
...
@@ -28,6 +28,7 @@ class ContainerOptions:
self
.
volume_binds
=
[]
self
.
volumes
=
[]
self
.
command
=
""
self
.
memory_limit
=
'2g'
def
add_env_variable
(
self
,
name
,
value
):
self
.
env
[
name
]
=
value
...
...
@@ -51,6 +52,12 @@ class ContainerOptions:
def
get_command
(
self
):
return
self
.
command
def
set_memory_limit
(
self
,
limit
):
self
.
memory_limit
=
limit
def
get_memory_limit
(
self
):
return
self
.
memory_limit
class
SwarmManager
:
def
__init__
(
self
):
...
...
@@ -127,6 +134,7 @@ class SwarmManager:
options
.
add_env_variable
(
"SPARK_MASTER_IP"
,
master_info
[
"docker_ip"
])
options
.
add_env_variable
(
"SPARK_WORKER_RAM"
,
cluster_descr
.
executor_ram_size
)
options
.
add_env_variable
(
"SPARK_WORKER_CORES"
,
cluster_descr
.
worker_cores
)
options
.
set_memory_limit
(
cluster_descr
.
executor_ram_size
)
info
=
self
.
_spawn_container
(
WORKER_IMAGE
,
options
)
cont_id
=
db
.
new_container
(
cluster_id
,
user_id
,
info
[
"docker_id"
],
info
[
"docker_ip"
],
"spark-worker-"
+
str
(
count
))
db
.
new_proxy_entry
(
get_uuid
(),
cluster_id
,
"http://"
+
info
[
"docker_ip"
]
+
":8081"
,
"web interface"
,
cont_id
)
...
...
@@ -139,6 +147,7 @@ class SwarmManager:
options
.
add_env_variable
(
"SPARK_MASTER_IP"
,
master_info
[
"docker_ip"
])
options
.
add_env_variable
(
"SPARK_EXECUTOR_RAM"
,
cluster_descr
.
executor_ram_size
)
options
.
add_env_variable
(
"PROXY_ID"
,
proxy_id
)
options
.
set_memory_limit
(
cluster_descr
.
executor_ram_size
)
info
=
self
.
_spawn_container
(
NOTEBOOK_IMAGE
,
options
)
cont_id
=
db
.
new_container
(
cluster_id
,
user_id
,
info
[
"docker_id"
],
info
[
"docker_ip"
],
"spark-notebook"
)
db
.
new_proxy_entry
(
proxy_id
,
cluster_id
,
"http://"
+
info
[
"docker_ip"
]
+
":9000/proxy/"
+
proxy_id
,
"notebook"
,
cont_id
)
...
...
@@ -155,13 +164,16 @@ class SwarmManager:
options
.
add_env_variable
(
"SPARK_OPTIONS"
,
app
[
"spark_options"
])
options
.
add_volume_bind
(
app
[
"path"
],
'/app'
,
True
)
options
.
set_command
(
"/opt/submit.sh /app "
+
app
[
"cmd"
])
options
.
set_memory_limit
(
cluster_descr
.
executor_ram_size
)
info
=
self
.
_spawn_container
(
SUBMIT_IMAGE
,
options
)
cont_id
=
state
.
new_container
(
cluster_id
,
user_id
,
info
[
"docker_id"
],
info
[
"docker_ip"
],
"spark-submit"
)
state
.
new_proxy_entry
(
get_uuid
(),
cluster_id
,
"http://"
+
info
[
"docker_ip"
]
+
":4040"
,
"spark application"
,
cont_id
)
return
info
def
_spawn_container
(
self
,
image
,
options
):
host_config
=
create_host_config
(
network_mode
=
"bridge"
,
binds
=
options
.
get_volume_binds
())
host_config
=
create_host_config
(
network_mode
=
"bridge"
,
binds
=
options
.
get_volume_binds
(),
mem_limit
=
options
.
get_memory_limit
())
cont
=
self
.
cli
.
create_container
(
image
=
image
,
environment
=
options
.
get_environment
(),
network_disabled
=
False
,
...
...
@@ -218,3 +230,6 @@ class SwarmManager:
return
False
ret
=
self
.
cli
.
inspect_container
(
container
=
cont
[
"docker_id"
])
return
ret
[
"State"
][
"Running"
]
def
swarm_status
(
self
):
return
self
.
cli
.
info
()
caaas/templates/apps.html
0 → 100644
View file @
dd9226c8
{% extends "base_user.html" %}
{% block title %}{{ user }} status{% endblock %}
{% block content %}
<h2>
Status overview for user {{ user }}
</h2>
{% if apps|length == 0 and not has_notebook %}
<p>
You have no clusters running at this time
</p>
{% endif %}
{% if has_notebook %}
<p>
You have a notebook running:
<a
href=
"{{ notebook_address }}"
>
open
</a>
<a
href=
"{{ url_for("
web_terminate
",
username=
user,
cluster_id=
notebook_cluster_id)
}}"
>
terminate
</a>
</p>
{% endif %}
<table
id=
"app_list"
>
<thead>
<tr>
<th>
Name
</th>
<th>
Command line
</th>
<th>
Spark options
</th>
<th>
Started
</th>
<th>
Finished
</th>
<th>
Status
</th>
<th>
Logs
</th>
</tr>
</thead>
{% for a in apps %}
<tr>
<td>
{{ a["execution_name"] }}
</td>
<td>
{{ a["cmd"] }}
</td>
<td>
{{ a["spark_options"] }}
</td>
<td>
{{ a["time_started"] }}
</td>
<td>
{{ a["time_finished"] }}
</td>
<td>
{{ a["status"] }}
</td>
<td>
{% if a["status"] == "finished" %}
<a
href=
"{{ url_for("
api_history_log_archive
",
username=
user,
app_id=
a["id"])
}}"
>
Zip
</a>
{% endif %}
{% if a["status"] == "running" %}
<a
href=
"{{ url_for("
web_inspect
",
username=
user,
app_id=
a["cluster_id"])
}}"
>
Inspect
</a>
{% endif %}
</td>
<td>
{% if a["status"] == "running" %}
{{ url_for("web_terminate", username=user, cluster_id=a["cluster_id"]) }}
{% endif %}
</td>
</tr>
<!-- FIXME: add a re-run link -->
{% endfor %}
</table>
{% endblock %}
\ No newline at end of file
caaas/templates/base.html
View file @
dd9226c8
...
...
@@ -17,6 +17,9 @@
.
done
(
function
(
data
)
{
$
(
"
#num_nodes
"
).
text
(
data
.
num_nodes
);
$
(
"
#num_containers
"
).
text
(
data
.
num_containers
);
}).
error
(
function
(
data
)
{
$
(
"
#num_nodes
"
).
text
(
"
N/A
"
);
$
(
"
#num_containers
"
).
text
(
"
N/A
"
);
});
}
update_status
();
...
...
@@ -26,7 +29,7 @@
<div
id=
"content"
>
{% block content %}{% endblock %}
</div>
<div
id=
"footer"
>
{% block footer %}
<div
class=
"caaas_status"
><p><em>
CAaaS
</em>
status:
<span
id=
"num_nodes"
>
0
</span>
available nodes,
<span
id=
"num_containers"
>
0
</span>
containers running
</p></div>
<div
class=
"caaas_status"
><p><em>
CAaaS
</em>
status:
<span
id=
"num_nodes"
>
N/A
</span>
available nodes,
<span
id=
"num_containers"
>
N/A
</span>
containers running
</p></div>
<h6>
©
Copyright 2015 by
<a
href=
"http://distsysgroup.wordpress.com/"
>
DSG
</a></h6>
{% endblock %}
</div>
...
...
caaas/templates/home.html
View file @
dd9226c8
...
...
@@ -3,18 +3,18 @@
{% block content %}
<p>
Start an activity:
</p>
<ul>
<li><a
href=
"
/web/{{ user }}/spark-notebook
"
>
Give me a Spark notebook
</a></li>
<li><a
href=
"
/web/{{ user }}/submit-spark-app
"
>
Submit a Spark application
</a></li>
<li><a
href=
"
{{ url_for("
web_notebook
",
username=
user)
}}
"
>
Give me a Spark notebook
</a></li>
<li><a
href=
"
{{ url_for("
web_spark_submit
",
username=
user)
}}
"
>
Submit a Spark application
</a></li>
</ul>
<p>
M
onitor m
y activities:
</p>
<p>
My activities:
</p>
<ul>
<li><a
href=
"
/web/{{ user }}/status"
>
Running a
pplications
</a></li>
<li><a
href=
"
{{ url_for("
web_user_apps
",
username=
user)
}}"
>
A
pplications
</a></li>
</ul>
<p>
Cluster status:
</p>
<ul>
<li><a
href=
"
/
web
/
status"
>
Overview
</a></li>
<li><a
href=
"
{{ url_for("
web
_
status
"
)
}}"
>
Overview
</a></li>
</ul>
...
...
caaas/templates/status.html
View file @
dd9226c8
{% extends "base_user.html" %}
{% block title %}Status{% endblock %}
{% block content %}
<script
type=
"application/javascript"
>
function
gen_one_user
(
i
,
item
)
{
var
block
=
$
(
"
<div></div>
"
);
block
.
append
(
"
<p></p>
"
).
text
(
i
);
var
ul
=
$
(
"
<ul></ul>
"
);
ul
.
append
(
$
(
"
<li></li>
"
).
text
(
"
Num. clusters:
"
+
item
.
num_clusters
));
ul
.
append
(
$
(
"
<li></li>
"
).
text
(
"
Num. containers:
"
+
item
.
num_containers
));
ul
.
append
(
$
(
"
<li></li>
"
).
text
(
"
Has a notebook:
"
+
item
.
has_notebook
));
block
.
append
(
ul
);
return
block
;
}
(
function
()
{
$
.
getJSON
(
api_endpoint
+
"
/full-status
"
)
.
done
(
function
(
data
)
{
$
.
each
(
data
,
function
(
i
,
item
)
{
$
(
"
#userlist
"
).
append
(
gen_one_user
(
i
,
item
))
})
});
})();
</script>
<h2>
System status overview
</h2>
<ul>
<li>
Containers: {{ Containers }}
</li>
<li>
Images: {{ Images }}
</li>
<li>
MemTotal: {{ MemTotal }}
</li>
<li>
Memory limit: {{ MemoryLimit }}
</li>
<li>
NCPU: {{ NCPU }}
</li>
<li>
Driver status:
</li>
<ul>
{% for entry in DriverStatus %}
<li>
{{ entry[0] }}: {{ entry[1] }}
</li>
{% endfor %}
</ul>
</ul>
<div
id=
"userlist"
></div>
{% endblock %}
\ No newline at end of file
caaas/templates/user-status.html
deleted
100644 → 0
View file @
c177a3fc
{% extends "base_user.html" %}
{% block title %}{{ user }} status{% endblock %}
{% block content %}
<h2>
Status overview for user {{ user }}
</h2>
<div
id=
"cluster_list"
>
{% if clusters|length == 0 %}
<p>
You have no clusters running at this time
</p>
{% endif %}
<ul>
{% for c in clusters %}
<li>
{{ clusters[c]["name"] }} (
<a
href=
"/web/{{ user }}/cluster/{{ c }}/inspect"
>
inspect
</a>
,
<a
href=
"/web/{{ user }}/cluster/{{ c }}/terminate"
>
terminate
</a>
)
</li>
{% endfor %}
</ul>
</div>
{% endblock %}
\ No newline at end of file
caaas/web.py
View file @
dd9226c8
...
...
@@ -11,7 +11,8 @@ def index():
@
app
.
route
(
"/web/status"
)
def
web_status
():
return
render_template
(
'status.html'
)
status
=
sm
.
swarm_status
()
return
render_template
(
'status.html'
,
**
status
)
@
app
.
route
(
"/web/<username>"
)
...
...
@@ -24,19 +25,19 @@ def web_index(username):
return
render_template
(
'home.html'
,
**
template_vars
)
@
app
.
route
(
"/web/<username>/status"
)
def
web_user_status
(
username
):
db
=
CAaaState
()
user_id
=
db
.
get_user_id
(
username
)
cluster_list
=
db
.
get_clusters
(
user_id
)
for
clid
in
cluster_list
:
cluster_list
[
clid
][
"is_notebook"
]
=
cluster_list
[
clid
][
"name"
]
==
"notebook"
cluster_list
[
clid
][
"num_containers"
]
=
db
.
count_containers
(
user_id
,
clid
)
@
app
.
route
(
"/web/<username>/apps"
)
def
web_user_apps
(
username
):
state
=
CAaaState
()
user_id
=
state
.
get_user_id
(
username
)
apps
=
state
.
get_applications
(
user_id
)
template_vars
=
{
"user"
:
username
,
"clusters"
:
cluster_list
"apps"
:
apps
,
"has_notebook"
:
state
.
has_notebook
(
user_id
),
"notebook_address"
:
sm
.
get_notebook
(
user_id
),
"notebook_cluster_id"
:
state
.
get_notebook
(
user_id
)
}
return
render_template
(
'
user-statu
s.html'
,
**
template_vars
)
return
render_template
(
'
app
s.html'
,
**
template_vars
)
@
app
.
route
(
"/web/<username>/spark-notebook"
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment