Start to build the app server

.gitignore

+# Created by .ignore support plugin (hsz.mobi)
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+
+
+caaas.ini
\ No newline at end of file

.idea/.name

+caaas
\ No newline at end of file

.idea/caaas.iml

+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 2.7.9 (E:\User Software\WinPython-64bit-2.7.9.3\python-2.7.9.amd64\python.exe)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TemplatesService">
+    <option name="TEMPLATE_FOLDERS">
+      <list>
+        <option value="$MODULE_DIR$/images/templates" />
+      </list>
+    </option>
+  </component>
+</module>
\ No newline at end of file

.idea/dataSources.ids

+<?xml version="1.0" encoding="UTF-8"?>
+<component name="dataSourceStorage">
+  <data-source name="MySQL - @m1" uuid="a32fd6de-3ffa-40c0-9ec8-8953a89c53e0">
+    <database-info product="MySQL" version="5.5.44-MariaDB-1ubuntu0.14.04.1" jdbc-version="4.0" driver-name="MySQL Connector Java" driver-version="mysql-connector-java-5.1.35 ( Revision: 5fb9c5849535c13917c2cf9baaece6ef9693ef27 )">
+      <extra-name-characters>#@</extra-name-characters>
+      <identifier-quote-string>`</identifier-quote-string>
+    </database-info>
+    <case-sensitivity plain-identifiers="exact" quoted-identifiers="exact"/>
+    <schema name="" catalog="caaas"/>
+    <table name="clusters" schema="" catalog="caaas" type="TABLE">
+      <column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
+      <column name="user_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
+      <column name="master_address" sqlType="VARCHAR" precision="512" scale="0" nullable="false" jdbcType="12"/>
+      <column name="name" sqlType="VARCHAR" precision="256" scale="0" nullable="false" jdbcType="12"/>
+      <primary-key name="PRIMARY" columns="id"/>
+    </table>
+    <table name="containers" schema="" catalog="caaas" type="TABLE">
+      <column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
+      <column name="docker_id" sqlType="VARCHAR" precision="1024" scale="0" nullable="false" jdbcType="12"/>
+      <column name="cluster_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
+      <column name="user_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
+      <primary-key name="PRIMARY" columns="id"/>
+    </table>
+    <table name="notebooks" schema="" catalog="caaas" type="TABLE">
+      <column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
+      <column name="cluster_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
+      <column name="time_created" sqlType="TIMESTAMP" precision="19" scale="0" nullable="false" jdbcType="93" def="Q1VSUkVOVF9USU1FU1RBTVA="/>
+      <column name="address" sqlType="VARCHAR" precision="512" scale="0" nullable="false" jdbcType="12"/>
+      <column name="user_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
+      <primary-key name="PRIMARY" columns="id"/>
+    </table>
+    <table name="users" schema="" catalog="caaas" type="TABLE">
+      <column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
+      <column name="username" sqlType="VARCHAR" precision="32" scale="0" nullable="false" jdbcType="12"/>
+      <primary-key name="PRIMARY" columns="id"/>
+    </table>
+  </data-source>
+</component>
\ No newline at end of file

.idea/dataSources.local.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="dataSourceStorageLocal">
+    <data-source name="MySQL - @m1" uuid="a32fd6de-3ffa-40c0-9ec8-8953a89c53e0">
+      <secret-storage>master_key</secret-storage>
+      <user-name>caaas</user-name>
+      <schema-pattern>caaas.*</schema-pattern>
+      <default-schemas>caaas.*</default-schemas>
+    </data-source>
+  </component>
+</project>
\ No newline at end of file

.idea/dataSources.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="DataSourceManagerImpl" format="xml" hash="3541978474">
+    <data-source source="LOCAL" name="MySQL - @m1" uuid="a32fd6de-3ffa-40c0-9ec8-8953a89c53e0">
+      <driver-ref>mysql</driver-ref>
+      <synchronize>true</synchronize>
+      <jdbc-driver>com.mysql.jdbc.Driver</jdbc-driver>
+      <jdbc-url>jdbc:mysql://localhost:3306</jdbc-url>
+      <driver-properties>
+        <property name="zeroDateTimeBehavior" value="convertToNull" />
+        <property name="tinyInt1isBit" value="false" />
+        <property name="characterEncoding" value="utf8" />
+        <property name="characterSetResults" value="utf8" />
+        <property name="yearIsDateType" value="false" />
+      </driver-properties>
+      <libraries />
+    </data-source>
+  </component>
+</project>
\ No newline at end of file

.idea/misc.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectLevelVcsManager" settingsEditedManually="false">
+    <OptionsSetting value="true" id="Add" />
+    <OptionsSetting value="true" id="Remove" />
+    <OptionsSetting value="true" id="Checkout" />
+    <OptionsSetting value="true" id="Update" />
+    <OptionsSetting value="true" id="Status" />
+    <OptionsSetting value="true" id="Edit" />
+    <ConfirmationsSetting value="0" id="Add" />
+    <ConfirmationsSetting value="0" id="Remove" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7.9 (E:\User Software\WinPython-64bit-2.7.9.3\python-2.7.9.amd64\python.exe)" project-jdk-type="Python SDK" />
+</project>
\ No newline at end of file

.idea/modules.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/caaas.iml" filepath="$PROJECT_DIR$/.idea/caaas.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file

.idea/sqldialects.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="SqlDialectMappings">
+    <file url="file://$PROJECT_DIR$" dialect="MySQL" />
+  </component>
+</project>
\ No newline at end of file

.idea/vcs.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>
\ No newline at end of file

caaas.ini.template

+[docker]
+swarm-manager = 127.0.0.1:2380
+
+[db]
+user = caaas
+pass = changeme
+server = 127.0.0.1
+db = caaas
\ No newline at end of file

caaas/__init__.py

+from caaas.sql import init_db
+from caaas.swarm import swarm
+
+from flask import Flask
+app = Flask(__name__)
+
+import caaas.web
+import caaas.api

caaas/api.py

+from flask import jsonify
+
+from caaas import app
+from caaas.swarm import swarm
+
+
+@app.route("/api/status")
+def api_status():
+    template_variables = {
+        'num_containers': int(swarm.status.num_containers),
+        'num_nodes': int(swarm.status.num_nodes)
+    }
+    return jsonify(**template_variables)

caaas/sql.py

+import mysql.connector
+
+db = None
+
+
+class CAaaSSQL:
+    def __init__(self):
+        self.cnx = None
+
+    def connect(self, user, passw, server, dbname):
+        assert self.cnx is None
+        self.cnx = mysql.connector.connect(user=user, password=passw, host=server, database=dbname)
+
+    def _check_user(self, username):
+        cursor = self.cnx.cursor(dictionary=True)
+        q = "SELECT id FROM users WHERE username=%s"
+
+        cursor.execute(q, (username,))
+        if cursor.rowcount == 0:
+            cursor.close()
+            return self._create_user(username)
+        else:
+            row = cursor.fetchone()
+            cursor.close()
+            return row["id"]
+
+    def _create_user(self, username):
+        cursor = self.cnx.cursor()
+        q = "INSERT INTO users (username) VALUES (%s)"
+        cursor.execute(q, username)
+        user_id = cursor.lastrowid
+        self.cnx.commit()
+        cursor.close()
+        return user_id
+
+
+def init_db(user, passw, server, dbname):
+    global db
+    db = CAaaSSQL()
+    db.connect(user, passw, server, dbname)

caaas/swarm.py

+from docker import Client
+from threading import Thread
+import time
+
+
+class SwarmStatus:
+    def __init__(self):
+        self.num_nodes = 0
+        self.num_containers = 0
+
+
+class Swarm:
+    def __init__(self):
+        self.status = SwarmStatus()
+        self.cli = None
+        self.manager = None
+
+    def connect(self, swarm_manager):
+        self.manager = swarm_manager
+        self.cli = Client(base_url="tcp://" + self.manager)
+
+    def start_update_thread(self):
+        assert self.manager is not None
+        th = Thread(target=self._thread_cb)
+        th.start()
+
+    def _thread_cb(self):
+        print("Stats update thread started")
+        while True:
+            self.update_status()
+            time.sleep(1)
+
+    def update_status(self):
+        assert self.cli is not None
+        info = self.cli.info()
+        self.status.num_containers = info["Containers"]
+        self.status.num_nodes = info["DriverStatus"][3][1]
+
+swarm = Swarm()

caaas/templates/base.html

+<!DOCTYPE html>
+<html lang="en">
+<head>
+    {% block head %}
+    <meta charset="UTF-8">
+    <title>{% block title %}{% endblock %} - CAAAS</title>
+    <script src="/static/jquery-2.1.4.min.js" type="application/javascript"></script>
+    <script type="application/javascript">
+        var api_endpoint = "/api";
+    </script>
+    {% endblock %}
+</head>
+<body>
+    <div class="header"><em>CAaaS</em> &dash; Available nodes: <span id="num_nodes">0</span> Containers: <span id="num_containers">0</span></div>
+    <script>
+        function update_status() {
+            $.getJSON( api_endpoint + "/status")
+                .done(function( data ) {
+                    $("#num_nodes").text(data.num_nodes);
+                    $("#num_containers").text(data.num_containers);
+                });
+        }
+        update_status();
+        window.setInterval(update_status, 5000);
+    </script>
+    <div id="content">{% block content %}{% endblock %}</div>
+    <div id="footer">
+        {% block footer %}
+        &copy; Copyright 2015 by <a href="http://distsysgroup.wordpress.com/">DSG</a>.
+        {% endblock %}
+    </div>
+</body>
+</html>
\ No newline at end of file

caaas/templates/index.html

+{% extends "base.html" %}
+{% block title %}Home{% endblock %}
+{% block content %}
+<h1>DSG CAaaS system - Analytics on demand</h1>
+<p>Start an activity:</p>
+<ul>
+    <li><a href="/web/{{ user }}/spark-notebook">Give me a Spark notebook</a></li>
+    <li>Submit a Spark application</li>
+</ul>
+
+<p>Monitor my activities:</p>
+<ul>
+    <li>Running applications</li>
+</ul>
+
+{% endblock %}
\ No newline at end of file

caaas/web.py

+from flask import render_template
+
+from caaas import app
+from caaas.swarm import swarm
+
+
+@app.route("/web/<username>")
+def web_index(username):
+    template_vars = {
+        "user": username
+    }
+    return render_template('index.html', **template_vars)
+
+
+@app.route("/web/<username>/spark-notebook")
+def web_notebook(username):
+    pass

main.py

+from configparser import ConfigParser
+from caaas import swarm, app, init_db
+
+
+def read_config():
+    conf = ConfigParser()
+    conf.read('caaas.ini')
+    return conf
+
+
+def main():
+    conf = read_config()
+
+    init_db(conf['db']['user'], conf['db']['pass'], conf['db']['server'], conf['db']['db'])
+
+    swarm.connect(conf['docker']['swarm-manager'])
+    swarm.start_update_thread()
+
+    app.debug = True
+    app.run()
+
+if __name__ == "__main__":
+    main()