Commit b526067b authored by Daniele Venzano's avatar Daniele Venzano
Browse files

Merge branch 'v2'

parents e128a1c5 23da8a1b
......@@ -58,7 +58,4 @@ docs/_build/
# PyBuilder
target/
caaas.ini
\ No newline at end of file
.idea/
caaas
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="BashSupportProjectSettings">
<option name="supportBash4" value="true" />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Remote Python 3.4.0 (sftp://ubuntu@192.168.45.25:22/usr/bin/python3)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TemplatesService">
<option name="TEMPLATE_CONFIGURATION" value="Jinja2" />
<option name="TEMPLATE_FOLDERS">
<list>
<option value="$MODULE_DIR$/caaas/templates" />
</list>
</option>
</component>
</module>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectCodeStyleSettingsManager">
<option name="PER_PROJECT_SETTINGS">
<value>
<XML>
<option name="XML_LEGACY_SETTINGS_IMPORTED" value="true" />
</XML>
</value>
</option>
<option name="PREFERRED_PROJECT_CODE_STYLE" value="Default (1)" />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<component name="dataSourceStorage">
<data-source name="MySQL - @m1" uuid="a32fd6de-3ffa-40c0-9ec8-8953a89c53e0">
<database-info product="MySQL" version="5.5.44-MariaDB-1ubuntu0.14.04.1" jdbc-version="4.0" driver-name="MySQL Connector Java" driver-version="mysql-connector-java-5.1.35 ( Revision: 5fb9c5849535c13917c2cf9baaece6ef9693ef27 )">
<extra-name-characters>#@</extra-name-characters>
<identifier-quote-string>`</identifier-quote-string>
</database-info>
<case-sensitivity plain-identifiers="exact" quoted-identifiers="exact"/>
<schema name="" catalog="caaas"/>
<table name="applications" schema="" catalog="caaas" type="TABLE">
<column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
<column name="execution_name" sqlType="VARCHAR" precision="512" scale="0" nullable="false" jdbcType="12"/>
<column name="cmd" sqlType="VARCHAR" precision="1024" scale="0" nullable="false" jdbcType="12"/>
<column name="spark_options" sqlType="VARCHAR" precision="1024" scale="0" nullable="false" jdbcType="12"/>
<column name="user_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<column name="time_started" sqlType="TIMESTAMP" precision="19" scale="0" nullable="true" jdbcType="93"/>
<column name="time_finished" sqlType="TIMESTAMP" precision="19" scale="0" nullable="true" jdbcType="93"/>
<column name="cluster_id" sqlType="INT" precision="10" scale="0" nullable="true" jdbcType="4"/>
<column name="status" sqlType="VARCHAR" precision="16" scale="0" nullable="false" jdbcType="12" def="J3NldHVwJw=="/>
<primary-key name="PRIMARY" columns="id"/>
</table>
<table name="clusters" schema="" catalog="caaas" type="TABLE">
<column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
<column name="user_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<column name="master_address" sqlType="VARCHAR" precision="512" scale="0" nullable="false" jdbcType="12"/>
<column name="name" sqlType="VARCHAR" precision="256" scale="0" nullable="false" jdbcType="12"/>
<column name="time_created" sqlType="TIMESTAMP" precision="19" scale="0" nullable="false" jdbcType="93" def="Q1VSUkVOVF9USU1FU1RBTVA="/>
<primary-key name="PRIMARY" columns="id"/>
</table>
<table name="containers" schema="" catalog="caaas" type="TABLE">
<column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
<column name="docker_id" sqlType="VARCHAR" precision="1024" scale="0" nullable="false" jdbcType="12"/>
<column name="cluster_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<column name="user_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<column name="ip_address" sqlType="VARCHAR" precision="16" scale="0" nullable="false" jdbcType="12"/>
<column name="contents" sqlType="VARCHAR" precision="512" scale="0" nullable="false" jdbcType="12"/>
<primary-key name="PRIMARY" columns="id"/>
</table>
<table name="proxy" schema="" catalog="caaas" type="TABLE">
<column name="id" sqlType="VARCHAR" precision="128" scale="0" nullable="false" jdbcType="12"/>
<column name="internal_url" sqlType="VARCHAR" precision="512" scale="0" nullable="true" jdbcType="12"/>
<column name="cluster_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<column name="service_name" sqlType="VARCHAR" precision="64" scale="0" nullable="true" jdbcType="12"/>
<column name="container_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<column name="last_access" sqlType="TIMESTAMP" precision="19" scale="0" nullable="true" jdbcType="93"/>
<primary-key name="PRIMARY" columns="id"/>
</table>
<table name="users" schema="" catalog="caaas" type="TABLE">
<column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
<column name="email" sqlType="VARCHAR" precision="128" scale="0" nullable="false" jdbcType="12"/>
<primary-key name="PRIMARY" columns="id"/>
</table>
</data-source>
</component>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="dataSourceStorageLocal">
<data-source name="MySQL - @m1" uuid="a32fd6de-3ffa-40c0-9ec8-8953a89c53e0">
<secret-storage>master_key</secret-storage>
<user-name>caaas</user-name>
<schema-pattern>caaas.*</schema-pattern>
<default-schemas>caaas.*</default-schemas>
</data-source>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="DataSourceManagerImpl" format="xml" hash="3694309854">
<data-source source="LOCAL" name="MySQL - @m1" uuid="a32fd6de-3ffa-40c0-9ec8-8953a89c53e0">
<driver-ref>mysql</driver-ref>
<synchronize>true</synchronize>
<jdbc-driver>com.mysql.jdbc.Driver</jdbc-driver>
<jdbc-url>jdbc:mysql://localhost:3306</jdbc-url>
<driver-properties>
<property name="zeroDateTimeBehavior" value="convertToNull" />
<property name="tinyInt1isBit" value="false" />
<property name="characterEncoding" value="utf8" />
<property name="characterSetResults" value="utf8" />
<property name="yearIsDateType" value="false" />
</driver-properties>
<libraries />
</data-source>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="PublishConfigData" autoUpload="Always" serverName="vm" deleteMissingItems="true" traceLevel="DETAILS">
<serverData>
<paths name="bfm2">
<serverdata>
<mappings>
<mapping deploy="/caaas" local="$PROJECT_DIR$" web="/" />
</mappings>
</serverdata>
</paths>
<paths name="vm">
<serverdata>
<mappings>
<mapping deploy="/caaas" local="$PROJECT_DIR$" web="/" />
</mappings>
</serverdata>
</paths>
</serverData>
<option name="myAutoUpload" value="ALWAYS" />
</component>
</project>
\ No newline at end of file
<component name="ProjectDictionaryState">
<dictionary name="venzano">
<words>
<w>caaas</w>
<w>jinja</w>
<w>venza</w>
</words>
</dictionary>
</component>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<file url="file://$PROJECT_DIR$/caaas/sql.py" charset="UTF-8" />
<file url="file://$PROJECT_DIR$/images/notebook/files/start-notebook.sh" charset="UTF-8" />
<file url="PROJECT" charset="UTF-8" />
</component>
</project>
\ No newline at end of file
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="1">
<item index="0" class="java.lang.String" itemvalue="mysql" />
</list>
</value>
</option>
</inspection_tool>
<inspection_tool class="SpellCheckingInspection" enabled="true" level="TYPO" enabled_by_default="true">
<option name="processCode" value="false" />
<option name="processLiterals" value="true" />
<option name="processComments" value="true" />
</inspection_tool>
</profile>
</component>
\ No newline at end of file
<component name="InspectionProjectProfileManager">
<settings>
<option name="PROJECT_PROFILE" value="Project Default" />
<option name="USE_PROJECT_PROFILE" value="true" />
<version value="1.0" />
</settings>
</component>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectLevelVcsManager" settingsEditedManually="false">
<OptionsSetting value="true" id="Add" />
<OptionsSetting value="true" id="Remove" />
<OptionsSetting value="true" id="Checkout" />
<OptionsSetting value="true" id="Update" />
<OptionsSetting value="true" id="Status" />
<OptionsSetting value="true" id="Edit" />
<ConfirmationsSetting value="0" id="Add" />
<ConfirmationsSetting value="0" id="Remove" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Remote Python 3.4.0 (sftp://ubuntu@192.168.45.25:22/usr/bin/python3)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/caaas.iml" filepath="$PROJECT_DIR$/.idea/caaas.iml" />
</modules>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="SqlDialectMappings">
<file url="file://$PROJECT_DIR$" dialect="MySQL" />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="WebResourcesPaths">
<contentEntries>
<entry url="file://$PROJECT_DIR$">
<entryData>
<resourceRoots>
<path value="file://$PROJECT_DIR$/caaas" />
</resourceRoots>
</entryData>
</entry>
</contentEntries>
</component>
</project>
\ No newline at end of file
This diff is collapsed.
# CAaaS - Container Analytics as a Service
# Zoe - Container Analytics as a Service
This web application uses a Docker Swarm cluster to run on-demand Spark clusters.
This application uses Docker Swarm to run on-demand Spark clusters.
It is composed of three components:
* zoectl: command-line client
* zoe-scheduler: the main daemon that performs application scheduling and talks to Swarm
* zoe-web: the web service
## Requirements
......@@ -9,9 +15,29 @@ This web application uses a Docker Swarm cluster to run on-demand Spark clusters
* A Docker registry containing Spark images
* Apache to act as a reverse proxy
## Apache configuration
## Configuration
Zoe configuration is kept, for now, in a Python file: `common/configuration.py`
### Swarm/Docker
For testing you can use also a single Docker instance, just set its endpoint in the configuration file mentioned above.
To use Swarm, we use an undocumented network configuration, with the docker bridges connected to a physical interface, so that
containers on different hosts can talk to each other on the same layer 2 domain.
### Docker registry
Use the scripts in the [zoe-docker-images](https://github.com/DistributedSystemsGroup/zoe-docker-images) repository to create
and populate a private registry with Spark images. The images are quite standard and can be used also without Zoe, for examples
on how to do that, see the `scripts/start_cluster.sh` script.
Put this in your virtual host entry:
### Apache configuration
Zoe generates dynamically proxy entries to let users access to the various web interfaces contained in the Spark containers.
To do this, it needs to be able to reload Apache and to write to a configuration file included in the VirtualHost directive.
Here is an example configuration for a virtual host:
```
ProxyHTMLLinks a href
ProxyHTMLLinks area href
......@@ -34,25 +60,17 @@ ProxyHTMLEvents onclick ondblclick onmousedown onmouseup \
onunload onsubmit onreset onselect onchange
ProxyRequests Off
IncludeOptional /tmp/caaas-proxy.conf*
```
If you need to proxy the web application itself, add also these directives:
```
<Location /web/>
ProxyHtmlEnable On
ProxyHTMLExtended On
ProxyPass http://192.168.45.25:5000/web/
ProxyPassReverse http://192.168.45.25:5000/web/
ProxyHTMLURLMap /web/ /web/
</Location>
<Location /api/>
ProxyHtmlEnable On
ProxyHTMLExtended On
ProxyPass http://192.168.45.25:5000/api/
ProxyPassReverse http://192.168.45.25:5000/api/
ProxyHTMLURLMap /api/ /api/
<Location />
ProxyHtmlEnable On
ProxyHTMLExtended On
ProxyPass http://127.0.0.1:5000/ retry=0
ProxyPassReverse http://127.0.0.1:5000/
</Location>
IncludeOptional /tmp/zoe-proxy.conf*
```
The script `apache-proxy.py` needs to be run to update apache whenever containers are created or destroyed.
\ No newline at end of file
This configuration will also proxy zoe-web, that starts on port 5000 by default.
Please note that putting the generated config file in /tmp can be a serious security problem, depending on your setup.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment