Commit 018a3f87 authored by Daniele Venzano's avatar Daniele Venzano
Browse files

Add support for SparkSubmit jobs, to be tested

Ignore IDEA project files
parent e96e1d18
......@@ -58,7 +58,3 @@ docs/_build/
# PyBuilder
target/
caaas.ini
\ No newline at end of file
caaas
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="BashSupportProjectSettings">
<option name="supportBash4" value="true" />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Remote Python 3.4.0 (sftp://ubuntu@192.168.45.25:22/usr/bin/python3)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TemplatesService">
<option name="TEMPLATE_CONFIGURATION" value="Jinja2" />
<option name="TEMPLATE_FOLDERS">
<list>
<option value="$MODULE_DIR$/caaas/templates" />
</list>
</option>
</component>
</module>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectCodeStyleSettingsManager">
<option name="PER_PROJECT_SETTINGS">
<value>
<XML>
<option name="XML_LEGACY_SETTINGS_IMPORTED" value="true" />
</XML>
</value>
</option>
<option name="PREFERRED_PROJECT_CODE_STYLE" value="Default (1)" />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<component name="dataSourceStorage">
<data-source name="MySQL - @m1" uuid="a32fd6de-3ffa-40c0-9ec8-8953a89c53e0">
<database-info product="MySQL" version="5.5.44-MariaDB-1ubuntu0.14.04.1" jdbc-version="4.0" driver-name="MySQL Connector Java" driver-version="mysql-connector-java-5.1.35 ( Revision: 5fb9c5849535c13917c2cf9baaece6ef9693ef27 )">
<extra-name-characters>#@</extra-name-characters>
<identifier-quote-string>`</identifier-quote-string>
</database-info>
<case-sensitivity plain-identifiers="exact" quoted-identifiers="exact"/>
<schema name="" catalog="caaas"/>
<table name="applications" schema="" catalog="caaas" type="TABLE">
<column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
<column name="execution_name" sqlType="VARCHAR" precision="512" scale="0" nullable="false" jdbcType="12"/>
<column name="cmd" sqlType="VARCHAR" precision="1024" scale="0" nullable="false" jdbcType="12"/>
<column name="spark_options" sqlType="VARCHAR" precision="1024" scale="0" nullable="false" jdbcType="12"/>
<column name="user_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<column name="time_started" sqlType="TIMESTAMP" precision="19" scale="0" nullable="true" jdbcType="93"/>
<column name="time_finished" sqlType="TIMESTAMP" precision="19" scale="0" nullable="true" jdbcType="93"/>
<column name="cluster_id" sqlType="INT" precision="10" scale="0" nullable="true" jdbcType="4"/>
<column name="status" sqlType="VARCHAR" precision="16" scale="0" nullable="false" jdbcType="12" def="J3NldHVwJw=="/>
<primary-key name="PRIMARY" columns="id"/>
</table>
<table name="clusters" schema="" catalog="caaas" type="TABLE">
<column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
<column name="user_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<column name="master_address" sqlType="VARCHAR" precision="512" scale="0" nullable="false" jdbcType="12"/>
<column name="name" sqlType="VARCHAR" precision="256" scale="0" nullable="false" jdbcType="12"/>
<column name="time_created" sqlType="TIMESTAMP" precision="19" scale="0" nullable="false" jdbcType="93" def="Q1VSUkVOVF9USU1FU1RBTVA="/>
<primary-key name="PRIMARY" columns="id"/>
</table>
<table name="containers" schema="" catalog="caaas" type="TABLE">
<column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
<column name="docker_id" sqlType="VARCHAR" precision="1024" scale="0" nullable="false" jdbcType="12"/>
<column name="cluster_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<column name="user_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<column name="ip_address" sqlType="VARCHAR" precision="16" scale="0" nullable="false" jdbcType="12"/>
<column name="contents" sqlType="VARCHAR" precision="512" scale="0" nullable="false" jdbcType="12"/>
<primary-key name="PRIMARY" columns="id"/>
</table>
<table name="proxy" schema="" catalog="caaas" type="TABLE">
<column name="id" sqlType="VARCHAR" precision="128" scale="0" nullable="false" jdbcType="12"/>
<column name="internal_url" sqlType="VARCHAR" precision="512" scale="0" nullable="true" jdbcType="12"/>
<column name="cluster_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<column name="service_name" sqlType="VARCHAR" precision="64" scale="0" nullable="true" jdbcType="12"/>
<column name="container_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<column name="last_access" sqlType="TIMESTAMP" precision="19" scale="0" nullable="true" jdbcType="93"/>
<primary-key name="PRIMARY" columns="id"/>
</table>
<table name="users" schema="" catalog="caaas" type="TABLE">
<column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
<column name="email" sqlType="VARCHAR" precision="128" scale="0" nullable="false" jdbcType="12"/>
<primary-key name="PRIMARY" columns="id"/>
</table>
</data-source>
<data-source name="MySQL - @m1 devel" uuid="33b1ec79-4374-4ff8-a8f8-26c89b418a79">
<database-info product="MySQL" version="5.5.44-MariaDB-1ubuntu0.14.04.1" jdbc-version="4.0" driver-name="MySQL Connector Java" driver-version="mysql-connector-java-5.1.35 ( Revision: 5fb9c5849535c13917c2cf9baaece6ef9693ef27 )">
<extra-name-characters>#@</extra-name-characters>
<identifier-quote-string>`</identifier-quote-string>
</database-info>
<case-sensitivity plain-identifiers="exact" quoted-identifiers="exact"/>
<schema name="" catalog="caaas_devel"/>
<table name="applications" schema="" catalog="caaas_devel" type="TABLE">
<column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
<column name="name" sqlType="VARCHAR" precision="64" scale="0" nullable="true" jdbcType="12"/>
<column name="required_resources" sqlType="BLOB" precision="65535" scale="0" nullable="true" jdbcType="-4"/>
<column name="user_id" sqlType="INT" precision="10" scale="0" nullable="true" jdbcType="4"/>
<column name="type" sqlType="VARCHAR" precision="20" scale="0" nullable="true" jdbcType="12"/>
<column name="master_image" sqlType="VARCHAR" precision="256" scale="0" nullable="true" jdbcType="12"/>
<column name="worker_image" sqlType="VARCHAR" precision="256" scale="0" nullable="true" jdbcType="12"/>
<primary-key name="PRIMARY" columns="id"/>
<foreign-key name="applications_ibfk_1" columns="user_id" ref-table="users" ref-schema="" ref-catalog="caaas_devel" ref-columns="id" update-rule="3" delete-rule="3" deferrability="2"/>
<exported-key name="executions_ibfk_1" table="executions" schema="" catalog="caaas_devel" columns="application_id"/>
</table>
<table name="clusters" schema="" catalog="caaas_devel" type="TABLE">
<column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
<column name="execution_id" sqlType="INT" precision="10" scale="0" nullable="true" jdbcType="4"/>
<primary-key name="PRIMARY" columns="id"/>
<foreign-key name="clusters_ibfk_1" columns="execution_id" ref-table="executions" ref-schema="" ref-catalog="caaas_devel" ref-columns="id" update-rule="3" delete-rule="3" deferrability="2"/>
<exported-key name="containers_ibfk_1" table="containers" schema="" catalog="caaas_devel" columns="cluster_id"/>
<exported-key name="proxies_ibfk_1" table="proxies" schema="" catalog="caaas_devel" columns="cluster_id"/>
</table>
<table name="containers" schema="" catalog="caaas_devel" type="TABLE">
<column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
<column name="docker_id" sqlType="VARCHAR" precision="128" scale="0" nullable="true" jdbcType="12"/>
<column name="cluster_id" sqlType="INT" precision="10" scale="0" nullable="true" jdbcType="4"/>
<column name="ip_address" sqlType="VARCHAR" precision="16" scale="0" nullable="true" jdbcType="12"/>
<column name="readable_name" sqlType="VARCHAR" precision="32" scale="0" nullable="true" jdbcType="12"/>
<primary-key name="PRIMARY" columns="id"/>
<foreign-key name="containers_ibfk_1" columns="cluster_id" ref-table="clusters" ref-schema="" ref-catalog="caaas_devel" ref-columns="id" update-rule="3" delete-rule="3" deferrability="2"/>
<exported-key name="proxies_ibfk_2" table="proxies" schema="" catalog="caaas_devel" columns="container_id"/>
</table>
<table name="executions" schema="" catalog="caaas_devel" type="TABLE">
<column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
<column name="name" sqlType="VARCHAR" precision="64" scale="0" nullable="true" jdbcType="12"/>
<column name="assigned_resources" sqlType="BLOB" precision="65535" scale="0" nullable="true" jdbcType="-4"/>
<column name="application_id" sqlType="INT" precision="10" scale="0" nullable="true" jdbcType="4"/>
<column name="time_scheduled" sqlType="DATETIME" precision="19" scale="0" nullable="true" jdbcType="93"/>
<column name="time_started" sqlType="DATETIME" precision="19" scale="0" nullable="true" jdbcType="93"/>
<column name="time_finished" sqlType="DATETIME" precision="19" scale="0" nullable="true" jdbcType="93"/>
<column name="status" sqlType="VARCHAR" precision="32" scale="0" nullable="true" jdbcType="12"/>
<column name="type" sqlType="VARCHAR" precision="20" scale="0" nullable="true" jdbcType="12"/>
<column name="commandline" sqlType="VARCHAR" precision="1024" scale="0" nullable="true" jdbcType="12"/>
<column name="spark_opts" sqlType="VARCHAR" precision="1024" scale="0" nullable="true" jdbcType="12"/>
<primary-key name="PRIMARY" columns="id"/>
<foreign-key name="executions_ibfk_1" columns="application_id" ref-table="applications" ref-schema="" ref-catalog="caaas_devel" ref-columns="id" update-rule="3" delete-rule="3" deferrability="2"/>
<exported-key name="clusters_ibfk_1" table="clusters" schema="" catalog="caaas_devel" columns="execution_id"/>
</table>
<table name="proxies" schema="" catalog="caaas_devel" type="TABLE">
<column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
<column name="internal_url" sqlType="VARCHAR" precision="1024" scale="0" nullable="true" jdbcType="12"/>
<column name="cluster_id" sqlType="INT" precision="10" scale="0" nullable="true" jdbcType="4"/>
<column name="container_id" sqlType="INT" precision="10" scale="0" nullable="true" jdbcType="4"/>
<column name="service_name" sqlType="VARCHAR" precision="32" scale="0" nullable="true" jdbcType="12"/>
<column name="last_access" sqlType="DATETIME" precision="19" scale="0" nullable="true" jdbcType="93"/>
<primary-key name="PRIMARY" columns="id"/>
<foreign-key name="proxies_ibfk_1" columns="cluster_id" ref-table="clusters" ref-schema="" ref-catalog="caaas_devel" ref-columns="id" update-rule="3" delete-rule="3" deferrability="2"/>
<foreign-key name="proxies_ibfk_2" columns="container_id" ref-table="containers" ref-schema="" ref-catalog="caaas_devel" ref-columns="id" update-rule="3" delete-rule="3" deferrability="2"/>
</table>
<table name="users" schema="" catalog="caaas_devel" type="TABLE">
<column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
<column name="email" sqlType="VARCHAR" precision="128" scale="0" nullable="true" jdbcType="12"/>
<primary-key name="PRIMARY" columns="id"/>
<exported-key name="applications_ibfk_1" table="applications" schema="" catalog="caaas_devel" columns="user_id"/>
</table>
</data-source>
</component>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="dataSourceStorageLocal">
<data-source name="MySQL - @m1" uuid="a32fd6de-3ffa-40c0-9ec8-8953a89c53e0">
<secret-storage>master_key</secret-storage>
<user-name>caaas</user-name>
<schema-pattern>caaas.*</schema-pattern>
<default-schemas>caaas.*</default-schemas>
</data-source>
<data-source name="MySQL - @m1 devel" uuid="33b1ec79-4374-4ff8-a8f8-26c89b418a79">
<secret-storage>master_key</secret-storage>
<user-name>caaas_devel</user-name>
<schema-pattern>caaas_devel.*</schema-pattern>
<default-schemas>caaas_devel.*</default-schemas>
</data-source>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="DataSourceManagerImpl" format="xml" hash="330654499">
<data-source source="LOCAL" name="MySQL - @m1" uuid="a32fd6de-3ffa-40c0-9ec8-8953a89c53e0">
<driver-ref>mysql</driver-ref>
<synchronize>true</synchronize>
<jdbc-driver>com.mysql.jdbc.Driver</jdbc-driver>
<jdbc-url>jdbc:mysql://localhost:3306</jdbc-url>
<driver-properties>
<property name="zeroDateTimeBehavior" value="convertToNull" />
<property name="tinyInt1isBit" value="false" />
<property name="characterEncoding" value="utf8" />
<property name="characterSetResults" value="utf8" />
<property name="yearIsDateType" value="false" />
</driver-properties>
<libraries />
</data-source>
<data-source source="LOCAL" name="MySQL - @m1 devel" uuid="33b1ec79-4374-4ff8-a8f8-26c89b418a79">
<driver-ref>mysql</driver-ref>
<synchronize>true</synchronize>
<jdbc-driver>com.mysql.jdbc.Driver</jdbc-driver>
<jdbc-url>jdbc:mysql://localhost:3306</jdbc-url>
<driver-properties>
<property name="zeroDateTimeBehavior" value="convertToNull" />
<property name="tinyInt1isBit" value="false" />
<property name="characterEncoding" value="utf8" />
<property name="characterSetResults" value="utf8" />
<property name="yearIsDateType" value="false" />
</driver-properties>
<libraries />
</data-source>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="PublishConfigData" autoUpload="Always" serverName="vm" deleteMissingItems="true" traceLevel="DETAILS">
<serverData>
<paths name="bfm2">
<serverdata>
<mappings>
<mapping deploy="/caaas" local="$PROJECT_DIR$" web="/" />
</mappings>
</serverdata>
</paths>
<paths name="vm">
<serverdata>
<mappings>
<mapping deploy="/caaas" local="$PROJECT_DIR$" web="/" />
</mappings>
</serverdata>
</paths>
</serverData>
<option name="myAutoUpload" value="ALWAYS" />
</component>
</project>
\ No newline at end of file
<component name="ProjectDictionaryState">
<dictionary name="venzano">
<words>
<w>caaas</w>
<w>jinja</w>
<w>venza</w>
</words>
</dictionary>
</component>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<file url="file://$PROJECT_DIR$/caaas_web/sql.py" charset="UTF-8" />
<file url="file://$PROJECT_DIR$/scripts/images/notebook/files/start-notebook.sh" charset="UTF-8" />
<file url="PROJECT" charset="UTF-8" />
</component>
</project>
\ No newline at end of file
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="1">
<item index="0" class="java.lang.String" itemvalue="mysql" />
</list>
</value>
</option>
</inspection_tool>
<inspection_tool class="SpellCheckingInspection" enabled="true" level="TYPO" enabled_by_default="true">
<option name="processCode" value="false" />
<option name="processLiterals" value="true" />
<option name="processComments" value="true" />
</inspection_tool>
</profile>
</component>
\ No newline at end of file
<component name="InspectionProjectProfileManager">
<settings>
<option name="PROJECT_PROFILE" value="Project Default" />
<option name="USE_PROJECT_PROFILE" value="true" />
<version value="1.0" />
</settings>
</component>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectLevelVcsManager" settingsEditedManually="false">
<OptionsSetting value="true" id="Add" />
<OptionsSetting value="true" id="Remove" />
<OptionsSetting value="true" id="Checkout" />
<OptionsSetting value="true" id="Update" />
<OptionsSetting value="true" id="Status" />
<OptionsSetting value="true" id="Edit" />
<ConfirmationsSetting value="0" id="Add" />
<ConfirmationsSetting value="0" id="Remove" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Remote Python 3.4.0 (sftp://ubuntu@192.168.45.25:22/usr/bin/python3)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/caaas.iml" filepath="$PROJECT_DIR$/.idea/caaas.iml" />
</modules>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="SqlDialectMappings">
<file url="file://$PROJECT_DIR$" dialect="MySQL" />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="WebResourcesPaths">
<contentEntries>
<entry url="file://$PROJECT_DIR$">
<entryData>
<resourceRoots>
<path value="file://$PROJECT_DIR$/caaas_web" />
</resourceRoots>
</entryData>
</entry>
</contentEntries>
</component>
</project>
\ No newline at end of file
This diff is collapsed.
# CAaaS - Container Analytics as a Service
# Zoe - Container Analytics as a Service
This web application uses a Docker Swarm cluster to run on-demand Spark clusters.
This application uses a Docker Swarm cluster to run on-demand Spark clusters.
IT is composed of three components:
* zoectl: command-line client
* zoe-scheduler: the main daemon that performs application scheduling and talks to Swarm
* zoe-web: the web service
## Requirements
......@@ -34,7 +40,7 @@ ProxyHTMLEvents onclick ondblclick onmousedown onmouseup \
onunload onsubmit onreset onselect onchange
ProxyRequests Off
IncludeOptional /tmp/caaas-proxy.conf*
IncludeOptional /tmp/zoe-proxy.conf*
```
If you need to proxy the web application itself, add also these directives:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment