Commit 7f93d95f by Daniele Venzano

Update categories and add resource parameters

parent ef068db5
Pipeline #4708 passed with stages
in 1 minute 9 seconds
Showing with 66 additions and 2 deletions
......@@ -2,7 +2,7 @@
"version": 1,
"zapps": [
{
"category": "Spark",
"category": "Data science",
"name": "Distributed PySpark notebook",
"description": "spark-jupyter.json",
"readable_descr": "README-jupyter.md",
......@@ -14,11 +14,43 @@
"description": "Hostname of the HDFS NameNode",
"type": "string",
"default": "hdfs-namenode.zoe"
},
{
"kind": "environment",
"name": "SPARK_EXECUTOR_RAM",
"readable_name": "Executor memory (bytes)",
"description": "Executor memory, must be less than the worker memory limit",
"type": "int",
"default": 11274289152
},
{
"kind": "environment",
"name": "SPARK_DRIVER_RAM",
"readable_name": "Spark driver memory (bytes)",
"description": "Driver memory, must be less than the memory limit for the jupyter service",
"type": "int",
"default": 2147483648
},
{
"kind": "environment",
"name": "SPARK_WORKER_RAM",
"readable_name": "Spark worker memory (bytes)",
"description": "Worker memory, must be less than the memory limit for the worker service",
"type": "int",
"default": 11274289152
},
{
"kind": "environment",
"name": "SPARK_WORKER_CORES",
"readable_name": "Spark worker cores",
"description": "Number of cores each worker has access to, must be equal to the core limit for the worker service",
"type": "int",
"default": 6
}
]
},
{
"category": "Spark",
"category": "Non-interactive",
"name": "Spark submit",
"description": "spark-submit.json",
"readable_descr": "README-submit.md",
......@@ -38,6 +70,38 @@
"description": "The Spark submit command line that tells Spark what to execute. Some options are already passed (like --master)",
"type": "string",
"default": "wordcount.py hdfs://192.168.45.157/datasets/gutenberg_big_2x.txt hdfs://192.168.45.157/tmp/wcount-out"
},
{
"kind": "environment",
"name": "SPARK_EXECUTOR_RAM",
"readable_name": "Executor memory (bytes)",
"description": "Executor memory, must be less than the worker memory limit",
"type": "int",
"default": 11274289152
},
{
"kind": "environment",
"name": "SPARK_DRIVER_RAM",
"readable_name": "Spark driver memory (bytes)",
"description": "Driver memory, must be less than the memory limit for the submit service",
"type": "int",
"default": 2147483648
},
{
"kind": "environment",
"name": "SPARK_WORKER_RAM",
"readable_name": "Spark worker memory (bytes)",
"description": "Worker memory, must be less than the memory limit for the worker service",
"type": "int",
"default": 11274289152
},
{
"kind": "environment",
"name": "SPARK_WORKER_CORES",
"readable_name": "Spark worker cores",
"description": "Number of cores each worker has access to, must be equal to the core limit for the worker service",
"type": "int",
"default": 6
}
]
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment