spark-jupyter.json 4.31 KB
Newer Older
1
{
2
    "name": "spark-jupyter",
3 4
    "services": [
        {
5
            "command": null,
6 7 8
            "environment": [
                [
                    "SPARK_MASTER_IP",
Daniele Venzano's avatar
Daniele Venzano committed
9 10 11 12 13 14 15 16 17
                    "{dns_name#self}"
                ],
                [
                    "HADOOP_USER_NAME",
                    "{user_name}"
                ],
                [
                    "PYTHONHASHSEED",
                    "42"
18 19
                ]
            ],
Daniele Venzano's avatar
Daniele Venzano committed
20
            "essential_count": 1,
21
            "image": "docker-registry:5000/zapps/spark2-master:4508",
22 23 24 25 26 27
            "monitor": false,
            "name": "spark-master",
            "ports": [
                {
                    "name": "Spark master web interface",
                    "port_number": 8080,
28 29
                    "protocol": "tcp",
                    "url_template": "http://HOST:PORT/"
30 31
                }
            ],
32
            "replicas": 1,
33 34
            "resources": {
                "cores": {
35 36 37 38 39 40
                    "max": null,
                    "min": null
                },
                "memory": {
                    "max": 536870912,
                    "min": 536870912
41
                }
Daniele Venzano's avatar
Daniele Venzano committed
42 43
            },
            "startup_order": 0,
44
            "total_count": 1,
45
            "volumes": []
46 47
        },
        {
48
            "command": null,
49 50 51 52 53 54 55 56 57 58 59
            "environment": [
                [
                    "SPARK_WORKER_CORES",
                    "6"
                ],
                [
                    "SPARK_WORKER_RAM",
                    "11273240064"
                ],
                [
                    "SPARK_MASTER_IP",
Daniele Venzano's avatar
Daniele Venzano committed
60
                    "{dns_name#spark-master0}"
61 62 63
                ],
                [
                    "SPARK_LOCAL_IP",
Daniele Venzano's avatar
Daniele Venzano committed
64 65 66 67 68 69 70 71 72
                    "{dns_name#self}"
                ],
                [
                    "PYTHONHASHSEED",
                    "42"
                ],
                [
                    "HADOOP_USER_NAME",
                    "{user_name}"
73 74
                ]
            ],
Daniele Venzano's avatar
Daniele Venzano committed
75
            "essential_count": 1,
76
            "image": "docker-registry:5000/zapps/spark2-worker:4508",
77
            "monitor": false,
Daniele Venzano's avatar
Daniele Venzano committed
78
            "name": "spark-worker",
79 80
            "ports": [],
            "replicas": 1,
81 82
            "resources": {
                "cores": {
83 84 85 86 87 88
                    "max": null,
                    "min": null
                },
                "memory": {
                    "max": 12884901888,
                    "min": 12884901888
89
                }
Daniele Venzano's avatar
Daniele Venzano committed
90 91
            },
            "startup_order": 1,
92
            "total_count": 2,
93
            "volumes": []
94 95
        },
        {
96
            "command": null,
97 98 99
            "environment": [
                [
                    "SPARK_MASTER",
Daniele Venzano's avatar
Daniele Venzano committed
100
                    "spark://{dns_name#spark-master0}:7077"
101 102 103 104 105 106 107 108 109
                ],
                [
                    "SPARK_EXECUTOR_RAM",
                    "11273240064"
                ],
                [
                    "SPARK_DRIVER_RAM",
                    "2147483648"
                ],
Daniele Venzano's avatar
Daniele Venzano committed
110 111 112 113
                [
                    "HADOOP_USER_NAME",
                    "{user_name}"
                ],
114 115 116 117
                [
                    "NB_USER",
                    "{user_name}"
                ],
Daniele Venzano's avatar
Daniele Venzano committed
118 119 120 121
                [
                    "PYTHONHASHSEED",
                    "42"
                ],
122 123
                [
                    "NAMENODE_HOST",
Daniele Venzano's avatar
Daniele Venzano committed
124
                    "hdfs-namenode.zoe"
125 126
                ]
            ],
Daniele Venzano's avatar
Daniele Venzano committed
127
            "essential_count": 1,
128
            "image": "docker-registry:5000/zapps/spark2-jupyter-notebook:4508",
129 130 131 132 133 134
            "monitor": true,
            "name": "spark-jupyter",
            "ports": [
                {
                    "name": "Jupyter Notebook interface",
                    "port_number": 8888,
135 136
                    "protocol": "tcp",
                    "url_template": "http://HOST:PORT/"
137 138
                }
            ],
139
            "replicas": 1,
140 141
            "resources": {
                "cores": {
142 143 144 145 146 147
                    "max": null,
                    "min": null
                },
                "memory": {
                    "max": 12884901888,
                    "min": 12884901888
148
                }
Daniele Venzano's avatar
Daniele Venzano committed
149 150
            },
            "startup_order": 0,
151
            "total_count": 1,
152
            "volumes": []
153 154
        }
    ],
155
    "size": 512,
156
    "version": 3,
157
    "will_end": false
158
}