spark-submit.json 4.06 KB
Newer Older
1
{
2
    "name": "spark-jupyter",
3 4
    "services": [
        {
5
            "command": null,
6 7 8
            "environment": [
                [
                    "SPARK_MASTER_IP",
Daniele Venzano's avatar
Daniele Venzano committed
9 10 11 12 13 14 15 16 17
                    "{dns_name#self}"
                ],
                [
                    "HADOOP_USER_NAME",
                    "{user_name}"
                ],
                [
                    "PYTHONHASHSEED",
                    "42"
18 19
                ]
            ],
Daniele Venzano's avatar
Daniele Venzano committed
20
            "essential_count": 1,
21
            "image": "docker-registry:5000/zapps/spark2-master:4508",
22 23 24 25 26 27
            "monitor": false,
            "name": "spark-master",
            "ports": [
                {
                    "name": "Spark master web interface",
                    "port_number": 8080,
28 29
                    "protocol": "tcp",
                    "url_template": "http://HOST:PORT/"
30 31
                }
            ],
32
            "replicas": 1,
Daniele Venzano's avatar
Daniele Venzano committed
33 34
            "resources": {
                "cores": {
35 36 37 38 39 40
                    "max": null,
                    "min": null
                },
                "memory": {
                    "max": 536870912,
                    "min": 536870912
Daniele Venzano's avatar
Daniele Venzano committed
41
                }
Daniele Venzano's avatar
Daniele Venzano committed
42 43
            },
            "startup_order": 0,
Daniele Venzano's avatar
Daniele Venzano committed
44
            "total_count": 1,
45
            "volumes": []
46 47
        },
        {
48
            "command": null,
49 50 51 52 53 54 55 56 57 58 59
            "environment": [
                [
                    "SPARK_WORKER_CORES",
                    "6"
                ],
                [
                    "SPARK_WORKER_RAM",
                    "11273240064"
                ],
                [
                    "SPARK_MASTER_IP",
Daniele Venzano's avatar
Daniele Venzano committed
60
                    "{dns_name#spark-master0}"
61 62 63
                ],
                [
                    "SPARK_LOCAL_IP",
Daniele Venzano's avatar
Daniele Venzano committed
64 65 66 67 68 69 70 71 72
                    "{dns_name#self}"
                ],
                [
                    "PYTHONHASHSEED",
                    "42"
                ],
                [
                    "HADOOP_USER_NAME",
                    "{user_name}"
73 74
                ]
            ],
Daniele Venzano's avatar
Daniele Venzano committed
75
            "essential_count": 1,
76
            "image": "docker-registry:5000/zapps/spark2-worker:4508",
77
            "monitor": false,
Daniele Venzano's avatar
Daniele Venzano committed
78
            "name": "spark-worker",
79 80
            "ports": [],
            "replicas": 1,
Daniele Venzano's avatar
Daniele Venzano committed
81 82
            "resources": {
                "cores": {
83 84 85 86 87 88
                    "max": null,
                    "min": null
                },
                "memory": {
                    "max": 12884901888,
                    "min": 12884901888
Daniele Venzano's avatar
Daniele Venzano committed
89
                }
Daniele Venzano's avatar
Daniele Venzano committed
90 91
            },
            "startup_order": 1,
Daniele Venzano's avatar
Daniele Venzano committed
92
            "total_count": 2,
93
            "volumes": []
94 95
        },
        {
96
            "command": "wordcount.py hdfs://192.168.45.157/datasets/gutenberg_big_2x.txt hdfs://192.168.45.157/tmp/wcount-out",
97 98 99
            "environment": [
                [
                    "SPARK_MASTER",
Daniele Venzano's avatar
Daniele Venzano committed
100
                    "spark://{dns_name#spark-master0}:7077"
101 102 103 104 105 106 107 108 109
                ],
                [
                    "SPARK_EXECUTOR_RAM",
                    "11273240064"
                ],
                [
                    "SPARK_DRIVER_RAM",
                    "2147483648"
                ],
Daniele Venzano's avatar
Daniele Venzano committed
110 111 112 113 114 115 116 117
                [
                    "HADOOP_USER_NAME",
                    "{user_name}"
                ],
                [
                    "PYTHONHASHSEED",
                    "42"
                ],
118 119
                [
                    "NAMENODE_HOST",
Daniele Venzano's avatar
Daniele Venzano committed
120
                    "hdfs-namenode.zoe"
121 122
                ]
            ],
Daniele Venzano's avatar
Daniele Venzano committed
123
            "essential_count": 1,
124
            "image": "docker-registry:5000/zapps/spark2-submit:4508",
125
            "monitor": true,
126 127 128
            "name": "spark-submit",
            "ports": [],
            "replicas": 1,
Daniele Venzano's avatar
Daniele Venzano committed
129 130
            "resources": {
                "cores": {
131 132 133 134 135 136
                    "max": null,
                    "min": null
                },
                "memory": {
                    "max": 12884901888,
                    "min": 12884901888
Daniele Venzano's avatar
Daniele Venzano committed
137
                }
Daniele Venzano's avatar
Daniele Venzano committed
138
            },
139
            "startup_order": 2,
Daniele Venzano's avatar
Daniele Venzano committed
140
            "total_count": 1,
141
            "volumes": []
142 143
        }
    ],
144
    "size": 512,
145
    "version": 3,
146
    "will_end": false
147
}