Commit 644ae5ed authored by Daniele Venzano's avatar Daniele Venzano

Delete outdated files

parent 66124e77
#!/usr/bin/python3
import os
import json
from zoe_lib.workflow import ZoeWorkFlow
WORKSPACE_BASE_PATH = '/mnt/cephfs/zoe-workspaces'
ZOE_WORKER_JSON = '''
{
"command": "",
"docker_image": "192.168.45.252:5000/zoerepo/openmpi-worker",
"environment": [],
"monitor": false,
"name": "mpiworker",
"ports": [],
"required_resources": {
"memory": 1073741824
},
"volumes": []
}
'''
ZOE_MPIRUN_JSON = '''
{
"command": "mpirun <options>",
"docker_image": "192.168.45.252:5000/zoerepo/openmpi-worker",
"environment": [],
"monitor": true,
"name": "mpirun",
"ports": [],
"required_resources": {
"memory": 1073741824
},
"volumes": []
}
'''
ZOE_APP_BASE_JSON = '''
{
"name": "openmpi-hello",
"priority": 512,
"requires_binary": true,
"services": [],
"version": 1,
"will_end": true
}
'''
def prepare_mpirun(wf):
count = 4
mpihosts = ''
for i in range(count):
mpihosts += wf.generate_hostname('openmpi-worker-{}'.format(i)) + '\n'
mpihosts += '\n'
wf.workspace.put_string(mpihosts, 'mpihosts')
cmdline = 'mpirun -np {} --hostfile mpihosts ./MPI_Hello'.format(count)
zoe_app = json.loads(ZOE_APP_BASE_JSON)
zoe_app['name'] = 'mpi-hello-world'
mpirun_service = json.loads(ZOE_MPIRUN_JSON)
mpirun_service['command'] = cmdline
for wc in range(count):
mpiworker = json.loads(ZOE_WORKER_JSON)
mpiworker['name'] = 'mpiworker{}'.format(wc)
zoe_app['services'].append(mpiworker)
zoe_app['services'].append(mpirun_service)
return zoe_app
if __name__ == "__main__":
identity = {
'username': os.getenv('ZOE_USER'),
'password': os.getenv('ZOE_PASS'),
'zoe_url': os.getenv('ZOE_URL')
}
with ZoeWorkFlow(WORKSPACE_BASE_PATH, identity, 'mpi-test') as my_wf:
my_wf.workspace.put('/home/ubuntu/mpi_test/MPI_Hello', 'MPI_Hello')
my_wf.workspace.chmod('MPI_Hello', 0o755)
app = prepare_mpirun(my_wf)
print('Starting mpi app')
exec_id = my_wf.execution_start(app)
print('Waiting for termination...')
my_wf.wait_termination(exec_id)
# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
from zoe_lib.workspace import ZoeWorkspace
from zoe_lib.executions import ZoeExecutionsAPI
from zoe_lib.services import ZoeServiceAPI
from zoe_lib.info import ZoeInfoAPI
class ZoeWorkFlow:
def __init__(self, workspace_base_path, identity, name):
self.identity = identity
self.name = name
self.workspace = ZoeWorkspace(workspace_base_path, identity, name)
self.exec_api = ZoeExecutionsAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password'])
self.cont_api = ZoeServiceAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password'])
info_api = ZoeInfoAPI(self.identity['zoe_url'], self.identity['username'], self.identity['password'])
zoe_info = info_api.info()
self.hostname_prefix = zoe_info['name_prefix']
self.exec_counter = 0
def generate_hostname(self, process_name: str) -> str:
return self.hostname_prefix + '-' + process_name + '-' + self.identity['username']
def start_workflow(self):
self.workspace.create()
def end_workflow(self):
self.workspace.destroy()
def execution_start(self, app):
self.exec_counter += 1
return self.exec_api.execution_start(self.name + '{}'.format(self.exec_counter), app)
def wait_termination(self, exec_id):
execution = self.exec_api.execution_get(exec_id)
while execution['status'] == 'submitted' or execution['status'] == 'running':
time.sleep(1)
execution = self.exec_api.execution_get(exec_id)
def __enter__(self):
self.start_workflow()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.end_workflow()
# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import shutil
import logging
log = logging.getLogger(__name__)
class ZoeWorkspace:
def __init__(self, base_path, identity, wk_name):
self._workspace_name = identity['username'] + '-' + wk_name
self._workspace_path = os.path.join(base_path, self._workspace_name)
self._workspace_volume = {
'host_path': self._workspace_path,
'cont_path': '/mnt/workspace',
'readonly': False
}
def create(self):
log.debug("Creating workspace {}".format(self._workspace_path))
os.makedirs(self._workspace_path, exist_ok=False)
def destroy(self):
log.debug("Destroying workspace {}".format(self._workspace_path))
shutil.rmtree(self._workspace_path)
def get_volume_definition(self):
return self._workspace_volume
def put(self, src_filepath, dst_rel_filepath):
dst_path = os.path.join(self._workspace_path, dst_rel_filepath)
log.debug("Copying file {} to {}".format(src_filepath, dst_path))
shutil.copyfile(src_filepath, dst_path)
def put_string(self, file_contents, dst_rel_filepath):
dst_path = os.path.join(self._workspace_path, dst_rel_filepath)
open(dst_path, 'wb').write(file_contents.encode('utf-8'))
def chmod(self, rel_path: str, octal_perms: int):
dst_path = os.path.join(self._workspace_path, rel_path)
os.chmod(dst_path, octal_perms)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment