Commit d1fbc62d authored by Daniele Venzano's avatar Daniele Venzano

Move the Object Storage server into its own repository

Update install docs
Start using the configargparse module to have options settable via environment
parent 25f8226f
This diff is collapsed.
......@@ -32,11 +32,6 @@ defaults = {
'ddns_keyfile': '/path/to/rndc.key',
'ddns_server': '127.0.0.1',
'ddns_domain': 'swarm.example.com'
},
'zoe_storage': {
'storage_path': "/var/lib/zoe/history",
'http_listen_address': '127.0.0.1',
'http_listen_port': 4390,
}
}
......@@ -117,18 +112,6 @@ class ZoeConfig(ConfigParser):
def ipc_listen_address(self) -> str:
return self.get('zoe_scheduler', 'ipc_listen_address')
@property
def storage_path(self) -> str:
return self.get('zoe_storage', 'storage_path')
@property
def http_listen_port(self) -> int:
return self.getint('zoe_storage', 'http_listen_port')
@property
def http_listen_address(self) -> str:
return self.get('zoe_storage', 'http_listen_address')
@property
def ddns_keyfile(self) -> str:
return self.get('zoe_scheduler', 'ddns_keyfile')
......
Installing Zoe
==============
Zoe install procedure is migrating from PIP to Docker containers. The codebase is being split in several components, each will live in its own repository with a Dockerfile
to build the container image.
Requirements
------------
* An SQL database to keep all the state
* An SQL database to keep all the state (sqlite is used by default)
* Docker Swarm
* A DNS server for service discovery, with DDNS support
......@@ -81,3 +84,12 @@ For developers, we recommend the following procedure:
6. Start running applications! By default Zoe web listens on the 5000 port
Zoe Object Storage
^^^^^^^^^^^^^^^^^^
Application binaries and execution logs are saved in a simple Object Storage server.
* Clone it from git: https://github.com/DistributedSystemsGroup/zoe-object-storage
* Use the Dockerfile to build a Docker image
* Run it
* Put the IP address of the container in Zoe's main configuration file (when the transition to Dockerfiles will be finished it will be possible to use linking instead)
\ No newline at end of file
#!/usr/bin/env python3
# This script is useful to run Zoe without going through the pip install process when developing
from zoe_storage_server.entrypoint import object_server
if __name__ == '__main__':
object_server()
# Copyright (c) 2015, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from argparse import ArgumentParser, Namespace
import logging
import http.server
from common.configuration import conf_init, zoe_conf
from zoe_storage_server.http_handler import ZoeObjectStoreHTTPRequestHandler
log = logging.getLogger(__name__)
def process_arguments() -> Namespace:
argparser = ArgumentParser(description="Zoe Storage - Container Analytics as a Service storage component")
argparser.add_argument('-d', '--debug', action='store_true', help='Enable debug output')
return argparser.parse_args()
def object_server():
args = process_arguments()
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
logging.getLogger('requests').setLevel(logging.WARNING)
conf_init()
log.info("Object server listening on %s:%d" % (zoe_conf().http_listen_address, zoe_conf().http_listen_port))
httpd = http.server.HTTPServer((zoe_conf().http_listen_address, zoe_conf().http_listen_port), ZoeObjectStoreHTTPRequestHandler)
httpd.timeout = 1
httpd.allow_reuse_address = True
httpd.serve_forever()
# Copyright (c) 2015, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import cgi
import http.server
from zoe_storage_server.object_storage import ZoePersistentObjectStore
from common.configuration import zoe_conf
class ZoeObjectStoreHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
"""
This class extends the BaseHTTPRequestHandler from the Python library to provide an HTTP interface to the ZoePersistentObjectStore.
"""
def _read_request(self):
zos = ZoePersistentObjectStore(zoe_conf().storage_path)
req = self.path.split("/")[1:]
try:
obj_type = req[0]
obj_id = int(req[1])
except ValueError:
self.send_error(400, "ID should be a number")
return None
except IndexError:
self.send_error(400, "Malformed URL")
return None
if obj_type == "apps":
fdata = zos.application_data_download(obj_id)
elif obj_type == "logs":
fdata = zos.logs_archive_download(obj_id)
else:
self.send_error(400, "Unknown object type")
return None
if fdata is None:
self.send_error(404, "Object not found")
return None
self.send_response(200)
self.send_header("Content-type", "application/zip")
self.send_header("Content-Length", str(len(fdata)))
self.end_headers()
return fdata
def do_HEAD(self):
self._read_request()
def do_GET(self):
fdata = self._read_request()
if fdata is not None:
self.wfile.write(fdata)
def do_POST(self):
ctype, pdict = cgi.parse_header(self.headers.get_content_type())
if ctype == 'multipart/form-data':
fs = cgi.FieldStorage(fp=self.rfile, headers=self.headers, environ={'REQUEST_METHOD': 'POST'})
else:
self.send_error(400, "Expected form-data POST request")
return
zos = ZoePersistentObjectStore(zoe_conf().storage_path)
req = self.path.split("/")[1:]
try:
obj_type = req[0]
obj_id = int(req[1])
except ValueError:
self.send_error(400, "ID should be a number")
return None
except IndexError:
self.send_error(400, "Malformed URL")
return None
if obj_type == "apps":
zos.application_data_upload(obj_id, fs['file'].file)
elif obj_type == "logs":
zos.logs_archive_upload(obj_id, fs['file'].file)
else:
self.send_error(400, "Unknown object type")
return None
self.send_response(200, "Object saved")
self.end_headers()
def do_DELETE(self):
zos = ZoePersistentObjectStore(zoe_conf().storage_path)
req = self.path.split("/")[1:]
try:
obj_type = req[0]
obj_id = int(req[1])
except ValueError:
self.send_error(400, "ID should be a number")
return None
except IndexError:
self.send_error(400, "Malformed URL")
return None
if obj_type == "apps":
zos.application_data_delete(obj_id)
elif obj_type == "logs":
zos.logs_archive_delete(obj_id)
else:
self.send_error(400, "Unknown object type")
return None
self.send_response(200, "Object saved")
self.end_headers()
# Copyright (c) 2015, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from io import BufferedIOBase
import logging
import os
from shutil import copyfileobj
log = logging.getLogger(__name__)
class ZoePersistentObjectStore:
def __init__(self, storage_path):
self.base_path = storage_path
self.objects_path = os.path.join(self.base_path, 'objects')
self._init_paths()
def _init_paths(self):
if not os.path.exists(self.base_path):
log.error("Storage path does not exist: {}".format(self.base_path))
if not os.path.exists(self.objects_path):
os.makedirs(self.objects_path)
def _write(self, key: str, data: BufferedIOBase):
fpath = os.path.join(self.objects_path, key)
with open(fpath, "wb") as fp:
copyfileobj(data, fp)
def _read(self, key: str):
fpath = os.path.join(self.objects_path, key)
try:
data = open(fpath, "rb").read()
except OSError:
return None
else:
return data
def _delete(self, key: str):
fpath = os.path.join(self.objects_path, key)
try:
os.unlink(fpath)
except OSError:
log.warning("cannot delete object {}".format(key))
def application_data_upload(self, application_id: int, data: BufferedIOBase):
key = 'app-{}.zip'.format(application_id)
self._write(key, data)
def application_data_download(self, application_id: int) -> bytes:
key = 'app-{}.zip'.format(application_id)
return self._read(key)
def application_data_delete(self, application_id: int):
key = 'app-{}.zip'.format(application_id)
self._delete(key)
def logs_archive_upload(self, execution_id: int, data: BufferedIOBase):
key = 'log-{}.zip'.format(execution_id)
self._write(key, data)
def logs_archive_download(self, execution_id: int) -> bytes:
key = 'log-{}.zip'.format(execution_id)
return self._read(key)
def logs_archive_delete(self, execution_id: int):
key = 'log-{}.zip'.format(execution_id)
self._delete(key)
# Copyright (c) 2015, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import pytest
from common.configuration import conf_init
@pytest.fixture(scope='session')
def configuration(request):
env = request.config.getoption("--test-environment")
if env == 'local':
return conf_init('tests/resources/zoe-local.conf')
else:
return conf_init('tests/resources/zoe-travis.conf')
@pytest.fixture(scope='session')
def notebook_test():
jsondata = open("tests/resources/spark-notebook-test.json", "r")
dictdata = json.load(jsondata)
return dictdata
# Copyright (c) 2015, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from io import BytesIO
import os
from zoe_storage_server.object_storage import ZoePersistentObjectStore
def test_application_data_upload(configuration):
fake_data = BytesIO(b"test" * 1024)
zos = ZoePersistentObjectStore(configuration.storage_path)
filepath = os.path.join(zos.objects_path, "app-1.zip")
zos.application_data_upload(1, fake_data)
assert os.path.exists(filepath)
data = zos.application_data_download(1)
assert data == fake_data.getvalue()
zos.application_data_delete(1)
assert not os.path.exists(filepath)
def test_logs_upload(configuration):
fake_data = BytesIO(b"test" * 1024)
zos = ZoePersistentObjectStore(configuration.storage_path)
filepath = os.path.join(zos.objects_path, "log-1.zip")
zos.logs_archive_upload(1, fake_data)
assert os.path.exists(filepath)
data = zos.logs_archive_download(1)
assert data == fake_data.getvalue()
zos.logs_archive_delete(1)
assert not os.path.exists(filepath)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment