Make the client-side API use the cookie for authentication

zoe_api/rest_api/login.py

@@ -31,7 +31,7 @@ class LoginAPI(ZoeAPIRequestHandler):
         self.set_secure_cookie('zoe', cookie_val)
 
         ret = {
-            'user': self.current_user,
+            'user': self.current_user.serialize(),
         }
 
         self.write(ret)

zoe_api/web/start.py

@@ -64,31 +64,31 @@ class HomeWeb(ZoeWebRequestHandler):
             return
 
         filters = {
-            "user_id": self.current_user,
+            "user_id": self.current_user.id,
             "limit": 5
         }
         last_executions = self.api_endpoint.execution_list(self.current_user, **filters)
 
         filters = {
-            "user_id": self.current_user,
+            "user_id": self.current_user.id,
             "status": "running"
         }
         last_running_executions = self.api_endpoint.execution_list(self.current_user, **filters)
 
         filters = {
-            "user_id": self.current_user,
+            "user_id": self.current_user.id,
             "status": "submitted"
         }
         last_running_executions += self.api_endpoint.execution_list(self.current_user, **filters)
 
         filters = {
-            "user_id": self.current_user,
+            "user_id": self.current_user.id,
             "status": "scheduled"
         }
         last_running_executions += self.api_endpoint.execution_list(self.current_user, **filters)
 
         filters = {
-            "user_id": self.current_user,
+            "user_id": self.current_user.id,
             "status": "starting"
         }
         last_running_executions += self.api_endpoint.execution_list(self.current_user, **filters)

zoe_cmd/api_lib/__init__.py

+# Copyright (c) 2016, Daniele Venzano
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This package contains the Zoe library, with modules used by more than one of the Zoe components. This library can also be used to write new clients for Zoe.
+"""
+
+import requests
+
+from .info import ZoeInfoAPI
+from .validation import ZoeValidationAPI
+from .executions import ZoeExecutionsAPI
+from .services import ZoeServiceAPI
+from .statistics import ZoeStatisticsAPI
+from zoe_lib.version import ZOE_VERSION, ZOE_API_VERSION
+from zoe_lib.exceptions import ZoeAPIException
+
+
+class ZoeAPI:
+    """The main class to use the Zoe REST API client-side."""
+    def __init__(self, url, user, password):
+        self.url = url
+        self.token, self.auth_user = self._login(user, password)
+        self.info = ZoeInfoAPI(url, self.token)
+        self.validation = ZoeValidationAPI(url, self.token)
+        self.executions = ZoeExecutionsAPI(url, self.token)
+        self.services = ZoeServiceAPI(url, self.token)
+        self.statistics = ZoeStatisticsAPI(url, self.token)
+        self._check_api_version()
+
+    def _check_api_version(self):
+        """Checks if there is a version mismatch between server and client."""
+        try:
+            self.info.info()
+            return True
+        except ZoeAPIException:
+            print('Error: this client can talk to ZOE v. {}, but server is reporting an error'.format(ZOE_VERSION,))
+            print('Error: your client is too old (or too new) to speak with the configured server')
+            print('Error: check the version this server is running at the bottom of this web page: {}'.format(self.info.url))
+            return False
+
+    def _login(self, user, password):
+        url = self.url + '/api/' + ZOE_API_VERSION + '/login'
+        try:
+            req = requests.get(url, auth=(user, password))
+        except requests.exceptions.Timeout:
+            raise ZoeAPIException('HTTP connection timeout')
+        except requests.exceptions.HTTPError:
+            raise ZoeAPIException('Invalid HTTP response')
+        except requests.exceptions.ConnectionError as e:
+            raise ZoeAPIException('Connection error: {}'.format(e))
+
+        return req.headers['Set-Cookie'], req.json()['user']

zoe_lib/api_base.py → zoe_cmd/api_lib/api_base.py

+# Copyright (c) 2016, Daniele Venzano
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 """
 This package contains the Zoe library, with modules used by more than one of the Zoe components. This library can also be used to write new clients for Zoe.
 """
@@ -55,10 +69,9 @@ def retry(exception_to_check, tries=4, delay=3, backoff=2):
 
 class ZoeAPIBase:
     """Base class for the Zoe Client API."""
-    def __init__(self, url, user, password):
+    def __init__(self, url, token):
         self.url = url
-        self.user = user
-        self.password = password
+        self.token = token
 
     @retry(ZoeAPIException)
     def _rest_get_stream(self, path):
@@ -68,7 +81,7 @@ class ZoeAPIBase:
         """
         url = self.url + '/api/' + ZOE_API_VERSION + path
         try:
-            req = requests.get(url, auth=(self.user, self.password), stream=True)
+            req = requests.get(url, stream=True, headers={'Cookie': self.token})
         except requests.exceptions.Timeout:
             raise ZoeAPIException('HTTP connection timeout')
         except requests.exceptions.HTTPError:
@@ -87,7 +100,7 @@ class ZoeAPIBase:
         """
         url = self.url + '/api/' + ZOE_API_VERSION + path
         try:
-            req = requests.get(url, auth=(self.user, self.password), params=payload)
+            req = requests.get(url, params=payload, headers={'Cookie': self.token})
         except requests.exceptions.Timeout:
             raise ZoeAPIException('HTTP connection timeout')
         except requests.exceptions.HTTPError:
@@ -109,7 +122,7 @@ class ZoeAPIBase:
         """
         url = self.url + '/api/' + ZOE_API_VERSION + path
         try:
-            req = requests.post(url, auth=(self.user, self.password), json=payload)
+            req = requests.post(url, json=payload, headers={'Cookie': self.token})
         except requests.exceptions.Timeout:
             raise ZoeAPIException('HTTP connection timeout')
         except requests.exceptions.HTTPError:
@@ -131,7 +144,7 @@ class ZoeAPIBase:
         """
         url = self.url + '/api/' + ZOE_API_VERSION + path
         try:
-            req = requests.delete(url, auth=(self.user, self.password))
+            req = requests.delete(url, headers={'Cookie': self.token})
         except requests.exceptions.Timeout:
             raise ZoeAPIException('HTTP connection timeout')
         except requests.exceptions.HTTPError:

zoe_lib/executions.py → zoe_cmd/api_lib/executions.py

@@ -19,7 +19,7 @@ This module contains all execution-related API calls for Zoe clients.
 
 import logging
 
-from zoe_lib.api_base import ZoeAPIBase
+from zoe_cmd.api_lib.api_base import ZoeAPIBase
 from zoe_lib.exceptions import ZoeAPIException
 
 log = logging.getLogger(__name__)

zoe_lib/info.py → zoe_cmd/api_lib/info.py

@@ -19,7 +19,7 @@ This module contains the Zoe Info API.
 
 import logging
 
-from zoe_lib.api_base import ZoeAPIBase
+from zoe_cmd.api_lib.api_base import ZoeAPIBase
 from zoe_lib.exceptions import ZoeAPIException
 
 log = logging.getLogger(__name__)

zoe_lib/services.py → zoe_cmd/api_lib/services.py

@@ -18,7 +18,7 @@ This module contains all service-related API calls that a Zoe client can use.
 """
 import logging
 
-from zoe_lib.api_base import ZoeAPIBase
+from zoe_cmd.api_lib.api_base import ZoeAPIBase
 from zoe_lib.exceptions import ZoeAPIException
 
 log = logging.getLogger(__name__)

zoe_lib/statistics.py → zoe_cmd/api_lib/statistics.py

@@ -19,7 +19,7 @@ This module contains the Zoe Statistics API.
 
 import logging
 
-from zoe_lib.api_base import ZoeAPIBase
+from zoe_cmd.api_lib.api_base import ZoeAPIBase
 from zoe_lib.exceptions import ZoeAPIException
 
 log = logging.getLogger(__name__)

zoe_lib/validation.py → zoe_cmd/api_lib/validation.py

@@ -19,7 +19,7 @@ This module contains the Zoe Info API.
 
 import logging
 
-from zoe_lib.api_base import ZoeAPIBase
+from zoe_cmd.api_lib.api_base import ZoeAPIBase
 
 log = logging.getLogger(__name__)
 

zoe_cmd/entrypoint.py

@@ -29,31 +29,13 @@ from typing import Tuple
 from tabulate import tabulate
 
 from zoe_cmd import utils
-from zoe_lib.info import ZoeInfoAPI
-from zoe_lib.services import ZoeServiceAPI
-from zoe_lib.statistics import ZoeStatisticsAPI
+from zoe_cmd.api_lib import ZoeAPI
 from zoe_lib.exceptions import ZoeAPIException
-from zoe_lib.executions import ZoeExecutionsAPI
-from zoe_lib.version import ZOE_VERSION
 
 
-def _check_api_version(auth):
-    """Checks if there is a version mismatch between server and client."""
-    info_api = ZoeInfoAPI(auth['url'], auth['user'], auth['pass'])
+def _log_stream_stdout(service_id, timestamps, api: ZoeAPI):
     try:
-        info_api.info()
-        return True
-    except ZoeAPIException:
-        print('Error: this client can talk to ZOE v. {}, but server is reporting an error'.format(ZOE_VERSION,))
-        print('Error: your client is too old (or too new) to speak with the configured server')
-        print('Error: check the version this server is running at the bottom of this web page: {}'.format(auth['url']))
-        return False
-
-
-def _log_stream_stdout(service_id, timestamps, auth):
-    service_api = ZoeServiceAPI(auth['url'], auth['user'], auth['pass'])
-    try:
-        for line in service_api.get_logs(service_id):
+        for line in api.services.get_logs(service_id):
             if timestamps:
                 print(line[0], line[1])
             else:
@@ -64,29 +46,27 @@ def _log_stream_stdout(service_id, timestamps, auth):
     return 'stream_end'
 
 
-def info_cmd(auth, args_):
+def info_cmd(api: ZoeAPI, args_):
     """Queries the info endpoint."""
-    info_api = ZoeInfoAPI(auth['url'], auth['user'], auth['pass'])
-    info = info_api.info()
+    info = api.info.info()
     print("Zoe version: ", info['version'])
     print("Zoe API version: ", info['api_version'])
     print("ZApp format version: ", info['application_format_version'])
     print("Deployment name: ", info['deployment_name'])
 
 
-def app_get_cmd(auth, args):
+def app_get_cmd(api: ZoeAPI, args):
     """Extract an application description from an execution."""
-    exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
-    execution = exec_api.get(args.id)
+    execution = api.executions.get(args.id)
     if execution is None:
         print("no such execution")
     else:
         json.dump(execution['description'], sys.stdout, sort_keys=True, indent=4)
 
 
-def exec_list_cmd(auth, args):
+def exec_list_cmd(api: ZoeAPI, args):
     """List executions"""
-    exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
+    print(api.auth_user)
     filter_names = [
         'status',
         'name',
@@ -99,12 +79,12 @@ def exec_list_cmd(auth, args):
         'later_than_end'
     ]
     filters = {
-        'user_id': auth['user']
+        'user_id': api.auth_user['id']
     }
     for key, value in vars(args).items():
         if key in filter_names:
             filters[key] = value
-    data = exec_api.list(**filters)
+    data = api.executions.list(**filters)
     if len(data) == 0:
         return
     tabular_data = [[e['id'], e['name'], e['user_id'], e['status']] for e in sorted(data.values(), key=lambda x: x['id'])]
@@ -112,18 +92,17 @@ def exec_list_cmd(auth, args):
     print(tabulate(tabular_data, headers))
 
 
-def exec_start_cmd(auth, args):
+def exec_start_cmd(api: ZoeAPI, args):
     """Submit an execution."""
     app_descr = json.load(args.jsonfile)
-    exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
-    exec_id = exec_api.start(args.name, app_descr)
+    exec_id = api.executions.start(args.name, app_descr)
     if not args.synchronous:
         print("Application scheduled successfully with ID {}, use the exec-get command to check its status".format(exec_id))
     else:
         print("Application scheduled successfully with ID {}, waiting for status change".format(exec_id))
         old_status = 'submitted'
         while True:
-            execution = exec_api.get(exec_id)
+            execution = api.executions.get(exec_id)
             current_status = execution['status']
             if old_status != current_status:
                 print('Execution is now {}'.format(current_status))
@@ -132,15 +111,14 @@ def exec_start_cmd(auth, args):
                 break
             time.sleep(1)
         monitor_service_id = None
-        service_api = ZoeServiceAPI(auth['url'], auth['user'], auth['pass'])
         for service_id in execution['services']:
-            service = service_api.get(service_id)
+            service = api.services.get(service_id)
             if service['description']['monitor']:
                 monitor_service_id = service['id']
                 break
 
         print('\n>------ start of log streaming -------<\n')
-        why_stop = _log_stream_stdout(monitor_service_id, False, auth)
+        why_stop = _log_stream_stdout(monitor_service_id, False, api)
         print('\n>------ end of log streaming -------<\n')
         if why_stop == 'stream_end':
             print('Execution finished')
@@ -150,11 +128,9 @@ def exec_start_cmd(auth, args):
             exit(1)
 
 
-def exec_get_cmd(auth, args):
+def exec_get_cmd(api: ZoeAPI, args):
     """Gather information about an execution."""
-    exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
-    cont_api = ZoeServiceAPI(auth['url'], auth['user'], auth['pass'])
-    execution = exec_api.get(args.id)
+    execution = api.executions.get(args.id)
     if execution is None:
         print('Execution not found')
     else:
@@ -177,7 +153,7 @@ def exec_get_cmd(auth, args):
             print('Time end: {}'.format(datetime.fromtimestamp(execution['time_end'], timezone.utc).astimezone()))
         print()
 
-        endpoints = exec_api.endpoints(execution['id'])
+        endpoints = api.executions.endpoints(execution['id'])
         if endpoints is not None and len(endpoints) > 0:
             print('Exposed endpoints:')
             for endpoint in endpoints:
@@ -188,38 +164,37 @@ def exec_get_cmd(auth, args):
         print()
         tabular_data = []
         for c_id in execution['services']:
-            service = cont_api.get(c_id)
+            service = api.services.get(c_id)
             service_data = [service['id'], service['name'], 'true' if service['essential'] else 'false', service['status'], service['backend_status'], service['error_message'] if service['error_message'] is not None else '']
             tabular_data.append(service_data)
         headers = ['ID', 'Name', 'Essential', 'Zoe status', 'Backend status', 'Error message']
         print(tabulate(tabular_data, headers))
 
 
-def exec_kill_cmd(auth, args):
+def exec_kill_cmd(api: ZoeAPI, args):
     """Kill an execution."""
-    exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
-    exec_api.terminate(args.id)
+    api.executions.terminate(args.id)
 
 
-def logs_cmd(auth, args):
+def logs_cmd(api: ZoeAPI, args):
     """Retrieves and streams the logs of a service."""
-    _log_stream_stdout(args.service_id, args.timestamps, auth)
+    _log_stream_stdout(args.service_id, args.timestamps, api)
 
 
-def stats_cmd(auth, args_):
+def stats_cmd(api: ZoeAPI, args_):
     """Prints statistics on Zoe internals."""
-    stats_api = ZoeStatisticsAPI(auth['url'], auth['user'], auth['pass'])
-    sched = stats_api.scheduler()
+    sched = api.statistics.scheduler()
     print('Scheduler queue length: {}'.format(sched['queue_length']))
     print('Scheduler running queue length: {}'.format(sched['running_length']))
     print('Termination threads count: {}'.format(sched['termination_threads_count']))
 
+
 ENV_HELP_TEXT = '''To authenticate with Zoe you need to define three environment variables:
 ZOE_URL: point to the URL of the Zoe Scheduler (ex.: http://localhost:5000/
 ZOE_USER: the username used for authentication
 ZOE_PASS: the password used for authentication
 
-or create a ~/.zoerc file (another location can be specified with --auth-file) like this:
+or create a ~/.zoerc file (another location can be specified with --api-file) like this:
 url = xxx
 user = yyy
 pass = zzz
@@ -299,10 +274,8 @@ def zoe():
         sys.exit(1)
 
     try:
-        ret = _check_api_version(auth)
-        if not ret:
-            sys.exit(0)
-        args.func(auth, args)
+        api = ZoeAPI(auth['url'], auth['user'], auth['pass'])
+        args.func(api, args)
     except ZoeAPIException as e:
         print(e.message)
     except KeyboardInterrupt:

zoe_cmd/entrypoint_admin.py

@@ -28,25 +28,22 @@ from typing import Tuple
 from tabulate import tabulate
 
 from zoe_cmd import utils
-from zoe_lib.info import ZoeInfoAPI
+from zoe_cmd.api_lib import ZoeAPI
 from zoe_lib.exceptions import ZoeAPIException, InvalidApplicationDescription
-from zoe_lib.executions import ZoeExecutionsAPI
-from zoe_lib.services import ZoeServiceAPI
 from zoe_lib.applications import app_validate
 from zoe_lib.version import ZOE_API_VERSION
 
 
-def _check_api_version(auth):
+def _check_api_version(api: ZoeAPI):
     """Checks if there is a version mismatch between server and client."""
-    info_api = ZoeInfoAPI(auth['url'], auth['user'], auth['pass'])
-    info = info_api.info()
+    info = api.info.info()
     if info['api_version'] != ZOE_API_VERSION:
         print('Warning: this client understands ZOE API v. {}, but server talks v. {}'.format(ZOE_API_VERSION, info['api_version']))
         print('Warning: certain commands may not work correctly')
         print('Warning: please upgrade or downgrade your client to match the server version')
 
 
-def app_validate_cmd(auth_, args):
+def app_validate_cmd(api_: ZoeAPI, args):
     """Validate an application description."""
     app_descr = json.load(args.jsonfile)
     try:
@@ -57,9 +54,8 @@ def app_validate_cmd(auth_, args):
         print("Static validation OK")
 
 
-def exec_list_cmd(auth, args):
+def exec_list_cmd(api: ZoeAPI, args):
     """List executions"""
-    exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
     filter_names = [
         'status',
         'name',
@@ -76,7 +72,7 @@ def exec_list_cmd(auth, args):
     for key, value in vars(args).items():
         if key in filter_names:
             filters[key] = value
-    data = exec_api.list(**filters)
+    data = api.executions.list(**filters)
     if len(data) == 0:
         return
     tabular_data = [[e['id'], e['name'], e['user_id'], e['status']] for e in sorted(data.values(), key=lambda x: x['id'])]
@@ -84,11 +80,9 @@ def exec_list_cmd(auth, args):
     print(tabulate(tabular_data, headers))
 
 
-def exec_get_cmd(auth, args):
+def exec_get_cmd(api: ZoeAPI, args):
     """Gather information about an execution."""
-    exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
-    cont_api = ZoeServiceAPI(auth['url'], auth['user'], auth['pass'])
-    execution = exec_api.get(args.id)
+    execution = api.executions.get(args.id)
     if execution is None:
         print('Execution not found')
     else:
@@ -111,7 +105,7 @@ def exec_get_cmd(auth, args):
             print('Time end: {}'.format(datetime.fromtimestamp(execution['time_end'], timezone.utc).astimezone()))
         print()
 
-        endpoints = exec_api.endpoints(execution['id'])
+        endpoints = api.executions.endpoints(execution['id'])
         if endpoints is not None and len(endpoints) > 0:
             print('Exposed endpoints:')
             for endpoint in endpoints:
@@ -122,30 +116,28 @@ def exec_get_cmd(auth, args):
         print()
         tabular_data = []
         for c_id in execution['services']:
-            service = cont_api.get(c_id)
+            service = api.services.get(c_id)
             service_data = [service['id'], service['name'], 'true' if service['essential'] else 'false', service['status'], service['backend_status'], service['backend_host'], service['error_message'] if service['error_message'] is not None else '']
             tabular_data.append(service_data)
         headers = ['ID', 'Name', 'Essential', 'Zoe status', 'Backend status', 'Host', 'Error message']
         print(tabulate(tabular_data, headers))
 
 
-def exec_rm_cmd(auth, args):
+def exec_rm_cmd(api: ZoeAPI, args):
     """Delete an execution and kill it if necessary."""
-    exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
-    exec_api.delete(args.id)
+    api.executions.delete(args.id)
 
 
-def exec_kill_user_cmd(auth, args):
+def exec_kill_user_cmd(api: ZoeAPI, args):
     """Terminates all executions for the given user."""
-    exec_api = ZoeExecutionsAPI(auth['url'], auth['user'], auth['pass'])
     filters = {
         'status': 'running',
         'user_id': args.user_id
     }
-    data = exec_api.list(**filters)
+    data = api.executions.list(**filters)
     print('Terminating {} executions belonging to user {}'.format(len(data), args.user_id))
     for execution in data:
-        exec_api.terminate(execution)
+        api.executions.terminate(execution)
         print('Execution {} terminated'.format(execution))
 
 
@@ -223,8 +215,8 @@ def zoe():
         sys.exit(1)
 
     try:
-        _check_api_version(auth)
-        args.func(auth, args)
+        api = ZoeAPI(auth['url'], auth['user'], auth['pass'])
+        args.func(api, args)
     except ZoeAPIException as e:
         print(e.message)
     except KeyboardInterrupt:

zoe_lib/state/execution.py

@@ -20,6 +20,8 @@ import logging
 import threading
 import functools
 
+import psycopg2
+
 from zoe_lib.state.base import BaseRecord, BaseTable
 
 log = logging.getLogger(__name__)
@@ -299,7 +301,15 @@ class ExecutionTable(BaseTable):
                 q_base += ' ORDER BY id DESC LIMIT {} OFFSET {}'.format(limit, base)
             query = self.cursor.mogrify(q_base)
 
-        self.cursor.execute(query)
+        try:
+            self.cursor.execute(query)
+        except psycopg2.Error as e:
+            log.error('db error: {}'.format(e))
+            if only_one:
+                return None
+            else:
+                return []
+
         if only_one:
             row = self.cursor.fetchone()
             if row is None:
@@ -341,6 +351,11 @@ class ExecutionTable(BaseTable):
         else:
             query = self.cursor.mogrify(q_base)
 
-        self.cursor.execute(query)
+        try:
+            self.cursor.execute(query)
+        except psycopg2.Error as e:
+            log.error('db error: {}'.format(e))
+            return 0
+
         row = self.cursor.fetchone()
         return row[0]

zoe_lib/state/quota.py

@@ -102,9 +102,9 @@ class QuotaTable(BaseTable):
             row = self.cursor.fetchone()
             if row is None:
                 return None
-            return Quota(row, self)
+            return Quota(row, self.sql_manager)
         else:
-            return [Quota(x, self) for x in self.cursor]
+            return [Quota(x, self.sql_manager) for x in self.cursor]
 
     def update(self, record_id, **kwargs):