From 37da97249aef33e5c6bbe697f8698d51d8e4e0ef Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Fri, 29 May 2020 08:16:44 +0200 Subject: [PATCH 01/13] - resource wrappers: module, data, infra - refactored deployment, added data bits - added client-server chit-chat printout - other refactors --- code/requirements.txt | 1 + code/src/nuvla/api/api.py | 122 ++++++-- code/src/nuvla/api/models.py | 6 +- code/src/nuvla/api/resources/__init__.py | 80 +++++ code/src/nuvla/api/resources/callback.py | 11 +- code/src/nuvla/api/resources/credential.py | 38 +-- code/src/nuvla/api/resources/data.py | 103 +++++++ code/src/nuvla/api/resources/deployment.py | 285 ++++++++++++++---- code/src/nuvla/api/resources/infra_service.py | 167 ++++++++++ code/src/nuvla/api/resources/module.py | 243 ++++++++++++++- code/src/nuvla/api/resources/notification.py | 10 +- code/src/nuvla/api/resources/user.py | 41 +++ code/src/nuvla/api/resources/utils.py | 28 -- code/tests/deployment_tests.py | 26 ++ code/tests/module_tests.py | 90 ++++++ 15 files changed, 1100 insertions(+), 151 deletions(-) create mode 100644 code/src/nuvla/api/resources/data.py create mode 100644 code/src/nuvla/api/resources/infra_service.py create mode 100644 code/src/nuvla/api/resources/user.py delete mode 100644 code/src/nuvla/api/resources/utils.py create mode 100644 code/tests/module_tests.py diff --git a/code/requirements.txt b/code/requirements.txt index f229360..33f05f7 100644 --- a/code/requirements.txt +++ b/code/requirements.txt @@ -1 +1,2 @@ requests +PyYAML diff --git a/code/src/nuvla/api/api.py b/code/src/nuvla/api/api.py index 44a1364..328bfcb 100644 --- a/code/src/nuvla/api/api.py +++ b/code/src/nuvla/api/api.py @@ -54,11 +54,11 @@ """ -from __future__ import absolute_import - import logging import os import stat +from typing import Optional +from pprint import pp import requests from requests.cookies import MockRequest @@ -66,7 +66,7 @@ from http.cookiejar import MozillaCookieJar from urllib.parse import urlparse -from . import models +from .models import CimiResource, CimiCollection, CimiResponse, CloudEntryPoint logger = logging.getLogger(__name__) @@ -75,25 +75,70 @@ DEFAULT_COOKIE_FILE = os.path.expanduser('~/.nuvla/cookies.txt') HREF_SESSION_TMPL_PASSWORD = 'session-template/password' HREF_SESSION_TMPL_APIKEY = 'session-template/api-key' +CLOUD_ENTRY_POINT_ID = 'cloud-entry-point' class NuvlaError(Exception): - def __init__(self, reason, response=None): + def __init__(self, reason, response: Optional[requests.Response] = None): super(NuvlaError, self).__init__(reason) self.reason = reason self.response = response +class NuvlaResourceOperationNotAvailable(KeyError): + def __init__(self, reason, operation): + super().__init__(reason) + self.operation = operation + + +def _request_debug(method, endpoint, params=None, json=None, + data=None, headers: Optional[dict]=None): + print('::: HTTP Request -> Response :::') + print('>>> Request') + url_res = urlparse(endpoint) + print('{0} {1}'.format(method, url_res.path)) + print('Host: {}'.format(url_res.netloc)) + for k, v in headers.items(): + print('{0}: {1}'.format(k, v)) + if any([params, json, data]): + print() + if params: + print('params:', params) + if json: + pp(json) + if data: + dlen = len(data) + for k, v in data.items(): + print('{0}={1}'.format(k, v)) + dlen -= 1 + if dlen > 0: + print('&') + print('<<< Request') + + +def _response_debug(response: requests.Response): + print('>>> Response') + print(response.status_code, response.reason) + for k, v in response.headers.items(): + print('{0}: {1}'.format(k, v)) + print() + print(response.text) + # pp(response.json()) + print('<<< Response') + + class SessionStore(requests.Session): """A ``requests.Session`` subclass implementing a file-based session store.""" - def __init__(self, endpoint, persist_cookie, cookie_file, reauthenticate, login_params, authn_header=None): + def __init__(self, endpoint, persist_cookie, cookie_file, reauthenticate, + login_params, authn_header=None, debug=False): super(SessionStore, self).__init__() self.session_base_url = '{0}/api/session'.format(endpoint) self.reauthenticate = reauthenticate self.persist_cookie = persist_cookie self.login_params = login_params self.authn_header = authn_header + self._debug = debug if persist_cookie: if cookie_file is None: cookie_file = DEFAULT_COOKIE_FILE @@ -138,10 +183,17 @@ def request(self, *args, **kwargs): def cimi_login(self, login_params): self.login_params = login_params if self.login_params: - return self.request('POST', self.session_base_url, - headers={'Content-Type': 'application/json', - 'Accept': 'application/json'}, - json={'template': login_params}) + method = 'POST' + endpoint = self.session_base_url + headers = {'Content-Type': 'application/json', + 'Accept': 'application/json'} + json = {'template': login_params} + if self._debug: + _request_debug(method, endpoint, json=json, headers=headers) + response = self.request(method, endpoint, headers=headers, json=json) + if self._debug: + _response_debug(response) + return response else: return None @@ -182,7 +234,7 @@ class Api(object): """ This class is a Python wrapper&helper of the native Nuvla REST API""" def __init__(self, endpoint=DEFAULT_ENDPOINT, insecure=False, persist_cookie=True, cookie_file=None, - reauthenticate=False, login_creds=None, authn_header=None): + reauthenticate=False, login_creds=None, authn_header=None, debug=False): """ :param endpoint: Nuvla endpoint (https://nuvla.io). :param insecure: Don't check server certificate or you are using a http connection. @@ -196,7 +248,8 @@ def __init__(self, endpoint=DEFAULT_ENDPOINT, insecure=False, persist_cookie=Tru self.endpoint = endpoint self.session = SessionStore(endpoint, persist_cookie, cookie_file, reauthenticate, login_params=to_login_params(login_creds), - authn_header=authn_header) + authn_header=authn_header, + debug=debug) self.session.verify = not insecure if insecure: try: @@ -208,6 +261,7 @@ def __init__(self, endpoint=DEFAULT_ENDPOINT, insecure=False, persist_cookie=Tru urllib3.exceptions.InsecureRequestWarning) self._username = None self._cimi_cloud_entry_point = None + self._debug = debug def login(self, login_params): """Uses given 'login_params' to log into the Nuvla server. The @@ -242,7 +296,7 @@ def login_internal(self, username, password): :param password: :return: see login() """ - logging.warn('Deprecated! Use instead login_password') + logging.warning('Deprecated! Use instead login_password') return self.login(to_login_params({'username': username, 'password': password})) @@ -294,8 +348,8 @@ def is_authenticated(self): return self.current_session() is not None def _cimi_get_cloud_entry_point(self): - cep_json = self._cimi_get('cloud-entry-point') - return models.CloudEntryPoint(cep_json) + cep_json = self._cimi_get(CLOUD_ENTRY_POINT_ID) + return CloudEntryPoint(cep_json) @property def cloud_entry_point(self): @@ -304,11 +358,12 @@ def cloud_entry_point(self): return self._cimi_cloud_entry_point @staticmethod - def _cimi_find_operation_href(cimi_resource, operation): + def _cimi_find_operation_href(cimi_resource: CimiResource, operation: str): operation_href = cimi_resource.operations.get(operation, {}).get('href') if not operation_href: - raise KeyError("Operation '{0}' not found.".format(operation)) + raise NuvlaResourceOperationNotAvailable( + "Operation '{0}' not found.".format(operation), operation) return operation_href @@ -332,12 +387,17 @@ def _cimi_request(self, method, uri, params=None, json=None, data=None, headers= headers.update(json_header) else: headers = json_header - response = self.session.request(method, '{0}/{1}/{2}'.format(self.endpoint, 'api', uri), + endpoint = '{0}/{1}/{2}'.format(self.endpoint, 'api', uri) + if self._debug and uri != CLOUD_ENTRY_POINT_ID: + _request_debug(method, endpoint, params, json, data, headers) + response = self.session.request(method, endpoint, headers=headers, allow_redirects=False, params=params, json=json, data=data) + if self._debug and uri != CLOUD_ENTRY_POINT_ID: + _response_debug(response) try: response.raise_for_status() except HTTPError as e: @@ -371,7 +431,7 @@ def _cimi_put(self, resource_id=None, resource_type=None, params=None, json=None def _cimi_delete(self, resource_id=None): return self._cimi_request('DELETE', resource_id) - def get(self, resource_id, **kwargs): + def get(self, resource_id, **kwargs) -> CimiResource: """ Retreive a CIMI resource by it's resource id :param resource_id: The id of the resource to retrieve @@ -384,9 +444,9 @@ def get(self, resource_id, **kwargs): :rtype: CimiResource """ resp_json = self._cimi_get(resource_id=resource_id, params=kwargs) - return models.CimiResource(resp_json) + return CimiResource(resp_json) - def edit(self, resource_id, data, **kwargs): + def edit(self, resource_id, data, **kwargs) -> CimiResource: """ Edit a CIMI resource by it's resource id :param resource_id: The id of the resource to edit @@ -400,13 +460,13 @@ def edit(self, resource_id, data, **kwargs): :type select: str or list of str :return: A CimiResponse object which should contain the attributes 'status', 'resource-id' and 'message' - :rtype: CimiResponse + :rtype: CimiResource """ resource = self.get(resource_id=resource_id) operation_href = self._cimi_find_operation_href(resource, 'edit') - return models.CimiResponse(self._cimi_put(resource_id=operation_href, json=data, params=kwargs)) + return CimiResource(self._cimi_put(resource_id=operation_href, json=data, params=kwargs)) - def delete(self, resource_id): + def delete(self, resource_id) -> CimiResponse: """ Delete a CIMI resource by it's resource id :param resource_id: The id of the resource to delete @@ -418,9 +478,9 @@ def delete(self, resource_id): """ resource = self.get(resource_id=resource_id) operation_href = self._cimi_find_operation_href(resource, 'delete') - return models.CimiResponse(self._cimi_delete(resource_id=operation_href)) + return CimiResponse(self._cimi_delete(resource_id=operation_href)) - def delete_bulk(self, resource_type, filter, **kwargs): + def delete_bulk(self, resource_type, filter, **kwargs) -> CimiResponse: """ Bulk delete CIMI resources of the given type (Collection). :param resource_type: Type of the resource (Collection name). @@ -435,7 +495,7 @@ def delete_bulk(self, resource_type, filter, **kwargs): if not isinstance(filter, str) or len(filter) == 0: raise NuvlaError("'filter' must be a non-empty string.") else: - return models.CimiResponse( + return CimiResponse( self._cimi_request('DELETE', resource_type, data={'filter': filter}, headers={'bulk': 'yes'})) @@ -453,9 +513,9 @@ def add(self, resource_type, data): """ collection = self.search(resource_type=resource_type, last=0) operation_href = self._cimi_find_operation_href(collection, 'add') - return models.CimiResponse(self._cimi_post(resource_id=operation_href, json=data)) + return CimiResponse(self._cimi_post(resource_id=operation_href, json=data)) - def search(self, resource_type, **kwargs): + def search(self, resource_type, **kwargs) -> CimiCollection: """ Search for CIMI resources of the given type (Collection). :param resource_type: Type of the resource (Collection name) @@ -487,9 +547,9 @@ def search(self, resource_type, **kwargs): :rtype: CimiCollection """ resp_json = self._cimi_put(resource_type=resource_type, data=kwargs) - return models.CimiCollection(resp_json) + return CimiCollection(resp_json) - def operation(self, resource, operation, data=None): + def operation(self, resource: CimiResource, operation, data=None) -> CimiResponse: """ Execute an operation on a CIMI resource :param resource: The resource to execute operation on. @@ -506,4 +566,4 @@ def operation(self, resource, operation, data=None): """ operation_href = self._cimi_find_operation_href(resource, operation) resp_json = self._cimi_post(operation_href, json=data) - return models.CimiResource(resp_json) + return CimiResponse(resp_json) diff --git a/code/src/nuvla/api/models.py b/code/src/nuvla/api/models.py index 4023a01..8894e4e 100644 --- a/code/src/nuvla/api/models.py +++ b/code/src/nuvla/api/models.py @@ -23,7 +23,7 @@ def __str__(self): class CimiResource(CimiResponse): def __init__(self, data): - super(CimiResource, self).__init__(data) + super().__init__(data) self.id = data.get('id') self.resource_type = data.get('resource-type') self.__operations = None @@ -38,7 +38,7 @@ def operations(self): class CimiCollection(CimiResource): def __init__(self, data): - super(CimiCollection, self).__init__(data) + super().__init__(data) self.count = self.data.get('count') self.__lock_iter = Lock() self.__lock_list = Lock() @@ -69,7 +69,7 @@ def __iter__(self): class CloudEntryPoint(CimiResource): def __init__(self, data): - super(CloudEntryPoint, self).__init__(data) + super().__init__(data) self.collections = self._extract_entry_points() self.base_uri = self.data['base-uri'] diff --git a/code/src/nuvla/api/resources/__init__.py b/code/src/nuvla/api/resources/__init__.py index a01a0a3..e11e463 100644 --- a/code/src/nuvla/api/resources/__init__.py +++ b/code/src/nuvla/api/resources/__init__.py @@ -1,6 +1,86 @@ + +from typing import Union + +from nuvla.api import Api as Nuvla +from nuvla.api.models import CimiResource + from .callback import Callback from .credential import Credential from .deployment import Deployment from .deployment_parameter import DeploymentParameter from .module import Module from .notification import Notification + + +class ResourceNotFound(Exception): + pass + + +class ResourceCreateError(Exception): + def __init__(self, reason, response=None): + super(ResourceCreateError, self).__init__(reason) + self.reason = reason + self.response = response + + +def check_created(resp, errmsg=''): + """ + Returns id of the created resource or raises ResourceCreateError. + :param resp: nuvla.api.models.CimiResponse + :param errmsg: error message + :return: str, resource id + """ + if resp.data['status'] == 201: + return resp.data['resource-id'] + else: + if errmsg: + msg = '{0} : {1}'.format(errmsg, resp.data['message']) + else: + msg = resp.data['message'] + raise ResourceCreateError(msg, resp) + + +class ResourceBase: + + resource = None + + @staticmethod + def id(resource: Union[dict, CimiResource]): + key = 'id' + if isinstance(resource, dict): + return resource[key] + else: + return resource.data[key] + + def __init__(self, nuvla: Nuvla): + self.nuvla = nuvla + if not self.resource: + raise Exception('resource type not set.') + + def add(self, data: dict) -> str: + """Creates resource of type `self.resource` using provided `data`. + Returns created resource ID. + """ + return check_created(self.nuvla.add(self.resource, data), + f'Failed to create {self.resource}.') + + def get(self, resource_id: str) -> dict: + """Returns document identified by `resource_id`. + """ + return self.nuvla.get(resource_id).data + + def delete(self, resource_id: str) -> str: + """Deletes resource identified by `resource_id`. Returns resource ID. + """ + response = self.nuvla.delete(resource_id) + return response.data['resource-id'] + + def id_by_name(self, name, filter=None) -> list: + """Returns list of resource IDs with name equals to `name`. In most + cases the length of the list will be 0 or 1. + """ + fltr = f"name='{name}'" + if filter: + fltr += f' and {filter}' + res = self.nuvla.search(self.resource, filter=fltr) + return [r.data['id'] for r in res.resources] diff --git a/code/src/nuvla/api/resources/callback.py b/code/src/nuvla/api/resources/callback.py index 61b4e6f..ff03bd4 100644 --- a/code/src/nuvla/api/resources/callback.py +++ b/code/src/nuvla/api/resources/callback.py @@ -1,13 +1,10 @@ -from .utils import check_created +from . import ResourceBase -class Callback: +class Callback(ResourceBase): resource = 'callback' - def __init__(self, nuvla): - self.nuvla = nuvla - def create(self, action_name, target_resource, data=None, expires=None, acl=None): """ :param action_name: name of the action @@ -28,6 +25,4 @@ def create(self, action_name, target_resource, data=None, expires=None, acl=None if acl: callback.update({"acl": acl}) - resource_id = check_created(self.nuvla.add(self.resource, callback), - 'Failed to create callback.') - return resource_id + return self.add(callback) diff --git a/code/src/nuvla/api/resources/credential.py b/code/src/nuvla/api/resources/credential.py index 78e24e3..ed9c5e4 100644 --- a/code/src/nuvla/api/resources/credential.py +++ b/code/src/nuvla/api/resources/credential.py @@ -1,27 +1,29 @@ -class Credential(object): +from . import ResourceBase - @staticmethod - def id(credential): - return credential['id'] - def __init__(self, nuvla): - self.nuvla = nuvla +class Credential(ResourceBase): + + resource = 'credential' def find_parent(self, parent): filters = "parent='{}'".format(parent) - creds = self.nuvla.search('credential', filter=filters, select="id") + creds = self.nuvla.search(self.resource, filter=filters, select="id") return creds.resources - def find(self, infra_service_id): - """ - Returns list of credentials for `infra_service_id` infrastructure service. - :param infra_service_id: str, URI - :return: list - """ - return self.find_parent(infra_service_id) - - def delete(self, resource_id): - return self.nuvla.delete(resource_id) - +class CredentialK8s(Credential): + + def create(self, ca, cert, key, infra_service_id, name, description=None): + iscred_k8s = { + "name": name, + "description": description or name, + "template": { + "href": "credential-template/infrastructure-service-swarm", + "parent": infra_service_id, + "ca": ca, + "cert": cert, + "key": key + } + } + return self.add(iscred_k8s) diff --git a/code/src/nuvla/api/resources/data.py b/code/src/nuvla/api/resources/data.py new file mode 100644 index 0000000..b4df674 --- /dev/null +++ b/code/src/nuvla/api/resources/data.py @@ -0,0 +1,103 @@ +import os +import requests +from typing import Optional, Union + +from nuvla.api import Api as Nuvla +from . import ResourceBase + + +class DataRecord(ResourceBase): + resource = 'data-record' + + def __init__(self, nuvla: Nuvla): + super().__init__(nuvla) + + +class DataObjectS3(ResourceBase): + resource = 'data-object' + + def __init__(self, nuvla: Nuvla): + super().__init__(nuvla) + + def add(self, content: Union[str, bytes], bucket, object_path, s3_cred_id, + content_type='text/plain', name=None, description=None, + tags: Optional[list]=None, md5sum: Optional[str]=None) -> str: + """Stores `content` in S3 defined by `s3_cred_id` and registers the + object as data-object in Nuvla. Returns data-object resource ID. + `content` and `content_type` should match (e.g. str and plain/text, + bytes and image/png). + """ + data = { + "template": { + "name": name or object_path, + "description": description or name or object_path, + "credential": s3_cred_id, + "subtype": "generic", + "resource-type": "data-object-template", + "content-type": content_type, + "object": object_path, + "bucket": bucket, + "bytes": len(content), + "href": "data-object-template/generic" + } + } + if tags: + data.update({'tags': tags}) + if md5sum: + data.update({'md5sum': md5sum}) + + data_object_id = super().add(data) + + # Upload data. + data_object = self.nuvla.get(data_object_id) + response = self.nuvla.operation(data_object, "upload") + upload_url = response.data['uri'] + headers = {"content-type": content_type} + response = requests.put(upload_url, data=content, headers=headers) + response.raise_for_status() + + # Set object is ready. + data_object = self.nuvla.get(data_object_id) + self.nuvla.operation(data_object, "ready") + + return data_object_id + + def get_content(self, object_id) -> Union[str, bytes]: + """Returns string or bytes array by downloading from S3 the object + identified by `object_id`. They type of the returned object + corresponds to the `content-type` of the object (text or binary). + """ + data_object = self.nuvla.get(object_id) + response = self.nuvla.operation(data_object, 'download') + download_url = response.data['uri'] + content_type = data_object.data['content-type'] + + headers = {'content-type': content_type} + response = requests.get(download_url, headers=headers) + return response.content + + def get_to_file(self, object_id, filename=''): + """Downloads from S3 and stores to a file the content of the S3 object + identified by `object_id`. If `filename` is not given, the base name + of 'object' attribute is taken instead. + """ + content = self.get_content(object_id) + if not filename: + data_object = self.nuvla.get(object_id) + filename = os.path.basename(data_object.data['object']) + if isinstance(content, bytes): + fmode = 'wb' + else: + fmode = 'w' + with open(filename, fmode) as fh: + fh.write(content) + + def delete(self, resource_id) -> str: + """Deletes object from S3 and its record from Nuvla identified by + `resource_id`. Returns ID of the deleted object. + """ + return super().delete(resource_id) + + +class DataSet: + pass diff --git a/code/src/nuvla/api/resources/deployment.py b/code/src/nuvla/api/resources/deployment.py index c74bd67..cc90b19 100644 --- a/code/src/nuvla/api/resources/deployment.py +++ b/code/src/nuvla/api/resources/deployment.py @@ -1,9 +1,19 @@ -from .utils import check_created, ResourceNotFound -from ..api import Api as Nuvla +import re +import time +from typing import Union, Optional, List, Dict +from datetime import datetime, timezone, timedelta +from . import ResourceBase, ResourceNotFound +from ..api import NuvlaResourceOperationNotAvailable +from ..models import CimiResource, CimiResponse -class Deployment(object): + +class DeploymentOperationNotAvailable(Exception): + pass + + +class Deployment(ResourceBase): """Stateless interface to Nuvla module deployment.""" STATE_STARTED = 'STARTED' @@ -12,10 +22,6 @@ class Deployment(object): resource = 'deployment' - @staticmethod - def id(deployment): - return deployment['id'] - @staticmethod def uuid(deployment): return Deployment.id(deployment).split('/')[1] @@ -37,8 +43,12 @@ def is_application_kubernetes(deployment): return Deployment.subtype(deployment) == 'application_kubernetes' @staticmethod - def module(deployment): - return deployment['module'] + def module(deployment: Union[dict, CimiResource]): + key = 'module' + if isinstance(deployment, dict): + return deployment[key] + else: + return deployment.data[key] @staticmethod def module_content(deployment): @@ -49,8 +59,12 @@ def owner(deployment): return deployment['acl']['owners'][0] @staticmethod - def state(deployment): - return deployment['state'] + def state(deployment: Union[dict, CimiResource]): + key = 'state' + if isinstance(deployment, dict): + return deployment[key] + else: + return deployment.data[key] @staticmethod def credential_id(deployment): @@ -69,10 +83,24 @@ def get_port_name_value(port_mapping): port_details = port_mapping.split(':') return '.'.join([port_details[0], port_details[2]]), port_details[1] - def __init__(self, nuvla: Nuvla): - self.nuvla = nuvla + @staticmethod + def logs(logs: Union[dict, CimiResource]): + key = 'log' + if isinstance(logs, dict): + return logs.get(key, []) + else: + return logs.data.get(key, []) - def _set_data(self, deployment, sets=None, records=None, objects=None): + @staticmethod + def urls(deployment: Union[dict, CimiResource]) -> dict: + if isinstance(deployment, dict): + d = deployment + else: + d = deployment.data + return dict(d.get('module', {}).get('content', {}).get('urls', [])) + + def _set_data(self, deployment: dict, sets=None, records=None, + objects=None): """ :param deployment: :param sets: [{id: , @@ -152,7 +180,7 @@ def _set_data(self, deployment, sets=None, records=None, objects=None): deployment.update({'data': data}) def create(self, module_id, infra_cred_id=None, data_sets=None, - data_records=None, data_objects=None): + data_records=None, data_objects=None) -> CimiResource: """ Returns deployment as dictionary created from `module_id`. Sets `infra_cred_id` infrastructure credentials on the deployment, if given. @@ -161,63 +189,177 @@ def create(self, module_id, infra_cred_id=None, data_sets=None, :param infra_cred_id: str, resource URI :param data_sets: dict, {id: , filter: , - data-type: , + data-type: data-, time-start: <2020-02-24T13:10:30Z>, time-end: <2020-02-24T13:11:30Z>} either `id` or `filter` and `data-type` are required. :param data_records: list of data-record resource URIs :param data_objects: list of data-object resource URIs - :return: dict + :return: CimiResource """ module = {"module": {"href": module_id}} - res = self.nuvla.add(self.resource, module) - deployment_id = check_created(res, 'Failed to create deployment.') + deployment_id = self.add(module) - deployment = self.get(deployment_id) + # Get created deployment and update if needed. + dpl = self.get(deployment_id) + updated = False if infra_cred_id: - deployment.update({'parent': infra_cred_id}) - self._set_data(deployment, data_sets, data_records, data_objects) - try: - return self.nuvla.edit(deployment_id, deployment).data - except Exception as ex: - raise Exception('ERROR: Failed to edit {0}: {1}'.format(deployment_id, ex)) - - def get(self, resource_id): + dpl.data.update({'parent': infra_cred_id}) + updated = True + if any([data_sets, data_records, data_objects]): + self._set_data(dpl.data, data_sets, data_records, data_objects) + updated = True + if updated: + try: + dpl = self.nuvla.edit(deployment_id, dpl.data) + except Exception as ex: + raise Exception('Failed editing {0}: {1}'.format(deployment_id, ex)) + + return dpl + + def get(self, resource_id) -> CimiResource: """ Returns deployment identified by `resource_id` as dictionary. :param resource_id: str - :return: dict + :return: CimiResource """ - return self.nuvla.get(resource_id).data - - def start(self, resource_id): - return self.nuvla.operation(resource_id, 'start') - - def stop(self, resource_id): - return self.nuvla.operation(resource_id, 'stop') - - def delete(self, resource_id): - return self.nuvla.delete(resource_id) + return self.nuvla.get(resource_id) + + def get_state(self, resource_id): + return self.state(self.get(resource_id)) + + def _operation(self, resource_id, operation, timeout=0, + data: Optional[dict]=None) -> CimiResponse: + time_max = time.time() + timeout + while True: + resource = self.get(resource_id) + try: + if operation == 'delete': + return self.nuvla.delete(resource_id) + else: + return self.nuvla.operation(resource, operation, data) + except NuvlaResourceOperationNotAvailable: + msg = "Operation '{0}' is not available on deployment in state {1}." \ + .format(operation, Deployment.state(resource)) + print(msg) + if time.time() >= time_max: + raise DeploymentOperationNotAvailable(msg) + else: + time.sleep(3) + + def start(self, resource_id, timeout=0): + return self._operation(resource_id, 'start', timeout) + + def stop(self, resource_id, timeout=0): + return self._operation(resource_id, 'stop', timeout) + + def delete(self, resource_id, timeout=0): + return self._operation(resource_id, 'delete', timeout) def launch(self, module_id, infra_cred_id=None, data_sets=None, - data_records=None, data_objects=None): + data_records=None, data_objects=None) -> CimiResource: dpl = self.create(module_id, infra_cred_id=infra_cred_id, data_sets=data_sets, data_records=data_records, data_objects=data_objects) - rid = self.id(dpl) - self.start(rid) - return rid + self.start(dpl.id) + return self.get(dpl.id) - def terminate(self, resource_id): + def terminate(self, resource_id: str, timeout=30) -> CimiResponse: + """Stops and deletes the deployment. Waits at max `timeout` seconds for + the deployment to be fully stoped before attemting deletion. + """ self.stop(resource_id) - self.delete(resource_id) + return self.delete(resource_id, timeout=timeout) - def _create_log(self, resource_id): - pass + def list(self) -> List[Dict]: + res = self.nuvla.search(self.resource) + if res.count < 1: + return [] + return res.data['resources'] - def get_logs(self, resource_id): - pass + def init_logs(self, deployment: CimiResource, service: str, + since: Optional[datetime] = None) -> CimiResource: + """ + Creates and returns `deployment-log` resource that corresponds to the + logs of `service` from `deployment` and starting from `since`. + :param deployment: CimiResource + :param service: str + :param since: datetime + :return: CimiResource + """ + if not since: + fmt = '%Y-%m-%dT%H:%M:%S.%fZ' + t = datetime.strptime(deployment.data['created'], fmt) + # Give a bit of slack. + td = timedelta(minutes=5) + since = t - td + since_str = since.replace(tzinfo=timezone.utc) \ + .replace(second=0, microsecond=0).strftime('%Y-%m-%dT%H:%M:%SZ') + request = {'service': 'Service/{}'.format(service), + 'since': since_str} + resp = self._operation(deployment.id, 'create-log', data=request) + logs_id = resp.data['resource-id'] + return self.nuvla.get(logs_id) + + def get_logs(self, logs: CimiResource) -> CimiResource: + """ + Get `service` logs from `since` till now. + :param logs: CimiResource + :return: list: list of lines of the log + """ + # Request fetching of logs from container. + self.nuvla.operation(logs, 'fetch') + # Wait for logs to be produced. + time.sleep(5) + # Get the resource. It may contain the logs. + return self.nuvla.get(logs.id) + + def _template_interpolation(self, string: str, params: dict) -> str: + """Returns `string` interpolated by the values from `params`. + Returns an empty string, if either of those is emtpy: `string`, `params` + or the value of the required interpolation key. + Throws ValueError if `params` is missing substitution keys defined in + `string`. + """ + if not string: + return '' + groups = re.findall(r'(\${.*?})', string) + if len(groups) > 0 and not params: + raise ValueError('no substitutions provided.') + groups_keys = list(map(lambda x: x.replace('${','').replace('}',''), + groups)) + for k in groups_keys: + # If the interpolant from the string (key in groups_keys) is not in + # the list of parameters (params.keys()), interpolation is not possible. + if k not in params: + raise ValueError(f'{k} is missing in params') + if not params[k]: + return '' + string = re.compile('\${' + k + '}').sub(params[k], string) + return string + + def get_url(self, deployment: CimiResource, name) -> Union[str, None]: + """Returns interpolated URL defined by `name` in deployment. + Returns an empty string if not all deployment parameters required + for interpolation are filled with values. + Returns None if either URL `name` is not present on the deployment or + interpolation is not possible due to missing deployment parameters. This + means obtaining URL will not be possible. + """ + url = self.urls(deployment).get(name) + if not url: + return None + params = self.get_parameters(self.id(deployment.data)) + # Flatten parameters map. + flat_params = {} + for v in params.values(): + flat_params.update(v) + try: + return self._template_interpolation(url, flat_params) + except ValueError as ex: + print(ex) + return None def create_parameter(self, resource_id, user_id, param_name, param_value=None, node_id=None, param_description=None): @@ -233,21 +375,56 @@ def create_parameter(self, resource_id, user_id, param_name, param_value=None, parameter['value'] = param_value return self.nuvla.add('deployment-parameter', parameter) - def _get_parameter(self, resource_id, node_id, name, select=None): - filters = "parent='{0}' and node-id='{1}' and name='{2}'".format(resource_id, node_id, name) + def _get_parameter(self, resource_id, name, node_id=None, select=None): + filters = f"parent='{resource_id}' and name='{name}'" + if node_id: + filters += " and node-id='{}'".format(node_id) res = self.nuvla.search("deployment-parameter", filter=filters, select=select) if res.count < 1: - raise ResourceNotFound('Deployment parameter "{0}" not found.'.format(filters)) + raise ResourceNotFound(f'Deployment parameter "{filters}" not found.') return res.resources[0] def get_parameter(self, resource_id, node_id, name): + """Returns value of deployment `resource_id` parameter `name`. To get + global level parameters (not belonging to a node), provide '' or None + as `node_id`. + Returns None if parameter is not found. + """ try: - param = self._get_parameter(resource_id, node_id, name) + param = self._get_parameter(resource_id, name, node_id) except ResourceNotFound: return None return param.data.get('value') - def update_port_parameters(self, deployment, ports_mapping): + def get_parameters(self, resource_id, node_id='') -> Union[list, dict]: + """When `node_id` is not provided, returns dictionary with all + parameters + {'global': {'': '', }, + '': {'': '', }, + ...} + 'global' key corresponds to node_id == None or ''. + Returns list with parameters for the provided `node_id`. + Returns empty list or dict if parameters were not found. + """ + fltr = f"parent='{resource_id}'" + if node_id: + fltr += f" and node-id='{node_id}'" + params = [] + else: + params = {} + res = self.nuvla.search('deployment-parameter', filter=fltr) + for p in res.resources: + d = {p.data['name']: p.data.get('value')} + if node_id: + params.append(d) + else: + if 'node-id' in p.data: + params.setdefault(p.data['node-id'], {}).update(d) + else: + params.setdefault('global', {}).update(d) + return params + + def update_port_parameters(self, deployment: dict, ports_mapping): if ports_mapping: for port_mapping in ports_mapping.split(): port_param_name, port_param_value = self.get_port_name_value(port_mapping) diff --git a/code/src/nuvla/api/resources/infra_service.py b/code/src/nuvla/api/resources/infra_service.py new file mode 100644 index 0000000..ce7149d --- /dev/null +++ b/code/src/nuvla/api/resources/infra_service.py @@ -0,0 +1,167 @@ + +import os +import base64 + +import yaml + +from nuvla.api import Api as Nuvla +from nuvla.api.resources.credential import CredentialK8s +from . import ResourceBase + +""" +Parent/child hierarchy of the infrastructure service related resources. + +infrastructure-service-group (no type) + \_ infrastructure-service (Docker, K8s, S3, Docker Registry) + \_ (infrastructure service) credential (Docker, K8s, S3, Docker Registry) +""" + + +class InfraServiceGroup(ResourceBase): + resource = 'infrastructure-service-group' + + def add(self, name, description=None) -> str: + return super().add({'name': name, + 'description': description or name}) + + +class InfraService(ResourceBase): + resource = 'infrastructure-service' + subtype = None + infra_service_template = "infrastructure-service-template/generic" + + def __init__(self, nuvla: Nuvla, subtype=None): + super().__init__(nuvla) + if not self.subtype and not subtype: + raise Exception('subtype must be defined.') + if subtype: + self.subtype = subtype + + def create(self, endpoint, infra_group_id, name=None, description=None) -> str: + infra_service = { + "template": { + "href": self.infra_service_template, + "subtype": self.subtype, + "endpoint": endpoint, + "name": name or endpoint, + "description": description or name or endpoint, + "parent": infra_group_id + } + } + return self.add(infra_service) + + +class InfraServiceK8s(InfraService): + subtype = 'kubernetes' + + @staticmethod + def read_config(fname) -> dict: + with open(os.path.expanduser(fname)) as fh: + config = yaml.safe_load(fh.read()) + return config + + @staticmethod + def get_cluster_and_user(config, context_name): + cluster_n = None + user_n = None + for cx in config.get('contexts', []): + if cx.get('name') == context_name: + cluster_n = cx.get('context').get('cluster') + user_n = cx.get('context').get('user') + return cluster_n, user_n + + @staticmethod + def get_cluster_cacert(config, cluster_name): + for c in config.get('clusters'): + if c.get('name') == cluster_name: + ca = c.get('cluster').get('certificate-authority-data') + return base64.b64decode(ca).decode() + + @staticmethod + def get_cluster_endpoint(config, cluster_name): + for c in config.get('clusters'): + if c.get('name') == cluster_name: + return c.get('cluster').get('server') + + @staticmethod + def get_client_creds(config, user_name): + cert = None + key = None + for u in config.get('users'): + if u.get('name') == user_name: + cert = u.get('user').get('client-certificate-data') + key = u.get('user').get('client-key-data') + return base64.b64decode(cert).decode(), base64.b64decode(key).decode() + + def add_from_config(self, config_fn, group_id, context=None, name=None, description=None): + """Create Kubernetes IS and the corresponding credential from + Kubernetes client `config_fn` file. `group_id` is the ID of the IS group + to add the IS. If `context` is not provided, the 'current-context' from + the `config_fn` is used. + """ + conf = self.read_config(config_fn) + if not context: + context = conf.get('current-context') + if not context: + raise Exception('K8s: Context must be provided or set in config.') + + cluster_name, user_name = self.get_cluster_and_user(conf, context) + if not cluster_name and not user_name: + raise Exception(f'K8s: failed to find cluster or user ' + f'in context {context}.') + + cluster_url = self.get_cluster_endpoint(conf, cluster_name) + cluster_ca_pem = self.get_cluster_cacert(conf, cluster_name) + if not cluster_url and not cluster_ca_pem: + raise Exception(f'K8s: failed to find cluster URL or CA data ' + f'for cluster {cluster_name}.') + + user_cert, user_key = self.get_client_creds(conf, user_name) + if not user_cert and not user_key: + raise Exception(f'K8s: failed to find user cert and key for ' + f'user {user_name}') + + if not name: + name = cluster_name + is_id = self.create(cluster_url, group_id, name, description or name) + + cred = CredentialK8s(self.nuvla) + isc_id = cred.create(cluster_ca_pem, user_cert, user_key, is_id, name) + return is_id, isc_id + + +class InfraServiceDocker(InfraService): + subtype = 'docker' + + +class InfraServiceS3(InfraService): + subtype = 's3' + + +class InfraServiceRegistry(InfraService): + subtype = 'registry' + + +class InfraServiceCred(ResourceBase): + resource = 'credential' + subtype = None + + def __init__(self, nuvla: Nuvla, subtype=None): + super().__init__(nuvla) + if not self.subtype and not subtype: + raise Exception('subtype must be defined.') + if subtype: + self.subtype = subtype + + def create(self, ): + pass + + def create_from_config(self, filename, context): + pass + + def id_by_name(self, name, filter=None) -> list: + return super().id_by_name(name, filter=f"subtype='{self.subtype}'") + + +class InfraServiceCredK8s(InfraServiceCred): + subtype = 'infrastructure-service-swarm' \ No newline at end of file diff --git a/code/src/nuvla/api/resources/module.py b/code/src/nuvla/api/resources/module.py index be64b9a..0d2ac58 100644 --- a/code/src/nuvla/api/resources/module.py +++ b/code/src/nuvla/api/resources/module.py @@ -1,18 +1,257 @@ +import os +import re +from ..api import Api as Nuvla, NuvlaError + + +APP_TYPE_K8S = 'application_kubernetes' +APP_TYPE_DOCKER = 'application_docker' + + class Module: resource = 'module' - def __init__(self, nuvla): + TYPE_PROJECT = 'project' + TYPE_COMPONENT = 'component' + TYPE_APPLICATION = 'application' + + def __init__(self, nuvla: Nuvla): self.nuvla = nuvla def find(self, **kvargs): return self.nuvla.search(self.resource, **kvargs) - def get(self, resource_id): + def get_by_path(self, path, version=-1) -> dict: + res = self.nuvla.search(self.resource, filter="path='{}'".format(path)) + if res.count == 0: + return {} + return self.get_by_id(res.resources[0].id) + + def add_module(self, module: dict, exist_ok=False) -> str: + """ + Adds new `module` of type project, component and application. + Modules of different types can be built by ProjectBuilder, AppBuilderK8s, + AppBuilderDocker, ComponentBuilder. + + :param module: dict + :return: str: + """ + no_author_in_apps = module['subtype'] != self.TYPE_PROJECT and \ + 'author' not in module['content'] + if no_author_in_apps: + module['content']['author'] = self.nuvla._username + try: + response = self.nuvla.add(self.resource, module) + return response.data['resource-id'] + except NuvlaError as ex: + if re.match('path.*already exist', ex.reason) and exist_ok: + res = self.nuvla.search(self.resource, + filter="path='{}'".format(module['path'])) + return res.resources[0].id + else: + raise ex + + def get_by_id(self, resource_id) -> dict: """ Returns module identified by `resource_id` as dictionary. :param resource_id: str :return: dict """ return self.nuvla.get(resource_id).data + + def update(self, module: dict) -> dict: + res = self.nuvla.edit(module['id'], module) + return res.data + + def get_urls(self, module_id) -> dict: + res = self.nuvla.get(module_id) + return dict(res.data.get('content', {}).get('urls')) + +class ModuleBuilder: + + module_requires = ['path'] + type = None + + _type_not_set_err = 'module type is not set (subtype)' + + def __init__(self, base=None): + if base: + self.module = base + else: + self.module = {} + + def name(self, name): + if name: + self.module['name'] = name + return self + + def description(self, desc): + if desc: + self.module['description'] = desc + return self + + def path(self, path): + """ + Required to build. + :param path: full path to the module + :return: ModuleBuilder + """ + path = path.lstrip('/') + self.module['parent-path'] = os.path.dirname(path) + self.module['path'] = path + return self + + def build(self): + if not self.type: + raise Exception(self._type_not_set_err) + self.module['subtype'] = self.type + for r in self.module_requires: + if r not in self.module: + raise Exception('{} is missing in module.'.format(r)) + if 'name' not in self.module: + self.module['name'] = os.path.basename(self.module['path']) + if 'description' not in self.module: + self.module['description'] = self.module['name'] + return self.module + + +class ProjectBuilder(ModuleBuilder): + type = 'project' + + def __init__(self, base=None): + super().__init__(base) + + +class ComponentBuidler(ModuleBuilder): + type = 'component' + + def __init__(self, base=None): + super().__init__(base) + + +class AppBuilder(ModuleBuilder): + + module_requires_app = ['content'] + + script_key = None + content_requires = [] + + def __init__(self, base=None): + super().__init__(base) + self.module_requires.extend(self.module_requires_app) + if not self.script_key: + raise Exception('') + self.content_requires.append(self.script_key) + + def logo_url(self, url): + self.module['logo-url'] = url + return self + + def author(self, name): + self.module.setdefault('content', {})['author'] = name + return self + + def commit(self, message): + self.module.setdefault('content', {})['commit'] = message + return self + + def env_var(self, name, value='', required=False, description=''): + """ + Can be called multiple times to set a list of environment variables. + :param name: + :param value: + :param required: + :param description: + :return: + """ + env_var = {'name': name, 'value': value, + 'required': required, 'description': description} + self.module.setdefault('content', {}) \ + .setdefault('environmental-variables', []).append(env_var) + return self + + def url(self, name, value): + """ + Can be called multiple times to set a list of URLs. + :param name: + :param value: + :return: + """ + self.module.setdefault('content', {}) \ + .setdefault('urls', []).append([name, value]) + return self + + def output_parm(self, name, description=''): + """ + Can be called multiple times to set a list of output parameters. + :param name: + :param description: + :return: + """ + param = {"name": name, "description": description or name} + self.module.setdefault('content', {}) \ + .setdefault('output-parameters', []).append(param) + return self + + def files(self, name, content): + """ + Can be called multiple times to set a list of files. + :param name: + :param content: + :return: + """ + self.module.setdefault('content', {}) \ + .setdefault('files', []).append([name, content]) + return self + + def script(self, content): + """ + Required to build. Content of the deployment script. + :param content: string. + :return: + """ + if not self.script_key: + raise Exception('script type should be set') + self.module.setdefault('content', {})[self.script_key] = content + return self + + def data_content_type(self, type): + self.module.setdefault('data-accept-content-types', []) \ + .append(type) + return self + + def registry(self, resource_id): + self.module.setdefault('content', []) \ + .setdefault('private-registries', []) \ + .append(resource_id) + return self + + def build(self) -> dict: + """ + Builds and returns module as a dictionary. + :return: dict + """ + if 'commit' not in self.module['content']: + self.module['content']['commit'] = 'no commit message' + for r in self.content_requires: + if r not in self.module['content']: + raise Exception('{} is missing in module["content"].'.format(r)) + module = super(AppBuilder, self).build() + return self.module + + +class AppBuilderK8s(AppBuilder): + type = APP_TYPE_K8S + script_key = 'docker-compose' + + def __index__(self, base=None): + super().__init__(base) + + +class AppBuilderDocker(AppBuilder): + type = APP_TYPE_DOCKER + script_key = 'docker-compose' + + def __index__(self, base=None): + super().__init__(base) diff --git a/code/src/nuvla/api/resources/notification.py b/code/src/nuvla/api/resources/notification.py index c215463..bbeea66 100644 --- a/code/src/nuvla/api/resources/notification.py +++ b/code/src/nuvla/api/resources/notification.py @@ -1,13 +1,10 @@ -from .utils import check_created +from . import ResourceBase -class Notification: +class Notification(ResourceBase): resource = 'notification' - def __init__(self, nuvla): - self.nuvla = nuvla - def create(self, message, category, content_unique_id, target_resource=None, not_before=None, expiry=None, callback_id=None, callback_msg=None, acl=None): @@ -29,8 +26,7 @@ def create(self, message, category, content_unique_id, if acl: notification.update({'acl': acl}) - return check_created(self.nuvla.add(self.resource, notification), - 'Failed to create notification.') + return self.add(notification) def find_by_content_unique_id(self, content_unique_id): resp = self.nuvla.search(self.resource, diff --git a/code/src/nuvla/api/resources/user.py b/code/src/nuvla/api/resources/user.py new file mode 100644 index 0000000..8866f15 --- /dev/null +++ b/code/src/nuvla/api/resources/user.py @@ -0,0 +1,41 @@ + +from requests.exceptions import HTTPError + +from . import ResourceBase +from ..api import NuvlaError + + +class User(ResourceBase): + resource = 'user' + + def create(self, template: dict) -> str: + """ + Creates new user according to provided template. Returns new user ID or + throws an exception. + :param template: + :return: string + """ + return self.add(template) + + def login_password(self, username, password): + response = self.nuvla.login_password(username, password) + try: + response.raise_for_status() + except HTTPError as e: + try: + json_msg = e.response.json() + message = json_msg.get('message') + if message is None: + error = json_msg.get('error') + message = error.get('code') + ' - ' + error.get('reason') + except: + try: + message = e.response.text + except: + message = str(e) + raise NuvlaError(message, response) + + return response.json().get('resource-id') + + def logout(self): + self.nuvla.logout() diff --git a/code/src/nuvla/api/resources/utils.py b/code/src/nuvla/api/resources/utils.py deleted file mode 100644 index 4b99f2b..0000000 --- a/code/src/nuvla/api/resources/utils.py +++ /dev/null @@ -1,28 +0,0 @@ -class ResourceNotFound(Exception): - pass - - -class ResourceCreateError(Exception): - def __init__(self, reason, response=None): - super(ResourceCreateError, self).__init__(reason) - self.reason = reason - self.response = response - - -def check_created(resp, errmsg=''): - """ - Returns id of the created resource or raises ResourceCreateError. - :param resp: nuvla.api.models.CimiResponse - :param errmsg: error message - :return: str, resource id - """ - if resp.data['status'] == 201: - return resp.data['resource-id'] - else: - if errmsg: - msg = '{0} : {1}'.format(errmsg, resp.data['message']) - else: - msg = resp.data['message'] - raise ResourceCreateError(msg, resp) - - diff --git a/code/tests/deployment_tests.py b/code/tests/deployment_tests.py index 8818dca..3d37f59 100644 --- a/code/tests/deployment_tests.py +++ b/code/tests/deployment_tests.py @@ -105,3 +105,29 @@ def data_ids_in(_dpl, names, size): data_sets = [data_set_recs_objs] dpl_api._set_data(dpl, data_sets) data_filters_in(dpl, ['records', 'objects']) + + def test_template_interpolation(self): + dpl_api = Deployment(Mock()) + + # Empty inputs. + assert '' == dpl_api._template_interpolation('', {}) + assert '' == dpl_api._template_interpolation('', {'foo': 'bar'}) + + string = 'http://${foo}:${bar.baz}/?${baz-foo}' + + # No substitution keys. + self.assertRaises(ValueError, dpl_api._template_interpolation, + *(string, {})) + + # Missing substitution key in params. + self.assertRaises(ValueError, dpl_api._template_interpolation, + *(string, {'foo': 'FOO', 'bar.baz': 'BAR.BAZ'})) + + # All required keys, but one empty substitution key in params. + params = {'foo': '', 'bar.baz': 'BAR.BAZ', 'baz-foo': 'BAZ-FOO'} + assert '' == dpl_api._template_interpolation(string, params) + + # Success. + params = {'foo': 'FOO', 'bar.baz': 'BAR.BAZ', 'baz-foo': 'BAZ-FOO'} + assert 'http://FOO:BAR.BAZ/?BAZ-FOO' == \ + dpl_api._template_interpolation(string, params) diff --git a/code/tests/module_tests.py b/code/tests/module_tests.py new file mode 100644 index 0000000..00083c0 --- /dev/null +++ b/code/tests/module_tests.py @@ -0,0 +1,90 @@ +from unittest import TestCase + +from nuvla.api.resources.module import (ModuleBuilder, AppBuilderK8s, + AppBuilderDocker) + + +class ModuleBuilderTest(TestCase): + + def test_minimal_build(self): + mb = ModuleBuilder() + self.assertRaises(Exception, mb.build) + + mb.path('path') + try: + mb.build() + except Exception as ex: + if ModuleBuilder._type_not_set_err != str(ex): + self.fail('build() should have failed with {}' + .format(ModuleBuilder._type_not_set_err)) + + mb.name('hello').description('world') + test_m = { + 'parent-path': '', + 'path': 'path', + 'name': 'hello', + 'description': 'world'} + self.assertEqual(mb.module, test_m) + + +class AppBuilderK8sTest(TestCase): + + def test_path_test(self): + mb = AppBuilderK8s() + + mb.path('path') + self.assertEquals(mb.module['path'], 'path') + self.assertEquals(mb.module['parent-path'], '') + + mb.path('path/sub-path') + self.assertEquals(mb.module['path'], 'path/sub-path') + self.assertEquals(mb.module['parent-path'], 'path') + + def test_minimal_build(self): + mb = AppBuilderK8s() + self.assertRaises(Exception, mb.build) + + mb.path('path') + self.assertRaises(Exception, mb.build) + + mb.script('script') + mb.commit('message') + module = mb.build() + + test_m = { + 'parent-path': '', + 'path': 'path', + 'content': { + AppBuilderK8s.script_key: 'script', + 'commit': 'message'}, + 'name': 'path', + 'description': 'path', + 'subtype': AppBuilderK8s.type + } + self.assertEqual(module, test_m) + + +class AppBuilderDockerTest(TestCase): + + def test_minimal_build(self): + mb = AppBuilderDocker() + self.assertRaises(Exception, mb.build) + + mb.path('path') + self.assertRaises(Exception, mb.build) + + mb.script('script') + mb.commit('message') + module = mb.build() + + test_m = { + 'parent-path': '', + 'path': 'path', + 'content': { + AppBuilderDocker.script_key: 'script', + 'commit': 'message'}, + 'name': 'path', + 'description': 'path', + 'subtype': AppBuilderDocker.type + } + self.assertEqual(module, test_m) From 64d37edcd2c60d322650508299890dfb32dfe42d Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Fri, 29 May 2020 12:41:54 +0200 Subject: [PATCH 02/13] added s3 cred wrapper --- code/src/nuvla/api/resources/credential.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/code/src/nuvla/api/resources/credential.py b/code/src/nuvla/api/resources/credential.py index ed9c5e4..bba413f 100644 --- a/code/src/nuvla/api/resources/credential.py +++ b/code/src/nuvla/api/resources/credential.py @@ -15,7 +15,7 @@ def find_parent(self, parent): class CredentialK8s(Credential): def create(self, ca, cert, key, infra_service_id, name, description=None): - iscred_k8s = { + cred = { "name": name, "description": description or name, "template": { @@ -26,4 +26,20 @@ def create(self, ca, cert, key, infra_service_id, name, description=None): "key": key } } - return self.add(iscred_k8s) + return self.add(cred) + + +class CredentialS3(Credential): + + def create(self, key, secret, infra_service_id, name, description=None): + cred = { + "name": name, + "description": description or name, + "template": { + "href": "credential-template/infrastructure-service-minio", + "parent": infra_service_id, + "access-key": key, + "secret-key": secret + } + } + return self.add(cred) From d4b81b047d9b63489b0ab89a0fea472f05ac4a31 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Sun, 31 May 2020 17:58:54 +0200 Subject: [PATCH 03/13] refactor and fix circular dep when importing base class from . --- code/src/nuvla/api/resources/__init__.py | 79 ------------------- code/src/nuvla/api/resources/base.py | 79 +++++++++++++++++++ code/src/nuvla/api/resources/callback.py | 3 +- code/src/nuvla/api/resources/credential.py | 2 +- code/src/nuvla/api/resources/data.py | 2 +- code/src/nuvla/api/resources/deployment.py | 2 +- code/src/nuvla/api/resources/infra_service.py | 2 +- code/src/nuvla/api/resources/notification.py | 3 +- 8 files changed, 87 insertions(+), 85 deletions(-) create mode 100644 code/src/nuvla/api/resources/base.py diff --git a/code/src/nuvla/api/resources/__init__.py b/code/src/nuvla/api/resources/__init__.py index e11e463..025efe4 100644 --- a/code/src/nuvla/api/resources/__init__.py +++ b/code/src/nuvla/api/resources/__init__.py @@ -1,86 +1,7 @@ -from typing import Union - -from nuvla.api import Api as Nuvla -from nuvla.api.models import CimiResource - from .callback import Callback from .credential import Credential from .deployment import Deployment from .deployment_parameter import DeploymentParameter from .module import Module from .notification import Notification - - -class ResourceNotFound(Exception): - pass - - -class ResourceCreateError(Exception): - def __init__(self, reason, response=None): - super(ResourceCreateError, self).__init__(reason) - self.reason = reason - self.response = response - - -def check_created(resp, errmsg=''): - """ - Returns id of the created resource or raises ResourceCreateError. - :param resp: nuvla.api.models.CimiResponse - :param errmsg: error message - :return: str, resource id - """ - if resp.data['status'] == 201: - return resp.data['resource-id'] - else: - if errmsg: - msg = '{0} : {1}'.format(errmsg, resp.data['message']) - else: - msg = resp.data['message'] - raise ResourceCreateError(msg, resp) - - -class ResourceBase: - - resource = None - - @staticmethod - def id(resource: Union[dict, CimiResource]): - key = 'id' - if isinstance(resource, dict): - return resource[key] - else: - return resource.data[key] - - def __init__(self, nuvla: Nuvla): - self.nuvla = nuvla - if not self.resource: - raise Exception('resource type not set.') - - def add(self, data: dict) -> str: - """Creates resource of type `self.resource` using provided `data`. - Returns created resource ID. - """ - return check_created(self.nuvla.add(self.resource, data), - f'Failed to create {self.resource}.') - - def get(self, resource_id: str) -> dict: - """Returns document identified by `resource_id`. - """ - return self.nuvla.get(resource_id).data - - def delete(self, resource_id: str) -> str: - """Deletes resource identified by `resource_id`. Returns resource ID. - """ - response = self.nuvla.delete(resource_id) - return response.data['resource-id'] - - def id_by_name(self, name, filter=None) -> list: - """Returns list of resource IDs with name equals to `name`. In most - cases the length of the list will be 0 or 1. - """ - fltr = f"name='{name}'" - if filter: - fltr += f' and {filter}' - res = self.nuvla.search(self.resource, filter=fltr) - return [r.data['id'] for r in res.resources] diff --git a/code/src/nuvla/api/resources/base.py b/code/src/nuvla/api/resources/base.py new file mode 100644 index 0000000..6881578 --- /dev/null +++ b/code/src/nuvla/api/resources/base.py @@ -0,0 +1,79 @@ + +from typing import Union + +from nuvla.api import Api as Nuvla +from nuvla.api.models import CimiResource + + +class ResourceNotFound(Exception): + pass + + +class ResourceCreateError(Exception): + def __init__(self, reason, response=None): + super(ResourceCreateError, self).__init__(reason) + self.reason = reason + self.response = response + + +def check_created(resp, errmsg=''): + """ + Returns id of the created resource or raises ResourceCreateError. + :param resp: nuvla.api.models.CimiResponse + :param errmsg: error message + :return: str, resource id + """ + if resp.data['status'] == 201: + return resp.data['resource-id'] + else: + if errmsg: + msg = '{0} : {1}'.format(errmsg, resp.data['message']) + else: + msg = resp.data['message'] + raise ResourceCreateError(msg, resp) + + +class ResourceBase: + + resource = None + + @staticmethod + def id(resource: Union[dict, CimiResource]): + key = 'id' + if isinstance(resource, dict): + return resource[key] + else: + return resource.data[key] + + def __init__(self, nuvla: Nuvla): + self.nuvla = nuvla + if not self.resource: + raise Exception('resource type not set.') + + def add(self, data: dict) -> str: + """Creates resource of type `self.resource` using provided `data`. + Returns created resource ID. + """ + return check_created(self.nuvla.add(self.resource, data), + f'Failed to create {self.resource}.') + + def get(self, resource_id: str) -> dict: + """Returns document identified by `resource_id`. + """ + return self.nuvla.get(resource_id).data + + def delete(self, resource_id: str) -> str: + """Deletes resource identified by `resource_id`. Returns resource ID. + """ + response = self.nuvla.delete(resource_id) + return response.data['resource-id'] + + def id_by_name(self, name, filter=None) -> list: + """Returns list of resource IDs with name equals to `name`. In most + cases the length of the list will be 0 or 1. + """ + fltr = f"name='{name}'" + if filter: + fltr += f' and {filter}' + res = self.nuvla.search(self.resource, filter=fltr) + return [r.data['id'] for r in res.resources] diff --git a/code/src/nuvla/api/resources/callback.py b/code/src/nuvla/api/resources/callback.py index ff03bd4..b93517b 100644 --- a/code/src/nuvla/api/resources/callback.py +++ b/code/src/nuvla/api/resources/callback.py @@ -1,4 +1,5 @@ -from . import ResourceBase + +from .base import ResourceBase class Callback(ResourceBase): diff --git a/code/src/nuvla/api/resources/credential.py b/code/src/nuvla/api/resources/credential.py index bba413f..999c776 100644 --- a/code/src/nuvla/api/resources/credential.py +++ b/code/src/nuvla/api/resources/credential.py @@ -1,5 +1,5 @@ -from . import ResourceBase +from .base import ResourceBase class Credential(ResourceBase): diff --git a/code/src/nuvla/api/resources/data.py b/code/src/nuvla/api/resources/data.py index b4df674..67abe29 100644 --- a/code/src/nuvla/api/resources/data.py +++ b/code/src/nuvla/api/resources/data.py @@ -3,7 +3,7 @@ from typing import Optional, Union from nuvla.api import Api as Nuvla -from . import ResourceBase +from .base import ResourceBase class DataRecord(ResourceBase): diff --git a/code/src/nuvla/api/resources/deployment.py b/code/src/nuvla/api/resources/deployment.py index cc90b19..5d0e44d 100644 --- a/code/src/nuvla/api/resources/deployment.py +++ b/code/src/nuvla/api/resources/deployment.py @@ -4,7 +4,7 @@ from typing import Union, Optional, List, Dict from datetime import datetime, timezone, timedelta -from . import ResourceBase, ResourceNotFound +from .base import ResourceBase, ResourceNotFound from ..api import NuvlaResourceOperationNotAvailable from ..models import CimiResource, CimiResponse diff --git a/code/src/nuvla/api/resources/infra_service.py b/code/src/nuvla/api/resources/infra_service.py index ce7149d..5e9abe9 100644 --- a/code/src/nuvla/api/resources/infra_service.py +++ b/code/src/nuvla/api/resources/infra_service.py @@ -6,7 +6,7 @@ from nuvla.api import Api as Nuvla from nuvla.api.resources.credential import CredentialK8s -from . import ResourceBase +from .base import ResourceBase """ Parent/child hierarchy of the infrastructure service related resources. diff --git a/code/src/nuvla/api/resources/notification.py b/code/src/nuvla/api/resources/notification.py index bbeea66..1cf00d3 100644 --- a/code/src/nuvla/api/resources/notification.py +++ b/code/src/nuvla/api/resources/notification.py @@ -1,4 +1,5 @@ -from . import ResourceBase + +from .base import ResourceBase class Notification(ResourceBase): From 537de7622f926e12d88954443176f0916475ac5a Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Sun, 31 May 2020 18:00:36 +0200 Subject: [PATCH 04/13] refactor add to create on module and data; fix: put tags and md5 to template on data-object --- code/src/nuvla/api/resources/data.py | 14 +++++++------- code/src/nuvla/api/resources/module.py | 9 ++++----- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/code/src/nuvla/api/resources/data.py b/code/src/nuvla/api/resources/data.py index 67abe29..3b44040 100644 --- a/code/src/nuvla/api/resources/data.py +++ b/code/src/nuvla/api/resources/data.py @@ -19,15 +19,15 @@ class DataObjectS3(ResourceBase): def __init__(self, nuvla: Nuvla): super().__init__(nuvla) - def add(self, content: Union[str, bytes], bucket, object_path, s3_cred_id, - content_type='text/plain', name=None, description=None, - tags: Optional[list]=None, md5sum: Optional[str]=None) -> str: + def create(self, content: Union[str, bytes], bucket, object_path, s3_cred_id, + content_type='text/plain', name=None, description=None, + tags: Optional[list]=None, md5sum: Optional[str]=None) -> str: """Stores `content` in S3 defined by `s3_cred_id` and registers the object as data-object in Nuvla. Returns data-object resource ID. `content` and `content_type` should match (e.g. str and plain/text, bytes and image/png). """ - data = { + doc = { "template": { "name": name or object_path, "description": description or name or object_path, @@ -42,11 +42,11 @@ def add(self, content: Union[str, bytes], bucket, object_path, s3_cred_id, } } if tags: - data.update({'tags': tags}) + doc["template"].update({'tags': tags}) if md5sum: - data.update({'md5sum': md5sum}) + doc["template"].update({'md5sum': md5sum}) - data_object_id = super().add(data) + data_object_id = self.add(doc) # Upload data. data_object = self.nuvla.get(data_object_id) diff --git a/code/src/nuvla/api/resources/module.py b/code/src/nuvla/api/resources/module.py index 0d2ac58..4f61951 100644 --- a/code/src/nuvla/api/resources/module.py +++ b/code/src/nuvla/api/resources/module.py @@ -2,13 +2,14 @@ import os import re from ..api import Api as Nuvla, NuvlaError +from .base import ResourceBase APP_TYPE_K8S = 'application_kubernetes' APP_TYPE_DOCKER = 'application_docker' -class Module: +class Module(ResourceBase): resource = 'module' @@ -16,9 +17,6 @@ class Module: TYPE_COMPONENT = 'component' TYPE_APPLICATION = 'application' - def __init__(self, nuvla: Nuvla): - self.nuvla = nuvla - def find(self, **kvargs): return self.nuvla.search(self.resource, **kvargs) @@ -28,7 +26,7 @@ def get_by_path(self, path, version=-1) -> dict: return {} return self.get_by_id(res.resources[0].id) - def add_module(self, module: dict, exist_ok=False) -> str: + def create(self, module: dict, exist_ok=False) -> str: """ Adds new `module` of type project, component and application. Modules of different types can be built by ProjectBuilder, AppBuilderK8s, @@ -68,6 +66,7 @@ def get_urls(self, module_id) -> dict: res = self.nuvla.get(module_id) return dict(res.data.get('content', {}).get('urls')) + class ModuleBuilder: module_requires = ['path'] From 1f61bb07a7b6e07e82c815f5b65abac2c393307e Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Sun, 31 May 2020 18:03:20 +0200 Subject: [PATCH 05/13] pass doc through json for proper json representation at debug printout --- code/src/nuvla/api/api.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/code/src/nuvla/api/api.py b/code/src/nuvla/api/api.py index 328bfcb..29e8533 100644 --- a/code/src/nuvla/api/api.py +++ b/code/src/nuvla/api/api.py @@ -54,6 +54,7 @@ """ +import json import logging import os import stat @@ -91,7 +92,7 @@ def __init__(self, reason, operation): self.operation = operation -def _request_debug(method, endpoint, params=None, json=None, +def _request_debug(method, endpoint, params=None, doc=None, data=None, headers: Optional[dict]=None): print('::: HTTP Request -> Response :::') print('>>> Request') @@ -104,8 +105,8 @@ def _request_debug(method, endpoint, params=None, json=None, print() if params: print('params:', params) - if json: - pp(json) + if doc: + print(json.dumps(doc, indent=2, sort_keys=True)) if data: dlen = len(data) for k, v in data.items(): @@ -123,7 +124,6 @@ def _response_debug(response: requests.Response): print('{0}: {1}'.format(k, v)) print() print(response.text) - # pp(response.json()) print('<<< Response') @@ -189,7 +189,7 @@ def cimi_login(self, login_params): 'Accept': 'application/json'} json = {'template': login_params} if self._debug: - _request_debug(method, endpoint, json=json, headers=headers) + _request_debug(method, endpoint, doc=json, headers=headers) response = self.request(method, endpoint, headers=headers, json=json) if self._debug: _response_debug(response) From e0b8675cf52760fc955ae5e1c6d75967cfe4c5fb Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Mon, 1 Jun 2020 11:20:55 +0200 Subject: [PATCH 06/13] load ResourceBase from base and minor cleanup in api --- code/src/nuvla/api/api.py | 3 ++- code/src/nuvla/api/resources/user.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/code/src/nuvla/api/api.py b/code/src/nuvla/api/api.py index 29e8533..fdc509d 100644 --- a/code/src/nuvla/api/api.py +++ b/code/src/nuvla/api/api.py @@ -59,7 +59,6 @@ import os import stat from typing import Optional -from pprint import pp import requests from requests.cookies import MockRequest @@ -94,6 +93,7 @@ def __init__(self, reason, operation): def _request_debug(method, endpoint, params=None, doc=None, data=None, headers: Optional[dict]=None): + # TODO: Use logging library. print('::: HTTP Request -> Response :::') print('>>> Request') url_res = urlparse(endpoint) @@ -118,6 +118,7 @@ def _request_debug(method, endpoint, params=None, doc=None, def _response_debug(response: requests.Response): + # TODO: Use logging library. print('>>> Response') print(response.status_code, response.reason) for k, v in response.headers.items(): diff --git a/code/src/nuvla/api/resources/user.py b/code/src/nuvla/api/resources/user.py index 8866f15..e024549 100644 --- a/code/src/nuvla/api/resources/user.py +++ b/code/src/nuvla/api/resources/user.py @@ -1,7 +1,7 @@ from requests.exceptions import HTTPError -from . import ResourceBase +from .base import ResourceBase from ..api import NuvlaError From 2b878e3e33d17b95b4619fe81f1b303da1c6c716 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Mon, 1 Jun 2020 12:38:07 +0200 Subject: [PATCH 07/13] public create method instead of add on infra service group --- code/src/nuvla/api/resources/infra_service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/src/nuvla/api/resources/infra_service.py b/code/src/nuvla/api/resources/infra_service.py index 5e9abe9..00d1e19 100644 --- a/code/src/nuvla/api/resources/infra_service.py +++ b/code/src/nuvla/api/resources/infra_service.py @@ -20,9 +20,9 @@ class InfraServiceGroup(ResourceBase): resource = 'infrastructure-service-group' - def add(self, name, description=None) -> str: - return super().add({'name': name, - 'description': description or name}) + def create(self, name, description=None) -> str: + return self.add({'name': name, + 'description': description or name}) class InfraService(ResourceBase): From 82972f2a941ab2f0437762812e667ae9c7f25bc0 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Mon, 1 Jun 2020 14:55:41 +0200 Subject: [PATCH 08/13] moved infra service creds creation implementation from infra service to credentials; refactored k8s infra and creds creation --- code/src/nuvla/api/resources/credential.py | 140 +++++++++++++++--- code/src/nuvla/api/resources/infra_service.py | 102 ++----------- 2 files changed, 134 insertions(+), 108 deletions(-) diff --git a/code/src/nuvla/api/resources/credential.py b/code/src/nuvla/api/resources/credential.py index 999c776..56f57ae 100644 --- a/code/src/nuvla/api/resources/credential.py +++ b/code/src/nuvla/api/resources/credential.py @@ -1,45 +1,149 @@ +import os +import base64 + +import yaml + from .base import ResourceBase +from nuvla.api import Api as Nuvla class Credential(ResourceBase): resource = 'credential' + subtype = None + + def __init__(self, nuvla: Nuvla, subtype=None): + super().__init__(nuvla) + if not self.subtype and not subtype: + raise Exception('subtype must be defined.') + if subtype: + self.subtype = subtype def find_parent(self, parent): filters = "parent='{}'".format(parent) creds = self.nuvla.search(self.resource, filter=filters, select="id") return creds.resources + def id_by_name(self, name, filter=None) -> list: + return super().id_by_name(name, filter=f"subtype='{self.subtype}'") -class CredentialK8s(Credential): - - def create(self, ca, cert, key, infra_service_id, name, description=None): + def create_from_template(self, cread_data, parent, name, description=None): cred = { "name": name, "description": description or name, "template": { - "href": "credential-template/infrastructure-service-swarm", - "parent": infra_service_id, - "ca": ca, - "cert": cert, - "key": key + "href": f"credential-template/{self.subtype}", + "parent": parent } } + cred['template'].update(cread_data) return self.add(cred) +class KubeConfig: + + @staticmethod + def read_config(fname) -> dict: + with open(os.path.expanduser(fname)) as fh: + config = yaml.safe_load(fh.read()) + return config + + @staticmethod + def get_cluster_and_user(config, context_name): + cluster_n = None + user_n = None + for cx in config.get('contexts', []): + if cx.get('name') == context_name: + cluster_n = cx.get('context').get('cluster') + user_n = cx.get('context').get('user') + return cluster_n, user_n + + @staticmethod + def get_cluster_cacert(config, cluster_name): + for c in config.get('clusters'): + if c.get('name') == cluster_name: + ca = c.get('cluster').get('certificate-authority-data') + return base64.b64decode(ca).decode() + + @staticmethod + def get_cluster_endpoint(config, cluster_name): + for c in config.get('clusters'): + if c.get('name') == cluster_name: + return c.get('cluster').get('server') + + @staticmethod + def get_client_creds(config, user_name): + cert = None + key = None + for u in config.get('users'): + if u.get('name') == user_name: + cert = u.get('user').get('client-certificate-data') + key = u.get('user').get('client-key-data') + return base64.b64decode(cert).decode(), base64.b64decode(key).decode() + + +class CredentialK8s(Credential): + + subtype = 'infrastructure-service-swarm' + + def create(self, ca, cert, key, infra_service_id, name, description=None): + cred_data = { + "ca": ca, + "cert": cert, + "key": key + } + return self.create_from_template(cred_data, infra_service_id, name, + description) + + def create_from_config(self, filename, infra_service_id, context, name=None, + description=None): + kube = KubeConfig() + conf = kube.read_config(filename) + cluster_name, user_name = kube.get_cluster_and_user(conf, context) + if not cluster_name and not user_name: + raise Exception(f'K8s: failed to find cluster or user ' + f'in context {context}.') + + cluster_ca_pem = kube.get_cluster_cacert(conf, cluster_name) + if not cluster_ca_pem: + raise Exception(f'K8s: failed to find cluster CA data ' + f'for cluster {cluster_name}.') + + user_cert, user_key = kube.get_client_creds(conf, user_name) + if not user_cert and not user_key: + raise Exception(f'K8s: failed to find user cert and key for ' + f'user {user_name}') + + return self.create(cluster_ca_pem, user_cert, user_key, infra_service_id, + name or cluster_name, description) + + +class CredentialDockerSwarm(Credential): + pass + + class CredentialS3(Credential): + subtype = 'infrastructure-service-minio' + def create(self, key, secret, infra_service_id, name, description=None): - cred = { - "name": name, - "description": description or name, - "template": { - "href": "credential-template/infrastructure-service-minio", - "parent": infra_service_id, - "access-key": key, - "secret-key": secret - } + cred_data = { + "access-key": key, + "secret-key": secret } - return self.add(cred) + return self.create_from_template(cred_data, infra_service_id, name, + description) + + +class CredentialDockerRegistry(Credential): + + subtype = 'infrastructure-service-registry' + + def create(self, username, password, infra_service_id, name, description=None): + cred_data = { + 'username': username, + 'password': password + } + return self.create_from_template(cred_data, infra_service_id, name, + description) diff --git a/code/src/nuvla/api/resources/infra_service.py b/code/src/nuvla/api/resources/infra_service.py index 00d1e19..dd9e64a 100644 --- a/code/src/nuvla/api/resources/infra_service.py +++ b/code/src/nuvla/api/resources/infra_service.py @@ -1,11 +1,6 @@ -import os -import base64 - -import yaml - from nuvla.api import Api as Nuvla -from nuvla.api.resources.credential import CredentialK8s +from nuvla.api.resources.credential import CredentialK8s, KubeConfig from .base import ResourceBase """ @@ -52,82 +47,34 @@ def create(self, endpoint, infra_group_id, name=None, description=None) -> str: class InfraServiceK8s(InfraService): + subtype = 'kubernetes' - @staticmethod - def read_config(fname) -> dict: - with open(os.path.expanduser(fname)) as fh: - config = yaml.safe_load(fh.read()) - return config - - @staticmethod - def get_cluster_and_user(config, context_name): - cluster_n = None - user_n = None - for cx in config.get('contexts', []): - if cx.get('name') == context_name: - cluster_n = cx.get('context').get('cluster') - user_n = cx.get('context').get('user') - return cluster_n, user_n - - @staticmethod - def get_cluster_cacert(config, cluster_name): - for c in config.get('clusters'): - if c.get('name') == cluster_name: - ca = c.get('cluster').get('certificate-authority-data') - return base64.b64decode(ca).decode() - - @staticmethod - def get_cluster_endpoint(config, cluster_name): - for c in config.get('clusters'): - if c.get('name') == cluster_name: - return c.get('cluster').get('server') - - @staticmethod - def get_client_creds(config, user_name): - cert = None - key = None - for u in config.get('users'): - if u.get('name') == user_name: - cert = u.get('user').get('client-certificate-data') - key = u.get('user').get('client-key-data') - return base64.b64decode(cert).decode(), base64.b64decode(key).decode() - - def add_from_config(self, config_fn, group_id, context=None, name=None, description=None): + def create_from_config(self, config_fn, group_id, context=None, name=None, description=None): """Create Kubernetes IS and the corresponding credential from Kubernetes client `config_fn` file. `group_id` is the ID of the IS group to add the IS. If `context` is not provided, the 'current-context' from the `config_fn` is used. """ - conf = self.read_config(config_fn) + kube = KubeConfig() + conf = kube.read_config(config_fn) if not context: context = conf.get('current-context') if not context: raise Exception('K8s: Context must be provided or set in config.') - cluster_name, user_name = self.get_cluster_and_user(conf, context) - if not cluster_name and not user_name: - raise Exception(f'K8s: failed to find cluster or user ' - f'in context {context}.') - - cluster_url = self.get_cluster_endpoint(conf, cluster_name) - cluster_ca_pem = self.get_cluster_cacert(conf, cluster_name) - if not cluster_url and not cluster_ca_pem: - raise Exception(f'K8s: failed to find cluster URL or CA data ' - f'for cluster {cluster_name}.') - - user_cert, user_key = self.get_client_creds(conf, user_name) - if not user_cert and not user_key: - raise Exception(f'K8s: failed to find user cert and key for ' - f'user {user_name}') + cluster_name, _ = kube.get_cluster_and_user(conf, context) + if not cluster_name: + raise Exception(f'K8s: failed to find cluster in context {context}.') + cluster_url = kube.get_cluster_endpoint(conf, cluster_name) if not name: name = cluster_name - is_id = self.create(cluster_url, group_id, name, description or name) + infra_id = self.create(cluster_url, group_id, name, description) cred = CredentialK8s(self.nuvla) - isc_id = cred.create(cluster_ca_pem, user_cert, user_key, is_id, name) - return is_id, isc_id + isc_id = cred.create_from_config(config_fn, infra_id, context, name, description) + return infra_id, isc_id class InfraServiceDocker(InfraService): @@ -140,28 +87,3 @@ class InfraServiceS3(InfraService): class InfraServiceRegistry(InfraService): subtype = 'registry' - - -class InfraServiceCred(ResourceBase): - resource = 'credential' - subtype = None - - def __init__(self, nuvla: Nuvla, subtype=None): - super().__init__(nuvla) - if not self.subtype and not subtype: - raise Exception('subtype must be defined.') - if subtype: - self.subtype = subtype - - def create(self, ): - pass - - def create_from_config(self, filename, context): - pass - - def id_by_name(self, name, filter=None) -> list: - return super().id_by_name(name, filter=f"subtype='{self.subtype}'") - - -class InfraServiceCredK8s(InfraServiceCred): - subtype = 'infrastructure-service-swarm' \ No newline at end of file From 5990b7f66e25194a9198ce29942b76128f107613 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Mon, 1 Jun 2020 16:34:23 +0200 Subject: [PATCH 09/13] add create data record --- code/src/nuvla/api/resources/data.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/code/src/nuvla/api/resources/data.py b/code/src/nuvla/api/resources/data.py index 3b44040..7366e29 100644 --- a/code/src/nuvla/api/resources/data.py +++ b/code/src/nuvla/api/resources/data.py @@ -9,8 +9,10 @@ class DataRecord(ResourceBase): resource = 'data-record' - def __init__(self, nuvla: Nuvla): - super().__init__(nuvla) + def create(self, data: dict, infra_service_id: str): + if isinstance(data, dict): + data.update({'infrastructure-service': infra_service_id}) + return self.add(data) class DataObjectS3(ResourceBase): From f13e22b91d2727214421e3cb9577fdf35d912d21 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Wed, 3 Jun 2020 13:34:29 +0200 Subject: [PATCH 10/13] IS docker is swarm --- code/src/nuvla/api/resources/infra_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/src/nuvla/api/resources/infra_service.py b/code/src/nuvla/api/resources/infra_service.py index dd9e64a..1d914fe 100644 --- a/code/src/nuvla/api/resources/infra_service.py +++ b/code/src/nuvla/api/resources/infra_service.py @@ -78,7 +78,7 @@ def create_from_config(self, config_fn, group_id, context=None, name=None, descr class InfraServiceDocker(InfraService): - subtype = 'docker' + subtype = 'swarm' class InfraServiceS3(InfraService): From 6c6c5bfa0d959e830e74a85460706624ab285cb5 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Wed, 3 Jun 2020 13:34:56 +0200 Subject: [PATCH 11/13] docker app type is application --- code/src/nuvla/api/resources/module.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/code/src/nuvla/api/resources/module.py b/code/src/nuvla/api/resources/module.py index 4f61951..a101907 100644 --- a/code/src/nuvla/api/resources/module.py +++ b/code/src/nuvla/api/resources/module.py @@ -1,12 +1,12 @@ import os import re -from ..api import Api as Nuvla, NuvlaError +from ..api import NuvlaError from .base import ResourceBase APP_TYPE_K8S = 'application_kubernetes' -APP_TYPE_DOCKER = 'application_docker' +APP_TYPE_DOCKER = 'application' class Module(ResourceBase): @@ -131,6 +131,8 @@ def __init__(self, base=None): class AppBuilder(ModuleBuilder): + type = 'application' + module_requires_app = ['content'] script_key = None @@ -236,21 +238,15 @@ def build(self) -> dict: for r in self.content_requires: if r not in self.module['content']: raise Exception('{} is missing in module["content"].'.format(r)) - module = super(AppBuilder, self).build() - return self.module + return super(AppBuilder, self).build() class AppBuilderK8s(AppBuilder): type = APP_TYPE_K8S script_key = 'docker-compose' - def __index__(self, base=None): - super().__init__(base) - class AppBuilderDocker(AppBuilder): type = APP_TYPE_DOCKER script_key = 'docker-compose' - def __index__(self, base=None): - super().__init__(base) From 0caaeb4d765924e52ca9c1ea465c9162a300c89b Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Wed, 3 Jun 2020 13:35:10 +0200 Subject: [PATCH 12/13] add swarm creds --- code/src/nuvla/api/resources/credential.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/code/src/nuvla/api/resources/credential.py b/code/src/nuvla/api/resources/credential.py index 56f57ae..6045d62 100644 --- a/code/src/nuvla/api/resources/credential.py +++ b/code/src/nuvla/api/resources/credential.py @@ -120,7 +120,18 @@ def create_from_config(self, filename, infra_service_id, context, name=None, class CredentialDockerSwarm(Credential): - pass + + subtype = 'infrastructure-service-swarm' + + def create(self, ca, cert, key, infra_service_id, name, description=None): + cred_data = { + "ca": ca, + "cert": cert, + "key": key + } + return self.create_from_template(cred_data, infra_service_id, name, + description) + class CredentialS3(Credential): From 26f8a0e271f7336115d9406a2e9ed888a162be74 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Fri, 5 Jun 2020 12:15:00 +0200 Subject: [PATCH 13/13] refactored COE creds creation and decoding of K8s creds --- code/src/nuvla/api/resources/credential.py | 54 ++++++++++------------ 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/code/src/nuvla/api/resources/credential.py b/code/src/nuvla/api/resources/credential.py index 6045d62..9572f9a 100644 --- a/code/src/nuvla/api/resources/credential.py +++ b/code/src/nuvla/api/resources/credential.py @@ -28,7 +28,7 @@ def find_parent(self, parent): def id_by_name(self, name, filter=None) -> list: return super().id_by_name(name, filter=f"subtype='{self.subtype}'") - def create_from_template(self, cread_data, parent, name, description=None): + def create_from_template(self, cred_data, parent, name, description=None): cred = { "name": name, "description": description or name, @@ -37,12 +37,27 @@ def create_from_template(self, cread_data, parent, name, description=None): "parent": parent } } - cred['template'].update(cread_data) + cred['template'].update(cred_data) return self.add(cred) +class CredentialCOE(Credential): + def create(self, ca, cert, key, infra_service_id, name, description=None): + cred_data = { + "ca": ca, + "cert": cert, + "key": key + } + return self.create_from_template(cred_data, infra_service_id, name, + description) + + class KubeConfig: + @staticmethod + def decode(data): + return base64.b64decode(data).decode() + @staticmethod def read_config(fname) -> dict: with open(os.path.expanduser(fname)) as fh: @@ -59,12 +74,12 @@ def get_cluster_and_user(config, context_name): user_n = cx.get('context').get('user') return cluster_n, user_n - @staticmethod - def get_cluster_cacert(config, cluster_name): + @classmethod + def get_cluster_cacert(cls, config, cluster_name): for c in config.get('clusters'): if c.get('name') == cluster_name: ca = c.get('cluster').get('certificate-authority-data') - return base64.b64decode(ca).decode() + return cls.decode(ca) @staticmethod def get_cluster_endpoint(config, cluster_name): @@ -72,30 +87,21 @@ def get_cluster_endpoint(config, cluster_name): if c.get('name') == cluster_name: return c.get('cluster').get('server') - @staticmethod - def get_client_creds(config, user_name): + @classmethod + def get_client_creds(cls, config, user_name): cert = None key = None for u in config.get('users'): if u.get('name') == user_name: cert = u.get('user').get('client-certificate-data') key = u.get('user').get('client-key-data') - return base64.b64decode(cert).decode(), base64.b64decode(key).decode() + return cls.decode(cert), cls.decode(key) -class CredentialK8s(Credential): +class CredentialK8s(CredentialCOE): subtype = 'infrastructure-service-swarm' - def create(self, ca, cert, key, infra_service_id, name, description=None): - cred_data = { - "ca": ca, - "cert": cert, - "key": key - } - return self.create_from_template(cred_data, infra_service_id, name, - description) - def create_from_config(self, filename, infra_service_id, context, name=None, description=None): kube = KubeConfig() @@ -119,20 +125,10 @@ def create_from_config(self, filename, infra_service_id, context, name=None, name or cluster_name, description) -class CredentialDockerSwarm(Credential): +class CredentialDockerSwarm(CredentialCOE): subtype = 'infrastructure-service-swarm' - def create(self, ca, cert, key, infra_service_id, name, description=None): - cred_data = { - "ca": ca, - "cert": cert, - "key": key - } - return self.create_from_template(cred_data, infra_service_id, name, - description) - - class CredentialS3(Credential):