From 42c510665e0cbf80d2f271974dcacb8982a22f8c Mon Sep 17 00:00:00 2001 From: Chris Veilleux Date: Mon, 22 Oct 2018 13:08:00 -0500 Subject: [PATCH] changed skill install status logic to be its own endpoint --- .../backend/v1/market-api/market_api/api.py | 16 +- .../market_api/endpoints/__init__.py | 3 +- .../market_api/endpoints/available_skills.py | 104 ++++++++++ .../market-api/market_api/endpoints/common.py | 52 ++++- .../endpoints/skill_install_status.py | 188 ++++++++++++++++++ .../market_api/endpoints/skill_summary.py | 179 ----------------- 6 files changed, 357 insertions(+), 185 deletions(-) create mode 100644 market/backend/v1/market-api/market_api/endpoints/available_skills.py create mode 100644 market/backend/v1/market-api/market_api/endpoints/skill_install_status.py delete mode 100644 market/backend/v1/market-api/market_api/endpoints/skill_summary.py diff --git a/market/backend/v1/market-api/market_api/api.py b/market/backend/v1/market-api/market_api/api.py index bc19f3b6..ddd49c88 100644 --- a/market/backend/v1/market-api/market_api/api.py +++ b/market/backend/v1/market-api/market_api/api.py @@ -3,9 +3,10 @@ from flask_restful import Api from .config import get_config_location from market_api.endpoints import ( - SkillSummaryEndpoint, + AvailableSkillsEndpoint, SkillDetailEndpoint, SkillInstallEndpoint, + SkillInstallationsEndpoint, UserEndpoint ) @@ -14,7 +15,14 @@ marketplace = Flask(__name__) marketplace.config.from_object(get_config_location()) marketplace_api = Api(marketplace) -marketplace_api.add_resource(SkillSummaryEndpoint, '/api/skills') -marketplace_api.add_resource(SkillDetailEndpoint, '/api/skill/') -marketplace_api.add_resource(SkillInstallEndpoint, '/api/install') +marketplace_api.add_resource(AvailableSkillsEndpoint, '/api/skill/available') +marketplace_api.add_resource( + SkillDetailEndpoint, + '/api/skill/detail/' +) +marketplace_api.add_resource(SkillInstallEndpoint, '/api/skill/install') +marketplace_api.add_resource( + SkillInstallationsEndpoint, + '/api/skill/installations' +) marketplace_api.add_resource(UserEndpoint, '/api/user') diff --git a/market/backend/v1/market-api/market_api/endpoints/__init__.py b/market/backend/v1/market-api/market_api/endpoints/__init__.py index 036e1e7c..5360877e 100644 --- a/market/backend/v1/market-api/market_api/endpoints/__init__.py +++ b/market/backend/v1/market-api/market_api/endpoints/__init__.py @@ -1,4 +1,5 @@ +from .available_skills import AvailableSkillsEndpoint from .skill_detail import SkillDetailEndpoint from .skill_install import SkillInstallEndpoint -from .skill_summary import SkillSummaryEndpoint +from .skill_install_status import SkillInstallationsEndpoint from .user import UserEndpoint diff --git a/market/backend/v1/market-api/market_api/endpoints/available_skills.py b/market/backend/v1/market-api/market_api/endpoints/available_skills.py new file mode 100644 index 00000000..b7784ea3 --- /dev/null +++ b/market/backend/v1/market-api/market_api/endpoints/available_skills.py @@ -0,0 +1,104 @@ +"""Endpoint to provide skill summary data to the marketplace.""" +from collections import defaultdict +from http import HTTPStatus +from logging import getLogger +from typing import List + +import requests as service_request + +from selene_util.api import APIError, SeleneEndpoint +from .common import RepositorySkill + +_log = getLogger(__package__) + + +class AvailableSkillsEndpoint(SeleneEndpoint): + authentication_required = False + + def __init__(self): + super(AvailableSkillsEndpoint, self).__init__() + self.available_skills: List[RepositorySkill] = [] + self.response_skills: List[dict] = [] + self.skills_in_manifests = defaultdict(list) + + def get(self): + try: + self._get_available_skills() + except APIError: + pass + else: + self._build_response_data() + self.response = (self.response_skills, HTTPStatus.OK) + + return self.response + + def _get_available_skills(self): + """Retrieve all skills in the skill repository. + + The data is retrieved from a database table that is populated with + the contents of a JSON object in the mycroft-skills-data Github + repository. The JSON object contains metadata about each skill. + """ + skill_service_response = service_request.get( + self.config['SELENE_BASE_URL'] + '/skill/all' + ) + if skill_service_response.status_code != HTTPStatus.OK: + self._check_for_service_errors(skill_service_response) + self.available_skills = [ + RepositorySkill(**skill) for skill in skill_service_response.json() + ] + + def _build_response_data(self): + """Build the data to include in the response.""" + if self.request.query_string: + skills_to_include = self._filter_skills() + else: + skills_to_include = self.available_skills + self._reformat_skills(skills_to_include) + self._sort_skills() + + def _filter_skills(self) -> list: + """If search criteria exist, only return those skills that match.""" + skills_to_include = [] + + query_string = self.request.query_string.decode() + search_term = query_string.lower().split('=')[1] + for skill in self.available_skills: + search_term_match = ( + search_term is None or + search_term in skill.title.lower() or + search_term in skill.description.lower() or + search_term in skill.summary.lower() or + search_term in [c.lower() for c in skill.categories] or + search_term in [t.lower() for t in skill.tags] or + search_term in [t.lower() for t in skill.triggers] + ) + if search_term_match: + skills_to_include.append(skill) + + return skills_to_include + + def _reformat_skills(self, skills_to_include: List[RepositorySkill]): + """Build the response data from the skill service response""" + for skill in skills_to_include: + skill_info = dict( + icon=skill.icon, + iconImage=skill.icon_image, + isMycroftMade=skill.is_mycroft_made, + isSystemSkill=skill.is_system_skill, + marketCategory=skill.market_category, + name=skill.skill_name, + summary=skill.summary, + title=skill.title, + trigger=skill.triggers[0] + ) + self.response_skills.append(skill_info) + + def _sort_skills(self): + """Sort the skills in alphabetical order""" + sorted_skills = sorted( + self.response_skills, + key=lambda skill: + skill['title'] + ) + self.response_skills = sorted_skills diff --git a/market/backend/v1/market-api/market_api/endpoints/common.py b/market/backend/v1/market-api/market_api/endpoints/common.py index 8676dc71..def73ab1 100644 --- a/market/backend/v1/market-api/market_api/endpoints/common.py +++ b/market/backend/v1/market-api/market_api/endpoints/common.py @@ -1,12 +1,62 @@ from collections import defaultdict -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, field from typing import List +from markdown import markdown + import requests as service_request +DEFAULT_ICON_COLOR = '#6C7A89' +DEFAULT_ICON_NAME = 'comment-alt' +SYSTEM_CATEGORY = 'System' +UNDEFINED_CATEGORY = 'Not Categorized' VALID_INSTALLATION_VALUES = ('failed', 'installed', 'installing', 'uninstalled') +@dataclass +class RepositorySkill(object): + """Represents a single skill defined in the Mycroft Skills repository.""" + branch: str + categories: List[str] + created: str + credits: List[dict] + description: str + icon: dict + id: str + is_mycroft_made: bool = field(init=False) + is_system_skill: bool = field(init=False) + last_update: str + market_category: str = field(init=False) + platforms: List[str] + repository_owner: str + repository_url: str + skill_name: str + summary: str + tags: List[str] + title: str + triggers: List[str] + icon_image: str = field(default=None) + + def __post_init__(self): + self.is_system_skill = False + if 'system' in self.tags: + self.is_system_skill = True + self.market_category = SYSTEM_CATEGORY + elif self.categories: + # a skill may have many categories. the first one in the + # list is considered the "primary" category. This is the + # category the marketplace will use to group the skill. + self.market_category = self.categories[0] + else: + self.market_category = UNDEFINED_CATEGORY + + if not self.icon: + self.icon = dict(icon=DEFAULT_ICON_NAME, color=DEFAULT_ICON_COLOR) + + self.is_mycroft_made = self.credits[0].get('name') == 'Mycroft AI' + self.summary = markdown(self.summary, output_format='html5') + + @dataclass class ManifestSkill(object): """Represents a single skill on a device's skill manifest. diff --git a/market/backend/v1/market-api/market_api/endpoints/skill_install_status.py b/market/backend/v1/market-api/market_api/endpoints/skill_install_status.py new file mode 100644 index 00000000..ef2a43b0 --- /dev/null +++ b/market/backend/v1/market-api/market_api/endpoints/skill_install_status.py @@ -0,0 +1,188 @@ +from collections import defaultdict +from dataclasses import asdict, dataclass +from http import HTTPStatus +from typing import List + +import requests as service_request + +from selene_util.api import APIError, SeleneEndpoint + +VALID_INSTALLATION_VALUES = ( + 'failed', + 'installed', + 'installing', + 'uninstalling' +) + + +@dataclass +class ManifestSkill(object): + """Represents a single skill on a device's skill manifest. + + Mycroft core keeps a manifest off all skills associated with a device. + This manifest shows the status of each skill as it relates to the device. + """ + failure_message: str + installation: str + name: str + + +class SkillInstallationsEndpoint(SeleneEndpoint): + authentication_required = False + + def __init__(self): + super(SkillInstallationsEndpoint, self).__init__() + self.skills_in_manifests = defaultdict(list) + + def get(self): + try: + self._get_install_statuses() + except APIError: + pass + else: + response_data = self._build_response_data() + self.response = (response_data, HTTPStatus.OK) + + return self.response + + def _get_install_statuses(self): + self._authenticate() + if self.authenticated: + skill_manifests = self._get_skill_manifests() + self._parse_skill_manifests(skill_manifests) + else: + self.response = ( + dict(installStatuses=[], failureReasons=[]), + HTTPStatus.OK + ) + + def _get_skill_manifests(self) -> dict: + """Get the skill manifests from each of a user's devices + + The skill manifests will be used to determine the status of each + skill as it relates to the marketplace. + """ + service_request_headers = { + 'Authorization': 'Bearer ' + self.tartarus_token + } + service_url = ( + self.config['TARTARUS_BASE_URL'] + + '/user/' + + self.user_uuid + + '/skillJson' + ) + response = service_request.get( + service_url, + headers=service_request_headers + ) + + self._check_for_service_errors(response) + + return response.json() + + def _parse_skill_manifests(self, skill_manifests: dict): + for device in skill_manifests.get('devices', []): + for skill in device['skills']: + manifest_skill = ManifestSkill( + failure_message=skill['failure_message'], + installation=skill['installation'], + name=skill['name'] + ) + self.skills_in_manifests[manifest_skill.name].append( + manifest_skill + ) + self.skills_in_manifests['mycroft-audio-record'].append(ManifestSkill( + failure_message='', + installation='installed', + name='mycroft-audio-record' + )) + + def _build_response_data(self) -> dict: + install_statuses = {} + failure_reasons = {} + for skill_name, manifest_skills in self.skills_in_manifests.items(): + skill_aggregator = SkillManifestAggregator(manifest_skills) + skill_aggregator.aggregate_manifest_skills() + if skill_aggregator.aggregate_skill.installation == 'failed': + failure_reasons[skill_name] = ( + skill_aggregator.aggregate_skill.failure_message + ) + install_statuses[skill_name] = ( + skill_aggregator.aggregate_skill.installation + ) + + return dict( + installStatuses=install_statuses, + failureReasons=failure_reasons + ) + + +class SkillManifestAggregator(object): + """Base class containing functionality shared by summary and detail""" + + def __init__(self, manifest_skills: List[ManifestSkill]): + self.manifest_skills = manifest_skills + self.aggregate_skill = ManifestSkill( + **asdict(manifest_skills[0])) + + def aggregate_manifest_skills(self): + """Aggregate skill data on all devices into a single skill. + + Each skill is represented once on the Marketplace, even though it can + be present on multiple devices. + """ + self._validate_install_status() + self._determine_install_status() + if self.aggregate_skill.installation == 'failed': + self._determine_failure_reason() + + def _validate_install_status(self): + for manifest_skill in self.manifest_skills: + if manifest_skill.installation not in VALID_INSTALLATION_VALUES: + raise ValueError( + '"{install_status}" is not a supported value of the ' + 'installation field in the skill manifest'.format( + install_status=manifest_skill.installation + ) + ) + + def _determine_install_status(self): + """Use skill data from all devices to determine install status. + + When a skill is installed via the Marketplace, it is installed to all + devices. The Marketplace will not mark a skill as "installed" until + install is complete on all devices. Until that point, the status will + be "installing". + + If the install fails on any device, the install will be flagged as a + failed install in the Marketplace. + """ + failed = [s.installation == 'failed' for s in + self.manifest_skills] + installing = [ + s.installation == 'installing' for s in self.manifest_skills + ] + uninstalling = [ + s.installation == 'uninstalling' for s in + self.manifest_skills + ] + installed = [ + s.installation == 'installed' for s in self.manifest_skills + ] + if any(failed): + self.aggregate_skill.installation = 'failed' + elif any(installing): + self.aggregate_skill.installation = 'installing' + elif any(uninstalling): + self.aggregate_skill.installation = 'uninstalling' + elif all(installed): + self.aggregate_skill.installation = 'installed' + + def _determine_failure_reason(self): + """When a skill fails to install, determine the reason""" + for manifest_skill in self.manifest_skills: + if manifest_skill.installation == 'failed': + self.aggregate_skill.failure_reason = ( + manifest_skill.failure_message + ) + break diff --git a/market/backend/v1/market-api/market_api/endpoints/skill_summary.py b/market/backend/v1/market-api/market_api/endpoints/skill_summary.py deleted file mode 100644 index 2b7c752b..00000000 --- a/market/backend/v1/market-api/market_api/endpoints/skill_summary.py +++ /dev/null @@ -1,179 +0,0 @@ -"""Endpoint to provide skill summary data to the marketplace.""" -from collections import defaultdict -from dataclasses import dataclass, field -from http import HTTPStatus -from logging import getLogger -from typing import List - -from markdown import markdown -import requests as service_request - -from .common import ( - aggregate_manifest_skills, - call_skill_manifest_endpoint, - parse_skill_manifest_response -) -from selene_util.api import APIError, SeleneEndpoint - -DEFAULT_ICON_COLOR = '#6C7A89' -DEFAULT_ICON_NAME = 'comment-alt' -SYSTEM_CATEGORY = 'System' -UNDEFINED_CATEGORY = 'Not Categorized' - -_log = getLogger(__package__) - - -@dataclass -class RepositorySkill(object): - """Represents a single skill defined in the Mycroft Skills repository.""" - branch: str - categories: List[str] - created: str - credits: List[str] - description: str - id: str - last_update: str - platforms: List[str] - repository_owner: str - repository_url: str - skill_name: str - summary: str - tags: List[str] - title: str - triggers: List[str] - icon: dict - icon_image: str = field(default=None) - marketplace_category: str = field(init=False, default=UNDEFINED_CATEGORY) - - def __post_init__(self): - if 'system' in self.tags: - self.marketplace_category = SYSTEM_CATEGORY - elif self.categories: - # a skill may have many categories. the first one in the - # list is considered the "primary" category. This is the - # category the marketplace will use to group the skill. - self.marketplace_category = self.categories[0] - if not self.icon: - self.icon = dict(icon=DEFAULT_ICON_NAME, color=DEFAULT_ICON_COLOR) - - -class SkillSummaryEndpoint(SeleneEndpoint): - authentication_required = False - - def __init__(self): - super(SkillSummaryEndpoint, self).__init__() - self.available_skills: List[RepositorySkill] = [] - self.response_skills = defaultdict(list) - self.skills_in_manifests = defaultdict(list) - - def get(self): - try: - self._authenticate() - self._get_skills() - except APIError: - pass - else: - self._build_response_data() - self.response = (self.response_skills, HTTPStatus.OK) - - return self.response - - def _get_skills(self): - """Retrieve the skill data that will be used to build the response.""" - self._get_available_skills() - # if self.authenticated: - # self._get_skill_manifests() - - def _get_available_skills(self): - """Retrieve all skills in the skill repository. - - The data is retrieved from a database table that is populated with - the contents of a JSON object in the mycroft-skills-data Github - repository. The JSON object contains metadata about each skill. - """ - skill_service_response = service_request.get( - self.config['SELENE_BASE_URL'] + '/skill/all' - ) - if skill_service_response.status_code != HTTPStatus.OK: - self._check_for_service_errors(skill_service_response) - self.available_skills = [ - RepositorySkill(**skill) for skill in skill_service_response.json() - ] - - def _get_skill_manifests(self): - service_response = call_skill_manifest_endpoint( - self.tartarus_token, - self.config['TARTARUS_BASE_URL'], - self.user_uuid - ) - if service_response.status_code != HTTPStatus.OK: - self._check_for_service_errors(service_response) - self.skills_in_manifest = parse_skill_manifest_response( - service_response - ) - - def _build_response_data(self): - """Build the data to include in the response.""" - if self.request.query_string: - skills_to_include = self._filter_skills() - else: - skills_to_include = self.available_skills - self._reformat_skills(skills_to_include) - self._sort_skills() - - def _filter_skills(self) -> list: - """If search criteria exist, only return those skills that match.""" - skills_to_include = [] - - query_string = self.request.query_string.decode() - search_term = query_string.lower().split('=')[1] - for skill in self.available_skills: - search_term_match = ( - search_term is None or - search_term in skill.title.lower() or - search_term in skill.description.lower() or - search_term in skill.summary.lower() - ) - if skill.categories and not search_term_match: - search_term_match = ( - search_term in skill.categories[0].lower() - ) - for trigger in skill.triggers: - if search_term in trigger.lower(): - search_term_match = True - if search_term_match: - skills_to_include.append(skill) - - return skills_to_include - - def _reformat_skills(self, skills_to_include: list): - """Build the response data from the skill service response""" - for skill in skills_to_include: - install_status = None - manifest_skills = self.skills_in_manifests.get(skill.skill_name) - if skill.marketplace_category == SYSTEM_CATEGORY: - install_status = SYSTEM_CATEGORY.lower() - elif manifest_skills is not None: - aggregated_manifest = aggregate_manifest_skills(manifest_skills) - install_status = aggregated_manifest.installation - - skill_summary = dict( - credits=skill.credits, - icon=skill.icon, - iconImage=skill.icon_image, - id=skill.id, - installStatus=install_status, - repositoryUrl=skill.repository_url, - summary=markdown(skill.summary, output_format='html5'), - title=skill.title, - triggers=skill.triggers - ) - self.response_skills[skill.marketplace_category].append( - skill_summary - ) - - def _sort_skills(self): - """Sort the skills in alphabetical order""" - for skill_category, skills in self.response_skills.items(): - sorted_skills = sorted(skills, key=lambda skill: skill['title']) - self.response_skills[skill_category] = sorted_skills