MycroftAI
/
selene-backend
mirror of https://github.com/MycroftAI/selene-backend.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

changed skill install status logic to be its own endpoint

pull/9/head
Chris Veilleux 2018-10-22 13:08:00 -05:00
parent 711ef460f9
commit 42c510665e
6 changed files with 357 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
market/backend/v1/market-api/market_api/api.py
View File

@ -3,9 +3,10 @@ from flask_restful import Api
from .config import get_config_location
from market_api.endpoints import (
SkillSummaryEndpoint,
AvailableSkillsEndpoint,
SkillDetailEndpoint,
SkillInstallEndpoint,
SkillInstallationsEndpoint,
UserEndpoint
)
@ -14,7 +15,14 @@ marketplace = Flask(__name__)
marketplace.config.from_object(get_config_location())
marketplace_api = Api(marketplace)
marketplace_api.add_resource(SkillSummaryEndpoint, '/api/skills')
marketplace_api.add_resource(SkillDetailEndpoint, '/api/skill/<skill_id>')
marketplace_api.add_resource(SkillInstallEndpoint, '/api/install')
marketplace_api.add_resource(AvailableSkillsEndpoint, '/api/skill/available')
marketplace_api.add_resource(
SkillDetailEndpoint,
'/api/skill/detail/<skill_name>'
)
marketplace_api.add_resource(SkillInstallEndpoint, '/api/skill/install')
marketplace_api.add_resource(
SkillInstallationsEndpoint,
'/api/skill/installations'
)
marketplace_api.add_resource(UserEndpoint, '/api/user')

3
market/backend/v1/market-api/market_api/endpoints/__init__.py
View File

@ -1,4 +1,5 @@
from .available_skills import AvailableSkillsEndpoint
from .skill_detail import SkillDetailEndpoint
from .skill_install import SkillInstallEndpoint
from .skill_summary import SkillSummaryEndpoint
from .skill_install_status import SkillInstallationsEndpoint
from .user import UserEndpoint

104
market/backend/v1/market-api/market_api/endpoints/available_skills.py Normal file
View File

@ -0,0 +1,104 @@
"""Endpoint to provide skill summary data to the marketplace."""
from collections import defaultdict
from http import HTTPStatus
from logging import getLogger
from typing import List
import requests as service_request
from selene_util.api import APIError, SeleneEndpoint
from .common import RepositorySkill
_log = getLogger(__package__)
class AvailableSkillsEndpoint(SeleneEndpoint):
authentication_required = False
def __init__(self):
super(AvailableSkillsEndpoint, self).__init__()
self.available_skills: List[RepositorySkill] = []
self.response_skills: List[dict] = []
self.skills_in_manifests = defaultdict(list)
def get(self):
try:
self._get_available_skills()
except APIError:
pass
else:
self._build_response_data()
self.response = (self.response_skills, HTTPStatus.OK)
return self.response
def _get_available_skills(self):
"""Retrieve all skills in the skill repository.
The data is retrieved from a database table that is populated with
the contents of a JSON object in the mycroft-skills-data Github
repository. The JSON object contains metadata about each skill.
"""
skill_service_response = service_request.get(
self.config['SELENE_BASE_URL'] + '/skill/all'
)
if skill_service_response.status_code != HTTPStatus.OK:
self._check_for_service_errors(skill_service_response)
self.available_skills = [
RepositorySkill(**skill) for skill in skill_service_response.json()
]
def _build_response_data(self):
"""Build the data to include in the response."""
if self.request.query_string:
skills_to_include = self._filter_skills()
else:
skills_to_include = self.available_skills
self._reformat_skills(skills_to_include)
self._sort_skills()
def _filter_skills(self) -> list:
"""If search criteria exist, only return those skills that match."""
skills_to_include = []
query_string = self.request.query_string.decode()
search_term = query_string.lower().split('=')[1]
for skill in self.available_skills:
search_term_match = (
search_term is None or
search_term in skill.title.lower() or
search_term in skill.description.lower() or
search_term in skill.summary.lower() or
search_term in [c.lower() for c in skill.categories] or
search_term in [t.lower() for t in skill.tags] or
search_term in [t.lower() for t in skill.triggers]
)
if search_term_match:
skills_to_include.append(skill)
return skills_to_include
def _reformat_skills(self, skills_to_include: List[RepositorySkill]):
"""Build the response data from the skill service response"""
for skill in skills_to_include:
skill_info = dict(
icon=skill.icon,
iconImage=skill.icon_image,
isMycroftMade=skill.is_mycroft_made,
isSystemSkill=skill.is_system_skill,
marketCategory=skill.market_category,
name=skill.skill_name,
summary=skill.summary,
title=skill.title,
trigger=skill.triggers[0]
)
self.response_skills.append(skill_info)
def _sort_skills(self):
"""Sort the skills in alphabetical order"""
sorted_skills = sorted(
self.response_skills,
key=lambda skill:
skill['title']
)
self.response_skills = sorted_skills

52
market/backend/v1/market-api/market_api/endpoints/common.py
View File

@ -1,12 +1,62 @@
from collections import defaultdict
from dataclasses import asdict, dataclass
from dataclasses import asdict, dataclass, field
from typing import List
from markdown import markdown
import requests as service_request
DEFAULT_ICON_COLOR = '#6C7A89'
DEFAULT_ICON_NAME = 'comment-alt'
SYSTEM_CATEGORY = 'System'
UNDEFINED_CATEGORY = 'Not Categorized'
VALID_INSTALLATION_VALUES = ('failed', 'installed', 'installing', 'uninstalled')
@dataclass
class RepositorySkill(object):
"""Represents a single skill defined in the Mycroft Skills repository."""
branch: str
categories: List[str]
created: str
credits: List[dict]
description: str
icon: dict
id: str
is_mycroft_made: bool = field(init=False)
is_system_skill: bool = field(init=False)
last_update: str
market_category: str = field(init=False)
platforms: List[str]
repository_owner: str
repository_url: str
skill_name: str
summary: str
tags: List[str]
title: str
triggers: List[str]
icon_image: str = field(default=None)
def __post_init__(self):
self.is_system_skill = False
if 'system' in self.tags:
self.is_system_skill = True
self.market_category = SYSTEM_CATEGORY
elif self.categories:
# a skill may have many categories. the first one in the
# list is considered the "primary" category. This is the
# category the marketplace will use to group the skill.
self.market_category = self.categories[0]
else:
self.market_category = UNDEFINED_CATEGORY
if not self.icon:
self.icon = dict(icon=DEFAULT_ICON_NAME, color=DEFAULT_ICON_COLOR)
self.is_mycroft_made = self.credits[0].get('name') == 'Mycroft AI'
self.summary = markdown(self.summary, output_format='html5')
@dataclass
class ManifestSkill(object):
"""Represents a single skill on a device's skill manifest.

188
market/backend/v1/market-api/market_api/endpoints/skill_install_status.py Normal file
View File

@ -0,0 +1,188 @@
from collections import defaultdict
from dataclasses import asdict, dataclass
from http import HTTPStatus
from typing import List
import requests as service_request
from selene_util.api import APIError, SeleneEndpoint
VALID_INSTALLATION_VALUES = (
'failed',
'installed',
'installing',
'uninstalling'
)
@dataclass
class ManifestSkill(object):
"""Represents a single skill on a device's skill manifest.
Mycroft core keeps a manifest off all skills associated with a device.
This manifest shows the status of each skill as it relates to the device.
"""
failure_message: str
installation: str
name: str
class SkillInstallationsEndpoint(SeleneEndpoint):
authentication_required = False
def __init__(self):
super(SkillInstallationsEndpoint, self).__init__()
self.skills_in_manifests = defaultdict(list)
def get(self):
try:
self._get_install_statuses()
except APIError:
pass
else:
response_data = self._build_response_data()
self.response = (response_data, HTTPStatus.OK)
return self.response
def _get_install_statuses(self):
self._authenticate()
if self.authenticated:
skill_manifests = self._get_skill_manifests()
self._parse_skill_manifests(skill_manifests)
else:
self.response = (
dict(installStatuses=[], failureReasons=[]),
HTTPStatus.OK
)
def _get_skill_manifests(self) -> dict:
"""Get the skill manifests from each of a user's devices
The skill manifests will be used to determine the status of each
skill as it relates to the marketplace.
"""
service_request_headers = {
'Authorization': 'Bearer ' + self.tartarus_token
}
service_url = (
self.config['TARTARUS_BASE_URL'] +
'/user/' +
self.user_uuid +
'/skillJson'
)
response = service_request.get(
service_url,
headers=service_request_headers
)
self._check_for_service_errors(response)
return response.json()
def _parse_skill_manifests(self, skill_manifests: dict):
for device in skill_manifests.get('devices', []):
for skill in device['skills']:
manifest_skill = ManifestSkill(
failure_message=skill['failure_message'],
installation=skill['installation'],
name=skill['name']
)
self.skills_in_manifests[manifest_skill.name].append(
manifest_skill
)
self.skills_in_manifests['mycroft-audio-record'].append(ManifestSkill(
failure_message='',
installation='installed',
name='mycroft-audio-record'
))
def _build_response_data(self) -> dict:
install_statuses = {}
failure_reasons = {}
for skill_name, manifest_skills in self.skills_in_manifests.items():
skill_aggregator = SkillManifestAggregator(manifest_skills)
skill_aggregator.aggregate_manifest_skills()
if skill_aggregator.aggregate_skill.installation == 'failed':
failure_reasons[skill_name] = (
skill_aggregator.aggregate_skill.failure_message
)
install_statuses[skill_name] = (
skill_aggregator.aggregate_skill.installation
)
return dict(
installStatuses=install_statuses,
failureReasons=failure_reasons
)
class SkillManifestAggregator(object):
"""Base class containing functionality shared by summary and detail"""
def __init__(self, manifest_skills: List[ManifestSkill]):
self.manifest_skills = manifest_skills
self.aggregate_skill = ManifestSkill(
**asdict(manifest_skills[0]))
def aggregate_manifest_skills(self):
"""Aggregate skill data on all devices into a single skill.
Each skill is represented once on the Marketplace, even though it can
be present on multiple devices.
"""
self._validate_install_status()
self._determine_install_status()
if self.aggregate_skill.installation == 'failed':
self._determine_failure_reason()
def _validate_install_status(self):
for manifest_skill in self.manifest_skills:
if manifest_skill.installation not in VALID_INSTALLATION_VALUES:
raise ValueError(
'"{install_status}" is not a supported value of the '
'installation field in the skill manifest'.format(
install_status=manifest_skill.installation
)
)
def _determine_install_status(self):
"""Use skill data from all devices to determine install status.
When a skill is installed via the Marketplace, it is installed to all
devices. The Marketplace will not mark a skill as "installed" until
install is complete on all devices. Until that point, the status will
be "installing".
If the install fails on any device, the install will be flagged as a
failed install in the Marketplace.
"""
failed = [s.installation == 'failed' for s in
self.manifest_skills]
installing = [
s.installation == 'installing' for s in self.manifest_skills
]
uninstalling = [
s.installation == 'uninstalling' for s in
self.manifest_skills
]
installed = [
s.installation == 'installed' for s in self.manifest_skills
]
if any(failed):
self.aggregate_skill.installation = 'failed'
elif any(installing):
self.aggregate_skill.installation = 'installing'
elif any(uninstalling):
self.aggregate_skill.installation = 'uninstalling'
elif all(installed):
self.aggregate_skill.installation = 'installed'
def _determine_failure_reason(self):
"""When a skill fails to install, determine the reason"""
for manifest_skill in self.manifest_skills:
if manifest_skill.installation == 'failed':
self.aggregate_skill.failure_reason = (
manifest_skill.failure_message
)
break

179
market/backend/v1/market-api/market_api/endpoints/skill_summary.py
View File

@ -1,179 +0,0 @@
"""Endpoint to provide skill summary data to the marketplace."""
from collections import defaultdict
from dataclasses import dataclass, field
from http import HTTPStatus
from logging import getLogger
from typing import List
from markdown import markdown
import requests as service_request
from .common import (
aggregate_manifest_skills,
call_skill_manifest_endpoint,
parse_skill_manifest_response
)
from selene_util.api import APIError, SeleneEndpoint
DEFAULT_ICON_COLOR = '#6C7A89'
DEFAULT_ICON_NAME = 'comment-alt'
SYSTEM_CATEGORY = 'System'
UNDEFINED_CATEGORY = 'Not Categorized'
_log = getLogger(__package__)
@dataclass
class RepositorySkill(object):
"""Represents a single skill defined in the Mycroft Skills repository."""
branch: str
categories: List[str]
created: str
credits: List[str]
description: str
id: str
last_update: str
platforms: List[str]
repository_owner: str
repository_url: str
skill_name: str
summary: str
tags: List[str]
title: str
triggers: List[str]
icon: dict
icon_image: str = field(default=None)
marketplace_category: str = field(init=False, default=UNDEFINED_CATEGORY)
def __post_init__(self):
if 'system' in self.tags:
self.marketplace_category = SYSTEM_CATEGORY
elif self.categories:
# a skill may have many categories. the first one in the
# list is considered the "primary" category. This is the
# category the marketplace will use to group the skill.
self.marketplace_category = self.categories[0]
if not self.icon:
self.icon = dict(icon=DEFAULT_ICON_NAME, color=DEFAULT_ICON_COLOR)
class SkillSummaryEndpoint(SeleneEndpoint):
authentication_required = False
def __init__(self):
super(SkillSummaryEndpoint, self).__init__()
self.available_skills: List[RepositorySkill] = []
self.response_skills = defaultdict(list)
self.skills_in_manifests = defaultdict(list)
def get(self):
try:
self._authenticate()
self._get_skills()
except APIError:
pass
else:
self._build_response_data()
self.response = (self.response_skills, HTTPStatus.OK)
return self.response
def _get_skills(self):
"""Retrieve the skill data that will be used to build the response."""
self._get_available_skills()
# if self.authenticated:
# self._get_skill_manifests()
def _get_available_skills(self):
"""Retrieve all skills in the skill repository.
The data is retrieved from a database table that is populated with
the contents of a JSON object in the mycroft-skills-data Github
repository. The JSON object contains metadata about each skill.
"""
skill_service_response = service_request.get(
self.config['SELENE_BASE_URL'] + '/skill/all'
)
if skill_service_response.status_code != HTTPStatus.OK:
self._check_for_service_errors(skill_service_response)
self.available_skills = [
RepositorySkill(**skill) for skill in skill_service_response.json()
]
def _get_skill_manifests(self):
service_response = call_skill_manifest_endpoint(
self.tartarus_token,
self.config['TARTARUS_BASE_URL'],
self.user_uuid
)
if service_response.status_code != HTTPStatus.OK:
self._check_for_service_errors(service_response)
self.skills_in_manifest = parse_skill_manifest_response(
service_response
)
def _build_response_data(self):
"""Build the data to include in the response."""
if self.request.query_string:
skills_to_include = self._filter_skills()
else:
skills_to_include = self.available_skills
self._reformat_skills(skills_to_include)
self._sort_skills()
def _filter_skills(self) -> list:
"""If search criteria exist, only return those skills that match."""
skills_to_include = []
query_string = self.request.query_string.decode()
search_term = query_string.lower().split('=')[1]
for skill in self.available_skills:
search_term_match = (
search_term is None or
search_term in skill.title.lower() or
search_term in skill.description.lower() or
search_term in skill.summary.lower()
)
if skill.categories and not search_term_match:
search_term_match = (
search_term in skill.categories[0].lower()
)
for trigger in skill.triggers:
if search_term in trigger.lower():
search_term_match = True
if search_term_match:
skills_to_include.append(skill)
return skills_to_include
def _reformat_skills(self, skills_to_include: list):
"""Build the response data from the skill service response"""
for skill in skills_to_include:
install_status = None
manifest_skills = self.skills_in_manifests.get(skill.skill_name)
if skill.marketplace_category == SYSTEM_CATEGORY:
install_status = SYSTEM_CATEGORY.lower()
elif manifest_skills is not None:
aggregated_manifest = aggregate_manifest_skills(manifest_skills)
install_status = aggregated_manifest.installation
skill_summary = dict(
credits=skill.credits,
icon=skill.icon,
iconImage=skill.icon_image,
id=skill.id,
installStatus=install_status,
repositoryUrl=skill.repository_url,
summary=markdown(skill.summary, output_format='html5'),
title=skill.title,
triggers=skill.triggers
)
self.response_skills[skill.marketplace_category].append(
skill_summary
)
def _sort_skills(self):
"""Sort the skills in alphabetical order"""
for skill_category, skills in self.response_skills.items():
sorted_skills = sorted(skills, key=lambda skill: skill['title'])
self.response_skills[skill_category] = sorted_skills