MycroftAI
/
selene-backend
mirror of https://github.com/MycroftAI/selene-backend.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'remotes/origin/dev' into feature/publish_repo 

# Conflicts:
#	api/account/Pipfile.lock
#	api/public/public_api/endpoints/device_skill.py
#	shared/selene/data/device/__init__.py
pull/192/head
Chris Veilleux 2019-09-30 11:54:54 -05:00
parent b326067f2e b2b99e9dbd
commit ad3eb20a78
26 changed files with 574 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/account/Pipfile
View File

@ -4,7 +4,7 @@ verify_ssl = true
name = "pypi"
[packages]
flask = "*"
flask = "<1.1"
uwsgi = "*"
schematics = "*"
stripe = "*"

95
api/account/account_api/endpoints/device.py
View File

@ -18,6 +18,7 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from dataclasses import asdict
from datetime import datetime, timedelta
from http import HTTPStatus
from logging import getLogger
@ -30,9 +31,12 @@ from selene.api import SeleneEndpoint
from selene.api.etag import ETagManager
from selene.api.public_endpoint import delete_device_login
from selene.data.device import DeviceRepository, Geography, GeographyRepository
from selene.util.cache import SeleneCache
from selene.util.cache import DEVICE_LAST_CONTACT_KEY, SeleneCache
ONE_DAY = 86400
CONNECTED = 'Connected'
DISCONNECTED = 'Disconnected'
DORMANT = 'Dormant'
_log = getLogger()
@ -84,22 +88,95 @@ class DeviceEndpoint(SeleneEndpoint):
)
response_data = []
for device in devices:
device_dict = asdict(device)
device_dict['voice'] = device_dict.pop('text_to_speech')
response_data.append(device_dict)
response_device = self._format_device_for_response(device)
response_data.append(response_device)
return response_data
def _get_device(self, device_id):
device_repository = DeviceRepository(self.db)
device = device_repository.get_device_by_id(
device_id
)
response_data = asdict(device)
response_data['voice'] = response_data.pop('text_to_speech')
device = device_repository.get_device_by_id(device_id)
response_data = self._format_device_for_response(device)
return response_data
def _format_device_for_response(self, device):
"""Convert device object into a response object for this endpoint."""
last_contact_age = self._get_device_last_contact(device)
device_status = self._determine_device_status(last_contact_age)
if device_status == DISCONNECTED:
disconnect_duration = self._determine_disconnect_duration(
last_contact_age
)
else:
disconnect_duration = None
device_dict = asdict(device)
device_dict['status'] = device_status
device_dict['disconnect_duration'] = disconnect_duration
device_dict['voice'] = device_dict.pop('text_to_speech')
return device_dict
def _get_device_last_contact(self, device):
"""Get the last time the device contacted the backend.
The timestamp returned by this method will be used to determine if a
device is active or not.
The device table has a last contacted column but it is only updated
daily via batch script. The real-time values are kept in Redis.
If the Redis query returns nothing, the device hasn't contacted the
backend yet. This could be because it was just activated. Give the
device a couple of minutes to make that first call to the backend.
"""
last_contact_ts = self.cache.get(
DEVICE_LAST_CONTACT_KEY.format(device_id=device.id)
)
if last_contact_ts is None:
if device.last_contact_ts is None:
last_contact_age = datetime.utcnow() - device.add_ts
else:
last_contact_age = datetime.utcnow() - device.last_contact_ts
else:
last_contact_ts = last_contact_ts.decode()
last_contact_ts = datetime.strptime(
last_contact_ts,
'%Y-%m-%d %H:%M:%S.%f'
)
last_contact_age = datetime.utcnow() - last_contact_ts
return last_contact_age
@staticmethod
def _determine_device_status(last_contact_age):
"""Derive device status from the last time device contacted servers."""
if last_contact_age <= timedelta(seconds=120):
device_status = CONNECTED
elif timedelta(seconds=120) < last_contact_age < timedelta(days=30):
device_status = DISCONNECTED
else:
device_status = DORMANT
return device_status
@staticmethod
def _determine_disconnect_duration(last_contact_age):
"""Derive device status from the last time device contacted servers."""
disconnect_duration = 'unknown'
days, _ = divmod(last_contact_age, timedelta(days=1))
if days:
disconnect_duration = str(days) + ' days'
else:
hours, remaining = divmod(last_contact_age, timedelta(hours=1))
if hours:
disconnect_duration = str(hours) + ' hours'
else:
minutes, _ = divmod(remaining, timedelta(minutes=1))
if minutes:
disconnect_duration = str(minutes) + ' minutes'
return disconnect_duration
def post(self):
self._authenticate()
device = self._validate_request()

12
api/account/account_api/endpoints/skill_settings.py
View File

@ -74,8 +74,8 @@ class SkillSettingsEndpoint(SeleneEndpoint):
the value is the value.
"""
for skill_settings in self.family_settings:
if skill_settings.settings_display is not None:
for section in skill_settings.settings_display['sections']:
if skill_settings.settings_definition is not None:
for section in skill_settings.settings_definition['sections']:
for field in section['fields']:
if field['type'] == 'select':
parsed_options = []
@ -95,10 +95,10 @@ class SkillSettingsEndpoint(SeleneEndpoint):
for skill_settings in self.family_settings:
# The UI will throw an error if settings display is null due to how
# the skill settings data structures are defined.
if skill_settings.settings_display is None:
skill_settings.settings_display = dict(sections=[])
if skill_settings.settings_definition is None:
skill_settings.settings_definition = dict(sections=[])
response_skill = dict(
settingsDisplay=skill_settings.settings_display,
settingsDisplay=skill_settings.settings_definition,
settingsValues=skill_settings.settings_values,
deviceNames=skill_settings.device_names
)
@ -117,7 +117,7 @@ class SkillSettingsEndpoint(SeleneEndpoint):
"""Update the value of the settings column on the device_skill table,"""
for new_skill_settings in self.request.json['skillSettings']:
account_skill_settings = AccountSkillSetting(
settings_display=new_skill_settings['settingsDisplay'],
settings_definition=new_skill_settings['settingsDisplay'],
settings_values=new_skill_settings['settingsValues'],
device_names=new_skill_settings['deviceNames']
)

25
api/public/public_api/api.py
View File

@ -21,7 +21,6 @@ import os
from flask import Flask
from public_api.endpoints.stripe_webhook import StripeWebHookEndpoint
from selene.api import SeleneResponse, selene_api
from selene.api.base_config import get_base_config
from selene.api.public_endpoint import check_oauth_token
@ -36,14 +35,16 @@ from .endpoints.device_metrics import DeviceMetricsEndpoint
from .endpoints.device_oauth import OauthServiceEndpoint
from .endpoints.device_refresh_token import DeviceRefreshTokenEndpoint
from .endpoints.device_setting import DeviceSettingEndpoint
from .endpoints.device_skill import DeviceSkillEndpoint
from .endpoints.device_skill import SkillSettingsMetaEndpoint
from .endpoints.device_skill_manifest import DeviceSkillManifestEndpoint
from .endpoints.device_skill_settings import DeviceSkillSettingsEndpoint
from .endpoints.device_skill_settings import DeviceSkillSettingsEndpointV2
from .endpoints.device_subscription import DeviceSubscriptionEndpoint
from .endpoints.google_stt import GoogleSTTEndpoint
from .endpoints.oauth_callback import OauthCallbackEndpoint
from .endpoints.open_weather_map import OpenWeatherMapEndpoint
from .endpoints.premium_voice import PremiumVoiceEndpoint
from .endpoints.stripe_webhook import StripeWebHookEndpoint
from .endpoints.wolfram_alpha import WolframAlphaEndpoint
from .endpoints.wolfram_alpha_spoken import WolframAlphaSpokenEndpoint
@ -70,11 +71,17 @@ public.add_url_rule(
)
public.add_url_rule(
'/v1/device/<string:device_id>/userSkill',
view_func=DeviceSkillEndpoint.as_view('device_user_skill_api'),
'/v1/device/<string:device_id>/skill/settings',
view_func=DeviceSkillSettingsEndpointV2.as_view('skill_settings_api'),
methods=['GET']
)
public.add_url_rule(
'/v1/device/<string:device_id>/settingsMeta',
view_func=SkillSettingsMetaEndpoint.as_view('device_user_skill_api'),
methods=['PUT']
)
public.add_url_rule(
'/v1/device/<string:device_id>',
view_func=DeviceEndpoint.as_view('device_api'),
@ -172,9 +179,9 @@ public.add_url_rule(
methods=['POST']
)
"""
This is a workaround to allow the API return 401 when we call a non existent path. Use case:
GET /device/{uuid} with empty uuid. Core today uses the 401 to validate if it needs to perform a pairing process
Whe should fix that in a future version because we have to return 404 when we call a non existent path
"""
# This is a workaround to allow the API return 401 when we call a non existent
# path. Use case: GET /device/{uuid} with empty uuid. Core today uses the 401
# to validate if it needs to perform a pairing process. We should fix that in a
# future version because we have to return 404 when we call a non existent path
public.before_request(check_oauth_token)

268
api/public/public_api/endpoints/device_skill.py
View File

@ -17,16 +17,276 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""Applies a skill settings definition to the database.
Whenever a change is made to a skill's settings definition on a device, this
endpoint is called to update the same on the database. If a skill has
settings, the device's settings are also updated.
This endpoint assumes that the skill manifest is sent when the a device is
paired or a skill is installed. A skill that is not installed on a device
cannot send it's settings... right? The skill and its relationship to the
device should already be known when this endpoint is called.
"""
from http import HTTPStatus
from logging import getLogger
from schematics import Model
from schematics.types import (
BooleanType,
ListType,
ModelType,
StringType
)
from schematics.exceptions import DataError
from selene.api import PublicEndpoint
from selene.data.account import AccountRepository
from selene.data.device import DeviceSkillRepository
from selene.data.skill import (
extract_family_from_global_id,
SettingsDisplay,
SettingsDisplayRepository,
SkillRepository
)
from selene.data.skill import SkillSettingRepository
_log = getLogger(__package__)
class DeviceSkillEndpoint(PublicEndpoint):
"""Return a skill setting using the API v1 format for a given device and version_hash"""
def _normalize_field_value(field):
"""The field values in skillMetadata are all strings, convert to native."""
normalized_value = field.get('value')
if field['type'].lower() == 'checkbox':
if field['value'] in ('false', 'False', '0'):
normalized_value = False
elif field['value'] in ('true', 'True', '1'):
normalized_value = True
elif field['type'].lower() == 'number' and isinstance(field['value'], str):
if field['value']:
normalized_value = float(field['value'])
if not normalized_value % 1:
normalized_value = int(field['value'])
else:
normalized_value = 0
elif field['value'] == "[]":
normalized_value = []
return normalized_value
class RequestSkillField(Model):
name = StringType()
type = StringType()
label = StringType()
hint = StringType()
placeholder = StringType()
hide = BooleanType()
value = StringType()
options = StringType()
class RequestSkillSection(Model):
name = StringType(required=True)
fields = ListType(ModelType(RequestSkillField))
class RequestSkillMetadata(Model):
sections = ListType(ModelType(RequestSkillSection))
class RequestSkillIcon(Model):
color = StringType()
icon = StringType()
class RequestDeviceSkill(Model):
display_name = StringType(required=True)
icon = ModelType(RequestSkillIcon)
icon_img = StringType()
skill_gid = StringType(required=True)
skillMetadata = ModelType(RequestSkillMetadata)
class SkillSettingsMetaEndpoint(PublicEndpoint):
def __init__(self):
super(DeviceSkillEndpoint, self).__init__()
super().__init__()
self.skill = None
self.default_settings = None
self.skill_has_settings = False
self.settings_definition_id = None
self._device_skill_repo = None
def get(self, device_id):
@property
def device_skill_repo(self):
if self._device_skill_repo is None:
self._device_skill_repo = DeviceSkillRepository(self.db)
return self._device_skill_repo
def put(self, device_id):
self._authenticate(device_id)
self._validate_request()
self._get_skill()
self._parse_skill_metadata()
self._ensure_settings_definition_exists()
self._update_device_skill(device_id)
return '', HTTPStatus.NO_CONTENT
def _validate_request(self):
"""Ensure the request is well-formed."""
request_model = RequestDeviceSkill(self.request.json)
request_model.validate()
def _get_skill(self):
"""Retrieve the skill associated with the request."""
skill_repo = SkillRepository(self.db)
self.skill = skill_repo.get_skill_by_global_id(
self.request.json['skill_gid']
)
if self.skill is None:
err_msg = (
'No skill on database for skill ' +
self.request.json['skill_gid']
)
_log.error(err_msg)
raise DataError(dict(skill_gid=[err_msg]))
def _parse_skill_metadata(self):
"""Inspect the contents of the skill settings definition.
Skill authors often write settings definition files with strings in
fields that should be boolean or numeric. Ensure all fields are cast
to the correct type before interacting with the database.
"""
self.skill_has_settings = 'skillMetadata' in self.request.json
if self.skill_has_settings:
skill_metadata = self.request.json['skillMetadata']
self.default_settings = {}
normalized_sections = []
for section in skill_metadata['sections']:
for field in section['fields']:
if field['type'] != 'label':
field['value'] = _normalize_field_value(field)
self.default_settings[field['name']] = field['value']
normalized_sections.append(section)
self.request.json['skillMetadata'].update(
sections=normalized_sections
)
def _ensure_settings_definition_exists(self):
"""Add a row to skill.settings_display if it doesn't already exist."""
self.settings_definition_id = None
self._check_for_existing_settings_definition()
if self.settings_definition_id is None:
self._add_settings_definition()
def _check_for_existing_settings_definition(self):
"""Look for an existing database row matching the request."""
settings_def_repo = SettingsDisplayRepository(self.db)
settings_defs = settings_def_repo.get_settings_definitions_by_gid(
self.skill.skill_gid
)
for settings_def in settings_defs:
if settings_def.display_data == self.request.json:
self.settings_definition_id = settings_def.id
break
def _add_settings_definition(self):
"""The settings definition does not exist on database so add it."""
settings_def_repo = SettingsDisplayRepository(self.db)
settings_definition = SettingsDisplay(
skill_id=self.skill.id,
display_data=self.request.json
)
self.settings_definition_id = settings_def_repo.add(
settings_definition
)
def _update_device_skill(self, device_id):
"""Update device.device_skill to match the new settings definition.
If the skill has settings and the device_skill table does not, either
use the default values in the settings definition or copy the settings
from another device under the same account.
"""
device_skill = self._get_device_skill(device_id)
device_skill.settings_display_id = self.settings_definition_id
if self.skill_has_settings:
if device_skill.settings_values is None:
new_settings_values = self._initialize_skill_settings(
device_id
)
else:
new_settings_values = self._reconcile_skill_settings(
device_skill.settings_values
)
device_skill.settings_values = new_settings_values
self.device_skill_repo.update_device_skill_settings(
device_id,
device_skill
)
def _get_device_skill(self, device_id):
"""Retrieve the device's skill entry from the database."""
device_skill = self.device_skill_repo.get_skill_settings_for_device(
device_id,
self.skill.id
)
if device_skill is None:
error_msg = (
'Received skill setting definition before manifest for '
'skill ' + self.skill.skill_gid
)
_log.error(error_msg)
raise DataError(dict(skill_gid=[error_msg]))
return device_skill
def _reconcile_skill_settings(self, settings_values):
"""Fix any new or removed settings."""
new_settings_values = {}
for name, value in self.default_settings.items():
if name in settings_values:
new_settings_values[name] = settings_values[name]
else:
new_settings_values[name] = self.default_settings[name]
for name, value in settings_values.items():
if name in self.default_settings:
new_settings_values[name] = settings_values[name]
return new_settings_values
def _initialize_skill_settings(self, device_id):
"""Use default settings or copy from another device in same account."""
_log.info('Initializing settings for skill ' + self.skill.skill_gid)
account_repo = AccountRepository(self.db)
account = account_repo.get_account_by_device_id(device_id)
skill_settings_repo = SkillSettingRepository(self.db)
skill_family = extract_family_from_global_id(self.skill.skill_gid)
family_settings = skill_settings_repo.get_family_settings(
account.id,
skill_family
)
new_settings_values = self.default_settings
if family_settings is not None:
for settings in family_settings:
if settings.settings_values is None:
continue
if settings.settings_values != self.default_settings:
field_names = settings.settings_values.keys()
if field_names == self.default_settings.keys():
_log.info(
'Copying settings from another device for skill' +
self.skill.skill_gid
)
new_settings_values = settings.settings_values
break
else:
_log.info(
'Using default skill settings for skill ' +
self.skill.skill_gid
)
return new_settings_values

101
api/public/public_api/endpoints/device_skill_settings.py
View File

@ -50,6 +50,27 @@ GLOBAL_ID_ANY_PATTERN = '(?:{})|(?:{})|(?:{})'.format(
)
def _normalize_field_value(field):
"""The field values in skillMetadata are all strings, convert to native."""
normalized_value = field.get('value')
if field['type'].lower() == 'checkbox':
if field['value'] in ('false', 'False', '0'):
normalized_value = False
elif field['value'] in ('true', 'True', '1'):
normalized_value = True
elif field['type'].lower() == 'number' and isinstance(field['value'], str):
if field['value']:
normalized_value = float(field['value'])
if not normalized_value % 1:
normalized_value = int(field['value'])
else:
normalized_value = 0
elif field['value'] == "[]":
normalized_value = []
return normalized_value
class SkillSettingUpdater(object):
"""Update the settings data for all devices with a skill
@ -105,6 +126,7 @@ class SkillSettingUpdater(object):
field_value = field.get('value')
if field_value is not None:
if field_name is not None:
field_value = _normalize_field_value(field)
self.settings_values[field_name] = field_value
del(field['value'])
sections_without_values.append(section_without_values)
@ -152,7 +174,7 @@ class SkillSettingUpdater(object):
account_repo = AccountRepository(self.db)
account = account_repo.get_account_by_device_id(self.device_id)
skill_settings = (
self.device_skill_repo.get_device_skill_settings_for_account(
self.device_skill_repo.get_skill_settings_for_account(
account.id,
self.skill.id
)
@ -315,7 +337,9 @@ class DeviceSkillSettingsEndpoint(PublicEndpoint):
if skill_gid is not None:
response_skill.update(skill_gid=skill_gid)
identifier = skill.settings_display.get('identifier')
if identifier is not None:
if identifier is None:
response_skill.update(identifier=skill_gid)
else:
response_skill.update(identifier=identifier)
response_data.append(response_skill)
@ -330,7 +354,7 @@ class DeviceSkillSettingsEndpoint(PublicEndpoint):
for field in section_with_values['fields']:
field_name = field.get('name')
if field_name is not None and field_name in settings_values:
field.update(value=settings_values[field_name])
field.update(value=str(settings_values[field_name]))
sections_with_values.append(section_with_values)
return sections_with_values
@ -362,30 +386,59 @@ class DeviceSkillSettingsEndpoint(PublicEndpoint):
return skill_setting_updater.skill.id
def delete(self, device_id, skill_gid):
self._authenticate(device_id)
skill = self.skill_repo.get_skill_by_global_id(skill_gid)
settings_display_id = self._delete_skill_from_device(device_id, skill)
self._delete_orphaned_settings_display(settings_display_id)
return '', HTTPStatus.OK
def _delete_skill_from_device(self, device_id, skill):
settings_display_id = None
device_skills = (
self.device_skill_repo.get_device_skill_settings_for_device(
device_id
)
)
for device_skill in device_skills:
if device_skill.skill_id == skill.id:
self.device_skill_repo.remove(device_id, skill.id)
settings_display_id = device_skill.settings_display_id
return settings_display_id
def _delete_orphaned_settings_display(self, settings_display_id):
skill_count = self.device_skill_repo.get_settings_display_usage(
settings_display_id
)
if not skill_count:
self.settings_display_repo.remove(settings_display_id)
class DeviceSkillSettingsEndpointV2(PublicEndpoint):
"""Replacement that decouples settings definition from values.
The older version of this class needs to be kept around for compatibility
with pre 19.08 versions of mycroft-core. Once those versions are no
longer supported, the older class can be deprecated.
"""
def get(self, device_id):
"""
Retrieve skills installed on device from the database.
:raises NotModifiedException: when etag in request matches cache
"""
self._authenticate(device_id)
self._validate_etag(DEVICE_SKILL_ETAG_KEY.format(device_id=device_id))
response_data = self._build_response_data(device_id)
response = self._build_response(device_id, response_data)
return response
def _build_response_data(self, device_id):
device_skill_repo = DeviceSkillRepository(self.db)
device_skills = device_skill_repo.get_skill_settings_for_device(
device_id
)
if device_skills is not None:
response_data = {}
for skill in device_skills:
response_data[skill.skill_gid] = skill.settings_values
return response_data
def _build_response(self, device_id, response_data):
if response_data is None:
response = Response(
'',
status=HTTPStatus.NO_CONTENT,
content_type='application/json'
)
else:
response = Response(
json.dumps(response_data),
status=HTTPStatus.OK,
content_type='application/json'
)
self._add_etag(DEVICE_SKILL_ETAG_KEY.format(device_id=device_id))
return response

7
api/public/tests/features/device_skill_settings.feature
View File

@ -55,10 +55,3 @@ Feature: Upload and fetch skills and their settings
And the field is no longer in the skill settings
And the device skill E-tag is expired
And device last contact timestamp is updated
Scenario: A device requests a skill to be deleted
Given an authorized device
When the device requests a skill to be deleted
Then the request will be successful
And the skill will be removed from the device skill list
And device last contact timestamp is updated

9
api/public/tests/features/steps/device_skill_settings.py
View File

@ -211,12 +211,3 @@ def validate_skill_setting_field_removed(context):
new_skill_definition = new_settings_display['skillMetadata']
new_section = new_skill_definition['sections'][0]
assert_that(context.removed_field, not is_in(new_section['fields']))
@then('the skill will be removed from the device skill list')
def validate_delete_skill(context):
device_skill_repo = DeviceSkillRepository(context.db)
device_skills = device_skill_repo.get_device_skill_settings_for_device(
context.device_id
)
assert_that(len(device_skills), equal_to(1))

8
batch/job_scheduler/jobs.py
View File

@ -99,6 +99,13 @@ def test_scheduler():
job_runner.run_job()
def load_19_02_skills():
"""Load the json file from the mycroft-skills-data repository to the DB"""
job_runner = JobRunner('load_skill_display_data.py --core-version 19.02')
job_runner.job_date = date.today() - timedelta(days=1)
job_runner.run_job()
def parse_core_metrics():
"""Copy rows from metric.core to de-normalized metric.core_interaction
@ -138,6 +145,7 @@ if os.environ['SELENE_ENVIRONMENT'] != 'prod':
schedule.every().day.at('00:00').do(partition_api_metrics)
schedule.every().day.at('00:05').do(update_device_last_contact)
schedule.every().day.at('00:10').do(parse_core_metrics)
schedule.every().day.at('00:15').do(load_19_02_skills())
# Run the schedule
while True:

2
shared/selene/data/device/__init__.py
View File

@ -17,7 +17,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from .entity.device_skill import ManifestSkill, DeviceSkillSettings
from .entity.device_skill import ManifestSkill, AccountSkillSettings
from .entity.geography import Geography
from .entity.preference import AccountPreferences
from .entity.text_to_speech import TextToSpeech

1
shared/selene/data/device/entity/device.py
View File

@ -42,3 +42,4 @@ class Device(object):
wake_word: WakeWord
last_contact_ts: datetime = None
placement: str = None
add_ts: datetime = None

12
shared/selene/data/device/entity/device_skill.py
View File

@ -36,9 +36,17 @@ class ManifestSkill(object):
@dataclass
class DeviceSkillSettings(object):
device_ids: List[str]
class AccountSkillSettings(object):
install_method: str
skill_id: str
device_ids: List[str] = None
settings_values: dict = None
settings_display_id: str = None
@dataclass
class DeviceSkillSettings(object):
skill_id: str
skill_gid: str
settings_values: dict = None
settings_display_id: str = None

49
shared/selene/data/device/repository/device_skill.py
View File

@ -23,7 +23,11 @@ from dataclasses import asdict
from typing import List
from selene.data.skill import SettingsDisplay
from ..entity.device_skill import ManifestSkill, DeviceSkillSettings
from ..entity.device_skill import (
AccountSkillSettings,
DeviceSkillSettings,
ManifestSkill
)
from ...repository_base import RepositoryBase
@ -31,23 +35,31 @@ class DeviceSkillRepository(RepositoryBase):
def __init__(self, db):
super(DeviceSkillRepository, self).__init__(db, __file__)
def get_device_skill_settings_for_account(
def get_skill_settings_for_account(
self, account_id: str, skill_id: str
) -> List[DeviceSkillSettings]:
) -> List[AccountSkillSettings]:
return self._select_all_into_dataclass(
DeviceSkillSettings,
sql_file_name='get_device_skill_settings_for_account.sql',
AccountSkillSettings,
sql_file_name='get_skill_settings_for_account.sql',
args=dict(account_id=account_id, skill_id=skill_id)
)
def get_device_skill_settings_for_device(
self, device_id: str
) -> List[DeviceSkillSettings]:
return self._select_all_into_dataclass(
def get_skill_settings_for_device(self, device_id, skill_id=None):
device_skills = self._select_all_into_dataclass(
DeviceSkillSettings,
sql_file_name='get_device_skill_settings_for_device.sql',
sql_file_name='get_skill_settings_for_device.sql',
args=dict(device_id=device_id)
)
if skill_id is None:
skill_settings = device_skills
else:
skill_settings = None
for skill in device_skills:
if skill.skill_id == skill_id:
skill_settings = skill
break
return skill_settings
def update_skill_settings(
self, account_id: str, device_names: tuple, skill_name: str
@ -84,6 +96,23 @@ class DeviceSkillRepository(RepositoryBase):
)
self.cursor.insert(db_request)
def update_device_skill_settings(self, device_id, device_skill):
"""Update the skill settings columns on the device_skill table."""
if device_skill.settings_values is None:
db_settings_values = None
else:
db_settings_values = json.dumps(device_skill.settings_values)
db_request = self._build_db_request(
sql_file_name='update_device_skill_settings.sql',
args=dict(
device_id=device_id,
skill_id=device_skill.skill_id,
settings_display_id=device_skill.settings_display_id,
settings_values=db_settings_values
)
)
self.cursor.update(db_request)
def get_skill_manifest_for_device(
self, device_id: str
) -> List[ManifestSkill]:

15
shared/selene/data/device/repository/sql/get_device_skill_settings_for_device.sql
View File

@ -1,15 +0,0 @@
SELECT
skill_settings_display_id AS settings_display_id,
settings::jsonb AS settings_values,
install_method,
skill_id,
array_agg(device_id::text) AS device_ids
FROM
device.device_skill
WHERE
device_id = %(device_id)s
GROUP BY
skill_settings_display_id,
settings::jsonb,
install_method,
skill_id

1
shared/selene/data/device/repository/sql/get_devices_by_account_id.sql
View File

@ -7,6 +7,7 @@ SELECT
d.core_version,
d.placement,
d.last_contact_ts,
d.insert_ts AS add_ts,
json_build_object(
'setting_name', ww.setting_name,
'display_name', ww.display_name,

0
shared/selene/data/device/repository/sql/get_device_skill_settings_for_account.sql → shared/selene/data/device/repository/sql/get_skill_settings_for_account.sql
View File

10
shared/selene/data/device/repository/sql/get_skill_settings_for_device.sql Normal file
View File

@ -0,0 +1,10 @@
SELECT
ds.skill_settings_display_id AS settings_display_id,
ds.settings::jsonb AS settings_values,
ds.skill_id,
s.skill_gid
FROM
device.device_skill ds
INNER JOIN skill.skill s ON ds.skill_id = s.id
WHERE
device_id = %(device_id)s

8
shared/selene/data/device/repository/sql/update_device_skill_settings.sql Normal file
View File

@ -0,0 +1,8 @@
UPDATE
device.device_skill
SET
skill_settings_display_id = %(settings_display_id)s,
settings = %(settings_values)s
WHERE
device_id = %(device_id)s
AND skill_id = %(skill_id)s

2
shared/selene/data/skill/__init__.py
View File

@ -27,4 +27,4 @@ from .entity.skill_setting import (
from .repository.display import SkillDisplayRepository
from .repository.setting import SkillSettingRepository
from .repository.settings_display import SettingsDisplayRepository
from .repository.skill import SkillRepository
from .repository.skill import extract_family_from_global_id, SkillRepository

2
shared/selene/data/skill/entity/skill_setting.py
View File

@ -23,7 +23,7 @@ from typing import List
@dataclass
class AccountSkillSetting(object):
settings_display: dict
settings_definition: dict
settings_values: dict
device_names: List[str]

29
shared/selene/data/skill/repository/setting.py
View File

@ -36,26 +36,11 @@ class SkillSettingRepository(RepositoryBase):
account_id: str,
family_name: str
) -> List[AccountSkillSetting]:
db_request = self._build_db_request(
'get_settings_for_skill_family.sql',
return self._select_all_into_dataclass(
AccountSkillSetting,
sql_file_name='get_settings_for_skill_family.sql',
args=dict(family_name=family_name, account_id=account_id)
)
db_result = self.cursor.select_all(db_request)
skill_settings = []
for row in db_result:
settings_display = row['settings_display']
if settings_display is not None:
settings_display = settings_display.get('skillMetadata')
skill_settings.append(
AccountSkillSetting(
settings_display=settings_display,
settings_values=row['settings_values'],
device_names=row['device_names'],
)
)
return skill_settings
def get_installer_settings(self, account_id) -> List[AccountSkillSetting]:
skill_repo = SkillRepository(self.db)
@ -81,11 +66,17 @@ class SkillSettingRepository(RepositoryBase):
new_skill_settings: AccountSkillSetting,
skill_ids: List[str]
):
if new_skill_settings.settings_values is None:
serialized_settings_values = None
else:
serialized_settings_values = json.dumps(
new_skill_settings.settings_values
)
db_request = self._build_db_request(
'update_device_skill_settings.sql',
args=dict(
account_id=account_id,
settings_values=json.dumps(new_skill_settings.settings_values),
settings_values=serialized_settings_values,
skill_id=tuple(skill_ids),
device_names=tuple(new_skill_settings.device_names)
)

16
shared/selene/data/skill/repository/settings_display.py
View File

@ -29,6 +29,7 @@ class SettingsDisplayRepository(RepositoryBase):
super(SettingsDisplayRepository, self).__init__(db, __file__)
def add(self, settings_display: SettingsDisplay) -> str:
"""Add a new row to the skill.settings_display table."""
db_request = self._build_db_request(
sql_file_name='add_settings_display.sql',
args=dict(
@ -41,6 +42,7 @@ class SettingsDisplayRepository(RepositoryBase):
return result['id']
def get_settings_display_id(self, settings_display: SettingsDisplay):
"""Get the ID of a skill's settings definition."""
db_request = self._build_db_request(
sql_file_name='get_settings_display_id.sql',
args=dict(
@ -52,7 +54,21 @@ class SettingsDisplayRepository(RepositoryBase):
return None if result is None else result['id']
def get_settings_definitions_by_gid(self, global_id):
"""Get all matching settings definitions for a global skill ID.
There can be more than one settings definition for a global skill ID.
An example of when this could happen is if a skill author changed the
settings definition and not all devices have updated to the latest.
"""
return self._select_all_into_dataclass(
SettingsDisplay,
sql_file_name='get_settings_definition_by_gid.sql',
args=dict(global_id=global_id)
)
def remove(self, settings_display_id: str):
"""Delete a settings definition that is no longer used by any device"""
db_request = self._build_db_request(
sql_file_name='delete_settings_display.sql',
args=dict(settings_display_id=settings_display_id)

4
shared/selene/data/skill/repository/skill.py
View File

@ -23,7 +23,7 @@ from ..entity.skill import Skill, SkillFamily
from ...repository_base import RepositoryBase
def _parse_skill_gid(skill_gid):
def extract_family_from_global_id(skill_gid):
id_parts = skill_gid.split('|')
if id_parts[0].startswith('@'):
family_name = id_parts[1]
@ -76,7 +76,7 @@ class SkillRepository(RepositoryBase):
def ensure_skill_exists(self, skill_global_id: str) -> str:
skill = self.get_skill_by_global_id(skill_global_id)
if skill is None:
family_name = _parse_skill_gid(skill_global_id)
family_name = extract_family_from_global_id(skill_global_id)
skill_id = self._add_skill(skill_global_id, family_name)
else:
skill_id = skill.id

9
shared/selene/data/skill/repository/sql/get_settings_definition_by_gid.sql Normal file
View File

@ -0,0 +1,9 @@
SELECT
sd.id,
sd.skill_id,
sd.settings_display as display_data
FROM
skill.settings_display sd
INNER JOIN skill.skill s ON sd.skill_id = s.id
WHERE
s.skill_gid = %(global_id)s

12
shared/selene/data/skill/repository/sql/get_settings_for_skill_family.sql
View File

@ -1,15 +1,15 @@
SELECT
sd.settings_display::jsonb AS settings_display,
sd.settings_display::jsonb -> 'skillMetadata' AS settings_definition,
ds.settings::jsonb AS settings_values,
array_agg(d.name) AS device_names
FROM
skill.skill s
LEFT JOIN skill.settings_display sd ON sd.skill_id = s.id
INNER JOIN device.device_skill ds ON sd.id = ds.skill_settings_display_id
device.device_skill ds
INNER JOIN device.device d ON ds.device_id = d.id
INNER JOIN skill.skill s ON ds.skill_id = s.id
LEFT JOIN skill.settings_display sd ON ds.skill_settings_display_id = sd.id
WHERE
s.family_name = %(family_name)s
AND d.account_id = %(account_id)s
GROUP BY
sd.settings_display::jsonb,
ds.settings::jsonb
settings_definition,
settings_values

2
shared/selene/data/skill/repository/sql/get_skill_setting_by_device.sql
View File

@ -1,5 +1,5 @@
SELECT
ss.id AS skill_id,
dds.skill_id,
dds.settings AS settings_values,
ssd.settings_display
FROM