MycroftAI
/
mycroft-core
mirror of https://github.com/MycroftAI/mycroft-core.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Revert "Remove reliance on hashed meta (#2076)" 

This reverts commit 38123a1fc3.
pull/2077/head
Åke 2019-04-05 17:57:06 +02:00 committed by GitHub
parent 38123a1fc3
commit 637166e624
1 changed files with 202 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

230
mycroft/skills/settings.py
View File

@ -59,6 +59,7 @@
"""
import json
import hashlib
import os
import time
import copy
@ -74,6 +75,7 @@ from mycroft.util import camel_case_split
from mycroft.configuration import ConfigurationManager
from .msm_wrapper import create_msm
# This is the base needed for sending a blank settings meta entry (Tartarus)
# To this a global id is added
# TODO reduce the needed boilerplate here
@ -118,7 +120,7 @@ def build_global_id(directory, config):
if s.meta_info != {}:
return s.meta_info['skill_gid'], s.meta_info['display_name']
else: # No skills meta data available, local or unsubmitted skill
return "@{}|{}".format(DeviceApi().identity.uuid, s.name), None
return "@{}_{}".format(DeviceApi().identity.uuid, s.name), None
def display_name(name):
@ -137,6 +139,7 @@ class SkillSettings(dict):
no_upload (bool): True if the upload to mycroft servers should be
disabled.
"""
def __init__(self, directory, name):
super(SkillSettings, self).__init__()
# when skills try to instantiate settings
@ -164,10 +167,6 @@ class SkillSettings(dict):
self._blank_poll_timer = None
self._is_alive = True
# Collect Information from msm
skill_gid, disp_name = build_global_id(self._directory, self.config)
self.skill_gid = skill_gid
self.display_name = disp_name
# if settingsmeta exist
if isfile(self._meta_path):
self._poll_skill_settings()
@ -223,12 +222,34 @@ class SkillSettings(dict):
except RequestException:
return
settings = self._request_my_settings(self.skill_gid)
if settings is None:
# metadata got deleted from Home, send up
self._upload_meta(settings_meta)
else:
self.save_skill_settings(settings)
hashed_meta = self._get_meta_hash(settings_meta)
skill_settings = self._request_other_settings(hashed_meta)
# if hash is new then there is a diff version of settingsmeta
if self._is_new_hash(hashed_meta):
# first look at all other devices on user account to see
# if the settings exist. if it does then sync with device
if skill_settings:
# not_owner flags that this settings is loaded from
# another device. If a skill settings doesn't have
# not_owner, then the skill is created from that device
self['not_owner'] = True
self.save_skill_settings(skill_settings)
else: # upload skill settings if
uuid = self._load_uuid()
if uuid is not None:
self._delete_metadata(uuid)
self._upload_meta(settings_meta, hashed_meta)
else: # hash is not new
if skill_settings is not None:
self['not_owner'] = True
self.save_skill_settings(skill_settings)
else:
settings = self._request_my_settings(hashed_meta)
if settings is None:
# metadata got deleted from Home, send up
self._upload_meta(settings_meta, hashed_meta)
else:
self.save_skill_settings(settings)
self._complete_intialization = True
@property
@ -266,9 +287,10 @@ class SkillSettings(dict):
# Add Information extracted from the skills-meta.json entry for the
# skill.
data['skill_gid'] = self.skill_gid
data['display_name'] = (self.display_name or data.get('name') or
display_name(self.name))
skill_gid, display_name = build_global_id(self._directory, self.config)
data['skill_gid'] = skill_gid
data['display_name'] = (display_name or data.get('name') or
display_name(name))
# Backwards compatibility:
if 'name' not in data:
@ -285,7 +307,8 @@ class SkillSettings(dict):
dict: uuid, a unique id for the setting meta data
"""
try:
return self._put_metadata(settings_meta)
uuid = self._put_metadata(settings_meta)
return uuid
except Exception as e:
LOG.error(e)
return None
@ -296,6 +319,9 @@ class SkillSettings(dict):
Args:
skill_settings (dict): skill
"""
if self._is_new_hash(skill_settings['identifier']):
self._save_uuid(skill_settings['uuid'])
self._save_hash(skill_settings['identifier'])
sections = skill_settings['skillMetadata']['sections']
for section in sections:
for field in section["fields"]:
@ -303,6 +329,48 @@ class SkillSettings(dict):
self[field['name']] = field['value']
self.store()
def _load_uuid(self):
""" Loads uuid
Returns:
str: uuid of the previous settingsmeta
"""
directory = self.config.get("skills")["directory"]
directory = join(directory, self.name)
directory = expanduser(directory)
uuid_file = join(directory, 'uuid')
uuid = None
if isfile(uuid_file):
with open(uuid_file, 'r') as f:
uuid = f.read()
return uuid
def _save_uuid(self, uuid):
""" Saves uuid.
Args:
uuid (str): uuid, unique id of new settingsmeta
"""
directory = self.config.get("skills")["directory"]
directory = join(directory, self.name)
directory = expanduser(directory)
uuid_file = join(directory, 'uuid')
os.makedirs(directory, exist_ok=True)
with open(uuid_file, 'w') as f:
f.write(str(uuid))
def _uuid_exist(self):
""" Checks if there is an uuid file.
Returns:
bool: True if uuid file exist False otherwise
"""
directory = self.config.get("skills")["directory"]
directory = join(directory, self.name)
directory = expanduser(directory)
uuid_file = join(directory, 'uuid')
return isfile(uuid_file)
def _migrate_settings(self, settings_meta):
""" sync settings.json and settingsmeta.json in memory """
meta = settings_meta.copy()
@ -317,14 +385,72 @@ class SkillSettings(dict):
meta['skillMetadata']['sections'] = sections
return meta
def _upload_meta(self, settings_meta):
def _upload_meta(self, settings_meta, hashed_meta):
""" uploads the new meta data to settings with settings migration
Args:
settings_meta (dict): settingsmeta.json
hashed_meta (str): {skill-folder}-settinsmeta.json
"""
meta = self._migrate_settings(settings_meta)
self._put_metadata(settings_meta)
meta['identifier'] = str(hashed_meta)
response = self._send_settings_meta(meta)
if response and 'uuid' in response:
self._save_uuid(response['uuid'])
if 'not_owner' in self:
del self['not_owner']
self._save_hash(hashed_meta)
def hash(self, string):
""" md5 hasher for consistency across cpu architectures """
return hashlib.md5(bytes(string, 'utf-8')).hexdigest()
def _get_meta_hash(self, settings_meta):
""" Gets the hash of skill
Args:
settings_meta (dict): settingsmeta object
Returns:
_hash (str): hashed to identify skills
"""
_hash = self.hash(json.dumps(settings_meta, sort_keys=True) +
self._user_identity)
return "{}--{}".format(self.name, _hash)
def _save_hash(self, hashed_meta):
""" Saves hashed_meta to settings directory.
Args:
hashed_meta (str): hash of new settingsmeta
"""
directory = self.config.get("skills")["directory"]
directory = join(directory, self.name)
directory = expanduser(directory)
hash_file = join(directory, 'hash')
os.makedirs(directory, exist_ok=True)
with open(hash_file, 'w') as f:
f.write(hashed_meta)
def _is_new_hash(self, hashed_meta):
""" Check if stored hash is the same as current.
If the hashed file does not exist, usually in the
case of first load, then the create it and return True
Args:
hashed_meta (str): hash of metadata and uuid of device
Returns:
bool: True if hash is new, otherwise False
"""
directory = self.config.get("skills")["directory"]
directory = join(directory, self.name)
directory = expanduser(directory)
hash_file = join(directory, 'hash')
if isfile(hash_file):
with open(hash_file, 'r') as f:
current_hash = f.read()
return False if current_hash == str(hashed_meta) else True
return True
def update_remote(self):
""" update settings state from server """
@ -332,13 +458,17 @@ class SkillSettings(dict):
settings_meta = self._load_settings_meta()
if settings_meta is None:
return
skills_settings = self._request_my_settings(self.skill_gid)
hashed_meta = self._get_meta_hash(settings_meta)
if self.get('not_owner'):
skills_settings = self._request_other_settings(hashed_meta)
if not skills_settings:
skills_settings = self._request_my_settings(hashed_meta)
if skills_settings is not None:
self.save_skill_settings(skills_settings)
self.store()
else:
settings_meta = self._load_settings_meta()
self._upload_meta(settings_meta)
self._upload_meta(settings_meta, hashed_meta)
def _init_blank_meta(self):
""" Send blank settingsmeta to remote. """
@ -451,10 +581,12 @@ class SkillSettings(dict):
meta['skillMetadata']['sections'] = sections
return meta
def _request_my_settings(self, skill_gid):
def _request_my_settings(self, identifier):
""" Get skill settings for this device associated
with the identifier
Args:
identifier (str): a hashed_meta
Returns:
skill_settings (dict or None): returns a dict if matches
"""
@ -462,7 +594,7 @@ class SkillSettings(dict):
if settings:
# this loads the settings into memory for use in self.store
for skill_settings in settings:
if skill_settings['skill_gid'] == skill_gid:
if skill_settings['identifier'] == identifier:
skill_settings = \
self._type_cast(skill_settings, to_platform='core')
self._remote_settings = skill_settings
@ -486,6 +618,27 @@ class SkillSettings(dict):
settings = [skills for skills in settings if skills is not None]
return settings
def _request_other_settings(self, identifier):
""" Retrieve skill settings from other devices by identifier
Args:
identifier (str): identifier for this skill
Returns:
settings (dict or None): the retrieved settings or None
"""
path = \
"/" + self._device_identity + "/userSkill?identifier=" + identifier
try:
user_skill = self.api.request({"method": "GET", "path": path})
except RequestException:
# Some kind of Timeout, connection HTTPError, etc.
user_skill = None
if not user_skill:
return None
else:
settings = self._type_cast(user_skill[0], to_platform='core')
return settings
def _put_metadata(self, settings_meta):
""" PUT settingsmeta to backend to be configured in server.
used in place of POST and PATCH.
@ -494,14 +647,29 @@ class SkillSettings(dict):
settings_meta (dict): dictionary of the current settings meta data
"""
settings_meta = self._type_cast(settings_meta, to_platform='web')
return self.api.request({
"method": "PUT",
"path": self._api_path,
"json": settings_meta
})
def _delete_metadata(self, uuid):
""" Delete the current skill metadata
Args:
uuid (str): unique id of the skill
"""
try:
return self.api.request({
"method": "PUT",
"path": self._api_path,
"json": settings_meta
LOG.debug("deleting metadata")
self.api.request({
"method": "DELETE",
"path": self._api_path + "/{}".format(uuid)
})
except Exception:
raise
except Exception as e:
LOG.error(e)
LOG.error(
"cannot delete metadata because this"
"device is not original uploader of skill")
@property
def _should_upload_from_change(self):
@ -519,6 +687,8 @@ class SkillSettings(dict):
self_val = self.get(field['name'])
if str(remote_val) != str(self_val):
changed = True
if self.get('not_owner'):
changed = False
return changed
def store(self, force=False):
@ -534,4 +704,8 @@ class SkillSettings(dict):
if self._should_upload_from_change:
settings_meta = self._load_settings_meta()
self._upload_meta(settings_meta)
hashed_meta = self._get_meta_hash(settings_meta)
uuid = self._load_uuid()
if uuid is not None:
self._delete_metadata(uuid)
self._upload_meta(settings_meta, hashed_meta)