From 637166e624b408013ab2b0325b6f5d55ba9a4770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85ke?= Date: Fri, 5 Apr 2019 17:57:06 +0200 Subject: [PATCH] Revert "Remove reliance on hashed meta (#2076)" This reverts commit 38123a1fc3f056875cc42734c3e130a73ddec726. --- mycroft/skills/settings.py | 230 ++++++++++++++++++++++++++++++++----- 1 file changed, 202 insertions(+), 28 deletions(-) diff --git a/mycroft/skills/settings.py b/mycroft/skills/settings.py index bd723fb062..b47661736a 100644 --- a/mycroft/skills/settings.py +++ b/mycroft/skills/settings.py @@ -59,6 +59,7 @@ """ import json +import hashlib import os import time import copy @@ -74,6 +75,7 @@ from mycroft.util import camel_case_split from mycroft.configuration import ConfigurationManager from .msm_wrapper import create_msm + # This is the base needed for sending a blank settings meta entry (Tartarus) # To this a global id is added # TODO reduce the needed boilerplate here @@ -118,7 +120,7 @@ def build_global_id(directory, config): if s.meta_info != {}: return s.meta_info['skill_gid'], s.meta_info['display_name'] else: # No skills meta data available, local or unsubmitted skill - return "@{}|{}".format(DeviceApi().identity.uuid, s.name), None + return "@{}_{}".format(DeviceApi().identity.uuid, s.name), None def display_name(name): @@ -137,6 +139,7 @@ class SkillSettings(dict): no_upload (bool): True if the upload to mycroft servers should be disabled. """ + def __init__(self, directory, name): super(SkillSettings, self).__init__() # when skills try to instantiate settings @@ -164,10 +167,6 @@ class SkillSettings(dict): self._blank_poll_timer = None self._is_alive = True - # Collect Information from msm - skill_gid, disp_name = build_global_id(self._directory, self.config) - self.skill_gid = skill_gid - self.display_name = disp_name # if settingsmeta exist if isfile(self._meta_path): self._poll_skill_settings() @@ -223,12 +222,34 @@ class SkillSettings(dict): except RequestException: return - settings = self._request_my_settings(self.skill_gid) - if settings is None: - # metadata got deleted from Home, send up - self._upload_meta(settings_meta) - else: - self.save_skill_settings(settings) + hashed_meta = self._get_meta_hash(settings_meta) + skill_settings = self._request_other_settings(hashed_meta) + # if hash is new then there is a diff version of settingsmeta + if self._is_new_hash(hashed_meta): + # first look at all other devices on user account to see + # if the settings exist. if it does then sync with device + if skill_settings: + # not_owner flags that this settings is loaded from + # another device. If a skill settings doesn't have + # not_owner, then the skill is created from that device + self['not_owner'] = True + self.save_skill_settings(skill_settings) + else: # upload skill settings if + uuid = self._load_uuid() + if uuid is not None: + self._delete_metadata(uuid) + self._upload_meta(settings_meta, hashed_meta) + else: # hash is not new + if skill_settings is not None: + self['not_owner'] = True + self.save_skill_settings(skill_settings) + else: + settings = self._request_my_settings(hashed_meta) + if settings is None: + # metadata got deleted from Home, send up + self._upload_meta(settings_meta, hashed_meta) + else: + self.save_skill_settings(settings) self._complete_intialization = True @property @@ -266,9 +287,10 @@ class SkillSettings(dict): # Add Information extracted from the skills-meta.json entry for the # skill. - data['skill_gid'] = self.skill_gid - data['display_name'] = (self.display_name or data.get('name') or - display_name(self.name)) + skill_gid, display_name = build_global_id(self._directory, self.config) + data['skill_gid'] = skill_gid + data['display_name'] = (display_name or data.get('name') or + display_name(name)) # Backwards compatibility: if 'name' not in data: @@ -285,7 +307,8 @@ class SkillSettings(dict): dict: uuid, a unique id for the setting meta data """ try: - return self._put_metadata(settings_meta) + uuid = self._put_metadata(settings_meta) + return uuid except Exception as e: LOG.error(e) return None @@ -296,6 +319,9 @@ class SkillSettings(dict): Args: skill_settings (dict): skill """ + if self._is_new_hash(skill_settings['identifier']): + self._save_uuid(skill_settings['uuid']) + self._save_hash(skill_settings['identifier']) sections = skill_settings['skillMetadata']['sections'] for section in sections: for field in section["fields"]: @@ -303,6 +329,48 @@ class SkillSettings(dict): self[field['name']] = field['value'] self.store() + def _load_uuid(self): + """ Loads uuid + + Returns: + str: uuid of the previous settingsmeta + """ + directory = self.config.get("skills")["directory"] + directory = join(directory, self.name) + directory = expanduser(directory) + uuid_file = join(directory, 'uuid') + uuid = None + if isfile(uuid_file): + with open(uuid_file, 'r') as f: + uuid = f.read() + return uuid + + def _save_uuid(self, uuid): + """ Saves uuid. + + Args: + uuid (str): uuid, unique id of new settingsmeta + """ + directory = self.config.get("skills")["directory"] + directory = join(directory, self.name) + directory = expanduser(directory) + uuid_file = join(directory, 'uuid') + os.makedirs(directory, exist_ok=True) + with open(uuid_file, 'w') as f: + f.write(str(uuid)) + + def _uuid_exist(self): + """ Checks if there is an uuid file. + + Returns: + bool: True if uuid file exist False otherwise + """ + directory = self.config.get("skills")["directory"] + directory = join(directory, self.name) + directory = expanduser(directory) + uuid_file = join(directory, 'uuid') + return isfile(uuid_file) + def _migrate_settings(self, settings_meta): """ sync settings.json and settingsmeta.json in memory """ meta = settings_meta.copy() @@ -317,14 +385,72 @@ class SkillSettings(dict): meta['skillMetadata']['sections'] = sections return meta - def _upload_meta(self, settings_meta): + def _upload_meta(self, settings_meta, hashed_meta): """ uploads the new meta data to settings with settings migration Args: settings_meta (dict): settingsmeta.json + hashed_meta (str): {skill-folder}-settinsmeta.json """ meta = self._migrate_settings(settings_meta) - self._put_metadata(settings_meta) + meta['identifier'] = str(hashed_meta) + response = self._send_settings_meta(meta) + if response and 'uuid' in response: + self._save_uuid(response['uuid']) + if 'not_owner' in self: + del self['not_owner'] + self._save_hash(hashed_meta) + + def hash(self, string): + """ md5 hasher for consistency across cpu architectures """ + return hashlib.md5(bytes(string, 'utf-8')).hexdigest() + + def _get_meta_hash(self, settings_meta): + """ Gets the hash of skill + + Args: + settings_meta (dict): settingsmeta object + Returns: + _hash (str): hashed to identify skills + """ + _hash = self.hash(json.dumps(settings_meta, sort_keys=True) + + self._user_identity) + return "{}--{}".format(self.name, _hash) + + def _save_hash(self, hashed_meta): + """ Saves hashed_meta to settings directory. + + Args: + hashed_meta (str): hash of new settingsmeta + """ + directory = self.config.get("skills")["directory"] + directory = join(directory, self.name) + directory = expanduser(directory) + hash_file = join(directory, 'hash') + os.makedirs(directory, exist_ok=True) + with open(hash_file, 'w') as f: + f.write(hashed_meta) + + def _is_new_hash(self, hashed_meta): + """ Check if stored hash is the same as current. + + If the hashed file does not exist, usually in the + case of first load, then the create it and return True + + Args: + hashed_meta (str): hash of metadata and uuid of device + Returns: + bool: True if hash is new, otherwise False + """ + directory = self.config.get("skills")["directory"] + directory = join(directory, self.name) + directory = expanduser(directory) + hash_file = join(directory, 'hash') + if isfile(hash_file): + with open(hash_file, 'r') as f: + current_hash = f.read() + return False if current_hash == str(hashed_meta) else True + return True def update_remote(self): """ update settings state from server """ @@ -332,13 +458,17 @@ class SkillSettings(dict): settings_meta = self._load_settings_meta() if settings_meta is None: return - skills_settings = self._request_my_settings(self.skill_gid) + hashed_meta = self._get_meta_hash(settings_meta) + if self.get('not_owner'): + skills_settings = self._request_other_settings(hashed_meta) + if not skills_settings: + skills_settings = self._request_my_settings(hashed_meta) if skills_settings is not None: self.save_skill_settings(skills_settings) self.store() else: settings_meta = self._load_settings_meta() - self._upload_meta(settings_meta) + self._upload_meta(settings_meta, hashed_meta) def _init_blank_meta(self): """ Send blank settingsmeta to remote. """ @@ -451,10 +581,12 @@ class SkillSettings(dict): meta['skillMetadata']['sections'] = sections return meta - def _request_my_settings(self, skill_gid): + def _request_my_settings(self, identifier): """ Get skill settings for this device associated with the identifier + Args: + identifier (str): a hashed_meta Returns: skill_settings (dict or None): returns a dict if matches """ @@ -462,7 +594,7 @@ class SkillSettings(dict): if settings: # this loads the settings into memory for use in self.store for skill_settings in settings: - if skill_settings['skill_gid'] == skill_gid: + if skill_settings['identifier'] == identifier: skill_settings = \ self._type_cast(skill_settings, to_platform='core') self._remote_settings = skill_settings @@ -486,6 +618,27 @@ class SkillSettings(dict): settings = [skills for skills in settings if skills is not None] return settings + def _request_other_settings(self, identifier): + """ Retrieve skill settings from other devices by identifier + + Args: + identifier (str): identifier for this skill + Returns: + settings (dict or None): the retrieved settings or None + """ + path = \ + "/" + self._device_identity + "/userSkill?identifier=" + identifier + try: + user_skill = self.api.request({"method": "GET", "path": path}) + except RequestException: + # Some kind of Timeout, connection HTTPError, etc. + user_skill = None + if not user_skill: + return None + else: + settings = self._type_cast(user_skill[0], to_platform='core') + return settings + def _put_metadata(self, settings_meta): """ PUT settingsmeta to backend to be configured in server. used in place of POST and PATCH. @@ -494,14 +647,29 @@ class SkillSettings(dict): settings_meta (dict): dictionary of the current settings meta data """ settings_meta = self._type_cast(settings_meta, to_platform='web') + return self.api.request({ + "method": "PUT", + "path": self._api_path, + "json": settings_meta + }) + + def _delete_metadata(self, uuid): + """ Delete the current skill metadata + + Args: + uuid (str): unique id of the skill + """ try: - return self.api.request({ - "method": "PUT", - "path": self._api_path, - "json": settings_meta + LOG.debug("deleting metadata") + self.api.request({ + "method": "DELETE", + "path": self._api_path + "/{}".format(uuid) }) - except Exception: - raise + except Exception as e: + LOG.error(e) + LOG.error( + "cannot delete metadata because this" + "device is not original uploader of skill") @property def _should_upload_from_change(self): @@ -519,6 +687,8 @@ class SkillSettings(dict): self_val = self.get(field['name']) if str(remote_val) != str(self_val): changed = True + if self.get('not_owner'): + changed = False return changed def store(self, force=False): @@ -534,4 +704,8 @@ class SkillSettings(dict): if self._should_upload_from_change: settings_meta = self._load_settings_meta() - self._upload_meta(settings_meta) + hashed_meta = self._get_meta_hash(settings_meta) + uuid = self._load_uuid() + if uuid is not None: + self._delete_metadata(uuid) + self._upload_meta(settings_meta, hashed_meta)