mycroft-core/mycroft/skills/settings.py

# Copyright 2017 Mycroft AI Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
    SkillSettings is a simple extension of the python dict which enables
    local storage of settings.  Additionally it can interact with a backend
    system to provide a GUI interface, described by meta-data described in
    an optional 'settingsmeta.json' file.

    Usage Example:
        from mycroft.skill.settings import SkillSettings

        s = SkillSettings('./settings.json', 'ImportantSettings')
        s['meaning of life'] = 42
        s['flower pot sayings'] = 'Not again...'
        s.store()

    Metadata format:
        TODO: see https://goo.gl/MY3i1S
"""

import json
import subprocess
import hashlib
from threading import Timer
from os.path import isfile, join, expanduser, basename

from mycroft.api import DeviceApi
from mycroft.util.log import LOG
from mycroft.configuration import ConfigurationManager


class SkillSettings(dict):
    """ A dictionary that can easily be save to a file, serialized as json. It
        also syncs to the backend for skill settings

        Args:
            directory (str): Path to storage directory
            name (str):      user readable name associated with the settings
    """

    def __init__(self, directory, name):
        super(SkillSettings, self).__init__()
        self.api = DeviceApi()
        self._device_identity = self.api.identity.uuid
        self.config = ConfigurationManager.get()
        self.name = name
        self.directory = directory
        # set file paths
        self._settings_path = join(directory, 'settings.json')
        self._meta_path = join(directory, 'settingsmeta.json')
        self._api_path = "/" + self._device_identity + "/skill"
        self.is_alive = True
        self.loaded_hash = hash(str(self))

        # if settingsmeta.json exists
        # this block of code is a control flow for
        # different scenarios that may arises with settingsmeta
        if isfile(self._meta_path):
            LOG.info("settingsmeta.json exist for {}".format(self.name))
            settings_meta = self._load_settings_meta()
            hashed_meta = self._get_meta_hash(str(settings_meta))
            skill_settings = self._request_other_settings(hashed_meta)
            # if hash is new then there is a diff version of settingsmeta
            if self._is_new_hash(hashed_meta):
                # first look at all other devices on user account to see
                # if the settings exist. if it does then sync with this device
                if skill_settings is not None:
                    # is_synced flags that this settings is loaded from
                    # another device. If a skill settings doesn't have
                    # is_synced, then the skill is created from that device
                    self['is_synced'] = True
                    self.save_skill_settings(skill_settings)
                else:  # upload skill settings if other devices do not have it
                    uuid = self._load_uuid()
                    if uuid is not None:
                        self._delete_metadata(uuid)
                    self._upload_meta(settings_meta, hashed_meta)
            else:  # hash is not new
                if skill_settings is not None:
                    self['is_synced'] = True
                    self.save_skill_settings(skill_settings)
                else:
                    settings = self._request_my_settings(hashed_meta)
                    if settings is None:
                        LOG.info("seems like it got deleted from home... "
                                 "sending settingsmeta.json for "
                                 "{}".format(self.name))
                        self._upload_meta(settings_meta, hashed_meta)
                    else:
                        self.save_skill_settings(settings)

            t = Timer(60, self._poll_skill_settings, [hashed_meta])
            t.daemon = True
            t.start()

        self.load_skill_settings()

    @property
    def _is_stored(self):
        return hash(str(self)) == self.loaded_hash

    def __getitem__(self, key):
        """ Get key """
        return super(SkillSettings, self).__getitem__(key)

    def __setitem__(self, key, value):
        """ Add/Update key. """
        return super(SkillSettings, self).__setitem__(key, value)

    def _load_settings_meta(self):
        """ Loads settings metadata from skills path. """
        with open(self._meta_path) as f:
            data = json.load(f)
        return data

    def _send_settings_meta(self, settings_meta):
        """ Send settingsmeta.json to the server.

            Args:
                settings_meta (dict): dictionary of the current settings meta

            Returns:
                str: uuid, a unique id for the setting meta data
        """
        try:
            uuid = self._put_metadata(settings_meta)
            return uuid
        except Exception as e:
            LOG.error(e)
            return None

    def save_skill_settings(self, skill_settings):
        """ takes skill object and save onto self

            Args:
                settings (dict): skill
        """
        if self._is_new_hash(skill_settings['identifier']):
            self._save_uuid(skill_settings['uuid'])
            self._save_hash(skill_settings['identifier'])
        sections = skill_settings['skillMetadata']['sections']
        for section in sections:
            for field in section["fields"]:
                if "name" in field:  # no name for 'label' fields
                    self[field['name']] = field['value']
        self.store()

    def _load_uuid(self):
        """ Loads uuid

            Returns:
                str: uuid of the previous settingsmeta
        """
        directory = self.config.get("skills")["directory"]
        directory = join(directory, self.name)
        directory = expanduser(directory)
        uuid_file = join(directory, 'uuid')
        uuid = None
        if isfile(uuid_file):
            with open(uuid_file, 'r') as f:
                uuid = f.read()
        return uuid

    def _save_uuid(self, uuid):
        """ Saves uuid.

            Args:
                str: uuid, unique id of new settingsmeta
        """
        LOG.info("saving uuid {}".format(str(uuid)))
        directory = self.config.get("skills")["directory"]
        directory = join(directory, self.name)
        directory = expanduser(directory)
        uuid_file = join(directory, 'uuid')
        with open(uuid_file, 'w') as f:
            f.write(str(uuid))

    def _uuid_exist(self):
        """ Checks if there is an uuid file.

            Returns:
                bool: True if uuid file exist False otherwise
        """
        directory = self.config.get("skills")["directory"]
        directory = join(directory, self.name)
        directory = expanduser(directory)
        uuid_file = join(directory, 'uuid')
        return isfile(uuid_file)

    def _migrate_settings(self, settings_meta):
        """ sync settings.json and settingsmeta.json in memory """
        meta = settings_meta.copy()
        sections = meta['skillMetadata']['sections']
        for i, section in enumerate(sections):
            for j, field in enumerate(section['fields']):
                if 'name' in field:
                    if field["name"] in self:
                        sections[i]['fields'][j]["value"] = self['name']
        meta['skillMetadata']['sections'] = sections
        return meta

    def _upload_meta(self, settings_meta, hashed_meta):
        """ uploads the new meta data to settings with settings migration

            Args:
                settings_meta (dict): settingsmeta.json
                hashed_meta (str): {skill-folder}-settinsmeta.json
        """
        LOG.info("sending settingsmeta.json for {}".format(self.name) +
                 " to servers")
        meta = self._migrate_settings(settings_meta)
        meta['identifier'] = str(hashed_meta)
        response = self._send_settings_meta(meta)
        self._save_uuid(response['uuid'])
        self._save_hash(hashed_meta)

    def _delete_old_meta(self):
        """" Deletes the old meta data """
        if self._uuid_exist():
            try:
                LOG.info("a uuid exist for {}".format(self.name) +
                         " deleting old one")
                old_uuid = self._load_uuid()
                self._delete_metatdata(old_uuid)
            except Exception as e:
                LOG.info(e)

    def hash(self, str):
        """ md5 hasher for consistency across cpu architectures """
        return hashlib.md5(str).hexdigest()

    def _get_meta_hash(self, settings_meta):
        """ Get's the hash of skill

            Args:
                settings_meta (str): stringified settingsmeta

            Returns:
                _hash (str): hashed to identify skills
        """
        return "{}-{}".format(
                        basename(self.directory),
                        self.hash(str(settings_meta)))

    def _save_hash(self, hashed_meta):
        """ Saves hashed_meta to settings directory.

            Args:
                hashed_meta (int): hash of new settingsmeta
        """
        LOG.info("saving hash {}".format(str(hashed_meta)))
        directory = self.config.get("skills")["directory"]
        directory = join(directory, self.name)
        directory = expanduser(directory)
        hash_file = join(directory, 'hash')
        with open(hash_file, 'w') as f:
            f.write(str(hashed_meta))

    def _is_new_hash(self, hashed_meta):
        """ checks if the stored hash is the same as current.
            if the hashed file does not exist, usually in the
            case of first load, then the create it and return True

            Args:
                hashed_meta (int): hash of metadata and uuid of device

            Returns:
                bool: True if hash is new, otherwise False
        """
        directory = self.config.get("skills")["directory"]
        directory = join(directory, self.name)
        directory = expanduser(directory)
        hash_file = join(directory, 'hash')
        if isfile(hash_file):
            with open(hash_file, 'r') as f:
                current_hash = f.read()
            return False if current_hash == str(hashed_meta) else True
        return True

    def update(self):
        """ update settings state from server """
        skills_settings = None
        if self.get('is_synced'):
            skills_settings = self._request_other_settings(hashed_meta)
        if not skills_settings:
            skills_settings = self._request_my_settings(hashed_meta)

        if skills_settings is not None:
            self.save_skill_settings(skills_settings)
            self.store()

    def _poll_skill_settings(self, hashed_meta):
        """ If identifier exists for this skill poll to backend to
            request settings and store it if it changes
            TODO: implement as websocket

            Args:
                hashed_meta (int): the hashed identifier
        """
        skills_settings = None
        if self.get('is_synced'):
            skills_settings = self._request_other_settings(hashed_meta)
        if not skills_settings:
            skills_settings = self._request_my_settings(hashed_meta)

        if skills_settings is not None:
            self.save_skill_settings(skills_settings)
            self.store()
        else:
            settings_meta = self._load_settings_meta()
            self._upload_meta(settings_meta, hashed_meta)

        if self.is_alive:
            # continues to poll settings every 60 seconds
            t = Timer(60, self._poll_skill_settings, [hashed_meta])
            t.daemon = True
            t.start()

    def load_skill_settings(self):
        """ If settings.json exist, open and read stored values into self """
        if isfile(self._settings_path):
            with open(self._settings_path) as f:
                try:
                    json_data = json.load(f)
                    for key in json_data:
                        self[key] = json_data[key]
                except Exception as e:
                    # TODO: Show error on webUI.  Dev will have to fix
                    # metadata to be able to edit later.
                    LOG.error(e)

    def _request_my_settings(self, identifier):
        """ Get skill settings for this device associated
            with the identifier

            Args:
                identifier (str): a hashed_meta

            Returns:
                skill_settings (dict or None): returns a dict if matches
        """
        LOG.info("getting skill settings from "
                 "server for {}".format(self.name))
        settings = self._request_settings()
        # this loads the settings into memory for use in self.store
        for skill_settings in settings:
            if skill_settings['identifier'] == identifier:
                self._remote_settings = skill_settings
                return skill_settings
        return None

    def _request_settings(self):
        """ Get all skill settings for this device from server.

            Returns:
                dict: dictionary with settings collected from the server.
        """
        settings = self.api.request({
            "method": "GET",
            "path": self._api_path
        })
        settings = [skills for skills in settings if skills is not None]
        return settings

    def _request_other_settings(self, identifier):
        """ Retrieves user skill from other devices by identifier (hashed_meta)

        Args:
            indentifier (str): identifier for this skill

        Returns:
            settings (dict or None): returns the settings if true else None
        """
        LOG.info(
            "syncing settings with other devices "
            "from server for {}".format(self.name))

        path = \
            "/" + self._device_identity + "/userSkill?identifier=" + identifier
        user_skill = self.api.request({
            "method": "GET",
            "path": path
        })
        if len(user_skill) == 0:
            return None
        else:
            return user_skill[0]

    def _put_metadata(self, settings_meta):
        """ PUT settingsmeta to backend to be configured in server.
            used in place of POST and PATCH.

            Args:
                settings_meta (dict): dictionary of the current settings meta
                                      data
        """
        return self.api.request({
            "method": "PUT",
            "path": self._api_path,
            "json": settings_meta
        })

    def _delete_metadata(self, uuid):
        """ Deletes the current skill metadata

            Args:
                uuid (str): unique id of the skill
        """
        try:
            LOG.info("deleting metadata")
            self.api.request({
                "method": "DELETE",
                "path": self._api_path + "/{}".format(uuid)
            })
        except Exception as e:
            LOG.error(e)
            LOG.info(
                "cannot delete metadata because this"
                "device is not original uploader of skill")

    @property
    def _should_upload_from_change(self):
        changed = False
        if hasattr(self, '_remote_settings'):
            sections = self._remote_settings['skillMetadata']['sections']
            for i, section in enumerate(sections):
                for j, field in enumerate(section['fields']):
                    if 'name' in field:
                        if field["name"] in self:
                            remote_val = sections[i]['fields'][j]["value"]
                            self_val = self.get(field['name'])
                            if str(remote_val) != str(self_val):
                                changed = True
        return changed

    def store(self, force=False):
        """ Store dictionary to file if a change has occured.

            Args:
                force:  Force write despite no change
        """

        if force or not self._is_stored:
            with open(self._settings_path, 'w') as f:
                json.dump(self, f)
            self.loaded_hash = hash(str(self))

        if self._should_upload_from_change:
            settings_meta = self._load_settings_meta()
            hashed_meta = self._get_meta_hash(settings_meta)
            uuid = self._load_uuid()
            if uuid is not None:
                LOG.info("deleting meata data for {}".format(self.name))
                self._delete_metadata(uuid)
            self._upload_meta(settings_meta, hashed_meta)