mycroft-core/mycroft/skills/skill_loader.py

# Copyright 2019 Mycroft AI Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Periodically run by skill manager to load skills into memory."""
import gc
import imp
import os
import sys
from time import time

from mycroft.configuration import Configuration
from mycroft.messagebus import Message
from mycroft.util.log import LOG

from .settings import SettingsMetaUploader

SKILL_MAIN_MODULE = '__init__.py'


def _get_last_modified_time(path):
    """Get the last modified date of the most recently updated file in a path.

    Exclude compiled python files, hidden directories and the settings.json
    file.

    Arguments:
        path: skill directory to check

    Returns:
        int: time of last change
    """
    all_files = []
    for root_dir, dirs, files in os.walk(path):
        dirs[:] = [d for d in dirs if not d.startswith('.')]
        for f in files:
            ignore_file = (
                    f.endswith('.pyc') or
                    f == 'settings.json' or
                    f.startswith('.') or
                    f.endswith('.qmlc')
            )
            if not ignore_file:
                all_files.append(os.path.join(root_dir, f))

    # check files of interest in the skill root directory
    return max(os.path.getmtime(f) for f in all_files)


class SkillLoader:
    def __init__(self, bus, skill_directory):
        self.bus = bus
        self.skill_directory = skill_directory
        self.skill_id = os.path.basename(skill_directory)
        self.load_attempted = False
        self.loaded = False
        self.last_modified = 0
        self.last_loaded = 0
        self.instance = None
        self.active = True
        self.config = Configuration.get()

    @property
    def is_blacklisted(self):
        """Boolean value representing whether or not a skill is blacklisted."""
        blacklist = self.config['skills'].get('blacklisted_skills', [])
        if self.skill_id in blacklist:
            return True
        else:
            return False

    def reload_needed(self):
        """Load an unloaded skill or reload unloaded/changed skill.

        Returns:
             bool: if the skill was loaded/reloaded
        """
        try:
            self.last_modified = _get_last_modified_time(self.skill_directory)
        except FileNotFoundError as e:
            LOG.error('Failed to get last_modification time '
                      '({})'.format(repr(e)))
            self.last_modified = self.last_loaded

        modified = self.last_modified > self.last_loaded

        # create local reference to avoid threading issues
        instance = self.instance

        reload_allowed = (
                self.active and
                (instance is None or instance.reload_skill)
        )
        return modified and reload_allowed

    def reload(self):
        LOG.info('ATTEMPTING TO RELOAD SKILL: ' + self.skill_id)
        if self.instance:
            self._unload()
        return self._load()

    def load(self):
        LOG.info('ATTEMPTING TO LOAD SKILL: ' + self.skill_id)
        return self._load()

    def _unload(self):
        """Remove listeners and stop threads before loading"""
        self._execute_instance_shutdown()
        if self.config.get("debug", False):
            self._garbage_collect()
        self.loaded = False
        self._emit_skill_shutdown_event()

    def unload(self):
        if self.instance:
            self._execute_instance_shutdown()
        self.loaded = False

    def activate(self):
        self.active = True
        self.load()

    def deactivate(self):
        self.active = False
        self.unload()

    def _execute_instance_shutdown(self):
        """Call the shutdown method of the skill being reloaded."""
        try:
            self.instance.default_shutdown()
        except Exception as e:
            log_msg = 'An error occurred while shutting down {}'
            LOG.exception(log_msg.format(self.instance.name))
        else:
            LOG.info('Skill {} shut down successfully'.format(self.skill_id))

    def _garbage_collect(self):
        """Invoke Python garbage collector to remove false references"""
        gc.collect()
        # Remove two local references that are known
        refs = sys.getrefcount(self.instance) - 2
        if refs > 0:
            log_msg = (
                "After shutdown of {} there are still {} references "
                "remaining. The skill won't be cleaned from memory."
            )
            LOG.warning(log_msg.format(self.instance.name, refs))

    def _emit_skill_shutdown_event(self):
        message = Message(
            "mycroft.skills.shutdown",
            data=dict(path=self.skill_directory, id=self.skill_id)
        )
        self.bus.emit(message)

    def _load(self):
        self._prepare_for_load()
        if self.is_blacklisted:
            self._skip_load()
        else:
            skill_module = self._load_skill_source()
            if skill_module and self._create_skill_instance(skill_module):
                self._check_for_first_run()
                self.loaded = True

        self.last_loaded = time()
        self._communicate_load_status()
        if self.loaded:
            self._prepare_settings_meta()
        return self.loaded

    def _prepare_settings_meta(self):
        settings_meta = SettingsMetaUploader(self.skill_directory,
                                             self.instance.name)
        self.instance.settings_meta = settings_meta

    def _prepare_for_load(self):
        self.load_attempted = True
        self.loaded = False
        self.instance = None

    def _skip_load(self):
        log_msg = 'Skill {} is blacklisted - it will not be loaded'
        LOG.info(log_msg.format(self.skill_id))

    def _load_skill_source(self):
        """Use Python's import library to load a skill's source code."""
        # TODO: Replace the deprecated "imp" library with the newer "importlib"
        module_name = self.skill_id.replace('.', '_')
        main_file_path = os.path.join(self.skill_directory, SKILL_MAIN_MODULE)
        try:
            with open(main_file_path, 'rb') as main_file:
                skill_module = imp.load_module(
                    module_name,
                    main_file,
                    main_file_path,
                    ('.py', 'rb', imp.PY_SOURCE)
                )
        except FileNotFoundError as f:
            error_msg = 'Failed to load {} due to a missing file.'
            LOG.exception(error_msg.format(self.skill_id))
        except Exception as e:
            LOG.exception('Failed to load skill: '
                          '{} ({})'.format(self.skill_id, repr(e)))
        else:
            module_is_skill = (
                hasattr(skill_module, 'create_skill') and
                callable(skill_module.create_skill)
            )
            if module_is_skill:
                return skill_module
        return None  # Module wasn't loaded

    def _create_skill_instance(self, skill_module):
        """Use v2 skills framework to create the skill."""
        try:
            self.instance = skill_module.create_skill()
        except Exception as e:
            log_msg = 'Skill __init__ failed with {}'
            LOG.exception(log_msg.format(repr(e)))
            self.instance = None

        if self.instance:
            self.instance.skill_id = self.skill_id
            self.instance.bind(self.bus)
            try:
                self.instance.load_data_files()
                # Set up intent handlers
                # TODO: can this be a public method?
                self.instance._register_decorated()
                self.instance.register_resting_screen()
                self.instance.initialize()
            except Exception as e:
                # If an exception occurs, make sure to clean up the skill
                self.instance.default_shutdown()
                self.instance = None
                log_msg = 'Skill initialization failed with {}'
                LOG.exception(log_msg.format(repr(e)))

        return self.instance is not None

    def _check_for_first_run(self):
        """The very first time a skill is run, speak the intro."""
        first_run = self.instance.settings.get(
            "__mycroft_skill_firstrun",
            True
        )
        if first_run:
            LOG.info("First run of " + self.skill_id)
            self.instance.settings["__mycroft_skill_firstrun"] = False
            self.instance.settings.store()
            intro = self.instance.get_intro_message()
            if intro:
                self.instance.speak(intro)

    def _communicate_load_status(self):
        if self.loaded:
            message = Message(
                'mycroft.skills.loaded',
                data=dict(
                    path=self.skill_directory,
                    id=self.skill_id,
                    name=self.instance.name,
                    modified=self.last_modified
                )
            )
            self.bus.emit(message)
            LOG.info('Skill {} loaded successfully'.format(self.skill_id))
        else:
            message = Message(
                'mycroft.skills.loading_failure',
                data=dict(path=self.skill_directory, id=self.skill_id)
            )
            self.bus.emit(message)
            LOG.error('Skill {} failed to load'.format(self.skill_id))