MycroftAI
/
mycroft-core
mirror of https://github.com/MycroftAI/mycroft-core.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Feature/skillsmeta gid (#2074) 

* Add global id basics to settings meta

- All skills will upload a blank settingsmeta
- a skill_gid will be appended to all settingsmeta upload-data
- Added basic function for generating skill_gid

* Use new skill_gid field.

Populate skill_gid directly from metadata

* Separate travis tmp-dirs

- Update travis script to use tempdir for each python version
- Update test script to handle nonstandard tempdirs
- Generate msm folder using tempdir when running create_msm test

* Add title field with pretty name

* Collect and expand "title" as needed

For title use market-place title or name in settings meta or skillname

* Switch skill_manager create_msm test to 19.02

* Remove leading / trailing Skill in display name

Also rename title displayname to match new mycroft-skills-data

* Lock msm_create and mock the name info test_settings
pull/2075/head
Åke 2019-04-02 18:14:49 +02:00 committed by Steve Penrod
parent 6706c37782
commit 82fa314ce9
8 changed files with 168 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.travis.yml
View File

@ -6,6 +6,7 @@ before_install:
- sudo apt-get install -qq mpg123 portaudio19-dev libglib2.0-dev swig bison libtool autoconf libglib2.0-dev libicu-dev libfann-dev realpath
- sudo apt-get install -y gcc-4.8 g++-4.8
- export CC="gcc-4.8"
- export TMPDIR="/tmp/${TRAVIS_PYTHON_VERSION}"
python:
- "3.4"
- "3.5"
@ -15,6 +16,9 @@ python:
cache: pocketsphinx-python
# command to install dependencies
install:
- rm -rf ${TMPDIR}
- mkdir ${TMPDIR}
- echo ${TMPDIR}
- VIRTUALENV_ROOT=${VIRTUAL_ENV} ./dev_setup.sh
- pip install -r requirements.txt
- pip install -r test-requirements.txt

28
mycroft/skills/msm_wrapper.py Normal file
View File

@ -0,0 +1,28 @@
import os
from os.path import join, expanduser, exists
from msm import MycroftSkillsManager, SkillRepo
from mycroft.util.combo_lock import ComboLock
mycroft_msm_lock = ComboLock('/tmp/mycroft-msm.lck')
def create_msm(config):
""" Create msm object from config. """
msm_config = config['skills']['msm']
repo_config = msm_config['repo']
data_dir = expanduser(config['data_dir'])
skills_dir = join(data_dir, msm_config['directory'])
repo_cache = join(data_dir, repo_config['cache'])
platform = config['enclosure'].get('platform', 'default')
with mycroft_msm_lock:
# Try to create the skills directory if it doesn't exist
if not exists(skills_dir):
os.makedirs(skills_dir)
return MycroftSkillsManager(
platform=platform, skills_dir=skills_dir,
repo=SkillRepo(repo_cache, repo_config['url'],
repo_config['branch']),
versioned=msm_config['versioned'])

114
mycroft/skills/settings.py
View File

@ -61,22 +61,83 @@
import json
import hashlib
import os
import time
import copy
import re
from threading import Timer
from os.path import isfile, join, expanduser
from requests.exceptions import RequestException
from msm import SkillEntry
from mycroft.api import DeviceApi, is_paired
from mycroft.util.log import LOG
from mycroft.util import camel_case_split
from mycroft.configuration import ConfigurationManager
from .msm_wrapper import create_msm
# This is the base needed for sending a blank settings meta entry (Tartarus)
# To this a global id is added
# TODO reduce the needed boilerplate here
BLANK_META = {
"skillMetadata": {
"sections": [
{
"name": "",
"fields": [
{
"type": "label",
"label": ""
}
]
}
]
}
}
msm = None
msm_creation_time = 0
def build_global_id(directory, config):
""" Create global id for the skill.
TODO: Handle dirty skill
Arguments:
directory: skill directory
config: config for the device to fetch msm setup
"""
# Update the msm object if it's more than an hour old
global msm
global msm_creation_time
if msm is None or time.time() - msm_creation_time > 60 * 60:
msm_creation_time = time.time()
msm = create_msm(config)
s = SkillEntry.from_folder(directory, msm)
if s.meta_info != {}:
return s.meta_info['skill_gid'], s.meta_info['display_name']
else: # No skills meta data available, local or unsubmitted skill
return "@{}_{}".format(DeviceApi().identity.uuid, s.name), None
def display_name(name):
""" Splits camelcase and removes leading/trailing Skill. """
name = re.sub(r'(^[Ss]kill|[Ss]kill$)', '', name)
return camel_case_split(name)
class SkillSettings(dict):
""" Dictionary that can easily be saved to a file, serialized as json. It
also syncs to the backend for skill settings
Args:
directory (str): Path to storage directory
name (str): user readable name associated with the settings
directory (str): Path to storage directory
name (str): user readable name associated with the settings
no_upload (bool): True if the upload to mycroft servers should be
disabled.
"""
def __init__(self, directory, name):
@ -93,6 +154,8 @@ class SkillSettings(dict):
# set file paths
self._settings_path = join(directory, 'settings.json')
self._meta_path = join(directory, 'settingsmeta.json')
self._directory = directory
self.is_alive = True
self.loaded_hash = hash(json.dumps(self, sort_keys=True))
self._complete_intialization = False
@ -101,11 +164,17 @@ class SkillSettings(dict):
self._user_identity = None
self.changed_callback = None
self._poll_timer = None
self._blank_poll_timer = None
self._is_alive = True
# if settingsmeta exist
if isfile(self._meta_path):
self._poll_skill_settings()
# if not disallowed by user upload an entry for all skills installed
elif self.config['skills']['upload_skill_manifest']:
self._blank_poll_timer = Timer(1, self._init_blank_meta)
self._blank_poll_timer.daemon = True
self._blank_poll_timer.start()
def __hash__(self):
""" Simple object unique hash. """
@ -121,6 +190,8 @@ class SkillSettings(dict):
self._is_alive = False
if self._poll_timer:
self._poll_timer.cancel()
if self._blank_poll_timer:
self._blank_poll_timer.cancel()
def set_changed_callback(self, callback):
""" Set callback to perform when server settings have changed.
@ -195,18 +266,37 @@ class SkillSettings(dict):
return super(SkillSettings, self).__setitem__(key, value)
def _load_settings_meta(self):
""" Loads settings metadata from skills path. """
""" Load settings metadata from the skill folder.
If no settingsmeta exists a basic settingsmeta will be created
containing a basic identifier.
Returns:
(dict) settings meta
"""
if isfile(self._meta_path):
try:
with open(self._meta_path, encoding='utf-8') as f:
data = json.load(f)
return data
except Exception as e:
LOG.error("Failed to load setting file: "+self._meta_path)
LOG.error("Failed to load setting file: " + self._meta_path)
LOG.error(repr(e))
return None
else:
return None
data = copy.copy(BLANK_META)
# Add Information extracted from the skills-meta.json entry for the
# skill.
skill_gid, display_name = build_global_id(self._directory, self.config)
data['skill_gid'] = skill_gid
data['display_name'] = (display_name or data.get('name') or
display_name(name))
# Backwards compatibility:
if 'name' not in data:
data['name'] = data['display_name']
return data
def _send_settings_meta(self, settings_meta):
""" Send settingsmeta.json to the server.
@ -380,6 +470,18 @@ class SkillSettings(dict):
settings_meta = self._load_settings_meta()
self._upload_meta(settings_meta, hashed_meta)
def _init_blank_meta(self):
""" Send blank settingsmeta to remote. """
try:
if not is_paired() and self.is_alive:
self._blank_poll_timer = Timer(60, self._init_blank_meta)
self._blank_poll_timer.daemon = True
self._blank_poll_timer.start()
else:
self.initialize_remote_settings()
except Exception as e:
LOG.exception('Failed to send blank meta: {}'.format(repr(e)))
def _poll_skill_settings(self):
""" If identifier exists for this skill poll to backend to
request settings and store it if it changes

20
mycroft/skills/skill_manager.py
View File

@ -33,7 +33,7 @@ from mycroft.util.log import LOG
from mycroft.api import DeviceApi, is_paired
from .core import load_skill, create_skill_descriptor, MainModule
from .msm_wrapper import create_msm as msm_creator
DEBUG = Configuration.get().get("debug", False)
skills_config = Configuration.get().get("skills")
@ -132,23 +132,7 @@ class SkillManager(Thread):
@staticmethod
def create_msm():
config = Configuration.get()
msm_config = config['skills']['msm']
repo_config = msm_config['repo']
data_dir = expanduser(config['data_dir'])
skills_dir = join(data_dir, msm_config['directory'])
# Try to create the skills directory if it doesn't exist
if not exists(skills_dir):
os.makedirs(skills_dir)
repo_cache = join(data_dir, repo_config['cache'])
platform = config['enclosure'].get('platform', 'default')
return MycroftSkillsManager(
platform=platform, skills_dir=skills_dir,
repo=SkillRepo(
repo_cache, repo_config['url'], repo_config['branch']
), versioned=msm_config['versioned']
)
return msm_creator(Configuration.get())
def schedule_now(self, message=None):
self.next_download = time.time() - 1

2
requirements.txt
View File

@ -22,7 +22,7 @@ pulsectl==17.7.4
google-api-python-client==1.6.4
fasteners==0.14.1
msm==0.7.3
msm==0.7.4
msk==0.3.12
adapt-parser==0.3.2
padatious==0.4.6

5
test/unittests/skills/test_settings.py
View File

@ -14,6 +14,7 @@
#
import json
import unittest
from mock import MagicMock
from os import remove
from os.path import join, dirname
@ -21,6 +22,10 @@ from os.path import join, dirname
from mycroft.skills.settings import SkillSettings
SkillSettings._poll_skill_settings = MagicMock()
SkillSettings._init_blank_meta = MagicMock()
class SkillSettingsTest(unittest.TestCase):
def setUp(self):
try:

18
test/unittests/skills/test_skill_manager.py
View File

@ -1,5 +1,6 @@
import unittest
import mock
import copy
import tempfile
from os.path import exists, join
from shutil import rmtree
@ -9,7 +10,6 @@ from mycroft.configuration import Configuration
from mycroft.skills.skill_manager import SkillManager
BASE_CONF = base_config()
BASE_CONF['data_dir'] = tempfile.mkdtemp()
BASE_CONF['skills'] = {
'msm': {
'directory': 'skills',
@ -17,7 +17,7 @@ BASE_CONF['skills'] = {
'repo': {
'cache': '.skills-repo',
'url': 'https://github.com/MycroftAI/mycroft-skills',
'branch': '18.08'
'branch': '19.02'
}
},
'update_interval': 3600,
@ -56,14 +56,16 @@ class MycroftSkillTest(unittest.TestCase):
self.emitter.reset()
self.temp_dir = tempfile.mkdtemp()
@mock.patch.dict(Configuration._Configuration__config, BASE_CONF)
def test_create_manager(self):
""" Verify that the skill manager and msm loads as expected and
that the skills dir is created as needed.
"""
SkillManager(self.emitter)
self.assertTrue(exists(join(BASE_CONF['data_dir'], 'skills')))
conf = copy.deepcopy(BASE_CONF)
conf['data_dir'] = self.temp_dir
with mock.patch.dict(Configuration._Configuration__config,
BASE_CONF):
SkillManager(self.emitter)
self.assertTrue(exists(join(BASE_CONF['data_dir'], 'skills')))
@classmethod
def tearDownClass(cls):
rmtree(BASE_CONF['data_dir'])
def tearDown(self):
rmtree(self.temp_dir)

17
test/unittests/util/test_signal.py
View File

@ -15,23 +15,25 @@
import unittest
from shutil import rmtree
from os.path import exists, isfile
from os.path import exists, isfile, join
from tempfile import gettempdir
from mycroft.util import create_signal, check_for_signal
class TestSignals(unittest.TestCase):
def setUp(self):
if exists('/tmp/mycroft'):
rmtree('/tmp/mycroft')
if exists(join(gettempdir(), 'mycroft')):
rmtree(join(gettempdir(), 'mycroft'))
def test_create_signal(self):
create_signal('test_signal')
self.assertTrue(isfile('/tmp/mycroft/ipc/signal/test_signal'))
self.assertTrue(isfile(join(gettempdir(),
'mycroft/ipc/signal/test_signal')))
def test_check_signal(self):
if exists('/tmp/mycroft'):
rmtree('/tmp/mycroft')
if exists(join(gettempdir(), 'mycroft')):
rmtree(join(gettempdir(), 'mycroft'))
# check that signal is not found if file does not exist
self.assertFalse(check_for_signal('test_signal'))
@ -39,7 +41,8 @@ class TestSignals(unittest.TestCase):
create_signal('test_signal')
self.assertTrue(check_for_signal('test_signal'))
# Check that the signal is removed after use
self.assertFalse(isfile('/tmp/mycroft/ipc/signal/test_signal'))
self.assertFalse(isfile(join(gettempdir(),
'mycroft/ipc/signal/test_signal')))
if __name__ == "__main__":