MycroftAI
/
selene-backend
mirror of https://github.com/MycroftAI/selene-backend.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'remotes/origin/dev' into test

pull/145/head^2^2
Chris Veilleux 2019-05-21 12:03:25 -05:00
parent 673f3e9252 45c6b1e4f8
commit 4e2f625082
18 changed files with 1 additions and 547 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/public/uwsgi.ini
View File

@ -1,7 +1,7 @@
[uwsgi]
master = true
module = public_api.api:public
processes = 4
processes = 10
socket = :5000
die-on-term = true
lazy = true

26
service/Dockerfile
View File

@ -1,26 +0,0 @@
# Docker config for the Selene skill service
# The selene-shared parent image contains all the common Docker configs for
# all Selene apps and services see the "shared" directory in this repository.
FROM docker.mycroft.ai/selene-shared:2018.4
LABEL description="Selene Skill Service API"
# Use pipenv to install the package's dependencies in the container
COPY Pipfile Pipfile
COPY Pipfile.lock Pipfile.lock
RUN pipenv install --system
# Now that pipenv has installed all the packages required by selene-util
# the Pipfile can be removed from the container.
RUN rm Pipfile
RUN rm Pipfile.lock
# Load the skill service application to the image
COPY skill/skill_service /opt/selene/skill_service
WORKDIR /opt/selene/
EXPOSE 7100
# Use uwsgi to serve the API
COPY uwsgi.ini uwsgi.ini
ENTRYPOINT ["uwsgi", "--ini", "uwsgi.ini"]

13
service/Pipfile
View File

@ -1,13 +0,0 @@
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[packages]
flask-restful = "*"
mongoengine = "*"
pygithub = "*"
uwsgi = "*"
[requires]
python_version = "3.7"

234
service/Pipfile.lock generated
View File

@ -1,234 +0,0 @@
{
"_meta": {
"hash": {
"sha256": "0249828591e0b2cebfad9c9ff2d00f8e931f5228061b151f12666add5cbd2d81"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.7"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"aniso8601": {
"hashes": [
"sha256:547e7bc88c19742e519fb4ca39f4b8113fdfb8fca322e325f16a8bfc6cfc553c",
"sha256:e7560de91bf00baa712b2550a2fdebf0188c5fce2fcd1162fbac75c19bb29c95"
],
"version": "==4.0.1"
},
"certifi": {
"hashes": [
"sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c",
"sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a"
],
"version": "==2018.10.15"
},
"chardet": {
"hashes": [
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
],
"version": "==3.0.4"
},
"click": {
"hashes": [
"sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
"sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"
],
"version": "==7.0"
},
"deprecated": {
"hashes": [
"sha256:8bfeba6e630abf42b5d111b68a05f7fe3d6de7004391b3cd614947594f87a4ff",
"sha256:b784e0ca85a8c1e694d77e545c10827bd99772392e79d5f5442e761515a1246e"
],
"version": "==1.2.4"
},
"flask": {
"hashes": [
"sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48",
"sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05"
],
"version": "==1.0.2"
},
"flask-restful": {
"hashes": [
"sha256:5795519501347e108c436b693ff9a4d7b373a3ac9069627d64e4001c05dd3407",
"sha256:e2f1b8063de3944b94c7f8be5cee4d2161db0267c54c5b757d875295061776fa"
],
"index": "pypi",
"version": "==0.3.6"
},
"idna": {
"hashes": [
"sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e",
"sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16"
],
"version": "==2.7"
},
"itsdangerous": {
"hashes": [
"sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19",
"sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749"
],
"version": "==1.1.0"
},
"jinja2": {
"hashes": [
"sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd",
"sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4"
],
"version": "==2.10"
},
"markupsafe": {
"hashes": [
"sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432",
"sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b",
"sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9",
"sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af",
"sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834",
"sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd",
"sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d",
"sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7",
"sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b",
"sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3",
"sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c",
"sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2",
"sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7",
"sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36",
"sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1",
"sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e",
"sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1",
"sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c",
"sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856",
"sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550",
"sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492",
"sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672",
"sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401",
"sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6",
"sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6",
"sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c",
"sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd",
"sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1"
],
"version": "==1.1.0"
},
"mongoengine": {
"hashes": [
"sha256:1c375c8bbae89fc3df672c619a5835a2bd15148f9e8939f214a0b398cf12525f",
"sha256:e8ed6d9d7f648205f4adcd773bb25ce626fdc6cd74b2959451e425441fb0e6d3"
],
"index": "pypi",
"version": "==0.16.0"
},
"pygithub": {
"hashes": [
"sha256:70d90139f61a3d88417ff15eaca6150d0b3ba7ef0dc59589ea3719c3ce518ef6"
],
"index": "pypi",
"version": "==1.43.3"
},
"pyjwt": {
"hashes": [
"sha256:30b1380ff43b55441283cc2b2676b755cca45693ae3097325dea01f3d110628c",
"sha256:4ee413b357d53fd3fb44704577afac88e72e878716116270d722723d65b42176"
],
"version": "==1.6.4"
},
"pymongo": {
"hashes": [
"sha256:025f94fc1e1364f00e50badc88c47f98af20012f23317234e51a11333ef986e6",
"sha256:02aa7fb282606331aefbc0586e2cf540e9dbe5e343493295e7f390936ad2738e",
"sha256:057210e831573e932702cf332012ed39da78edf0f02d24a3f0b213264a87a397",
"sha256:0d946b79c56187fe139276d4c8ed612a27a616966c8b9779d6b79e2053587c8b",
"sha256:104790893b928d310aae8a955e0bdbaa442fb0ac0a33d1bbb0741c791a407778",
"sha256:15527ef218d95a8717486106553b0d54ff2641e795b65668754e17ab9ca6e381",
"sha256:1826527a0b032f6e20e7ac7f72d7c26dd476a5e5aa82c04aa1c7088a59fded7d",
"sha256:22e3aa4ce1c3eebc7f70f9ca7fd4ce1ea33e8bdb7b61996806cd312f08f84a3a",
"sha256:244e1101e9a48615b9a16cbd194f73c115fdfefc96894803158608115f703b26",
"sha256:24b8c04fdb633a84829d03909752c385faef249c06114cc8d8e1700b95aae5c8",
"sha256:2c276696350785d3104412cbe3ac70ab1e3a10c408e7b20599ee41403a3ed630",
"sha256:2d8474dc833b1182b651b184ace997a7bd83de0f51244de988d3c30e49f07de3",
"sha256:3119b57fe1d964781e91a53e81532c85ed1701baaddec592e22f6b77a9fdf3df",
"sha256:3bee8e7e0709b0fcdaa498a3e513bde9ffc7cd09dbceb11e425bd91c89dbd5b6",
"sha256:436c071e01a464753d30dbfc8768dd93aecf2a8e378e5314d130b95e77b4d612",
"sha256:46635e3f19ad04d5a7d7cf23d232388ddbfccf46d9a3b7436b6abadda4e84813",
"sha256:4772e0b679717e7ac4608d996f57b6f380748a919b457cb05bb941467b888b22",
"sha256:4e2cd80e16f481a62c3175b607373200e714ed29025f21559ebf7524f295689f",
"sha256:52732960efa0e003ca1c092dc0a3c65276e897681287a788a01ca78dda3b41f0",
"sha256:55a7de51ec7d1731b2431886d0349146645f2816e5b8eb982d7c49f89472c9f3",
"sha256:5f8ed5934197a2d4b2087646e98de3e099a237099dcf498b9e38dd3465f74ef4",
"sha256:64b064124fcbc8eb04a155117dc4d9a336e3cda3f069958fbc44fe70c3c3d1e9",
"sha256:65958b8e4319f992e85dad59d8081888b97fcdbde5f0d14bc28f2848b92d3ef1",
"sha256:7683428862e20c6a790c19e64f8ccf487f613fbc83d47e3d532df9c81668d451",
"sha256:78566d5570c75a127c2491e343dc006798a384f06be588fe9b0cbe5595711559",
"sha256:7d1cb00c093dbf1d0b16ccf123e79dee3b82608e4a2a88947695f0460eef13ff",
"sha256:8c74e2a9b594f7962c62cef7680a4cb92a96b4e6e3c2f970790da67cc0213a7e",
"sha256:8e60aa7699170f55f4b0f56ee6f8415229777ac7e4b4b1aa41fc61eec08c1f1d",
"sha256:9447b561529576d89d3bf973e5241a88cf76e45bd101963f5236888713dea774",
"sha256:970055bfeb0be373f2f5299a3db8432444bad3bc2f198753ee6c2a3a781e0959",
"sha256:a6344b8542e584e140dc3c651d68bde51270e79490aa9320f9e708f9b2c39bd5",
"sha256:ce309ca470d747b02ba6069d286a17b7df8e9c94d10d727d9cf3a64e51d85184",
"sha256:cfbd86ed4c2b2ac71bbdbcea6669bf295def7152e3722ddd9dda94ac7981f33d",
"sha256:d7929c513732dff093481f4a0954ed5ff16816365842136b17caa0b4992e49d3"
],
"version": "==3.7.2"
},
"pytz": {
"hashes": [
"sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca",
"sha256:8e0f8568c118d3077b46be7d654cc8167fa916092e28320cde048e54bfc9f1e6"
],
"version": "==2018.7"
},
"requests": {
"hashes": [
"sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54",
"sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263"
],
"version": "==2.20.1"
},
"six": {
"hashes": [
"sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
"sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
],
"version": "==1.11.0"
},
"urllib3": {
"hashes": [
"sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
"sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
],
"version": "==1.24.1"
},
"uwsgi": {
"hashes": [
"sha256:d2318235c74665a60021a4fc7770e9c2756f9fc07de7b8c22805efe85b5ab277"
],
"index": "pypi",
"version": "==2.0.17.1"
},
"werkzeug": {
"hashes": [
"sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c",
"sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b"
],
"version": "==0.14.1"
},
"wrapt": {
"hashes": [
"sha256:d4d560d479f2c21e1b5443bbd15fe7ec4b37fe7e53d335d3b9b0a7b1226fe3c6"
],
"version": "==1.10.11"
}
},
"develop": {}
}

0
service/skill/skill_service/__init__.py
View File

0
service/skill/skill_service/api/__init__.py
View File

15
service/skill/skill_service/api/api.py
View File

@ -1,15 +0,0 @@
"""Define the API Flask application and its endpoints"""
from flask import Flask
from flask_restful import Api
from .endpoints import AllSkillsEndpoint, SkillDetailEndpoint
# from .config import get_config_location
skill = Flask(__name__)
# skill.config.from_object(get_config_location())
skill_api = Api(skill)
skill_api.add_resource(AllSkillsEndpoint, '/skill/all')
skill_api.add_resource(SkillDetailEndpoint, '/skill/name/<string:skill_name>')

38
service/skill/skill_service/api/config.py
View File

@ -1,38 +0,0 @@
import os
class LoginConfigException(Exception):
pass
class BaseConfig:
"""Base configuration."""
DEBUG = False
class DevelopmentConfig(BaseConfig):
"""Development configuration."""
DEBUG = True
TARTARUS_BASE_URL = 'https://api-test.mycroft.ai/v1'
def get_config_location():
environment_configs = dict(
dev='skill_service.api.config.DevelopmentConfig',
# test=TestConfig,
# prod=ProdConfig
)
try:
environment_name = os.environ['SELENE_ENVIRONMENT']
except KeyError:
raise LoginConfigException('the SELENE_ENVIRONMENT variable is not set')
try:
configs_location = environment_configs[environment_name]
except KeyError:
raise LoginConfigException(
'no configuration defined for the "{}" environment'.format(environment_name)
)
return configs_location

2
service/skill/skill_service/api/endpoints/__init__.py
View File

@ -1,2 +0,0 @@
from .all_skills import AllSkillsEndpoint
from .skill_detail import SkillDetailEndpoint

21
service/skill/skill_service/api/endpoints/all_skills.py
View File

@ -1,21 +0,0 @@
"""Define the view to return summary information for all available skills."""
from http import HTTPStatus
from flask_restful import Resource
from .skill_formatter import format_skill_for_response
from ...repository.skill import select_all_skills
class AllSkillsEndpoint(Resource):
"""All skills available for use on devices running Mycroft core."""
def get(self):
"""Handle a HTTP GET request for available skills."""
response = []
for skill in select_all_skills():
formatted_skill = format_skill_for_response(skill)
response.append(formatted_skill)
return response, HTTPStatus.OK

16
service/skill/skill_service/api/endpoints/skill_detail.py
View File

@ -1,16 +0,0 @@
"""Define the view to get the detailed information about a particular skill."""
from http import HTTPStatus
from flask_restful import Resource
from .skill_formatter import format_skill_for_response
from ...repository.skill import select_skill_by_name
class SkillDetailEndpoint(Resource):
def get(self, skill_name):
"""Handle HTP GET request for detailed information about a skill."""
skill = select_skill_by_name(skill_name)
response = format_skill_for_response(skill)
return response, HTTPStatus.OK

12
service/skill/skill_service/api/endpoints/skill_formatter.py
View File

@ -1,12 +0,0 @@
def format_skill_for_response(skill):
"""Manipulate a mongoengine object into a serializable object."""
formatted_skill = skill.to_mongo().to_dict()
formatted_skill['id'] = str(formatted_skill['_id'])
formatted_skill['created'] = formatted_skill['_id'].generation_time
del formatted_skill['_id']
for datetime_attr in ('created', 'last_update'):
formatted_skill[datetime_attr] = formatted_skill[datetime_attr].timestamp()
return formatted_skill

2
service/skill/skill_service/repository/__init__.py
View File

@ -1,2 +0,0 @@
from .db import connect_to_skill_db
from .skill import select_all_skills, Skill, upsert_skill

11
service/skill/skill_service/repository/db.py
View File

@ -1,11 +0,0 @@
"""Database access utility functions"""
from os import environ
from mongoengine import connect
def connect_to_skill_db():
"""Establish a connection to the Mongo skills database."""
host = environ['SKILL_DB_HOST']
port = int(environ['SKILL_DB_PORT'])
database = 'skillDB'
connect(database, host=host, port=port)

67
service/skill/skill_service/repository/skill.py
View File

@ -1,67 +0,0 @@
"""
Queries and manipulations of the skill collection in the marketplaceDB
"""
from datetime import datetime
from mongoengine import (
DateTimeField,
DictField,
Document,
ListField,
StringField
)
from .db import connect_to_skill_db
class Skill(Document):
"""
Represents the schema of documents in the skill collection
"""
branch = StringField(required=True)
categories = ListField(StringField())
credits = ListField(DictField())
description = StringField()
icon = DictField()
icon_image = StringField()
last_update = DateTimeField(default=datetime.now(), required=True)
platforms = ListField(StringField(), required=True, default=['all'])
repository_owner = StringField(required=True)
repository_url = StringField(required=True)
skill_name = StringField(required=True, unique=True)
summary = StringField()
tags = ListField(StringField())
title = StringField(required=True)
triggers = ListField(StringField())
def select_all_skills() -> list:
"""
Map the repository name to the skill object
Skill repositories in Github must be uniquely named
:return: dictionary of skill objects keyed by the repository name
"""
connect_to_skill_db()
return Skill.objects
def select_skill_by_name(skill_name: str) -> Skill:
"""
Query the database for a specified skill ID
:return: the Skill object with an ID matching the argument
"""
connect_to_skill_db()
return Skill.objects(skill_name=skill_name).first()
def upsert_skill(skill: Skill):
"""
An upsert will update a document if it exists or insert it if not.
:param skill: The skill to update or insert
"""
connect_to_skill_db()
skill.save()

0
service/skill/skill_service/script/__init__.py
View File

84
service/skill/skill_service/script/load_skill_metadata.py
View File

@ -1,84 +0,0 @@
from datetime import datetime
import json
from os import environ
from skill_service.repository import (
connect_to_skill_db,
select_all_skills,
Skill,
upsert_skill
)
from selene_util.github import download_repository_file, log_into_github
class SkillRefresher(object):
"""
Reconcile a skill repository's README.md with the database
"""
def __init__(self, skill: Skill, github_metadata):
self.skill = skill
self.skill_metadata = github_metadata
def _skill_meta_changed(self) -> bool:
"""
Determine if any of the skill metadata fields will be updated.
This is important to know whether or not the last update timestamp
needs a new value. Changes in the metadata will result in an update
of the timestamp whereas the timestamp will stay the same if nothing
has changed.
"""
return (
self.skill.categories != self.skill_metadata.get('categories', []) or
self.skill.credits != self.skill_metadata.get('credits', []) or
self.skill.description != self.skill_metadata['description'] or
self.skill.icon != self.skill_metadata.get('icon') or
self.skill.icon_image != self.skill_metadata.get('icon_img') or
self.skill.platforms != self.skill_metadata['platforms'] or
self.skill.repository_owner != self.skill_metadata['github_username'] or
self.skill.repository_url != self.skill_metadata['repo'] or
self.skill.summary != self.skill_metadata.get('short_desc') or
self.skill.tags != self.skill_metadata['tags'] or
self.skill.title != self.skill_metadata['title'] or
self.skill.triggers != self.skill_metadata.get('examples')
)
def refresh(self):
"""
Refresh the skill database with the repository README.md file
"""
if self._skill_meta_changed():
self.skill.branch = environ['SKILL_BRANCH']
self.skill.categories = self.skill_metadata.get('categories')
self.skill.credits = self.skill_metadata.get('credits')
self.skill.description = self.skill_metadata['description']
self.skill.last_update = datetime.now()
self.skill.icon = self.skill_metadata.get('icon')
self.skill.icon_image = self.skill_metadata.get('icon_img')
self.skill.platforms = self.skill_metadata['platforms']
self.skill.repository_owner = self.skill_metadata['github_username']
self.skill.repository_url = self.skill_metadata['repo']
self.skill.skill_name = self.skill_metadata['name']
self.skill.summary = self.skill_metadata.get('short_desc')
self.skill.tags = self.skill_metadata.get('tags')
self.skill.title = self.skill_metadata['title']
self.skill.triggers = self.skill_metadata.get('examples')
upsert_skill(self.skill)
connect_to_skill_db()
skills_in_db = {skill.skill_name: skill for skill in select_all_skills()}
# TODO figure out a way to paramaterize these
github = log_into_github('dev@mycroft.ai', 'pFuG8z5ngmqVDla1aaED2rKl3yke5vZ7')
file_contents = download_repository_file(
github,
'mycroft-skills-data',
'18.08',
'skill-metadata.json'
)
skills_metadata = json.loads(file_contents)
for skill_identifier, skill_metadata in skills_metadata.items():
skill_in_db = skills_in_db.get(skill_identifier, Skill())
skill_refresher = SkillRefresher(skill_in_db, skill_metadata)
skill_refresher.refresh()

5
service/uwsgi.ini
View File

@ -1,5 +0,0 @@
[uwsgi]
master = true
module = skill_service.api.api:skill
processes = 4
socket = :7100