From e23d9ab8407f43006e79195e50b03f48ee871316 Mon Sep 17 00:00:00 2001 From: Matheus Lima Date: Tue, 26 Mar 2019 22:09:41 -0300 Subject: [PATCH] Created endpoint to upload skills manifest --- api/public/public_api/api.py | 9 ++- .../endpoints/device_skill_manifest.py | 39 +++++++++++ .../public_api/endpoints/device_skills.py | 8 +++ .../features/device_skill_manifest.feature | 6 ++ .../features/steps/device_skill_manifest.py | 70 +++++++++++++++++++ shared/selene/data/repository_base.py | 10 ++- shared/selene/data/skill/repository/skill.py | 18 +++++ .../repository/sql/update_skill_manifest.sql | 21 ++++++ shared/selene/util/db/__init__.py | 2 +- shared/selene/util/db/cursor.py | 16 +++++ 10 files changed, 196 insertions(+), 3 deletions(-) create mode 100644 api/public/public_api/endpoints/device_skill_manifest.py create mode 100644 api/public/tests/features/device_skill_manifest.feature create mode 100644 api/public/tests/features/steps/device_skill_manifest.py create mode 100644 shared/selene/data/skill/repository/sql/update_skill_manifest.sql diff --git a/api/public/public_api/api.py b/api/public/public_api/api.py index dcf71302..164ce914 100644 --- a/api/public/public_api/api.py +++ b/api/public/public_api/api.py @@ -2,7 +2,6 @@ import os from flask import Flask -from public_api.endpoints.device_location import DeviceLocationEndpoint from selene.api import SeleneResponse, selene_api from selene.api.base_config import get_base_config from selene.api.public_endpoint import check_oauth_token @@ -21,6 +20,8 @@ from .endpoints.google_stt import GoogleSTTEndpoint from .endpoints.open_weather_map import OpenWeatherMapEndpoint from .endpoints.wolfram_alpha import WolframAlphaEndpoint from .endpoints.wolfram_alpha_spoken import WolframAlphaSpokenEndpoint +from .endpoints.device_location import DeviceLocationEndpoint +from .endpoints.device_skill_manifest import DeviceSkillManifest public = Flask(__name__) public.config.from_object(get_base_config()) @@ -121,6 +122,12 @@ public.add_url_rule( view_func=DeviceLocationEndpoint.as_view('device_location_api'), methods=['GET'] ) +public.add_url_rule( + '/v1/device//skillJson', + view_func=DeviceSkillManifest.as_view('skill_manifest_api'), + methods=['PUT'] +) + """ This is a workaround to allow the API return 401 when we call a non existent path. Use case: diff --git a/api/public/public_api/endpoints/device_skill_manifest.py b/api/public/public_api/endpoints/device_skill_manifest.py new file mode 100644 index 00000000..d33d9623 --- /dev/null +++ b/api/public/public_api/endpoints/device_skill_manifest.py @@ -0,0 +1,39 @@ +import json +from http import HTTPStatus + +from schematics import Model +from schematics.types import StringType, ModelType, ListType, DateTimeType + +from selene.api import PublicEndpoint +from selene.data.skill import SkillRepository +from selene.util.db import get_db_connection + + +class SkillManifest(Model): + name = StringType(default='') + origin = StringType(default='') + installation = StringType(default='') + failure_message = StringType(default='') + status = StringType(default='') + beta = StringType(default='') + installed = DateTimeType() + updated = DateTimeType() + + +class SkillJson(Model): + blacklist = ListType(StringType) + skills = ListType(ModelType(SkillManifest, required=True)) + + +class DeviceSkillManifest(PublicEndpoint): + def __init__(self): + super(DeviceSkillManifest, self).__init__() + + def put(self, device_id): + payload = json.loads(self.request.data) + skill_json = SkillJson(payload) + skill_json.validate() + with get_db_connection(self.config['DB_CONNECTION_POOL']) as db: + SkillRepository(db).update_skills_manifest(device_id, payload['skills']) + return '', HTTPStatus.OK + diff --git a/api/public/public_api/endpoints/device_skills.py b/api/public/public_api/endpoints/device_skills.py index 649d3357..c09433e1 100644 --- a/api/public/public_api/endpoints/device_skills.py +++ b/api/public/public_api/endpoints/device_skills.py @@ -29,11 +29,19 @@ class SkillMetadata(Model): sections = ListType(ModelType(SkillSection)) +class SkillIcon(Model): + color = StringType() + icon = StringType() + + class Skill(Model): name = StringType(required=True) identifier = StringType(required=True) skillMetadata = ModelType(SkillMetadata) color = StringType() + icon_img = StringType() + icon = ModelType(SkillIcon) + class DeviceSkillsEndpoint(PublicEndpoint): """Fetch all skills associated with a given device using the API v1 format""" diff --git a/api/public/tests/features/device_skill_manifest.feature b/api/public/tests/features/device_skill_manifest.feature new file mode 100644 index 00000000..2209eb18 --- /dev/null +++ b/api/public/tests/features/device_skill_manifest.feature @@ -0,0 +1,6 @@ +Feature: Upload and fetch skills manifest + + Scenario: A skill manifest is successfully uploaded + Given a device with a skill + When a skill manifest is uploaded + Then the skill manifest endpoint should return 200 status code \ No newline at end of file diff --git a/api/public/tests/features/steps/device_skill_manifest.py b/api/public/tests/features/steps/device_skill_manifest.py new file mode 100644 index 00000000..466b9e28 --- /dev/null +++ b/api/public/tests/features/steps/device_skill_manifest.py @@ -0,0 +1,70 @@ +import json +from http import HTTPStatus + +from behave import given, when, then +from hamcrest import assert_that, equal_to + +skill_manifest = { + 'skills': [ + { + 'name': 'skill-name-1', + 'origin': 'voice', + 'installation': 'installed', + 'failure_message': '', + 'status': 'active', + 'installed': 1553610007, + 'updated': 1553610007 + } + ] +} + +skill = { + 'name': 'skill-name-1', + 'identifier': 'skill-name-13-123', + 'skillMetadata': { + 'sections': [ + { + 'name': 'section1', + 'fields': [ + { + 'label': 'test' + } + ] + } + ] + } +} + + +@given('a device with a skill') +def device_skill(context): + login = context.device_login + device_id = login['uuid'] + access_token = login['accessToken'] + headers = dict(Authorization='Bearer {token}'.format(token=access_token)) + context.client.put( + '/v1/device/{uuid}/skill'.format(uuid=device_id), + data=json.dumps(skill), + content_type='application_json', + headers=headers + ) + + +@when('a skill manifest is uploaded') +def device_skill_manifest(context): + login = context.device_login + device_id = login['uuid'] + access_token = login['accessToken'] + headers = dict(Authorization='Bearer {token}'.format(token=access_token)) + context.upload_skill_manifest_response = context.client.put( + '/v1/device/{uuid}/skillJson'.format(uuid=device_id), + data=json.dumps(skill_manifest), + content_type='application_json', + headers=headers + ) + + +@then('the skill manifest endpoint should return 200 status code') +def validate_skill_manifest_upload(context): + response = context.upload_skill_manifest_response + assert_that(response.status_code, equal_to(HTTPStatus.OK)) diff --git a/shared/selene/data/repository_base.py b/shared/selene/data/repository_base.py index 9d6b9e76..ebb78e83 100644 --- a/shared/selene/data/repository_base.py +++ b/shared/selene/data/repository_base.py @@ -1,5 +1,7 @@ from os import path -from selene.util.db import Cursor, DatabaseRequest, get_sql_from_file +from typing import List + +from selene.util.db import Cursor, DatabaseRequest, get_sql_from_file, DatabaseBatchRequest class RepositoryBase(object): @@ -12,3 +14,9 @@ class RepositoryBase(object): sql=get_sql_from_file(path.join(self.sql_dir, sql_file_name)), args=args ) + + def _build_db_batch_request(self, sql_file_name: str, args: List[dict]): + return DatabaseBatchRequest( + sql=get_sql_from_file(path.join(self.sql_dir, sql_file_name)), + args=args + ) diff --git a/shared/selene/data/skill/repository/skill.py b/shared/selene/data/skill/repository/skill.py index d272440c..61279591 100644 --- a/shared/selene/data/skill/repository/skill.py +++ b/shared/selene/data/skill/repository/skill.py @@ -1,4 +1,5 @@ import json +from datetime import datetime from typing import List from selene.util.db import use_transaction @@ -114,3 +115,20 @@ class SkillRepository(RepositoryBase): settings[field['name']] = field['value'] field.pop('value', None) return settings, skill + + def update_skills_manifest(self, device_id: str, skill_manifest: List[dict]): + for skill in skill_manifest: + skill['device_id'] = device_id + installed = skill.get('installed') + if installed: + installed = datetime.fromtimestamp(installed) + skill['installed'] = installed + updated = skill.get('updated') + if updated: + updated = datetime.fromtimestamp(updated) + skill['updated'] = updated + db_batch_request = self._build_db_batch_request( + 'update_skill_manifest.sql', + args=skill_manifest + ) + self.cursor.batch_update(db_batch_request) diff --git a/shared/selene/data/skill/repository/sql/update_skill_manifest.sql b/shared/selene/data/skill/repository/sql/update_skill_manifest.sql new file mode 100644 index 00000000..7b7a78ca --- /dev/null +++ b/shared/selene/data/skill/repository/sql/update_skill_manifest.sql @@ -0,0 +1,21 @@ +UPDATE + device.device_skill dev_skill +SET + install_method = %(origin)s, + install_status = %(status)s, + install_failure_reason = %(failure_message)s, + install_ts = %(installed)s, + update_ts = %(updated)s +WHERE + id = ( + SELECT + dev_skill.id + FROM + device.device dev + INNER JOIN + device.device_skill dev_skill ON dev.id = dev_skill.device_id + INNER JOIN + skill.skill skill ON dev_skill.skill_id = skill.id + WHERE + dev.id = %(device_id)s AND skill.name = %(name)s + ) \ No newline at end of file diff --git a/shared/selene/util/db/__init__.py b/shared/selene/util/db/__init__.py index 0d916286..47941649 100644 --- a/shared/selene/util/db/__init__.py +++ b/shared/selene/util/db/__init__.py @@ -1,4 +1,4 @@ from .connection import connect_to_db, DatabaseConnectionConfig from .connection_pool import allocate_db_connection_pool, get_db_connection -from .cursor import DatabaseRequest, Cursor, get_sql_from_file +from .cursor import DatabaseRequest, DatabaseBatchRequest, Cursor, get_sql_from_file from .transaction import use_transaction diff --git a/shared/selene/util/db/cursor.py b/shared/selene/util/db/cursor.py index 1e56d69a..029e5cd6 100644 --- a/shared/selene/util/db/cursor.py +++ b/shared/selene/util/db/cursor.py @@ -9,6 +9,9 @@ Example Usage: from dataclasses import dataclass, field from logging import getLogger from os import path +from typing import List + +from psycopg2.extras import execute_batch _log = getLogger(__package__) @@ -38,6 +41,12 @@ class DatabaseRequest(object): args: dict = field(default=None) +@dataclass +class DatabaseBatchRequest(object): + sql: str + args: List[dict] + + class Cursor(object): def __init__(self, db): self.db = db @@ -92,6 +101,10 @@ class Cursor(object): _log.debug(str(cursor.rowcount) + 'rows affected') return cursor.rowcount + def _execute_batch(self, db_request: DatabaseBatchRequest): + with self.db.cursor() as cursor: + execute_batch(cursor, db_request.sql, db_request.args) + def delete(self, db_request: DatabaseRequest): """Helper function for SQL delete statements""" deleted_rows = self._execute(db_request) @@ -109,3 +122,6 @@ class Cursor(object): """Helper function for SQL update statements.""" updated_rows = self._execute(db_request) return updated_rows + + def batch_update(self, db_request: DatabaseBatchRequest): + self._execute_batch(db_request)