From f6934e075e8e63f7c500a4d3383b78a0a9fc82d6 Mon Sep 17 00:00:00 2001 From: Matheus Lima Date: Fri, 7 Jun 2019 17:03:17 -0300 Subject: [PATCH 1/3] Migrating DailyReport script to use the SeleneScript base class --- batch/script/daily_report.py | 52 ++++++--- .../selene/data/account/repository/account.py | 105 +++++++++--------- 2 files changed, 86 insertions(+), 71 deletions(-) diff --git a/batch/script/daily_report.py b/batch/script/daily_report.py index 6a4a8c96..f7b3633a 100644 --- a/batch/script/daily_report.py +++ b/batch/script/daily_report.py @@ -5,8 +5,9 @@ from os import environ import schedule import time +from selene.batch import SeleneScript from selene.data.account import AccountRepository -from selene.util.db import DatabaseConnectionConfig, connect_to_db +from selene.util.db import DatabaseConnectionConfig from selene.util.email import EmailMessage, SeleneMailer mycroft_db = DatabaseConnectionConfig( @@ -19,24 +20,39 @@ mycroft_db = DatabaseConnectionConfig( ) -def build_report(): - with connect_to_db(mycroft_db) as db: - user_metrics = AccountRepository(db).daily_report(datetime.now()) +class DailyReport(SeleneScript): + def __init__(self): + super(DailyReport, self).__init__(__file__) + self._arg_parser.add_argument( + '--run-mode', + help='If the script should run as a job or just once', + choices=['job', 'once'], + type=str, + default='job' + ) - email = EmailMessage( - sender='reports@mycroft.ai', - recipient=os.environ['REPORT_RECIPIENT'], - subject='Mycroft Daily Report', - template_file_name='metrics.html', - template_variables=dict(user_metrics=user_metrics) - ) + def _run(self): + if self.args.run_mode == 'job': + schedule.every().day.at('00:00').do(self._build_report) + while True: + schedule.run_pending() + time.sleep(1) + else: + self._build_report(self.args.date) - mailer = SeleneMailer(email) - mailer.send(True) + def _build_report(self, date: datetime = datetime.now()): + user_metrics = AccountRepository(self.db).daily_report(date) + + email = EmailMessage( + sender='reports@mycroft.ai', + recipient=os.environ['REPORT_RECIPIENT'], + subject='Mycroft Daily Report', + template_file_name='metrics.html', + template_variables=dict(user_metrics=user_metrics) + ) + + mailer = SeleneMailer(email) + mailer.send(True) -schedule.every().day.at('00:00').do(build_report) - -while True: - schedule.run_pending() - time.sleep(1) +DailyReport().run() diff --git a/shared/selene/data/account/repository/account.py b/shared/selene/data/account/repository/account.py index 9d66a238..71359197 100644 --- a/shared/selene/data/account/repository/account.py +++ b/shared/selene/data/account/repository/account.py @@ -204,73 +204,72 @@ class AccountRepository(RepositoryBase): report_table = [{ 'type': 'User', - 'current': report_1_day['total'], - 'oneDay': report_1_day['total'] - report_1_day['total_new'], - 'oneDayDelta': report_1_day['total_new'], + 'current': report_1_day.total, + 'oneDay': report_1_day.total - report_1_day.total_new, + 'oneDayDelta': report_1_day.total_new, 'oneDayMinus': 0, - 'fifteenDays': report_15_days['total'] - report_15_days['total_new'], - 'fifteenDaysDelta': report_15_days['total_new'], + 'fifteenDays': report_15_days.total - report_15_days.total_new, + 'fifteenDaysDelta': report_15_days.total_new, 'fifteenDaysMinus': 0, - 'thirtyDays': report_30_days['total'] - report_30_days['total_new'], - 'thirtyDaysDelta': report_30_days['total_new'], + 'thirtyDays': report_30_days.total - report_30_days.total_new, + 'thirtyDaysDelta': report_30_days.total_new, 'thirtyDaysMinus': 0 }, { 'type': 'Free Account', - 'current': report_1_day['total'] - report_1_day['paid_total'], - 'oneDay': report_1_day['total'] - report_1_day['paid_total'] - report_1_day['total_new'] + report_1_day[ - 'paid_new'], - 'oneDayDelta': report_1_day['total_new'] - report_1_day['paid_new'], + 'current': report_1_day.total - report_1_day.paid_total, + 'oneDay': report_1_day.total - report_1_day.paid_total - report_1_day.total_new + report_1_day.paid_new, + 'oneDayDelta': report_1_day.total_new - report_1_day.paid_new, 'oneDayMinus': 0, - 'fifteenDays': report_15_days['total'] - report_15_days['paid_total'] - report_15_days['total_new'] + - report_15_days['paid_new'], - 'fifteenDaysDelta': report_15_days['total_new'] - report_15_days['paid_new'], + 'fifteenDays': report_15_days.total - report_15_days.paid_total - report_15_days.total_new + + report_15_days.paid_new, + 'fifteenDaysDelta': report_15_days.total_new - report_15_days.paid_new, 'fifteenDaysMinus': 0, - 'thirtyDays': report_30_days['total'] - report_30_days['paid_total'] - report_30_days['total_new'] + - report_30_days['paid_new'], - 'thirtyDaysDelta': report_30_days['total_new'] - report_30_days['paid_new'], + 'thirtyDays': report_30_days.total - report_30_days.paid_total - report_30_days.total_new + + report_30_days.paid_new, + 'thirtyDaysDelta': report_30_days.total_new - report_30_days.paid_new, 'thirtyDaysMinus': 0 }, { 'type': 'Monthly Account', - 'current': report_1_day['monthly_total'], - 'oneDay': report_1_day['monthly_total'] - report_1_day['monthly_new'] + report_1_day['monthly_minus'], - 'oneDayDelta': report_1_day['monthly_new'], - 'oneDayMinus': report_1_day['monthly_minus'], - 'fifteenDays': report_15_days['monthly_total'] - report_15_days['monthly_new'] + - report_15_days['monthly_minus'], - 'fifteenDaysDelta': report_15_days['monthly_new'], - 'fifteenDaysMinus': report_15_days['monthly_minus'], - 'thirtyDays': report_30_days['monthly_total'] - report_30_days['monthly_new'] + - report_30_days['monthly_minus'], - 'thirtyDaysDelta': report_30_days['monthly_new'], - 'thirtyDaysMinus': report_30_days['monthly_minus'] + 'current': report_1_day.monthly_total, + 'oneDay': report_1_day.monthly_total - report_1_day.monthly_new + report_1_day.monthly_minus, + 'oneDayDelta': report_1_day.monthly_new, + 'oneDayMinus': report_1_day.monthly_minus, + 'fifteenDays': report_15_days.monthly_total - report_15_days.monthly_new + + report_15_days.monthly_minus, + 'fifteenDaysDelta': report_15_days.monthly_new, + 'fifteenDaysMinus': report_15_days.monthly_minus, + 'thirtyDays': report_30_days.monthly_total - report_30_days.monthly_new + + report_30_days.monthly_minus, + 'thirtyDaysDelta': report_30_days.monthly_new, + 'thirtyDaysMinus': report_30_days.monthly_minus }, { 'type': 'Yearly Account', - 'current': report_1_day['yearly_total'], - 'oneDay': report_1_day['yearly_total'] - report_1_day['yearly_new'] + report_1_day['yearly_minus'], - 'oneDayDelta': report_1_day['yearly_new'], - 'oneDayMinus': report_1_day['yearly_minus'], - 'fifteenDays': report_15_days['yearly_total'] - report_15_days['yearly_new'] + - report_15_days['yearly_minus'], - 'fifteenDaysDelta': report_15_days['yearly_new'], - 'fifteenDaysMinus': report_15_days['yearly_minus'], - 'thirtyDays': report_30_days['yearly_total'] - report_30_days['yearly_new'] + - report_30_days['yearly_minus'], - 'thirtyDaysDelta': report_30_days['yearly_new'], - 'thirtyDaysMinus': report_30_days['yearly_minus'] + 'current': report_1_day.yearly_total, + 'oneDay': report_1_day.yearly_total - report_1_day.yearly_new + report_1_day.yearly_minus, + 'oneDayDelta': report_1_day.yearly_new, + 'oneDayMinus': report_1_day.yearly_minus, + 'fifteenDays': report_15_days.yearly_total - report_15_days.yearly_new + + report_15_days.yearly_minus, + 'fifteenDaysDelta': report_15_days.yearly_new, + 'fifteenDaysMinus': report_15_days.yearly_minus, + 'thirtyDays': report_30_days.yearly_total - report_30_days.yearly_new + + report_30_days.yearly_minus, + 'thirtyDaysDelta': report_30_days.yearly_new, + 'thirtyDaysMinus': report_30_days.yearly_minus }, { 'type': 'Paid Account', - 'current': report_1_day['paid_total'], - 'oneDay': report_1_day['paid_total'] - report_1_day['paid_new'] + report_1_day['paid_minus'], - 'oneDayDelta': report_1_day['paid_new'], - 'oneDayMinus': report_1_day['paid_minus'], - 'fifteenDays': report_15_days['paid_total'] - report_15_days['paid_new'] + - report_15_days['paid_minus'], - 'fifteenDaysDelta': report_15_days['paid_new'], - 'fifteenDaysMinus': report_15_days['paid_minus'], - 'thirtyDays': report_30_days['paid_total'] - report_30_days['paid_new'] + - report_30_days['paid_minus'], - 'thirtyDaysDelta': report_30_days['paid_new'], - 'thirtyDaysMinus': report_30_days['paid_minus'] + 'current': report_1_day.paid_total, + 'oneDay': report_1_day.paid_total - report_1_day.paid_new + report_1_day.paid_minus, + 'oneDayDelta': report_1_day.paid_new, + 'oneDayMinus': report_1_day.paid_minus, + 'fifteenDays': report_15_days.paid_total - report_15_days.paid_new + + report_15_days.paid_minus, + 'fifteenDaysDelta': report_15_days.paid_new, + 'fifteenDaysMinus': report_15_days.paid_minus, + 'thirtyDays': report_30_days.paid_total - report_30_days.paid_new + + report_30_days.paid_minus, + 'thirtyDaysDelta': report_30_days.paid_new, + 'thirtyDaysMinus': report_30_days.paid_minus }] return report_table From ab0a14faea8974e739f67012fb5c185e15272b6d Mon Sep 17 00:00:00 2001 From: Matheus Lima Date: Fri, 14 Jun 2019 18:01:22 -0300 Subject: [PATCH 2/3] Adding date to daily report script --- batch/script/daily_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batch/script/daily_report.py b/batch/script/daily_report.py index f7b3633a..6d2eab31 100644 --- a/batch/script/daily_report.py +++ b/batch/script/daily_report.py @@ -46,7 +46,7 @@ class DailyReport(SeleneScript): email = EmailMessage( sender='reports@mycroft.ai', recipient=os.environ['REPORT_RECIPIENT'], - subject='Mycroft Daily Report', + subject='Mycroft Daily Report - {}'.format(self.args.date), template_file_name='metrics.html', template_variables=dict(user_metrics=user_metrics) ) From 4bbd078987209d3c3baa388c20801ce1b2deaaab Mon Sep 17 00:00:00 2001 From: Matheus Lima Date: Mon, 10 Jun 2019 15:33:21 -0300 Subject: [PATCH 3/3] Created endpoint to delete a given skill from a device --- api/public/public_api/api.py | 7 ++++ .../public_api/endpoints/device_skills.py | 6 +++ .../tests/features/device_skills.feature | 7 ++++ api/public/tests/features/get_device.feature | 1 - .../tests/features/steps/device_skills.py | 38 +++++++++++++++++-- shared/selene/data/skill/repository/skill.py | 2 +- 6 files changed, 56 insertions(+), 5 deletions(-) diff --git a/api/public/public_api/api.py b/api/public/public_api/api.py index 1b299482..189fea00 100644 --- a/api/public/public_api/api.py +++ b/api/public/public_api/api.py @@ -41,11 +41,18 @@ public.register_blueprint(selene_api) _log = configure_logger('public_api') +public.add_url_rule( + '/v1/device//skill/', + view_func=DeviceSkillsEndpoint.as_view('device_skill_delete_api'), + methods=['DELETE'] +) + public.add_url_rule( '/v1/device//skill', view_func=DeviceSkillsEndpoint.as_view('device_skill_api'), methods=['GET', 'PUT'] ) + public.add_url_rule( '/v1/device//userSkill', view_func=DeviceSkillEndpoint.as_view('device_user_skill_api'), diff --git a/api/public/public_api/endpoints/device_skills.py b/api/public/public_api/endpoints/device_skills.py index bea9e11a..1385eafb 100644 --- a/api/public/public_api/endpoints/device_skills.py +++ b/api/public/public_api/endpoints/device_skills.py @@ -9,6 +9,7 @@ from schematics.types import StringType, BooleanType, ListType, ModelType from selene.api import PublicEndpoint from selene.api.etag import device_skill_etag_key from selene.data.skill import SkillRepository +from selene.data.skill.repository.device_skill import DeviceSkillRepository global_id_pattern = '^([^\|@]+)\|([^\|]+$)' # matches | global_id_dirt_pattern = '^@(.*)\|(.*)\|(.*)$' # matches @|| @@ -94,3 +95,8 @@ class DeviceSkillsEndpoint(PublicEndpoint): skill.validate() skill_id = SkillRepository(self.db).add(device_id, payload) return {'uuid': skill_id}, HTTPStatus.OK + + def delete(self, device_id, skill_id): + self._authenticate(device_id) + DeviceSkillRepository(self.db).delete(device_id, skill_id) + return '', HTTPStatus.OK diff --git a/api/public/tests/features/device_skills.feature b/api/public/tests/features/device_skills.feature index e9280b52..62b6934e 100644 --- a/api/public/tests/features/device_skills.feature +++ b/api/public/tests/features/device_skills.feature @@ -24,3 +24,10 @@ Feature: Upload and fetch skills When a skill with empty settings is uploaded Then the endpoint to retrieve the skill should return 200 And device last contact timestamp is updated + + Scenario: A skill setting is successfully deleted + Given a device with skill settings + When the skill settings is deleted + And the skill settings is fetched + Then the endpoint to delete the skills settings should return 200 + And device last contact timestamp is updated diff --git a/api/public/tests/features/get_device.feature b/api/public/tests/features/get_device.feature index 9a37ef4d..23c940ee 100644 --- a/api/public/tests/features/get_device.feature +++ b/api/public/tests/features/get_device.feature @@ -14,7 +14,6 @@ Feature: Get device's information Scenario: Try to get a device without passing the access token When try to fetch a device without the authorization header Then a 401 status code should be returned - And device last contact timestamp is updated Scenario: Update device information When the device is updated diff --git a/api/public/tests/features/steps/device_skills.py b/api/public/tests/features/steps/device_skills.py index d46a1407..0a4d39fe 100644 --- a/api/public/tests/features/steps/device_skills.py +++ b/api/public/tests/features/steps/device_skills.py @@ -2,7 +2,7 @@ import json from http import HTTPStatus from behave import when, then, given -from hamcrest import assert_that, equal_to, not_none, is_not +from hamcrest import assert_that, equal_to, not_none, is_not, has_key from selene.api.etag import ETagManager, device_skill_etag_key from selene.data.skill import AccountSkillSetting, SkillSettingRepository @@ -136,6 +136,7 @@ def validate_get_skill_updated_response(context): skills_response = json.loads(response.data) assert_that(len(skills_response), equal_to(1)) response = skills_response[0] + assert_that(response, has_key('uuid')) assert_that(response['skill_gid'], equal_to(skill['skill_gid'])) assert_that(response['identifier'], equal_to(skill['identifier'])) assert_that(response['skillMetadata'], equal_to(skill['skillMetadata'])) @@ -227,5 +228,36 @@ def validate_empty_skill_uploading(context): assert_that(response.status_code, equal_to(HTTPStatus.OK)) new_etag = response.headers.get('ETag') assert_that(new_etag, not_none()) - retrieved_skill = json.loads(context.get_skill_response.data) - assert_that([skill_empty_settings], equal_to(retrieved_skill)) + retrieved_skill = json.loads(context.get_skill_response.data)[0] + assert_that(skill_empty_settings['skill_gid'], retrieved_skill['skill_gid']) + assert_that(skill_empty_settings['identifier'], retrieved_skill['identifier']) + + +@when('the skill settings is deleted') +def delete_skill(context): + skills = json.loads(context.get_skill_response.data) + skill_fetched = skills[0] + skill_uuid = skill_fetched['uuid'] + login = context.device_login + device_id = login['uuid'] + access_token = login['accessToken'] + headers = dict(Authorization='Bearer {token}'.format(token=access_token)) + context.delete_skill_response = context.client.delete( + '/v1/device/{device_uuid}/skill/{skill_uuid}'.format(device_uuid=device_id, skill_uuid=skill_uuid), + headers=headers + ) + context.get_skill_after_delete_response = context.client.get( + '/v1/device/{uuid}/skill'.format(uuid=device_id), + headers=headers + ) + + +@then('the endpoint to delete the skills settings should return 200') +def validate_delete_skill(context): + # Validating that the deletion happened successfully + response = context.delete_skill_response + assert_that(response.status_code, HTTPStatus.OK) + + # Validating that the skill is not listed after we fetch the device's skills + response = context.get_skill_after_delete_response + assert_that(response.status_code, equal_to(HTTPStatus.NO_CONTENT)) diff --git a/shared/selene/data/skill/repository/skill.py b/shared/selene/data/skill/repository/skill.py index 052c99eb..eb76b685 100644 --- a/shared/selene/data/skill/repository/skill.py +++ b/shared/selene/data/skill/repository/skill.py @@ -38,7 +38,7 @@ class SkillRepository(RepositoryBase): skills = [] for result in sql_results: sections = self._fill_setting_with_values(result['settings'], result['settings_display']) - skill = {} + skill = {'uuid': result['id']} if sections: skill['skillMetadata'] = {'sections': sections} display = result['settings_display']