diff --git a/api/public/Pipfile.lock b/api/public/Pipfile.lock index c30db728..ab5909fd 100644 --- a/api/public/Pipfile.lock +++ b/api/public/Pipfile.lock @@ -53,11 +53,11 @@ }, "flask": { "hashes": [ - "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48", - "sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05" + "sha256:ad7c6d841e64296b962296c2c2dabc6543752985727af86a975072dea984b6f3", + "sha256:e7d32475d1de5facaa55e3958bc4ec66d3762076b074296aa50ef8fdc5b9df61" ], "index": "pypi", - "version": "==1.0.2" + "version": "==1.0.3" }, "idna": { "hashes": [ @@ -188,11 +188,18 @@ }, "requests": { "hashes": [ - "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", - "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" ], "index": "pypi", - "version": "==2.21.0" + "version": "==2.22.0" + }, + "schedule": { + "hashes": [ + "sha256:3f895a1036799a25ab9c335de917073e63cf8256920917e932777382f101f08f", + "sha256:f9fb5181283de4db6e701d476dd01b6a3dd81c38462a54991ddbb9d26db857c9" + ], + "version": "==0.6.0" }, "schematics": { "hashes": [ @@ -207,10 +214,10 @@ }, "sendgrid": { "hashes": [ - "sha256:351a7fc501d2b9d5afdcbc70a02490917057d6ce5cc22c558cadfc16229f157b", - "sha256:e1f93c72b3db3bd00d86f79ee926a093ee7e65533936a140855916569b08e0b0" + "sha256:297d33363a70df9b39419210e1273b165d487730e85c495695e0015bc626db71", + "sha256:8b82c8c801dde8180a567913a9f80d8a63f38e39f209edde302b6df899b4bca1" ], - "version": "==6.0.4" + "version": "==6.0.5" }, "six": { "hashes": [ @@ -228,18 +235,18 @@ }, "stripe": { "hashes": [ - "sha256:1686b4b2fe2ba77bf5e14f431e6ee5fa926e92c01a568709c16469b9c53beeb4", - "sha256:6013eab7961b409a323207bef9afc92af8939ffe4f127adc756ea3f907568d4c" + "sha256:362472a0b69f1791629d75c4829be3af92dc8c9254812af5670dcbcde704bb1f", + "sha256:92d6691382e0abf314759863c48a4830b5bfd3a935193ee6c3e5621ab25740ba" ], "index": "pypi", - "version": "==2.24.1" + "version": "==2.29.4" }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", + "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" ], - "version": "==1.24.1" + "version": "==1.25.3" }, "uwsgi": { "hashes": [ @@ -250,10 +257,10 @@ }, "werkzeug": { "hashes": [ - "sha256:0a73e8bb2ff2feecfc5d56e6f458f5b99290ef34f565ffb2665801ff7de6af7a", - "sha256:7fad9770a8778f9576693f0cc29c7dcc36964df916b83734f4431c0e612a7fbc" + "sha256:865856ebb55c4dcd0630cdd8f3331a1847a819dda7e8c750d3db6f2aa6c0209c", + "sha256:a0b915f0815982fb2a09161cb8f31708052d0951c3ba433ccc5e1aa276507ca6" ], - "version": "==0.15.2" + "version": "==0.15.4" }, "wrapt": { "hashes": [ diff --git a/api/public/public_api/api.py b/api/public/public_api/api.py index 97a92b8a..ea9314ad 100644 --- a/api/public/public_api/api.py +++ b/api/public/public_api/api.py @@ -12,7 +12,7 @@ from .endpoints.device_activate import DeviceActivateEndpoint from .endpoints.device_code import DeviceCodeEndpoint from .endpoints.device_email import DeviceEmailEndpoint from .endpoints.device_location import DeviceLocationEndpoint -from .endpoints.device_metrics import DeviceMetricsEndpoint, MetricsService +from .endpoints.device_metrics import DeviceMetricsEndpoint from .endpoints.device_oauth import OauthServiceEndpoint from .endpoints.device_refresh_token import DeviceRefreshTokenEndpoint from .endpoints.device_setting import DeviceSettingEndpoint @@ -34,13 +34,9 @@ public.config.from_object(get_base_config()) public.config['GOOGLE_STT_KEY'] = os.environ['GOOGLE_STT_KEY'] public.config['SELENE_CACHE'] = SeleneCache() -public.config['METRICS_SERVICE'] = MetricsService() - public.response_class = SeleneResponse public.register_blueprint(selene_api) -_log = configure_logger('public_api') - public.add_url_rule( '/v1/device//skill/', view_func=DeviceSkillsEndpoint.as_view('device_skill_delete_api'), diff --git a/api/public/public_api/endpoints/device_metrics.py b/api/public/public_api/endpoints/device_metrics.py index ba7f38f5..531d726b 100644 --- a/api/public/public_api/endpoints/device_metrics.py +++ b/api/public/public_api/endpoints/device_metrics.py @@ -1,41 +1,24 @@ -import json -import os +from datetime import datetime from http import HTTPStatus -import requests - from selene.api import PublicEndpoint -from selene.data.account import AccountRepository - - -class MetricsService(object): - def __init__(self): - self.metrics_service_host = os.environ['METRICS_SERVICE_HOST'] - - def send_metric(self, metric: str, user_id: str, device_id: str, data: dict): - body = dict( - userUuid=user_id, - deviceUuid=device_id, - data=data - ) - url = '{host}/metric/{metric}'.format(host=self.metrics_service_host, metric=metric) - requests.post(url, body) +from selene.data.metrics import CoreMetric, CoreMetricRepository class DeviceMetricsEndpoint(PublicEndpoint): """Endpoint to communicate with the metrics service""" - def __init__(self): - super(DeviceMetricsEndpoint, self).__init__() - self.metrics_service: MetricsService = self.config['METRICS_SERVICE'] - def post(self, device_id, metric): self._authenticate(device_id) - payload = json.loads(self.request.data) - account = AccountRepository(self.db).get_account_by_device_id(device_id) - if account: - self.metrics_service.send_metric(metric, account.id, device_id, payload) - response = '', HTTPStatus.OK - else: - response = '', HTTPStatus.NO_CONTENT - return response + core_metric = CoreMetric( + device_id=device_id, + metric_type=metric, + insert_ts=datetime.now(), + metric_value=self.request.json + ) + self._add_metric(core_metric) + return '', HTTPStatus.NO_CONTENT + + def _add_metric(self, metric: CoreMetric): + core_metrics_repo = CoreMetricRepository(self.db) + core_metrics_repo.add(metric) diff --git a/api/public/tests/features/device_metrics.feature b/api/public/tests/features/device_metrics.feature index 7cd75ad2..0f04f853 100644 --- a/api/public/tests/features/device_metrics.feature +++ b/api/public/tests/features/device_metrics.feature @@ -1,11 +1,14 @@ -Feature: Send a metric to the metric service +Feature: Save metrics sent to selene from mycroft core - Scenario: a metric is sent to the metric endpoint by a valid device - When the metric is sent - Then 200 status code should be returned + Scenario: Metric sent by device saved to database + Given an authorized device + When the metrics endpoint is called + Then the metric is saved to the database + And the request will be successful And device last contact timestamp is updated - Scenario: a metric is sent by a not allowed device - When the metric is sent by a not allowed device - Then metrics endpoint should return 401 + Scenario: Metric endpoint fails for unauthorized device + Given an unauthorized device + When the metrics endpoint is called + Then the request will fail with an unauthorized error And device last contact timestamp is updated diff --git a/api/public/tests/features/environment.py b/api/public/tests/features/environment.py index 5f4809ba..816d0a35 100644 --- a/api/public/tests/features/environment.py +++ b/api/public/tests/features/environment.py @@ -68,6 +68,7 @@ def before_scenario(context, _): cache = context.client_config['SELENE_CACHE'] context.etag_manager = ETagManager(cache, context.client_config) db = connect_to_db(context.client_config['DB_CONNECTION_CONFIG']) + context.db = db try: _add_agreements(context, db) _add_account(context, db) diff --git a/api/public/tests/features/steps/common.py b/api/public/tests/features/steps/common.py index 8170556f..4bf4dd07 100644 --- a/api/public/tests/features/steps/common.py +++ b/api/public/tests/features/steps/common.py @@ -1,7 +1,8 @@ +from http import HTTPStatus from datetime import datetime from behave import then -from hamcrest import assert_that, equal_to, not_none +from hamcrest import assert_that, equal_to, is_in, not_none from selene.util.cache import DEVICE_LAST_CONTACT_KEY @@ -14,3 +15,27 @@ def check_device_last_contact(context): last_contact_ts = datetime.strptime(value, '%Y-%m-%d %H:%M:%S.%f') assert_that(last_contact_ts.date(), equal_to(datetime.utcnow().date())) + + +@then('the request will be successful') +def check_request_success(context): + assert_that( + context.response.status_code, + is_in([HTTPStatus.OK, HTTPStatus.NO_CONTENT]) + ) + + +@then('the request will fail with {error_type} error') +def check_for_bad_request(context, error_type): + if error_type == 'a bad request': + assert_that( + context.response.status_code, + equal_to(HTTPStatus.BAD_REQUEST) + ) + elif error_type == 'an unauthorized': + assert_that( + context.response.status_code, + equal_to(HTTPStatus.UNAUTHORIZED) + ) + else: + raise ValueError('unsupported error_type') diff --git a/api/public/tests/features/steps/device_metrics.py b/api/public/tests/features/steps/device_metrics.py index e14dfdd0..425a4a87 100644 --- a/api/public/tests/features/steps/device_metrics.py +++ b/api/public/tests/features/steps/device_metrics.py @@ -1,63 +1,51 @@ import json import uuid -from http import HTTPStatus -from unittest import mock -from unittest.mock import patch, MagicMock, call -from behave import when, then +from behave import given, then, when from hamcrest import assert_that, equal_to -payload = dict( +from selene.data.metrics import CoreMetricRepository + +METRIC_TYPE_TIMING = 'timing' +metric_value = dict( type='timing', start='123' ) -@when('the metric is sent') -@patch('public_api.endpoints.device_metrics.MetricsService') -def send_metric(context, metrics_service): - context.client_config['METRICS_SERVICE'] = metrics_service - login = context.device_login - device_id = login['uuid'] - access_token = login['accessToken'] - headers = dict(Authorization='Bearer {token}'.format(token=access_token)) +@given('an authorized device') +def define_authorized_device(context): + context.metric_device_id = context.device_login['uuid'] + + +@given('an unauthorized device') +def define_unauthorized_device(context): + context.metric_device_id = str(uuid.uuid4()) + + +@when('the metrics endpoint is called') +def call_metrics_endpoint(context): + headers = dict(Authorization='Bearer {token}'.format( + token=context.device_login['accessToken']) + ) + url = '/v1/device/{device_id}/metric/{metric}'.format( + device_id=context.metric_device_id, metric='timing' + ) + context.client.content_type = 'application/json' context.response = context.client.post( - '/v1/device/{uuid}/metric/{metric}'.format(uuid=device_id, metric='timing'), - data=json.dumps(payload), - content_type='application_json', + url, + data=json.dumps(metric_value), + content_type='application/json', headers=headers ) -@then('200 status code should be returned') -def validate_response(context): - response = context.response - assert_that(response.status_code, equal_to(HTTPStatus.OK)) - metrics_service: MagicMock = context.client_config['METRICS_SERVICE'] - metrics_service.send_metric.assert_has_calls([ - call('timing', - context.account.id, - context.device_login['uuid'], - mock.ANY) - ]) - - -@when('the metric is sent by a not allowed device') -@patch('public_api.endpoints.device_metrics.MetricsService') -def send_metrics_invalid(context, metrics_service): - context.client_config['METRICS_SERVICE'] = metrics_service - headers = dict(Authorization='Bearer {token}'.format(token=context.device_login['accessToken'])) - context.response = context.client.post( - '/v1/device/{uuid}/metric/{metric}'.format(uuid=str(uuid.uuid4()), metric='timing'), - data=json.dumps(payload), - content_type='application_json', - headers=headers +@then('the metric is saved to the database') +def validate_metric_in_db(context): + core_metric_repo = CoreMetricRepository(context.db) + device_metrics = core_metric_repo.get_metrics_by_device( + context.device_login['uuid'] ) - - -@then('metrics endpoint should return 401') -def validate_invalid_device(context): - response = context.response - assert_that(response.status_code, equal_to(HTTPStatus.UNAUTHORIZED)) - metrics_service: MagicMock = context.client_config['METRICS_SERVICE'] - metrics_service.send_metric.assert_not_called() + device_metric = device_metrics[0] + assert_that(device_metric.metric_type, equal_to(METRIC_TYPE_TIMING)) + assert_that(device_metric.metric_value, equal_to(metric_value)) diff --git a/db/mycroft/metrics_schema/tables/core.sql b/db/mycroft/metrics_schema/tables/core.sql new file mode 100644 index 00000000..9446a840 --- /dev/null +++ b/db/mycroft/metrics_schema/tables/core.sql @@ -0,0 +1,12 @@ +CREATE TABLE metrics.core ( + id uuid PRIMARY KEY + DEFAULT gen_random_uuid(), + device_id uuid NOT NULL + REFERENCES device.device + ON DELETE CASCADE, + metric_type text NOT NULL, + insert_ts TIMESTAMP NOT NULL + DEFAULT now(), + metric_value json NOT NULL, + UNIQUE (device_id, insert_ts) +) diff --git a/db/scripts/bootstrap_mycroft_db.py b/db/scripts/bootstrap_mycroft_db.py index 0b94f5bb..f260ec59 100644 --- a/db/scripts/bootstrap_mycroft_db.py +++ b/db/scripts/bootstrap_mycroft_db.py @@ -47,7 +47,8 @@ GEOGRAPHY_TABLE_ORDER = ( METRICS_TABLE_ORDER = ( 'api', - 'job' + 'job', + 'core' ) schema_directory = '{}_schema' diff --git a/shared/Pipfile b/shared/Pipfile index 789a2726..a979779c 100644 --- a/shared/Pipfile +++ b/shared/Pipfile @@ -9,7 +9,6 @@ pyjwt = "*" pygithub = "*" psycopg2-binary = "*" passlib = "*" -pyhamcrest = "*" schematics = "*" redis = "*" sendgrid = "*" @@ -17,6 +16,7 @@ stripe = "*" schedule = "*" [dev-packages] +pyhamcrest = "*" [requires] python_version = "3.7" diff --git a/shared/Pipfile.lock b/shared/Pipfile.lock index bef7f341..fac429cc 100644 --- a/shared/Pipfile.lock +++ b/shared/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "65e2515ef76796070383f0f57a17182678a98576c253f6bae2f5c53d263f8111" + "sha256": "28abea4ca1dea11dcbac266294be1b114e46aef685e562465864a7a357fa3f62" }, "pipfile-spec": 6, "requires": { @@ -101,14 +101,6 @@ "index": "pypi", "version": "==1.43.7" }, - "pyhamcrest": { - "hashes": [ - "sha256:6b672c02fdf7470df9674ab82263841ce8333fb143f32f021f6cb26f0e512420", - "sha256:8ffaa0a53da57e89de14ced7185ac746227a8894dbd5a3c718bf05ddbd1d56cd" - ], - "index": "pypi", - "version": "==1.9.0" - }, "pyjwt": { "hashes": [ "sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e", @@ -136,7 +128,7 @@ "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" ], - "markers": "python_version >= '3.4' and python_version < '4.0'", + "markers": "python_version >= '3.0'", "version": "==2.22.0" }, "schedule": { @@ -163,27 +155,13 @@ "index": "pypi", "version": "==6.0.5" }, - "six": { - "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" - ], - "version": "==1.12.0" - }, "stripe": { "hashes": [ - "sha256:67c906ea533c1ddfb80579a7efa0bd5e59e2b8c6422d58ee8237b592d955c81a", - "sha256:73f9af72ef8125e0d1c713177d006f1cbe95602beb3e10cb0b0a4ae358d1ae86" + "sha256:362472a0b69f1791629d75c4829be3af92dc8c9254812af5670dcbcde704bb1f", + "sha256:92d6691382e0abf314759863c48a4830b5bfd3a935193ee6c3e5621ab25740ba" ], "index": "pypi", - "version": "==2.29.3" - }, - "toml": { - "hashes": [ - "sha256:380178cde50a6a79f9d2cf6f42a62a5174febe5eea4126fe4038785f1d888d42", - "sha256:a7901919d3e4f92ffba7ff40a9d697e35bbbc8a8049fe8da742f34c83606d957" - ], - "version": "==0.9.6" + "version": "==2.29.4" }, "urllib3": { "hashes": [ @@ -199,5 +177,42 @@ "version": "==1.11.1" } }, - "develop": {} + "develop": { + "behave": { + "hashes": [ + "sha256:b9662327aa53294c1351b0a9c369093ccec1d21026f050c3bd9b3e5cccf81a86", + "sha256:ebda1a6c9e5bfe95c5f9f0a2794e01c7098b3dde86c10a95d8621c5907ff6f1c" + ], + "index": "pypi", + "version": "==1.2.6" + }, + "parse": { + "hashes": [ + "sha256:1b68657434d371e5156048ca4a0c5aea5afc6ca59a2fea4dd1a575354f617142" + ], + "version": "==1.12.0" + }, + "parse-type": { + "hashes": [ + "sha256:6e906a66f340252e4c324914a60d417d33a4bea01292ea9bbf68b4fc123be8c9", + "sha256:f596bdc75d3dd93036fbfe3d04127da9f6df0c26c36e01e76da85adef4336b3c" + ], + "version": "==0.4.2" + }, + "pyhamcrest": { + "hashes": [ + "sha256:6b672c02fdf7470df9674ab82263841ce8333fb143f32f021f6cb26f0e512420", + "sha256:8ffaa0a53da57e89de14ced7185ac746227a8894dbd5a3c718bf05ddbd1d56cd" + ], + "index": "pypi", + "version": "==1.9.0" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + } + } } diff --git a/shared/selene/api/endpoints/account.py b/shared/selene/api/endpoints/account.py index eee68f43..911783c2 100644 --- a/shared/selene/api/endpoints/account.py +++ b/shared/selene/api/endpoints/account.py @@ -25,12 +25,14 @@ from selene.util.auth import ( get_facebook_account_email, get_google_account_email ) +from selene.util.cache import SeleneCache from selene.util.payment import ( cancel_stripe_subscription, create_stripe_account, create_stripe_subscription ) from ..base_endpoint import SeleneEndpoint +from ..etag import ETagManager MONTHLY_MEMBERSHIP = 'Monthly Membership' YEARLY_MEMBERSHIP = 'Yearly Membership' @@ -232,11 +234,17 @@ class AccountEndpoint(SeleneEndpoint): response_data = dict(errors=errors) response_status = HTTPStatus.BAD_REQUEST else: + self._expire_device_setting_cache() response_data = '' response_status = HTTPStatus.NO_CONTENT return response_data, response_status + def _expire_device_setting_cache(self): + cache = SeleneCache() + etag_manager = ETagManager(cache, self.config) + etag_manager.expire_device_setting_etag_by_account_id(self.account.id) + def _update_account(self): errors = [] for key, value in self.request.json.items(): diff --git a/shared/selene/data/metrics/__init__.py b/shared/selene/data/metrics/__init__.py index 44dc2373..8a0c3d04 100644 --- a/shared/selene/data/metrics/__init__.py +++ b/shared/selene/data/metrics/__init__.py @@ -1,4 +1,6 @@ from .entity.api import ApiMetric +from .entity.core import CoreMetric from .entity.job import JobMetric from .repository.api import ApiMetricsRepository +from .repository.core import CoreMetricRepository from .repository.job import JobRepository diff --git a/shared/selene/data/metrics/entity/core.py b/shared/selene/data/metrics/entity/core.py new file mode 100644 index 00000000..0f68ce71 --- /dev/null +++ b/shared/selene/data/metrics/entity/core.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass +from datetime import datetime + + +@dataclass +class CoreMetric(object): + device_id: str + metric_type: str + insert_ts: datetime + metric_value: dict + id: str = None diff --git a/shared/selene/data/metrics/repository/core.py b/shared/selene/data/metrics/repository/core.py new file mode 100644 index 00000000..d44ead41 --- /dev/null +++ b/shared/selene/data/metrics/repository/core.py @@ -0,0 +1,26 @@ +import json +from dataclasses import asdict + +from ..entity.core import CoreMetric +from ...repository_base import RepositoryBase + + +class CoreMetricRepository(RepositoryBase): + def __init__(self, db): + super(CoreMetricRepository, self).__init__(db, __file__) + + def add(self, metric: CoreMetric): + db_request_args = asdict(metric) + db_request_args['metric_value'] = json.dumps(db_request_args['metric_value']) + db_request = self._build_db_request( + sql_file_name='add_core_metric.sql', + args=db_request_args + ) + self.cursor.insert(db_request) + + def get_metrics_by_device(self, device_id): + return self._select_all_into_dataclass( + CoreMetric, + sql_file_name='get_core_metric_by_device.sql', + args=dict(device_id=device_id) + ) diff --git a/shared/selene/data/metrics/repository/sql/add_core_metric.sql b/shared/selene/data/metrics/repository/sql/add_core_metric.sql new file mode 100644 index 00000000..bf4a83bd --- /dev/null +++ b/shared/selene/data/metrics/repository/sql/add_core_metric.sql @@ -0,0 +1,4 @@ +INSERT INTO + metrics.core (device_id, metric_type, metric_value) +VALUES + (%(device_id)s, %(metric_type)s, %(metric_value)s) diff --git a/shared/selene/data/metrics/repository/sql/get_core_metric_by_device.sql b/shared/selene/data/metrics/repository/sql/get_core_metric_by_device.sql new file mode 100644 index 00000000..05118a2c --- /dev/null +++ b/shared/selene/data/metrics/repository/sql/get_core_metric_by_device.sql @@ -0,0 +1,10 @@ +SELECT + id, + device_id, + metric_type, + insert_ts, + metric_value +FROM + metrics.core +WHERE + device_id = %(device_id)s