MycroftAI
/
selene-backend
mirror of https://github.com/MycroftAI/selene-backend.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'remotes/origin/test'

pull/191/head
Chris Veilleux 2019-06-18 15:37:23 -05:00
parent d222756020 557bda246d
commit a13b52d37e
17 changed files with 231 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
api/public/Pipfile.lock generated
View File

@ -53,11 +53,11 @@
},
"flask": {
"hashes": [
"sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48",
"sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05"
"sha256:ad7c6d841e64296b962296c2c2dabc6543752985727af86a975072dea984b6f3",
"sha256:e7d32475d1de5facaa55e3958bc4ec66d3762076b074296aa50ef8fdc5b9df61"
],
"index": "pypi",
"version": "==1.0.2"
"version": "==1.0.3"
},
"idna": {
"hashes": [
@ -188,11 +188,18 @@
},
"requests": {
"hashes": [
"sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e",
"sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b"
"sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
"sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
],
"index": "pypi",
"version": "==2.21.0"
"version": "==2.22.0"
},
"schedule": {
"hashes": [
"sha256:3f895a1036799a25ab9c335de917073e63cf8256920917e932777382f101f08f",
"sha256:f9fb5181283de4db6e701d476dd01b6a3dd81c38462a54991ddbb9d26db857c9"
],
"version": "==0.6.0"
},
"schematics": {
"hashes": [
@ -207,10 +214,10 @@
},
"sendgrid": {
"hashes": [
"sha256:351a7fc501d2b9d5afdcbc70a02490917057d6ce5cc22c558cadfc16229f157b",
"sha256:e1f93c72b3db3bd00d86f79ee926a093ee7e65533936a140855916569b08e0b0"
"sha256:297d33363a70df9b39419210e1273b165d487730e85c495695e0015bc626db71",
"sha256:8b82c8c801dde8180a567913a9f80d8a63f38e39f209edde302b6df899b4bca1"
],
"version": "==6.0.4"
"version": "==6.0.5"
},
"six": {
"hashes": [
@ -228,18 +235,18 @@
},
"stripe": {
"hashes": [
"sha256:1686b4b2fe2ba77bf5e14f431e6ee5fa926e92c01a568709c16469b9c53beeb4",
"sha256:6013eab7961b409a323207bef9afc92af8939ffe4f127adc756ea3f907568d4c"
"sha256:362472a0b69f1791629d75c4829be3af92dc8c9254812af5670dcbcde704bb1f",
"sha256:92d6691382e0abf314759863c48a4830b5bfd3a935193ee6c3e5621ab25740ba"
],
"index": "pypi",
"version": "==2.24.1"
"version": "==2.29.4"
},
"urllib3": {
"hashes": [
"sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
"sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
"sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1",
"sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
],
"version": "==1.24.1"
"version": "==1.25.3"
},
"uwsgi": {
"hashes": [
@ -250,10 +257,10 @@
},
"werkzeug": {
"hashes": [
"sha256:0a73e8bb2ff2feecfc5d56e6f458f5b99290ef34f565ffb2665801ff7de6af7a",
"sha256:7fad9770a8778f9576693f0cc29c7dcc36964df916b83734f4431c0e612a7fbc"
"sha256:865856ebb55c4dcd0630cdd8f3331a1847a819dda7e8c750d3db6f2aa6c0209c",
"sha256:a0b915f0815982fb2a09161cb8f31708052d0951c3ba433ccc5e1aa276507ca6"
],
"version": "==0.15.2"
"version": "==0.15.4"
},
"wrapt": {
"hashes": [

6
api/public/public_api/api.py
View File

@ -12,7 +12,7 @@ from .endpoints.device_activate import DeviceActivateEndpoint
from .endpoints.device_code import DeviceCodeEndpoint
from .endpoints.device_email import DeviceEmailEndpoint
from .endpoints.device_location import DeviceLocationEndpoint
from .endpoints.device_metrics import DeviceMetricsEndpoint, MetricsService
from .endpoints.device_metrics import DeviceMetricsEndpoint
from .endpoints.device_oauth import OauthServiceEndpoint
from .endpoints.device_refresh_token import DeviceRefreshTokenEndpoint
from .endpoints.device_setting import DeviceSettingEndpoint
@ -34,13 +34,9 @@ public.config.from_object(get_base_config())
public.config['GOOGLE_STT_KEY'] = os.environ['GOOGLE_STT_KEY']
public.config['SELENE_CACHE'] = SeleneCache()
public.config['METRICS_SERVICE'] = MetricsService()
public.response_class = SeleneResponse
public.register_blueprint(selene_api)
_log = configure_logger('public_api')
public.add_url_rule(
'/v1/device/<string:device_id>/skill/<string:skill_gid>',
view_func=DeviceSkillsEndpoint.as_view('device_skill_delete_api'),

45
api/public/public_api/endpoints/device_metrics.py
View File

@ -1,41 +1,24 @@
import json
import os
from datetime import datetime
from http import HTTPStatus
import requests
from selene.api import PublicEndpoint
from selene.data.account import AccountRepository
class MetricsService(object):
def __init__(self):
self.metrics_service_host = os.environ['METRICS_SERVICE_HOST']
def send_metric(self, metric: str, user_id: str, device_id: str, data: dict):
body = dict(
userUuid=user_id,
deviceUuid=device_id,
data=data
)
url = '{host}/metric/{metric}'.format(host=self.metrics_service_host, metric=metric)
requests.post(url, body)
from selene.data.metrics import CoreMetric, CoreMetricRepository
class DeviceMetricsEndpoint(PublicEndpoint):
"""Endpoint to communicate with the metrics service"""
def __init__(self):
super(DeviceMetricsEndpoint, self).__init__()
self.metrics_service: MetricsService = self.config['METRICS_SERVICE']
def post(self, device_id, metric):
self._authenticate(device_id)
payload = json.loads(self.request.data)
account = AccountRepository(self.db).get_account_by_device_id(device_id)
if account:
self.metrics_service.send_metric(metric, account.id, device_id, payload)
response = '', HTTPStatus.OK
else:
response = '', HTTPStatus.NO_CONTENT
return response
core_metric = CoreMetric(
device_id=device_id,
metric_type=metric,
insert_ts=datetime.now(),
metric_value=self.request.json
)
self._add_metric(core_metric)
return '', HTTPStatus.NO_CONTENT
def _add_metric(self, metric: CoreMetric):
core_metrics_repo = CoreMetricRepository(self.db)
core_metrics_repo.add(metric)

17
api/public/tests/features/device_metrics.feature
View File

@ -1,11 +1,14 @@
Feature: Send a metric to the metric service
Feature: Save metrics sent to selene from mycroft core
Scenario: a metric is sent to the metric endpoint by a valid device
When the metric is sent
Then 200 status code should be returned
Scenario: Metric sent by device saved to database
Given an authorized device
When the metrics endpoint is called
Then the metric is saved to the database
And the request will be successful
And device last contact timestamp is updated
Scenario: a metric is sent by a not allowed device
When the metric is sent by a not allowed device
Then metrics endpoint should return 401
Scenario: Metric endpoint fails for unauthorized device
Given an unauthorized device
When the metrics endpoint is called
Then the request will fail with an unauthorized error
And device last contact timestamp is updated

1
api/public/tests/features/environment.py
View File

@ -68,6 +68,7 @@ def before_scenario(context, _):
cache = context.client_config['SELENE_CACHE']
context.etag_manager = ETagManager(cache, context.client_config)
db = connect_to_db(context.client_config['DB_CONNECTION_CONFIG'])
context.db = db
try:
_add_agreements(context, db)
_add_account(context, db)

27
api/public/tests/features/steps/common.py
View File

@ -1,7 +1,8 @@
from http import HTTPStatus
from datetime import datetime
from behave import then
from hamcrest import assert_that, equal_to, not_none
from hamcrest import assert_that, equal_to, is_in, not_none
from selene.util.cache import DEVICE_LAST_CONTACT_KEY
@ -14,3 +15,27 @@ def check_device_last_contact(context):
last_contact_ts = datetime.strptime(value, '%Y-%m-%d %H:%M:%S.%f')
assert_that(last_contact_ts.date(), equal_to(datetime.utcnow().date()))
@then('the request will be successful')
def check_request_success(context):
assert_that(
context.response.status_code,
is_in([HTTPStatus.OK, HTTPStatus.NO_CONTENT])
)
@then('the request will fail with {error_type} error')
def check_for_bad_request(context, error_type):
if error_type == 'a bad request':
assert_that(
context.response.status_code,
equal_to(HTTPStatus.BAD_REQUEST)
)
elif error_type == 'an unauthorized':
assert_that(
context.response.status_code,
equal_to(HTTPStatus.UNAUTHORIZED)
)
else:
raise ValueError('unsupported error_type')

82
api/public/tests/features/steps/device_metrics.py
View File

@ -1,63 +1,51 @@
import json
import uuid
from http import HTTPStatus
from unittest import mock
from unittest.mock import patch, MagicMock, call
from behave import when, then
from behave import given, then, when
from hamcrest import assert_that, equal_to
payload = dict(
from selene.data.metrics import CoreMetricRepository
METRIC_TYPE_TIMING = 'timing'
metric_value = dict(
type='timing',
start='123'
)
@when('the metric is sent')
@patch('public_api.endpoints.device_metrics.MetricsService')
def send_metric(context, metrics_service):
context.client_config['METRICS_SERVICE'] = metrics_service
login = context.device_login
device_id = login['uuid']
access_token = login['accessToken']
headers = dict(Authorization='Bearer {token}'.format(token=access_token))
@given('an authorized device')
def define_authorized_device(context):
context.metric_device_id = context.device_login['uuid']
@given('an unauthorized device')
def define_unauthorized_device(context):
context.metric_device_id = str(uuid.uuid4())
@when('the metrics endpoint is called')
def call_metrics_endpoint(context):
headers = dict(Authorization='Bearer {token}'.format(
token=context.device_login['accessToken'])
)
url = '/v1/device/{device_id}/metric/{metric}'.format(
device_id=context.metric_device_id, metric='timing'
)
context.client.content_type = 'application/json'
context.response = context.client.post(
'/v1/device/{uuid}/metric/{metric}'.format(uuid=device_id, metric='timing'),
data=json.dumps(payload),
content_type='application_json',
url,
data=json.dumps(metric_value),
content_type='application/json',
headers=headers
)
@then('200 status code should be returned')
def validate_response(context):
response = context.response
assert_that(response.status_code, equal_to(HTTPStatus.OK))
metrics_service: MagicMock = context.client_config['METRICS_SERVICE']
metrics_service.send_metric.assert_has_calls([
call('timing',
context.account.id,
context.device_login['uuid'],
mock.ANY)
])
@when('the metric is sent by a not allowed device')
@patch('public_api.endpoints.device_metrics.MetricsService')
def send_metrics_invalid(context, metrics_service):
context.client_config['METRICS_SERVICE'] = metrics_service
headers = dict(Authorization='Bearer {token}'.format(token=context.device_login['accessToken']))
context.response = context.client.post(
'/v1/device/{uuid}/metric/{metric}'.format(uuid=str(uuid.uuid4()), metric='timing'),
data=json.dumps(payload),
content_type='application_json',
headers=headers
@then('the metric is saved to the database')
def validate_metric_in_db(context):
core_metric_repo = CoreMetricRepository(context.db)
device_metrics = core_metric_repo.get_metrics_by_device(
context.device_login['uuid']
)
@then('metrics endpoint should return 401')
def validate_invalid_device(context):
response = context.response
assert_that(response.status_code, equal_to(HTTPStatus.UNAUTHORIZED))
metrics_service: MagicMock = context.client_config['METRICS_SERVICE']
metrics_service.send_metric.assert_not_called()
device_metric = device_metrics[0]
assert_that(device_metric.metric_type, equal_to(METRIC_TYPE_TIMING))
assert_that(device_metric.metric_value, equal_to(metric_value))

12
db/mycroft/metrics_schema/tables/core.sql Normal file
View File

@ -0,0 +1,12 @@
CREATE TABLE metrics.core (
id uuid PRIMARY KEY
DEFAULT gen_random_uuid(),
device_id uuid NOT NULL
REFERENCES device.device
ON DELETE CASCADE,
metric_type text NOT NULL,
insert_ts TIMESTAMP NOT NULL
DEFAULT now(),
metric_value json NOT NULL,
UNIQUE (device_id, insert_ts)
)

3
db/scripts/bootstrap_mycroft_db.py
View File

@ -47,7 +47,8 @@ GEOGRAPHY_TABLE_ORDER = (
METRICS_TABLE_ORDER = (
'api',
'job'
'job',
'core'
)
schema_directory = '{}_schema'

2
shared/Pipfile
View File

@ -9,7 +9,6 @@ pyjwt = "*"
pygithub = "*"
psycopg2-binary = "*"
passlib = "*"
pyhamcrest = "*"
schematics = "*"
redis = "*"
sendgrid = "*"
@ -17,6 +16,7 @@ stripe = "*"
schedule = "*"
[dev-packages]
pyhamcrest = "*"
[requires]
python_version = "3.7"

71
shared/Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "65e2515ef76796070383f0f57a17182678a98576c253f6bae2f5c53d263f8111"
"sha256": "28abea4ca1dea11dcbac266294be1b114e46aef685e562465864a7a357fa3f62"
},
"pipfile-spec": 6,
"requires": {
@ -101,14 +101,6 @@
"index": "pypi",
"version": "==1.43.7"
},
"pyhamcrest": {
"hashes": [
"sha256:6b672c02fdf7470df9674ab82263841ce8333fb143f32f021f6cb26f0e512420",
"sha256:8ffaa0a53da57e89de14ced7185ac746227a8894dbd5a3c718bf05ddbd1d56cd"
],
"index": "pypi",
"version": "==1.9.0"
},
"pyjwt": {
"hashes": [
"sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e",
@ -136,7 +128,7 @@
"sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
"sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
],
"markers": "python_version >= '3.4' and python_version < '4.0'",
"markers": "python_version >= '3.0'",
"version": "==2.22.0"
},
"schedule": {
@ -163,27 +155,13 @@
"index": "pypi",
"version": "==6.0.5"
},
"six": {
"hashes": [
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
],
"version": "==1.12.0"
},
"stripe": {
"hashes": [
"sha256:67c906ea533c1ddfb80579a7efa0bd5e59e2b8c6422d58ee8237b592d955c81a",
"sha256:73f9af72ef8125e0d1c713177d006f1cbe95602beb3e10cb0b0a4ae358d1ae86"
"sha256:362472a0b69f1791629d75c4829be3af92dc8c9254812af5670dcbcde704bb1f",
"sha256:92d6691382e0abf314759863c48a4830b5bfd3a935193ee6c3e5621ab25740ba"
],
"index": "pypi",
"version": "==2.29.3"
},
"toml": {
"hashes": [
"sha256:380178cde50a6a79f9d2cf6f42a62a5174febe5eea4126fe4038785f1d888d42",
"sha256:a7901919d3e4f92ffba7ff40a9d697e35bbbc8a8049fe8da742f34c83606d957"
],
"version": "==0.9.6"
"version": "==2.29.4"
},
"urllib3": {
"hashes": [
@ -199,5 +177,42 @@
"version": "==1.11.1"
}
},
"develop": {}
"develop": {
"behave": {
"hashes": [
"sha256:b9662327aa53294c1351b0a9c369093ccec1d21026f050c3bd9b3e5cccf81a86",
"sha256:ebda1a6c9e5bfe95c5f9f0a2794e01c7098b3dde86c10a95d8621c5907ff6f1c"
],
"index": "pypi",
"version": "==1.2.6"
},
"parse": {
"hashes": [
"sha256:1b68657434d371e5156048ca4a0c5aea5afc6ca59a2fea4dd1a575354f617142"
],
"version": "==1.12.0"
},
"parse-type": {
"hashes": [
"sha256:6e906a66f340252e4c324914a60d417d33a4bea01292ea9bbf68b4fc123be8c9",
"sha256:f596bdc75d3dd93036fbfe3d04127da9f6df0c26c36e01e76da85adef4336b3c"
],
"version": "==0.4.2"
},
"pyhamcrest": {
"hashes": [
"sha256:6b672c02fdf7470df9674ab82263841ce8333fb143f32f021f6cb26f0e512420",
"sha256:8ffaa0a53da57e89de14ced7185ac746227a8894dbd5a3c718bf05ddbd1d56cd"
],
"index": "pypi",
"version": "==1.9.0"
},
"six": {
"hashes": [
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
],
"version": "==1.12.0"
}
}
}

8
shared/selene/api/endpoints/account.py
View File

@ -25,12 +25,14 @@ from selene.util.auth import (
get_facebook_account_email,
get_google_account_email
)
from selene.util.cache import SeleneCache
from selene.util.payment import (
cancel_stripe_subscription,
create_stripe_account,
create_stripe_subscription
)
from ..base_endpoint import SeleneEndpoint
from ..etag import ETagManager
MONTHLY_MEMBERSHIP = 'Monthly Membership'
YEARLY_MEMBERSHIP = 'Yearly Membership'
@ -232,11 +234,17 @@ class AccountEndpoint(SeleneEndpoint):
response_data = dict(errors=errors)
response_status = HTTPStatus.BAD_REQUEST
else:
self._expire_device_setting_cache()
response_data = ''
response_status = HTTPStatus.NO_CONTENT
return response_data, response_status
def _expire_device_setting_cache(self):
cache = SeleneCache()
etag_manager = ETagManager(cache, self.config)
etag_manager.expire_device_setting_etag_by_account_id(self.account.id)
def _update_account(self):
errors = []
for key, value in self.request.json.items():

2
shared/selene/data/metrics/__init__.py
View File

@ -1,4 +1,6 @@
from .entity.api import ApiMetric
from .entity.core import CoreMetric
from .entity.job import JobMetric
from .repository.api import ApiMetricsRepository
from .repository.core import CoreMetricRepository
from .repository.job import JobRepository

11
shared/selene/data/metrics/entity/core.py Normal file
View File

@ -0,0 +1,11 @@
from dataclasses import dataclass
from datetime import datetime
@dataclass
class CoreMetric(object):
device_id: str
metric_type: str
insert_ts: datetime
metric_value: dict
id: str = None

26
shared/selene/data/metrics/repository/core.py Normal file
View File

@ -0,0 +1,26 @@
import json
from dataclasses import asdict
from ..entity.core import CoreMetric
from ...repository_base import RepositoryBase
class CoreMetricRepository(RepositoryBase):
def __init__(self, db):
super(CoreMetricRepository, self).__init__(db, __file__)
def add(self, metric: CoreMetric):
db_request_args = asdict(metric)
db_request_args['metric_value'] = json.dumps(db_request_args['metric_value'])
db_request = self._build_db_request(
sql_file_name='add_core_metric.sql',
args=db_request_args
)
self.cursor.insert(db_request)
def get_metrics_by_device(self, device_id):
return self._select_all_into_dataclass(
CoreMetric,
sql_file_name='get_core_metric_by_device.sql',
args=dict(device_id=device_id)
)

4
shared/selene/data/metrics/repository/sql/add_core_metric.sql Normal file
View File

@ -0,0 +1,4 @@
INSERT INTO
metrics.core (device_id, metric_type, metric_value)
VALUES
(%(device_id)s, %(metric_type)s, %(metric_value)s)

10
shared/selene/data/metrics/repository/sql/get_core_metric_by_device.sql Normal file
View File

@ -0,0 +1,10 @@
SELECT
id,
device_id,
metric_type,
insert_ts,
metric_value
FROM
metrics.core
WHERE
device_id = %(device_id)s