Merge pull request #57 from MycroftAI/skills-setting

Updating the neo4j-postgres converter

api/public/public_api/endpoints/device_skill.py

@@ -1,3 +1,5 @@
+from http import HTTPStatus
+
 from selene.api import SeleneEndpoint
 from selene.data.skill import SkillRepository
 from selene.util.db import get_db_connection
@@ -12,4 +14,6 @@ class DeviceSkillEndpoint(SeleneEndpoint):
         version_hash = self.request.args.get('identifier')
         if version_hash:
             with get_db_connection(self.config['DB_CONNECTION_POOL']) as db:
-                return SkillRepository(db).get_skill_settings_by_device_id_and_version_hash(device_id, version_hash)
+                skill = SkillRepository(db).get_skill_settings_by_device_id_and_version_hash(device_id, version_hash)
+            response = (skill, HTTPStatus.OK) if skill is not None else ('', HTTPStatus.NO_CONTENT)
+            return response

api/public/public_api/endpoints/device_skills.py

@@ -1,5 +1,6 @@
-from selene.api import SeleneEndpoint
+from http import HTTPStatus
 
+from selene.api import SeleneEndpoint
 from selene.data.skill import SkillRepository
 from selene.util.db import get_db_connection
 
@@ -12,5 +13,8 @@ class DeviceSkillsEndpoint(SeleneEndpoint):
 
     def get(self, device_id):
         with get_db_connection(self.config['DB_CONNECTION_POOL']) as db:
-            return SkillRepository(db).get_skill_settings_by_device_id(device_id)
+            skills = SkillRepository(db).get_skill_settings_by_device_id(device_id)
+        response = (skills, HTTPStatus.OK) if skills is not None else ('', HTTPStatus.NO_CONTENT)
+        return response
+
 

api/public/public_api/endpoints/device_subscription.py

@@ -13,8 +13,8 @@ class DeviceSubscriptionEndpoint(SeleneEndpoint):
         with get_db_connection(self.config['DB_CONNECTION_POOL']) as db:
             account = AccountRepository(db).get_account_by_device_id(device_id)
         if account:
-            subscription = account.subscription
-            response = {'@type': subscription.type if subscription is not None else 'free'}, HTTPStatus.OK
+            membership = account.membership
+            response = {'@type': membership.type if membership is not None else 'free'}, HTTPStatus.OK
         else:
             response = '', HTTPStatus.NO_CONTENT
         return response

api/public/tests/features/environment.py

@@ -24,7 +24,7 @@ account = Account(
                 AccountAgreement(type=PRIVACY_POLICY, accept_date=date.today()),
                 AccountAgreement(type=TERMS_OF_USE, accept_date=date.today())
             ],
-    subscription=None
+    membership=None
 )
 
 wake_word = WakeWord(

api/public/tests/features/steps/get_device_subscription.py

@@ -6,7 +6,7 @@ from http import HTTPStatus
 from behave import when, then
 from hamcrest import assert_that, has_entry, equal_to
 
-from selene.data.account import AccountRepository, AccountSubscription
+from selene.data.account import AccountRepository, AccountMembership
 from selene.util.db import get_db_connection
 
 
@@ -25,9 +25,14 @@ def validate_response(context):
 
 @when('the subscription endpoint is called for a monthly account')
 def get_device_subscription(context):
-    membership = AccountSubscription(start_date=date.today(), type='Monthly Supporter', stripe_customer_id='test_monthly')
+    membership = AccountMembership(
+        start_date=date.today(),
+        type='Monthly Membership',
+        payment_method='Stripe',
+        payment_account_id='test_monthly'
+    )
     with get_db_connection(context.client_config['DB_CONNECTION_POOL']) as db:
-        AccountRepository(db).add_membership(context.account.id, membership)
+        AccountRepository(db)._add_membership(context.account.id, membership)
     context.subscription_response = context.client.get('/device/{uuid}/subscription'.format(uuid=context.device_id))
 
 
@@ -36,7 +41,7 @@ def validate_response_monthly(context):
     response = context.subscription_response
     assert_that(response.status_code, HTTPStatus.OK)
     subscription = json.loads(response.data)
-    assert_that(subscription, has_entry('@type', 'Monthly Supporter'))
+    assert_that(subscription, has_entry('@type', 'Monthly Membership'))
 
 
 @when('try to get the subscription for a nonexistent device')

db/mycroft/account_schema/tables/account.sql

@@ -1,7 +1,7 @@
 CREATE TABLE account.account (
     id                  uuid        PRIMARY KEY DEFAULT gen_random_uuid(),
     email_address       text        NOT NULL UNIQUE,
-    username            text        NOT NULL UNIQUE,
+    username            text        UNIQUE,
     password            text,
     insert_ts           TIMESTAMP   NOT NULL DEFAULT CURRENT_TIMESTAMP
 );

db/scripts/bootstrap_mycroft_db.py

@@ -146,7 +146,8 @@ for schema in SCHEMAS:
 template_db.close_db()
 
 print('Copying template to new database.')
-postgres_db = PostgresDB(dbname='postgres', user='chrisveilleux')
+# Copy template to new database.
+postgres_db = PostgresDB(dbname='postgres', user='postgres')
 postgres_db.execute_sql(get_sql_from_file('create_mycroft_db.sql'))
 postgres_db.close_db()
 

db/scripts/neo4j-postgres.py

@@ -1,8 +1,12 @@
 import csv
 import datetime
-
-from psycopg2 import connect
+import json
 import uuid
+from collections import defaultdict
+
+import time
+from psycopg2 import connect
+from psycopg2.extras import execute_batch
 
 users = {}
 user_settings = {}
@@ -28,6 +32,8 @@ def load_csv():
             users[row[0]] = {}
             users[row[0]]['email'] = row[1]
             users[row[0]]['password'] = row[2]
+            users[row[0]]['terms'] = row[3]
+            users[row[0]]['privacy'] = row[4]
 
     with open('user_settings.csv') as user_setting_csv:
         user_setting_reader = csv.reader(user_setting_csv)
@@ -84,7 +90,6 @@ def load_csv():
             skills[skill]['device_uuid'] = dev_uuid
             skills[skill]['name'] = row[2]
             skills[skill]['description'] = row[3]
-            skills[skill]['identifier'] = row[4]
             if dev_uuid in device_to_skill:
                 device_to_skill[dev_uuid].add(skill)
             else:
@@ -112,7 +117,7 @@ def load_csv():
             field_uuid = row[0]
             skill_fields[field_uuid] = {}
             section_uuid = row[1]
-            skill_fields[field_uuid]['section_uuid'] = section_uuid
+            #skill_fields[field_uuid]['section_uuid'] = section_uuid
             skill_fields[field_uuid]['name'] = row[2]
             skill_fields[field_uuid]['type'] = row[3]
             skill_fields[field_uuid]['label'] = row[4]
@@ -120,7 +125,7 @@ def load_csv():
             skill_fields[field_uuid]['placeholder'] = row[6]
             skill_fields[field_uuid]['hide'] = row[7]
             skill_fields[field_uuid]['options'] = row[8]
-            skill_fields[field_uuid]['order'] = row[9]
+            #skill_fields[field_uuid]['order'] = row[9]
             if section_uuid in section_to_field:
                 section_to_field[section_uuid].add(field_uuid)
             else:
@@ -142,51 +147,6 @@ def load_csv():
                 device_to_field[device_uuid] = {field_uuid}
 
 
-def parse_user_setting(user_uuid) -> (str, str, str, str):
-    if user_uuid in user_settings:
-        user_setting = user_settings[user_uuid]
-        date_format = user_setting['date_format']
-        if date_format == 'DMY':
-            date_format = 'DD/MM/YYYY'
-        else:
-            date_format = 'MM/DD/YYYY'
-        time_format = user_setting['time_format']
-        if time_format == 'full':
-            time_format = '24 Hour'
-        else:
-            time_format = '12 Hour'
-        measurement_system = user_setting['measurement_system']
-        if measurement_system == 'metric':
-            measurement_system = 'Metric'
-        elif measurement_system == 'imperial':
-            measurement_system = 'Imperial'
-        wake_word = user_setting['wake_word']
-        tts_type = user_setting['tts_type']
-        tts_voice = user_setting['tts_voice']
-        if tts_type == 'MimicSetting':
-            if tts_voice == 'ap':
-                tts = 'ap'
-            elif tts_voice == 'trinity':
-                tts = 'amy'
-            else:
-                tts = 'ap'
-        elif tts_type == 'Mimic2Setting':
-            tts = 'kusal'
-        elif tts_type == 'GoogleTTSSetting':
-            tts = 'google'
-        else:
-            tts = 'ap'
-        sample_rate = user_setting['sample_rate']
-        channels = user_setting['channels']
-        pronunciation = user_setting['pronunciation']
-        threshold = user_setting['threshold']
-        threshold_multiplier = user_setting['threshold_multiplier']
-        dynamic_energy_ratio = user_setting['dynamic_energy_ratio']
-        return date_format, time_format, measurement_system, tts, wake_word, sample_rate, channels, pronunciation, threshold, threshold_multiplier, dynamic_energy_ratio
-    else:
-        return 'MM/DD/YYYY', '12 Hour', 'Imperial', 'ap', 'Hey Mycroft', '16000', '1', 'HH EY . M AY K R AO F T', '1e-90', '1.0', '1.5'
-
-
 def format_date(value):
     value = int(value)
     value = datetime.datetime.fromtimestamp(value//1000)
@@ -199,21 +159,6 @@ def format_timestamp(value):
     return f'{value:%Y-%m-%d %H:%M:%S}'
 
 
-def parse_subscription(user_uuid):
-    if user_uuid in subscription:
-        subscr = subscription[user_uuid]
-        stripe_customer_id = subscr['stripe_customer_id']
-        start = format_timestamp(subscr['last_payment_ts'])
-        subscription_ts_range = '[{},)'.format(start)
-        subscription_type = subscr['type']
-        if subscription_type == 'MonthlyAccount':
-            subscription_type = 'month'
-        elif subscription_type == 'YearlyAccount':
-            subscription_type = 'year'
-        return subscription_ts_range, stripe_customer_id, subscription_type
-    else:
-        return '', '', ''
-
 
 db = connect(dbname='mycroft', user='postgres', host='127.0.0.1')
 db.autocommit = True
@@ -226,7 +171,7 @@ def get_subscription_uuid(subs):
         return subscription_uuids[subs]
     else:
         cursor = db.cursor()
-        cursor.execute(f'select id from account.subscription s where s.rate_period = \'{subs}\'')
+        cursor.execute(f'select id from account.membership s where s.rate_period = \'{subs}\'')
         result = cursor.fetchone()
         subscription_uuids[subs] = result
         return result
@@ -246,35 +191,46 @@ def get_tts_uuid(tts):
         return result
 
 
-def create_account(user_uuid):
-    user = users[user_uuid]
-    email = user['email']
-    password = user['password']
-    date_format, time_format, measurement_system, tts, wake_word, sample_rate, channels, pronunciation, threshold, threshold_multiplier, dynamic_energy_ratio = parse_user_setting(user_uuid)
-    subscription_ts_range, stripe_customer_id, subscription_type = parse_subscription(user_uuid)
-    cursor = db.cursor()
-
+def fill_account_table():
     query = 'insert into account.account(' \
             'id, ' \
             'email_address, ' \
             'password) ' \
             'values (%s, %s, %s)'
-    params = (user_uuid, email, password)
-    cursor.execute(query, params)
+    with db.cursor() as cur:
+        accounts = ((uuid, account['email'], account['password']) for uuid, account in users.items())
+        execute_batch(cur, query, accounts, page_size=1000)
 
-    wake_word_id = str(uuid.uuid4())
-    user['wake_word_id'] = wake_word_id
+
+def fill_account_agreement_table():
+    query = 'insert into account.agreement(account_id, agreement_id, accept_date)' \
+            'values (%s, select id from account.agreement where agreement = %s, %s)'
+    with db.cursor() as cur:
+        terms = [(uuid, format_timestamp(account['terms'])) for uuid, account in users.items()]
+        privacy = [(uuid, format_timestamp(account['privacy'])) for uuid, account in users.items()]
+        execute_batch(cur, query, terms+privacy, page_size=1000)
+
+
+def fill_wake_word_table():
     query = 'insert into device.wake_word (' \
             'id,' \
             'wake_word,' \
+            'engine,' \
             'account_id)' \
-            'values (%s, %s, %s)'
-    params = (wake_word_id, wake_word, user_uuid)
-    cursor.execute(query, params)
+            'values (%s, %s, %s, %s)'
 
-    text_to_speech_id = get_tts_uuid(tts)
-    user['text_to_speech_id'] = text_to_speech_id
+    def map_wake_word(user_id):
+        wake_word_id = str(uuid.uuid4())
+        wake_word = user_settings[user_id]['wake_word'] if user_id in user_settings else 'Hey Mycroft'
+        users[user_id]['wake_word_id'] = wake_word_id
+        return wake_word_id, wake_word, 'precise', user_id
 
+    with db.cursor() as cur:
+        wake_words = (map_wake_word(account_id) for account_id in users)
+        execute_batch(cur, query, wake_words, page_size=1000)
+
+
+def fill_account_preferences_table():
     query = 'insert into device.account_preferences(' \
             'account_id, ' \
             'date_format, ' \
@@ -283,20 +239,80 @@ def create_account(user_uuid):
             'wake_word_id,' \
             'text_to_speech_id)' \
             'values (%s, %s, %s, %s, %s, %s)'
-    params = (user_uuid, date_format, time_format, measurement_system, wake_word_id, text_to_speech_id)
-    cursor.execute(query, params)
 
-    if subscription_ts_range != '':
+    def map_account_preferences(user_uuid):
+        if user_uuid in user_settings:
+            user_setting = user_settings[user_uuid]
+            date_format = user_setting['date_format']
+            if date_format == 'DMY':
+                date_format = 'DD/MM/YYYY'
+            else:
+                date_format = 'MM/DD/YYYY'
+            time_format = user_setting['time_format']
+            if time_format == 'full':
+                time_format = '24 Hour'
+            else:
+                time_format = '12 Hour'
+            measurement_system = user_setting['measurement_system']
+            if measurement_system == 'metric':
+                measurement_system = 'Metric'
+            elif measurement_system == 'imperial':
+                measurement_system = 'Imperial'
+            tts_type = user_setting['tts_type']
+            tts_voice = user_setting['tts_voice']
+            if tts_type == 'MimicSetting':
+                if tts_voice == 'ap':
+                    tts = 'ap'
+                elif tts_voice == 'trinity':
+                    tts = 'amy'
+                else:
+                    tts = 'ap'
+            elif tts_type == 'Mimic2Setting':
+                tts = 'kusal'
+            elif tts_type == 'GoogleTTSSetting':
+                tts = 'google'
+            else:
+                tts = 'ap'
+            text_to_speech_id = get_tts_uuid(tts)
+            users[user_uuid]['text_to_speech_id'] = text_to_speech_id
+            return user_uuid, date_format, time_format, measurement_system, users[user_uuid]['wake_word_id'], text_to_speech_id
+        else:
+            text_to_speech_id = get_tts_uuid('ap')
+            users[user_uuid]['text_to_speech_id'] = text_to_speech_id
+            return user_uuid, 'MM/DD/YYYY', '12 Hour', 'Imperial', users[user_uuid]['wake_word_id'], text_to_speech_id
+
+    with db.cursor() as cur:
+        account_preferences = (map_account_preferences(user_uuid) for user_uuid in users)
+        execute_batch(cur, query, account_preferences, page_size=1000)
+
+
+def fill_subscription_table():
+    query = 'insert into account.account_membership(' \
+            'account_id, ' \
+            'membership_id, ' \
+            'membership_ts_range, ' \
+            'payment_account_id,' \
+            'payment_method) ' \
+            'values (%s, %s, %s, %s, %s)'
+
+    def map_subscription(user_uuid):
+        subscr = subscription[user_uuid]
+        stripe_customer_id = subscr['stripe_customer_id']
+        start = format_timestamp(subscr['last_payment_ts'])
+        subscription_ts_range = '[{},)'.format(start)
+        subscription_type = subscr['type']
+        if subscription_type == 'MonthlyAccount':
+            subscription_type = 'month'
+        elif subscription_type == 'YearlyAccount':
+            subscription_type = 'year'
         subscription_uuid = get_subscription_uuid(subscription_type)
-        query = 'insert into account.account_subscription(' \
-                'account_id, ' \
-                'subscription_id, ' \
-                'subscription_ts_range, ' \
-                'stripe_customer_id) ' \
-                'values (%s, %s, %s, %s)'
-        params = (user_uuid, subscription_uuid, subscription_ts_range, stripe_customer_id)
-        cursor.execute(query, params)
+        return user_uuid, subscription_uuid, subscription_ts_range, stripe_customer_id, 'Stripe'
+    with db.cursor() as cur:
+        account_subscriptions = (map_subscription(user_uuid) for user_uuid in subscription)
+        execute_batch(cur, query, account_subscriptions, page_size=1000)
 
+
+def fill_wake_word_settings_table():
     query = 'insert into device.wake_word_settings(' \
             'wake_word_id,' \
             'sample_rate,' \
@@ -306,123 +322,125 @@ def create_account(user_uuid):
             'threshold_multiplier,' \
             'dynamic_energy_ratio)' \
             'values (%s, %s, %s, %s, %s, %s, %s)'
-    params = (wake_word_id, sample_rate, channels, pronunciation, threshold, threshold_multiplier, dynamic_energy_ratio)
-    cursor.execute(query, params)
 
-load_csv()
-
-for account in users:
-    print('Creating user {}'.format(account))
-    create_account(account)
+    def map_wake_word_settings(user_uuid):
+        user_setting = user_settings[user_uuid]
+        wake_word_id = users[user_uuid]['wake_word_id']
+        sample_rate = user_setting['sample_rate']
+        channels = user_setting['channels']
+        pronunciation = user_setting['pronunciation']
+        threshold = user_setting['threshold']
+        threshold_multiplier = user_setting['threshold_multiplier']
+        dynamic_energy_ratio = user_setting['dynamic_energy_ratio']
+        return wake_word_id, sample_rate, channels, pronunciation, threshold, threshold_multiplier, dynamic_energy_ratio
+    with db.cursor() as cur:
+        account_wake_word_settings = (map_wake_word_settings(user_uuid) for user_uuid in users if user_uuid in user_settings)
+        execute_batch(cur, query, account_wake_word_settings, page_size=1000)
 
 
-def create_device(device_uuid, category):
-    print('Creating device {} with category {}'.format(device_uuid, category))
-    device = devices[device_uuid]
-    account_id = device['user_uuid']
-    device_name = device['name']
-    description = device['description']
-    platform = device['platform']
-    enclosure_version = device['enclosure_version']
-    core_version = device['core_version']
+def change_device_name():
+    for user in user_devices:
+        if user in users:
+            device_names = defaultdict(list)
+            for device_uuid, name in user_devices[user]:
+                device_names[name].append(device_uuid)
+            for name in device_names:
+                uuids = device_names[name]
+                if len(uuids) > 1:
+                    count = 1
+                    for uuid in uuids:
+                        devices[uuid]['name'] = '{name}-{uuid}'.format(name=name, uuid=uuid)
+                        count += 1
 
-    wake_word_id = users[account_id]['wake_word_id']
-    text_to_speech_id = users[account_id]['text_to_speech_id']
-
-    cursor = db.cursor()
-    query = 'select cat.id from device.category cat left join account.account acc on cat.account_id = acc.id where acc.id = %s and cat.category = %s'
-    params = (account_id, category)
-    cursor.execute(query, params)
-    category_id = cursor.fetchone()
-    if category_id is None:
-        query = 'insert into device.category(id, account_id, category) values (%s, %s, %s)'
-        category_id = str(uuid.uuid4())
-        params = (category_id, account_id, category)
-        cursor.execute(query, params)
 
+def fill_device_table():
     query = 'insert into device.device(' \
             'id, ' \
             'account_id, ' \
             'name, ' \
-            'category_id,' \
             'placement,' \
             'platform,' \
             'enclosure_version,' \
             'core_version,' \
             'wake_word_id,' \
             'text_to_speech_id) ' \
-            'values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)'
-    params = (device_uuid, account_id, device_name, category_id, description, platform, enclosure_version, core_version, wake_word_id, text_to_speech_id)
-    cursor.execute(query, params)
+            'values (%s, %s, %s, %s, %s, %s, %s, %s, %s)'
+
+    def map_device(device_id):
+        device = devices[device_id]
+        account_id = device['user_uuid']
+        name = device['name']
+        placement = device['description']
+        platform = device['platform']
+        enclosure_version = device['enclosure_version']
+        core_version = device['core_version']
+        wake_word_id = users[account_id]['wake_word_id']
+        text_to_speech_id = users[account_id]['text_to_speech_id']
+        return device_id, account_id, name, placement, platform, enclosure_version, core_version, wake_word_id, text_to_speech_id
+    with db.cursor() as cur:
+        devices_batch = (map_device(device_id) for user in user_devices if user in users for device_id, name in user_devices[user])
+        execute_batch(cur, query, devices_batch, page_size=1000)
 
 
-def create_device_skills(device_uuid):
-    cursor = db.cursor()
-    if device_uuid in device_to_skill:
-        for skill_uuid in device_to_skill[device_uuid]:
-            skill = skills[skill_uuid]
-            version_hash = skill['identifier']
-            skill_name = skill['name']
-            print('Creating skill with id {}'.format(skill_uuid))
-            query = 'insert into skill.skill(id, name) values (%s, %s)'
-            params = (skill_uuid, skill_name)
-            cursor.execute(query, params)
-
-            skill_version_id = str(uuid.uuid4())
-            query = 'insert into skill.setting_version(id, skill_id, version_hash) values (%s, %s, %s)'
-            params = (skill_version_id, skill_uuid, version_hash)
-            cursor.execute(query, params)
-
-            device_skill_id = str(uuid.uuid4())
-            query = 'insert into device.device_skill (id, device_id, skill_id) values (%s, %s, %s)'
-            params = (device_skill_id, device_uuid, skill_uuid)
-            cursor.execute(query, params)
-            for section_uuid in skill_to_section[skill_uuid]:
-                print('Creating section with id {}'.format(section_uuid))
-                query = 'insert into skill.setting_section(id, skill_version_id, section, display_order) values (%s, %s, %s, %s)'
-                section = skill_sections[section_uuid]
-                section_name = section['section']
-                display_order = section['display_order']
-                params = (section_uuid, skill_version_id, section_name, display_order)
-                cursor.execute(query, params)
-                for field_uuid in section_to_field[section_uuid]:
-                    print('Creating field with id {}'.format(field_uuid))
-                    query = 'insert into skill.setting(id, setting_section_id, setting, setting_type, hint, label, placeholder, hidden, options, display_order) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)'
-                    field = skill_fields[field_uuid]
-                    setting = field['name']
-                    setting_type = field['type']
-                    hint = field['hint']
-                    label = field['label']
-                    placeholder = field['placeholder']
-                    hidden = field['hide']
-                    options = field['options']
-                    display_order = field['order']
-                    params = (field_uuid, section_uuid, setting, setting_type, hint, label, placeholder, hidden == 'true', options, display_order)
-                    cursor.execute(query, params)
-
-                    field_value = skill_field_values[field_uuid]['field_value']
-                    query = 'insert into device.skill_setting(device_skill_id, setting_id, value) values (%s, %s, %s)'
-                    params = (device_skill_id, field_uuid, field_value)
-                    cursor.execute(query, params)
-
-
-def create_devices():
+def fill_skills_table():
+    skills_batch = []
+    settings_meta_batch = []
+    device_skill_batch = []
     for user in user_devices:
         if user in users:
-            category = {}
             for device_uuid, name in user_devices[user]:
-                if name in category:
-                    category[name].append(device_uuid)
-                else:
-                    category[name] = [device_uuid]
-            print('User {} Categories: {}'.format(user, category))
-            for name in category:
-                group = 1
-                for device_uuid in category[name]:
-                    create_device(device_uuid, 'Group {}'.format(group))
-                    create_device_skills(device_uuid)
-                    group += 1
+                if device_uuid in device_to_skill:
+                    for skill_uuid in device_to_skill[device_uuid]:
+                        skill = skills[skill_uuid]
+                        skill_name = skill['name']
+                        sections = []
+                        settings = {}
+                        if skill_uuid in skill_to_section:
+                            for section_uuid in skill_to_section[skill_uuid]:
+                                section = skill_sections[section_uuid]
+                                section_name = section['section']
+                                fields = []
+                                if section_uuid in section_to_field:
+                                    for field_uuid in section_to_field[section_uuid]:
+                                        fields.append(skill_fields[field_uuid])
+                                        settings[skill_fields[field_uuid]['name']] = skill_field_values[field_uuid]['field_value']
+                                sections.append({'name': section_name, 'fields': fields})
+                        skill_setting_meta = {'name': skill_name, 'skillMetadata': {'sections': sections}}
+                        skills_batch.append((skill_uuid, skill_name))
+                        meta_id = str(uuid.uuid4())
+                        settings_meta_batch.append((meta_id, skill_uuid, json.dumps(skill_setting_meta)))
+                        device_skill_batch.append((device_uuid, skill_uuid, meta_id, json.dumps(settings)))
 
+    with db.cursor() as curr:
+        query = 'insert into skill.skill(id, name) values (%s, %s)'
+        execute_batch(curr, query, skills_batch, page_size=1000)
+        query = 'insert into skill.settings_display(id, skill_id, settings_display) values (%s, %s, %s)'
+        execute_batch(curr, query, settings_meta_batch, page_size=1000)
+        query = 'insert into device.device_skill(device_id, skill_id, skill_settings_display_id, settings) ' \
+                'values (%s, %s, %s, %s)'
+        execute_batch(curr, query, device_skill_batch, page_size=1000)
 
-create_devices()
+start = time.time()
+load_csv()
+end = time.time()
 
+print('Time to load CSVs {}'.format(end - start))
+
+start = time.time()
+print('Importing account table')
+fill_account_table()
+print('Importing wake word table')
+fill_wake_word_table()
+print('Importing account preferences table')
+fill_account_preferences_table()
+print('Importing subscription table')
+fill_subscription_table()
+print('Importing wake word settings table')
+fill_wake_word_settings_table()
+print('Importing device table')
+change_device_name()
+fill_device_table()
+print('Filling skills table')
+fill_skills_table()
+end = time.time()
+print('Time to import: {}'.format(end-start))

db/scripts/queries.cypher

@@ -1,5 +1,5 @@
 users.csv
-match (n:User) return n.uuid, n.email, n.password
+match (n:User) return n.uuid, n.email, n.password, n.termsOfUseDate, n.privacyPolicyDate
 
 subscription.csv
 match (n) where ((n:MonthlyAccount) or (n:YearlyAccount)) and n.expiratesAt is not null set n.expiresAt = n.expiratesAt
@@ -23,7 +23,7 @@ devices_location.csv
 match (n:Device)-[:PLACED_AT]->()-[:COORDINATE]->(coord) return n.uuid, coord.latitude, coord.longitude
 
 skill.csv
-match (n:Skill) return n.uuid, n.name, n.description, n.identifier
+match (dev:Device)-[:SKILL_MAPPING]->()-[:SKILL]->(n:Skill) return n.uuid, dev.uuid, n.name, n.description
 
 skill_section.csv
 match (skill:Skill)-[:METADATA]->()-[:SECTION]->(section) return section.uuid, skill.uuid, section.name, section.order order by skill.uuid

shared/selene/data/account/repository/sql/get_account_by_device_id.sql

@@ -31,32 +31,33 @@ WITH
         WHERE
             dev.id = %(device_id)s
     ),
-    subscription AS (
+    membership AS (
         SELECT
             json_build_object(
-                'id', asub.id,
-                'type', s.subscription,
-                'start_date', lower(asub.subscription_ts_range)::DATE,
-                'stripe_customer_id', asub.stripe_customer_id
+                'id', acc_mem.id,
+                'type', mem.type,
+                'start_date', lower(acc_mem.membership_ts_range)::DATE,
+                'payment_method', acc_mem.payment_method,
+                'payment_account_id', acc_mem.payment_account_id
             )
         FROM
-            account.account_subscription asub
+            account.account_membership acc_mem
         INNER JOIN
-            account.subscription s ON asub.subscription_id = s.id
+            account.membership mem ON acc_mem.membership_id = mem.id
         INNER JOIN
-            account.account acc ON asub.account_id = acc.id
+            account.account acc ON acc_mem.account_id = acc.id
         INNER JOIN
             device.device dev ON acc.id = dev.account_id
         WHERE
             dev.id = %(device_id)s
-            AND upper(asub.subscription_ts_range) IS NULL
+            AND upper(acc_mem.membership_ts_range) IS NULL
     )
 SELECT
     json_build_object(
         'id', acc.id,
         'email_address', acc.email_address,
         'username', acc.username,
-        'subscription', (SELECT * FROM subscription),
+        'membership', (SELECT * FROM membership),
         'refresh_tokens', (SELECT * FROM refresh_tokens),
         'agreements', (SELECT * FROM agreements)
     ) as account

shared/selene/data/skill/repository/skill.py

@@ -19,7 +19,19 @@ class SkillRepository(RepositoryBase):
         )
         sql_results = self.cursor.select_all(query)
         if sql_results:
-            return [result['skill'] for result in sql_results]
+            skills = []
+            for result in sql_results:
+                sections = self._fill_setting_with_values(result['settings'], result['settings_display'])
+                skill = {
+                    'uuid': result['id'],
+                    'name': result['name'],
+                    'identifier': result['name'],
+                    'skillMetadata': {
+                        'sections': sections
+                    }
+                }
+                skills.append(skill)
+            return skills
 
     def get_skill_settings_by_device_id_and_version_hash(self, device_id, version_hash):
         """Return a skill setting for a given device id and skill version hash
@@ -32,9 +44,26 @@ class SkillRepository(RepositoryBase):
             'get_skill_setting_by_device_id_and_version_hash.sql',
             args=dict(device_id=device_id, version_hash=version_hash)
         )
-        sql_results = self.cursor.select_all(query)
+        sql_results = self.cursor.select_one(query)
         if sql_results:
-            return sql_results[0]['skill']
+            sections = self._fill_setting_with_values(sql_results['settings'], sql_results['settings_meta'])
+            skill = {
+                'uuid': sql_results['id'],
+                'name': sql_results['name'],
+                'identifier': sql_results['name'],
+                'skillMetadata': {
+                    'sections': sections
+                }
+            }
+            return skill
+
+    def _fill_setting_with_values(self, settings: dict, setting_meta: dict):
+        sections = setting_meta['skillMetadata']['sections']
+        for section in sections:
+            section_fields = section['fields']
+            for field in section_fields:
+                field['value'] = settings[field['name']]
+        return setting_meta
 
     def get_skills_for_account(self, account_id) -> List[Skill]:
         skills = []

shared/selene/data/skill/repository/sql/get_skill_setting_by_device_id.sql

@@ -1,40 +1,8 @@
-/* Build an array with all skill setting blocks belonging to a device using the API v1 domain format.
-Aggregate the skill sections from a skill and the settings from a section */
 SELECT
-    json_build_object(
-        'uuid', skill.id,
-        'name', skill.name,
-        'identifier', ver.version_hash,
-        'skillMetadata', json_build_object(
-            'sections', (
-                select json_agg(json_build_object(
-                    'uuid', sec.id,
-                    'section', sec.section,
-                    'display', sec.display_order,
-                    'fields', (
-                        select json_agg(json_build_object(
-                            'name', setting.setting,
-                            'type', setting.setting_type,
-                            'label', setting.label,
-                            'hint', setting.hint,
-                            'placeholder', setting.placeholder,
-                            'hide', setting.hidden,
-                            'value', (select value from device.skill_setting where device_skill_id = dev_skill.id and setting_id = setting.id),
-                            'options', setting.options,
-                            'order', setting.display_order
-                        ))
-                        FROM
-                            skill.setting setting
-                        WHERE
-                            setting.setting_section_id = sec.id
-                    )
-                ))
-                FROM
-                    skill.setting_section sec
-                WHERE
-                    sec.skill_version_id = ver.id
-        ))
-    ) as skill
+    skill.id,
+    skill.name,
+    dev_skill.settings,
+    display.settings_display
 FROM
     device.device dev
 INNER JOIN
@@ -42,7 +10,6 @@ INNER JOIN
 INNER JOIN
     skill.skill skill ON dev_skill.skill_id = skill.id
 INNER JOIN
-    skill.setting_version ver ON skill.id = ver.skill_id
+    skill.settings_display display ON dev_skill.skill_settings_display_id = display.id
 WHERE
-    dev.id = %(device_id)s;
-
+    dev.id = %(device_id)s

shared/selene/data/skill/repository/sql/get_skill_setting_by_device_id_and_version_hash.sql

@@ -1,40 +1,9 @@
-/* Build a skill setting block using the API v1 model format. Aggregate the sections for a skill into an array,
-as well the settings for a section*/
 SELECT
-    json_build_object(
-        'uuid', skill.id,
-        'name', skill.name,
-        'identifier', ver.version_hash,
-        'skillMetadata', json_build_object(
-            'sections', (
-                select json_agg(json_build_object(
-                    'uuid', sec.id,
-                    'section', sec.section,
-                    'display', sec.display_order,
-                    'fields', (
-                        select json_agg(json_build_object(
-                            'name', setting.setting,
-                            'type', setting.setting_type,
-                            'label', setting.label,
-                            'hint', setting.hint,
-                            'placeholder', setting.placeholder,
-                            'hide', setting.hidden,
-                            'value', (select value from device.skill_setting where device_skill_id = dev_skill.id and setting_id = setting.id),
-                            'options', setting.options,
-                            'order', setting.display_order
-                        ))
-                        FROM
-                            skill.setting setting
-                        WHERE
-                            setting.setting_section_id = sec.id
-                    )
-                ))
-                FROM
-                    skill.setting_section sec
-                WHERE
-                    sec.skill_version_id = ver.id
-        ))
-    ) as skill
+    skill.id,
+    skill.name,
+    meta.version,
+    dev_skill.settings,
+    meta.settings_meta
 FROM
     device.device dev
 INNER JOIN
@@ -42,6 +11,6 @@ INNER JOIN
 INNER JOIN
     skill.skill skill ON dev_skill.skill_id = skill.id
 INNER JOIN
-    skill.setting_version ver ON skill.id = ver.skill_id
+    skill.setting_meta meta ON dev_skill.skill_setting_meta_id = meta.id
 WHERE
-    dev.id = %(device_id)s AND ver.version_hash = %(version_hash)s
+    dev.id = %(device_id)s AND meta.version = %(version_hash)s