MycroftAI
/
mycroft-core
mirror of https://github.com/MycroftAI/mycroft-core.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #2736 from forslund/docs/plugin-apis 

Add Plugin base classes APIs to readthedocs
pull/2762/head
Kris Gesling 2020-12-03 20:33:59 +09:30 committed by GitHub
parent b0884301a3 e552910660
commit 6f8bae6ba1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 285 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
doc/index.rst
View File

@ -1,12 +1,24 @@
.. Mycroft documentation master file
Mycroft-core technical documentation
====================================
Mycroft Skills API
==================
------------------
*Reference for the Mycroft Skills API*
*Reference for use during Skill creation*
.. toctree::
:maxdepth: 4
:caption: Contents:
source/mycroft
Mycroft plugin API
------------------
*Reference for use during Plugin creation*
.. toctree::
:maxdepth: 4
:caption: Contents:
source/plugins

36
doc/source/plugins.rst Normal file
View File

@ -0,0 +1,36 @@
Mycroft plugins
===============
Mycroft is extendable by plugins. These plugins can add support for new Speech To Text engines, Text To Speech engines, wake word engines and add new audio playback options.
TTS - Base for TTS plugins
--------------------------
.. autoclass:: mycroft.tts.TTS
:members:
STT - base for STT plugins
--------------------------
.. autoclass:: mycroft.stt.STT
:members:
|
|
.. autoclass:: mycroft.stt.StreamingSTT
:members:
|
|
.. autoclass:: mycroft.stt.StreamThread
:members:
HotWordEngine - Base for Hotword engine plugins
-----------------------------------------------
.. autoclass:: mycroft.client.speech.hotword_factory.HotWordEngine
:members:
AudioBackend - Base for audioservice backend plugins
------------------
.. autoclass:: mycroft.audio.services.AudioBackend
:members:
|
|
.. autoclass:: mycroft.audio.services.RemoteAudioBackend
:members:

137
mycroft/audio/services/__init__.py
View File

@ -12,132 +12,129 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Definition of the audio service backends base classes.
These classes can be used to create an Audioservice plugin extending
Mycroft's media playback options.
"""
from abc import ABCMeta, abstractmethod
class AudioBackend(metaclass=ABCMeta):
"""
Base class for all audio backend implementations.
"""Base class for all audio backend implementations.
Args:
config: configuration dict for the instance
bus: Mycroft messagebus emitter
Arguments:
config (dict): configuration dict for the instance
bus (MessageBusClient): Mycroft messagebus emitter
"""
def __init__(self, config, bus):
self._track_start_callback = None
self.supports_mime_hints = False
self.config = config
self.bus = bus
@abstractmethod
def supported_uris(self):
"""List of supported uri types.
Returns:
list: Supported uri's
"""
Returns: list of supported uri types.
"""
pass
@abstractmethod
def clear_list(self):
"""
Clear playlist
"""
pass
"""Clear playlist."""
@abstractmethod
def add_list(self, tracks):
"""
Add tracks to backend's playlist.
"""Add tracks to backend's playlist.
Args:
tracks: list of tracks.
Arguments:
tracks (list): list of tracks.
"""
pass
@abstractmethod
def play(self, repeat=False):
"""
Start playback.
"""Start playback.
Args:
repeat: Repeat playlist, defaults to False
Starts playing the first track in the playlist and will contiune
until all tracks have been played.
Arguments:
repeat (bool): Repeat playlist, defaults to False
"""
pass
@abstractmethod
def stop(self):
"""
Stop playback.
"""Stop playback.
Returns: (bool) True if playback was stopped, otherwise False
Stops the current playback.
Returns:
bool: True if playback was stopped, otherwise False
"""
pass
def set_track_start_callback(self, callback_func):
"""
Register callback on track start, should be called as each track
in a playlist is started.
"""Register callback on track start.
This method should be called as each track in a playlist is started.
"""
self._track_start_callback = callback_func
def pause(self):
"""Pause playback.
Stops playback but may be resumed at the exact position the pause
occured.
"""
Pause playback.
"""
pass
def resume(self):
"""Resume paused playback.
Resumes playback after being paused.
"""
Resume paused playback.
"""
pass
def next(self):
"""
Skip to next track in playlist.
"""
pass
"""Skip to next track in playlist."""
def previous(self):
"""
Skip to previous track in playlist.
"""
pass
"""Skip to previous track in playlist."""
def lower_volume(self):
"""Lower volume.
This method is used to implement audio ducking. It will be called when
Mycroft is listening or speaking to make sure the media playing isn't
interfering.
"""
Lower volume.
"""
pass
def restore_volume(self):
"""Restore normal volume.
Called when to restore the playback volume to previous level after
Mycroft has lowered it using lower_volume().
"""
Restore normal volume.
"""
pass
def seek_forward(self, seconds=1):
"""
Skip X seconds
"""Skip X seconds.
Args:
seconds (int): number of seconds to seek, if negative rewind
Arguments:
seconds (int): number of seconds to seek, if negative rewind
"""
pass
def seek_backward(self, seconds=1):
"""
Rewind X seconds
"""Rewind X seconds.
Args:
seconds (int): number of seconds to seek, if negative rewind
Arguments:
seconds (int): number of seconds to seek, if negative jump forward.
"""
pass
def track_info(self):
"""
Fetch info about current playing track.
"""Get info about current playing track.
Returns:
Dict with track info.
Returns:
dict: Track info containing atleast the keys artist and album.
"""
ret = {}
ret['artist'] = ''
@ -145,13 +142,19 @@ class AudioBackend(metaclass=ABCMeta):
return ret
def shutdown(self):
""" Perform clean shutdown """
"""Perform clean shutdown.
Implements any audio backend specific shutdown procedures.
"""
self.stop()
class RemoteAudioBackend(AudioBackend):
""" Base class for remote audio backends.
"""Base class for remote audio backends.
These may be things like Chromecasts, mopidy servers, etc.
RemoteAudioBackends will always be checked after the normal
AudioBackends to make playback start locally by default.
An example of a RemoteAudioBackend would be things like Chromecasts,
mopidy servers, etc.
"""
pass

52
mycroft/client/speech/hotword_factory.py
View File

@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Factory functions for loading hotword engines - both internal and plugins.
"""
from time import time, sleep
import os
import platform
@ -54,12 +56,19 @@ def msec_to_sec(msecs):
msecs: milliseconds
Returns:
input converted from milliseconds to seconds
int: input converted from milliseconds to seconds
"""
return msecs / 1000
class HotWordEngine:
"""Hotword/Wakeword base class to be implemented by all wake word plugins.
Arguments:
key_phrase (str): string representation of the wake word
config (dict): Configuration block for the specific wake word
lang (str): language code (BCP-47)
"""
def __init__(self, key_phrase="hey mycroft", config=None, lang="en-us"):
self.key_phrase = str(key_phrase).lower()
@ -77,25 +86,43 @@ class HotWordEngine:
self.lang = str(self.config.get("lang", lang)).lower()
def found_wake_word(self, frame_data):
"""Check if wake word has been found.
Checks if the wake word has been found. Should reset any internal
tracking of the wake word state.
Arguments:
frame_data (binary data): Deprecated. Audio data for large chunk
of audio to be processed. This should not
be used to detect audio data instead
use update() to incrementaly update audio
Returns:
bool: True if a wake word was detected, else False
"""
return False
def update(self, chunk):
pass
"""Updates the hotword engine with new audio data.
The engine should process the data and update internal trigger state.
Arguments:
chunk (bytes): Chunk of audio data to process
"""
def stop(self):
""" Perform any actions needed to shut down the hot word engine.
"""Perform any actions needed to shut down the wake word engine.
This may include things such as unload loaded data or shutdown
external processess.
This may include things such as unloading data or shutdown
external processess.
"""
pass
class PocketsphinxHotWord(HotWordEngine):
"""Hotword engine using PocketSphinx.
"""Wake word engine using PocketSphinx.
PocketSphinx is very general purpose but has a somewhat high error rate.
The key advantage is to be able to specify the wakeword with phonemes.
The key advantage is to be able to specify the wake word with phonemes.
"""
def __init__(self, key_phrase="hey mycroft", config=None, lang="en-us"):
super().__init__(key_phrase, config, lang)
@ -154,7 +181,7 @@ class PocketsphinxHotWord(HotWordEngine):
class PreciseHotword(HotWordEngine):
"""Precice is the default wakeword engine for mycroft.
"""Precise is the default wake word engine for Mycroft.
Precise is developed by Mycroft AI and produces quite good wake word
spotting when trained on a decent dataset.
@ -301,7 +328,7 @@ class PreciseHotword(HotWordEngine):
class SnowboyHotWord(HotWordEngine):
"""Snowboy is a thirdparty hotword engine providing an easy training and
"""Snowboy is a thirdparty wake word engine providing an easy training and
testing interface.
"""
def __init__(self, key_phrase="hey mycroft", config=None, lang="en-us"):
@ -410,6 +437,11 @@ def load_wake_word_plugin(module_name):
class HotWordFactory:
"""Factory class instantiating the configured Hotword engine.
The factory can select between a range of built-in Hotword engines and also
from Hotword engine plugins.
"""
CLASSES = {
"pocketsphinx": PocketsphinxHotWord,
"precise": PreciseHotword,

80
mycroft/stt/__init__.py
View File

@ -27,7 +27,7 @@ from mycroft.util.plugins import load_plugin
class STT(metaclass=ABCMeta):
""" STT Base class, all STT backends derives from this one. """
"""STT Base class, all STT backends derive from this one. """
def __init__(self):
config_core = Configuration.get()
self.lang = str(self.init_language(config_core))
@ -39,6 +39,7 @@ class STT(metaclass=ABCMeta):
@staticmethod
def init_language(config_core):
"""Helper method to get language code from Mycroft config."""
lang = config_core.get("lang", "en-US")
langs = lang.split("-")
if len(langs) == 2:
@ -47,7 +48,21 @@ class STT(metaclass=ABCMeta):
@abstractmethod
def execute(self, audio, language=None):
pass
"""Implementation of STT functionallity.
This method needs to be implemented by the derived class to implement
the specific STT engine connection.
The method gets passed audio and optionally a language code and is
expected to return a text string.
Arguments:
audio (AudioData): audio recorded by mycroft.
language (str): optional language code
Returns:
str: parsed text
"""
class TokenSTT(STT, metaclass=ABCMeta):
@ -322,8 +337,14 @@ class DeepSpeechServerSTT(STT):
class StreamThread(Thread, metaclass=ABCMeta):
"""
ABC class to be used with StreamingSTT class implementations.
"""ABC class to be used with StreamingSTT class implementations.
This class reads audio chunks from a queue and sends it to a parsing
STT engine.
Arguments:
queue (Queue): Input Queue
language (str): language code for the current language.
"""
def __init__(self, queue, language):
@ -333,6 +354,7 @@ class StreamThread(Thread, metaclass=ABCMeta):
self.text = None
def _get_data(self):
"""Generator reading audio data from queue."""
while True:
d = self.queue.get()
if d is None:
@ -341,23 +363,38 @@ class StreamThread(Thread, metaclass=ABCMeta):
self.queue.task_done()
def run(self):
"""Thread entry point."""
return self.handle_audio_stream(self._get_data(), self.language)
@abstractmethod
def handle_audio_stream(self, audio, language):
pass
"""Handling of audio stream.
Needs to be implemented by derived class to process audio data and
optionally update `self.text` with the current hypothesis.
Argumens:
audio (bytes): raw audio data.
language (str): language code for the current session.
"""
class StreamingSTT(STT, metaclass=ABCMeta):
"""
ABC class for threaded streaming STT implemenations.
"""
"""ABC class for threaded streaming STT implemenations."""
def __init__(self):
super().__init__()
self.stream = None
self.can_stream = True
def stream_start(self, language=None):
"""Indicate start of new audio stream.
This creates a new thread for handling the incomming audio stream as
it's collected by Mycroft.
Arguments:
language (str): optional language code for the new stream.
"""
self.stream_stop()
language = language or self.lang
self.queue = Queue()
@ -365,9 +402,21 @@ class StreamingSTT(STT, metaclass=ABCMeta):
self.stream.start()
def stream_data(self, data):
"""Receiver of audio data.
Arguments:
data (bytes): raw audio data.
"""
self.queue.put(data)
def stream_stop(self):
"""Indicate that the audio stream has ended.
This will tear down the processing thread and collect the result
Returns:
str: parsed text
"""
if self.stream is not None:
self.queue.put(None)
self.stream.join()
@ -380,11 +429,20 @@ class StreamingSTT(STT, metaclass=ABCMeta):
return None
def execute(self, audio, language=None):
"""End the parsing thread and collect data."""
return self.stream_stop()
@abstractmethod
def create_streaming_thread(self):
pass
"""Create thread for parsing audio chunks.
This method should be implemented by the derived class to return an
instance derived from StreamThread to handle the audio stream and
send it to the STT engine.
Returns:
StreamThread: Thread to handle audio data.
"""
class DeepSpeechStreamThread(StreamThread):
@ -550,7 +608,9 @@ def load_stt_plugin(module_name):
"""Wrapper function for loading stt plugin.
Arguments:
(str) Mycroft stt module name from config
module_name (str): Mycroft stt module name from config
Returns:
class: STT plugin class
"""
return load_plugin('mycroft.plugin.stt', module_name)

77
mycroft/tts/tts.py
View File

@ -128,10 +128,10 @@ class PlaybackThread(Thread):
"""Send viseme data to enclosure
Arguments:
pairs(list): Visime and timing pair
pairs (list): Visime and timing pair
Returns:
True if button has been pressed.
bool: True if button has been pressed.
"""
if self.enclosure:
self.enclosure.mouth_viseme(time(), pairs)
@ -187,7 +187,7 @@ class TTS(metaclass=ABCMeta):
self.tts_name = type(self).__name__
def load_spellings(self):
"""Load phonetic spellings of words as dictionary"""
"""Load phonetic spellings of words as dictionary."""
path = join('text', self.lang.lower(), 'phonetic_spellings.txt')
spellings_file = resolve_resource_file(path)
if not spellings_file:
@ -202,7 +202,7 @@ class TTS(metaclass=ABCMeta):
return {}
def begin_audio(self):
"""Helper function for child classes to call in execute()"""
"""Helper function for child classes to call in execute()."""
# Create signals informing start of speech
self.bus.emit(Message("recognizer_loop:audio_output_start"))
@ -254,11 +254,23 @@ class TTS(metaclass=ABCMeta):
pass
def modify_tag(self, tag):
"""Override to modify each supported ssml tag"""
"""Override to modify each supported ssml tag.
Arguments:
tag (str): SSML tag to check and possibly transform.
"""
return tag
@staticmethod
def remove_ssml(text):
"""Removes SSML tags from a string.
Arguments:
text (str): input string
Returns:
str: input string stripped from tags.
"""
return re.sub('<[^>]*>', '', text).replace(' ', ' ')
def validate_ssml(self, utterance):
@ -267,10 +279,10 @@ class TTS(metaclass=ABCMeta):
Remove unsupported / invalid tags
Arguments:
utterance(str): Sentence to validate
utterance (str): Sentence to validate
Returns:
validated_sentence (str)
str: validated_sentence
"""
# if ssml is not supported by TTS engine remove all tags
if not self.ssml_tags:
@ -306,14 +318,14 @@ class TTS(metaclass=ABCMeta):
def execute(self, sentence, ident=None, listen=False):
"""Convert sentence to speech, preprocessing out unsupported ssml
The method caches results if possible using the hash of the
sentence.
The method caches results if possible using the hash of the
sentence.
Arguments:
sentence: Sentence to be spoken
ident: Id reference to current interaction
listen: True if listen should be triggered at the end
of the utterance.
Arguments:
sentence: (str) Sentence to be spoken
ident: (str) Id reference to current interaction
listen: (bool) True if listen should be triggered at the end
of the utterance.
"""
sentence = self.validate_ssml(sentence)
@ -357,11 +369,16 @@ class TTS(metaclass=ABCMeta):
self.queue.put((self.audio_ext, wav_file, vis, ident, l))
def viseme(self, phonemes):
"""Create visemes from phonemes. Needs to be implemented for all
tts backends.
"""Create visemes from phonemes.
Arguments:
phonemes(str): String with phoneme data
May be implemented to convert TTS phonemes into Mycroft mouth
visuals.
Arguments:
phonemes (str): String with phoneme data
Returns:
list: visemes
"""
return None
@ -384,8 +401,8 @@ class TTS(metaclass=ABCMeta):
"""Cache phonemes
Arguments:
key: Hash key for the sentence
phonemes: phoneme string to save
key (str): Hash key for the sentence
phonemes (str): phoneme string to save
"""
cache_dir = mycroft.util.get_cache_directory("tts/" + self.tts_name)
pho_file = os.path.join(cache_dir, key + ".pho")
@ -400,7 +417,7 @@ class TTS(metaclass=ABCMeta):
"""Load phonemes from cache file.
Arguments:
Key: Key identifying phoneme cache
key (str): Key identifying phoneme cache
"""
pho_file = os.path.join(
mycroft.util.get_cache_directory("tts/" + self.tts_name),
@ -436,6 +453,7 @@ class TTSValidator(metaclass=ABCMeta):
self.validate_connection()
def validate_dependencies(self):
"""Determine if all the TTS's external dependencies are satisfied."""
pass
def validate_instance(self):
@ -454,15 +472,19 @@ class TTSValidator(metaclass=ABCMeta):
@abstractmethod
def validate_lang(self):
pass
"""Ensure the TTS supports current language."""
@abstractmethod
def validate_connection(self):
pass
"""Ensure the TTS can connect to it's backend.
This can mean for example being able to launch the correct executable
or contact a webserver.
"""
@abstractmethod
def get_tts_class(self):
pass
"""Return TTS class that this validator is for."""
def load_tts_plugin(module_name):
@ -470,11 +492,18 @@ def load_tts_plugin(module_name):
Arguments:
(str) Mycroft tts module name from config
Returns:
class: found tts plugin class
"""
return load_plugin('mycroft.plugin.tts', module_name)
class TTSFactory:
"""Factory class instantiating the configured TTS engine.
The factory can select between a range of built-in TTS engines and also
from TTS engine plugins.
"""
from mycroft.tts.festival_tts import Festival
from mycroft.tts.espeak_tts import ESpeak
from mycroft.tts.fa_tts import FATTS