Matthew D. Scholefield
parent
04053a5839
commit
94d3087bbb
|
|
@ -0,0 +1,130 @@
|
|||
# Copyright 2020 Mycroft AI Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from collections import namedtuple
|
||||
from glob import glob
|
||||
from os.path import join
|
||||
|
||||
import numpy as np
|
||||
|
||||
from precise.params import pr
|
||||
from precise.util import load_audio
|
||||
from precise.vectorization import vectorize_raw
|
||||
|
||||
AnnoyanceEstimate = namedtuple(
|
||||
'AnnoyanceEstimate',
|
||||
'annoyance ww_annoyance nww_annoyance threshold'
|
||||
)
|
||||
|
||||
|
||||
class AnnoyanceEstimator:
|
||||
"""
|
||||
This class attempts to estimate the "annoyance" of a user
|
||||
of a given network. It models annoyance as follows:
|
||||
|
||||
Annoyance from false negatives (not activating when it should):
|
||||
We assume that the annoyance incurred by each subsequent failed
|
||||
activation attempt is double that of the previous attempt. ie.
|
||||
two failed activations causes 1 + 2 = 3 annoyance units but three
|
||||
failed activations causes 1 + 2 + 4 = 7 annoyance units.
|
||||
|
||||
Annoyance from false positives (activating when it should not):
|
||||
We assume that each false positive incurs some constant annoyance
|
||||
|
||||
With this, we can compute net annoyance from false positives
|
||||
and negatives individually, combine them for the total annoyance.
|
||||
|
||||
Finally, we can recompute this annoyance for each threshold
|
||||
value to find the threshold that yields the lowest net annoyance
|
||||
"""
|
||||
|
||||
def __init__(self, model, interaction_estimate, ambient_annoyance):
|
||||
self.thresholds = 1 / (1 + np.exp(-np.linspace(-20, 20, 1000)))
|
||||
self.interaction_estimate = interaction_estimate
|
||||
self.ambient_annoyance = ambient_annoyance
|
||||
|
||||
def compute_nww_annoyances(self, model, noise_folder, batch_size):
|
||||
"""
|
||||
Given some number, x, of ambient activations per hour, we can
|
||||
compute the annoyance per day from false positives as 24 * x
|
||||
times the annoyance incurred per ambient activation.
|
||||
"""
|
||||
nww_seconds = 0.0
|
||||
nww_buckets = np.zeros_like(self.thresholds)
|
||||
for i in glob(join(noise_folder, '*.wav')):
|
||||
print('Evaluating ambient activations on {}...'.format(i))
|
||||
inputs, audio_len = self._load_inputs(i)
|
||||
nww_seconds += audio_len / pr.sample_rate
|
||||
ambient_predictions = model.predict(inputs, batch_size=batch_size)
|
||||
del inputs
|
||||
nww_buckets += (ambient_predictions.reshape((-1, 1))
|
||||
> self.thresholds.reshape((1, -1))).sum(axis=0)
|
||||
nww_acts_per_hour = nww_buckets * 60 * 60 / nww_seconds
|
||||
return self.ambient_annoyance * nww_acts_per_hour * 24
|
||||
|
||||
def compute_ww_annoyances(self, ww_predictions):
|
||||
"""
|
||||
Given some proportion, p, of not recognizing the wake word, our
|
||||
total annoyance per interaction is modelled as p^1 * 2^0 + p^2 * 2^1
|
||||
+ ... + p^i * 2^(i - 1) which converges to 1 / (1 - 2p) - 1.
|
||||
Given some number of interactions per day we can then find the
|
||||
expected annoyance per day from false negatives.
|
||||
"""
|
||||
ww_buckets = (ww_predictions.reshape((-1, 1)) >
|
||||
self.thresholds.reshape((1, -1))).sum(axis=0)
|
||||
ww_fail_ratios = 1 - ww_buckets / len(ww_predictions)
|
||||
# Performs 1 / (1 - 2 * ww_fail_ratios) - 1, handling edge case
|
||||
ann_per_interaction = np.divide(
|
||||
1, 1 - 2 * ww_fail_ratios,
|
||||
where=ww_fail_ratios < 0.5
|
||||
) - 1
|
||||
ann_per_interaction[ww_fail_ratios >= 0.5] = float('inf')
|
||||
return self.interaction_estimate * ann_per_interaction
|
||||
|
||||
def estimate(self, model, predictions, targets, noise_folder, batch_size):
|
||||
"""
|
||||
Estimates the annoyance a model incurs according to the model
|
||||
described in the class documentation
|
||||
"""
|
||||
ww_predictions = predictions[np.where(targets > 0.5)]
|
||||
ww_annoyances = self.compute_ww_annoyances(ww_predictions)
|
||||
nww_annoyances = self.compute_nww_annoyances(
|
||||
model, noise_folder, batch_size
|
||||
)
|
||||
annoyance_by_threshold = ww_annoyances + nww_annoyances
|
||||
best_threshold_id = np.argmin(annoyance_by_threshold)
|
||||
min_annoyance = annoyance_by_threshold[best_threshold_id]
|
||||
return AnnoyanceEstimate(
|
||||
annoyance=min_annoyance,
|
||||
ww_annoyance=ww_annoyances[best_threshold_id],
|
||||
nww_annoyance=nww_annoyances[best_threshold_id],
|
||||
threshold=self.thresholds[best_threshold_id]
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _load_inputs(audio_file, chunk_size=4096):
|
||||
"""
|
||||
Loads network inputs from an audio file without caching
|
||||
Handles data conservatively in case the audio file is large
|
||||
Args:
|
||||
audio_file: Filename to load
|
||||
chunk_size: Samples to skip forward when loading network inpus
|
||||
"""
|
||||
audio = load_audio(audio_file)
|
||||
audio_len = len(audio)
|
||||
mfccs = vectorize_raw(audio)
|
||||
del audio
|
||||
mfcc_hops = chunk_size // pr.hop_samples
|
||||
return np.array([
|
||||
mfccs[i - pr.n_features:i] for i in range(pr.n_features, len(mfccs), mfcc_hops)
|
||||
]), audio_len
|
||||
|
|
@ -13,116 +13,122 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Use black box optimization to tune model hyperparameters
|
||||
Use black box optimization to tune model hyperparameters. Call
|
||||
this script in a loop to iteratively tune parameters
|
||||
|
||||
:-t --trials-name str -
|
||||
:trials_name str
|
||||
Filename to save hyperparameter optimization trials in
|
||||
'.bbopt.json' will automatically be appended
|
||||
|
||||
:-c --cycles int 20
|
||||
Number of cycles of optimization to run
|
||||
:noise_folder str
|
||||
Folder with random noise to evaluate ambient activations
|
||||
|
||||
:-m --model str .cache/optimized.net
|
||||
Model to load from
|
||||
:-ie --interaction-estimate int 100
|
||||
Estimated number of interactions per day
|
||||
|
||||
:-aaa --ambient-activation-annoyance float 1.0
|
||||
An ambient activation is X times as annoying as a failed
|
||||
activation when the wake word is spoken
|
||||
|
||||
:-bp --base-params str {}
|
||||
Json string containing base ListenerParams for all models
|
||||
|
||||
...
|
||||
"""
|
||||
import numpy
|
||||
# Optimizer blackhat
|
||||
from glob import glob
|
||||
from os import remove
|
||||
from os.path import isfile, splitext, join
|
||||
from pprint import pprint
|
||||
from prettyparse import Usage
|
||||
from shutil import rmtree
|
||||
from typing import Any
|
||||
import json
|
||||
from math import exp
|
||||
from uuid import uuid4
|
||||
|
||||
from keras.models import save_model
|
||||
from prettyparse import Usage
|
||||
|
||||
from precise.annoyance_estimator import AnnoyanceEstimator
|
||||
from precise.model import ModelParams, create_model
|
||||
from precise.params import pr, save_params
|
||||
from precise.scripts.train import TrainScript
|
||||
from precise.train_data import TrainData
|
||||
from precise.stats import Stats
|
||||
|
||||
|
||||
class TrainOptimizeScript(TrainScript):
|
||||
Usage(__doc__) | TrainScript.usage
|
||||
usage = Usage(__doc__) | TrainScript.usage
|
||||
del usage.arguments['model'] # Remove 'model' argument from original TrainScript
|
||||
|
||||
def __init__(self, args):
|
||||
super().__init__(args)
|
||||
from bbopt import BlackBoxOptimizer
|
||||
pr.__dict__.update(json.loads(args.base_params))
|
||||
args.model = args.trials_name + '-cur'
|
||||
save_params(args.model)
|
||||
super().__init__(args)
|
||||
self.bb = BlackBoxOptimizer(file=self.args.trials_name)
|
||||
if not self.test:
|
||||
data = TrainData.from_both(self.args.tags_file, self.args.tags_folder, self.args.folder)
|
||||
_, self.test = data.load(False, True)
|
||||
|
||||
from keras.callbacks import ModelCheckpoint
|
||||
for i in list(self.callbacks):
|
||||
if isinstance(i, ModelCheckpoint):
|
||||
self.callbacks.remove(i)
|
||||
def calc_params_cost(self, model):
|
||||
"""
|
||||
Models the real world cost of additional model parameters
|
||||
Up to a certain point, having more parameters isn't worse.
|
||||
However, at a certain point more parameters will risk
|
||||
running slower than realtime and become unfeasible. This
|
||||
is why it's modelled exponentially with some reasonable
|
||||
number of acceptable parameters.
|
||||
|
||||
def process_args(self, args: Any):
|
||||
model_parts = glob(splitext(args.model)[0] + '.*')
|
||||
if len(model_parts) < 5:
|
||||
for name in model_parts:
|
||||
if isfile(name):
|
||||
remove(name)
|
||||
else:
|
||||
rmtree(name)
|
||||
args.trials_name = args.trials_name.replace('.bbopt.json', '').replace('.json', '')
|
||||
if not args.trials_name:
|
||||
if isfile(join('.cache', 'trials.bbopt.json')):
|
||||
remove(join('.cache', 'trials.bbopt.json'))
|
||||
args.trials_name = join('.cache', 'trials')
|
||||
Ideally, this would be replaced with floating point
|
||||
computations and the numbers would be configurable
|
||||
rather than chosen relatively arbitrarily
|
||||
"""
|
||||
return 1.0 + exp((model.count_params() - 11000) / 10000)
|
||||
|
||||
def run(self):
|
||||
print('Writing to:', self.args.trials_name + '.bbopt.json')
|
||||
for i in range(self.args.cycles):
|
||||
self.bb.run(backend="random")
|
||||
print("\n= %d = (example #%d)" % (i + 1, len(self.bb.get_data()["examples"]) + 1))
|
||||
self.bb.run(alg='tree_structured_parzen_estimator')
|
||||
|
||||
params = ModelParams(
|
||||
recurrent_units=self.bb.randint("units", 1, 70, guess=50),
|
||||
dropout=self.bb.uniform("dropout", 0.1, 0.9, guess=0.6),
|
||||
extra_metrics=self.args.extra_metrics,
|
||||
skip_acc=self.args.no_validation,
|
||||
loss_bias=1.0 - self.args.sensitivity
|
||||
)
|
||||
print('Testing with:', params)
|
||||
model = create_model(self.args.model, params)
|
||||
model.fit(
|
||||
*self.sampled_data, batch_size=self.args.batch_size,
|
||||
epochs=self.epoch + self.args.epochs,
|
||||
validation_data=self.test * (not self.args.no_validation),
|
||||
callbacks=self.callbacks, initial_epoch=self.epoch
|
||||
)
|
||||
resp = model.evaluate(*self.test, batch_size=self.args.batch_size)
|
||||
if not isinstance(resp, (list, tuple)):
|
||||
resp = [resp, None]
|
||||
test_loss, test_acc = resp
|
||||
predictions = model.predict(self.test[0], batch_size=self.args.batch_size)
|
||||
model = create_model(None, ModelParams(
|
||||
recurrent_units=self.bb.randint("units", 1, 120, guess=30),
|
||||
dropout=self.bb.uniform("dropout", 0.05, 0.9, guess=0.2),
|
||||
extra_metrics=self.args.extra_metrics,
|
||||
skip_acc=self.args.no_validation,
|
||||
loss_bias=self.bb.uniform(
|
||||
'loss_bias', 0.01, 0.99, guess=1.0 - self.args.sensitivity
|
||||
),
|
||||
freeze_till=0
|
||||
))
|
||||
model.fit(
|
||||
*self.sampled_data, batch_size=self.args.batch_size,
|
||||
epochs=self.args.epochs,
|
||||
validation_data=self.test * (not self.args.no_validation),
|
||||
callbacks=[]
|
||||
)
|
||||
test_in, test_out = self.test
|
||||
test_pred = model.predict(test_in, batch_size=self.args.batch_size)
|
||||
stats_dict = Stats(test_pred, test_out, []).to_dict()
|
||||
|
||||
num_false_positive = numpy.sum(predictions * (1 - self.test[1]) > 0.5)
|
||||
num_false_negative = numpy.sum((1 - predictions) * self.test[1] > 0.5)
|
||||
false_positives = num_false_positive / numpy.sum(self.test[1] < 0.5)
|
||||
false_negatives = num_false_negative / numpy.sum(self.test[1] > 0.5)
|
||||
ann_est = AnnoyanceEstimator(
|
||||
model, self.args.interaction_estimate,
|
||||
self.args.ambient_activation_annoyance
|
||||
).estimate(
|
||||
model, test_pred, test_out,
|
||||
self.args.noise_folder, self.args.batch_size
|
||||
)
|
||||
params_cost = self.calc_params_cost(model)
|
||||
cost = ann_est.annoyance + params_cost
|
||||
|
||||
from math import exp
|
||||
param_score = 1.0 / (1.0 + exp((model.count_params() - 11000) / 2000))
|
||||
fitness = param_score * (1.0 - 0.2 * false_negatives - 0.8 * false_positives)
|
||||
model_name = '{}-{}.net'.format(self.args.trials_name, str(uuid4()))
|
||||
save_model(model, model_name)
|
||||
save_params(model_name)
|
||||
|
||||
self.bb.remember({
|
||||
"test loss": test_loss,
|
||||
"test accuracy": test_acc,
|
||||
"false positive%": false_positives,
|
||||
"false negative%": false_negatives,
|
||||
"fitness": fitness
|
||||
})
|
||||
|
||||
print("False positive: ", false_positives * 100, "%")
|
||||
|
||||
self.bb.maximize(fitness)
|
||||
pprint(self.bb.get_current_run())
|
||||
best_example = self.bb.get_optimal_run()
|
||||
print("\n= BEST = (example #%d)" % self.bb.get_data()["examples"].index(best_example))
|
||||
pprint(best_example)
|
||||
self.bb.remember({
|
||||
'test_stats': stats_dict,
|
||||
'best_threshold': ann_est.threshold,
|
||||
'cost': cost,
|
||||
'cost_info': {
|
||||
'params_cost': params_cost,
|
||||
'annoyance': ann_est.annoyance,
|
||||
'ww_annoyance': ann_est.ww_annoyance,
|
||||
'nww_annoyance': ann_est.nww_annoyance,
|
||||
},
|
||||
'model': model_name
|
||||
})
|
||||
print('Current Run: {}'.format(json.dumps(
|
||||
self.bb.get_current_run(), indent=4
|
||||
)))
|
||||
self.bb.minimize(cost)
|
||||
|
||||
|
||||
main = TrainOptimizeScript.run_main
|
||||
|
|
|
|||
Loading…
Reference in New Issue