2016-03-07 22:20:48 +00:00
|
|
|
"""Helper methods for various modules."""
|
2018-03-10 03:38:51 +00:00
|
|
|
import asyncio
|
2018-07-23 08:24:39 +00:00
|
|
|
from datetime import datetime, timedelta
|
2014-04-15 06:48:00 +00:00
|
|
|
import enum
|
2019-12-09 15:42:10 +00:00
|
|
|
from functools import wraps
|
2015-01-18 05:55:33 +00:00
|
|
|
import random
|
2019-12-09 15:42:10 +00:00
|
|
|
import re
|
|
|
|
import socket
|
2015-01-18 05:55:33 +00:00
|
|
|
import string
|
2019-12-09 15:42:10 +00:00
|
|
|
import threading
|
2016-02-10 07:27:01 +00:00
|
|
|
from types import MappingProxyType
|
2019-07-30 23:59:12 +00:00
|
|
|
from typing import (
|
|
|
|
Any,
|
|
|
|
Callable,
|
2019-12-09 15:42:10 +00:00
|
|
|
Coroutine,
|
|
|
|
Iterable,
|
2019-07-30 23:59:12 +00:00
|
|
|
KeysView,
|
2019-12-09 15:42:10 +00:00
|
|
|
Optional,
|
|
|
|
TypeVar,
|
2019-11-16 09:22:07 +00:00
|
|
|
Union,
|
2019-07-30 23:59:12 +00:00
|
|
|
)
|
2016-07-23 18:07:08 +00:00
|
|
|
|
2018-12-17 06:51:13 +00:00
|
|
|
import slugify as unicode_slug
|
|
|
|
|
2016-04-16 07:55:35 +00:00
|
|
|
from .dt import as_local, utcnow
|
2015-04-29 02:12:05 +00:00
|
|
|
|
2019-07-30 23:59:12 +00:00
|
|
|
T = TypeVar("T")
|
2020-05-09 11:08:40 +00:00
|
|
|
U = TypeVar("U") # pylint: disable=invalid-name
|
|
|
|
ENUM_T = TypeVar("ENUM_T", bound=enum.Enum) # pylint: disable=invalid-name
|
2016-08-07 23:26:35 +00:00
|
|
|
|
2019-07-30 23:59:12 +00:00
|
|
|
RE_SANITIZE_FILENAME = re.compile(r"(~|\.\.|/|\\)")
|
|
|
|
RE_SANITIZE_PATH = re.compile(r"(~|\.(\.)+)")
|
2013-12-07 20:54:19 +00:00
|
|
|
|
2013-11-11 00:46:48 +00:00
|
|
|
|
2016-08-07 23:26:35 +00:00
|
|
|
def sanitize_filename(filename: str) -> str:
|
2021-01-23 17:28:57 +00:00
|
|
|
"""Check if a filename is safe.
|
|
|
|
|
|
|
|
Only to be used to compare to original filename to check if changed.
|
|
|
|
If result changed, the given path is not safe and should not be used,
|
|
|
|
raise an error.
|
|
|
|
|
|
|
|
DEPRECATED.
|
|
|
|
"""
|
|
|
|
# Backwards compatible fix for misuse of method
|
|
|
|
if RE_SANITIZE_FILENAME.sub("", filename) != filename:
|
|
|
|
return ""
|
|
|
|
return filename
|
2013-12-07 20:54:19 +00:00
|
|
|
|
|
|
|
|
2016-08-07 23:26:35 +00:00
|
|
|
def sanitize_path(path: str) -> str:
|
2021-01-23 17:28:57 +00:00
|
|
|
"""Check if a path is safe.
|
|
|
|
|
|
|
|
Only to be used to compare to original path to check if changed.
|
|
|
|
If result changed, the given path is not safe and should not be used,
|
|
|
|
raise an error.
|
|
|
|
|
|
|
|
DEPRECATED.
|
|
|
|
"""
|
|
|
|
# Backwards compatible fix for misuse of method
|
|
|
|
if RE_SANITIZE_PATH.sub("", path) != path:
|
|
|
|
return ""
|
|
|
|
return path
|
2014-10-22 06:52:24 +00:00
|
|
|
|
|
|
|
|
2020-02-25 19:18:21 +00:00
|
|
|
def slugify(text: str, *, separator: str = "_") -> str:
|
2016-03-07 22:20:48 +00:00
|
|
|
"""Slugify a given text."""
|
2020-10-13 00:17:30 +00:00
|
|
|
return unicode_slug.slugify(text, separator=separator)
|
2014-01-20 03:10:40 +00:00
|
|
|
|
|
|
|
|
2016-07-23 18:07:08 +00:00
|
|
|
def repr_helper(inp: Any) -> str:
|
2016-03-07 22:20:48 +00:00
|
|
|
"""Help creating a more readable string representation of objects."""
|
2016-02-10 07:27:01 +00:00
|
|
|
if isinstance(inp, (dict, MappingProxyType)):
|
2014-04-14 07:10:24 +00:00
|
|
|
return ", ".join(
|
2020-04-07 21:14:28 +00:00
|
|
|
f"{repr_helper(key)}={repr_helper(item)}" for key, item in inp.items()
|
2019-07-30 23:59:12 +00:00
|
|
|
)
|
2018-07-23 08:16:05 +00:00
|
|
|
if isinstance(inp, datetime):
|
2016-04-16 07:55:35 +00:00
|
|
|
return as_local(inp).isoformat()
|
2017-07-06 03:02:16 +00:00
|
|
|
|
|
|
|
return str(inp)
|
2014-01-27 02:44:36 +00:00
|
|
|
|
|
|
|
|
2019-07-30 23:59:12 +00:00
|
|
|
def convert(
|
|
|
|
value: Optional[T], to_type: Callable[[T], U], default: Optional[U] = None
|
|
|
|
) -> Optional[U]:
|
2016-03-07 22:20:48 +00:00
|
|
|
"""Convert value to to_type, returns default if fails."""
|
2014-03-16 22:00:59 +00:00
|
|
|
try:
|
2014-03-26 07:08:50 +00:00
|
|
|
return default if value is None else to_type(value)
|
2016-02-21 19:23:16 +00:00
|
|
|
except (ValueError, TypeError):
|
2014-03-16 22:00:59 +00:00
|
|
|
# If value could not be converted
|
|
|
|
return default
|
|
|
|
|
|
|
|
|
2019-07-30 23:59:12 +00:00
|
|
|
def ensure_unique_string(
|
|
|
|
preferred_string: str, current_strings: Union[Iterable[str], KeysView[str]]
|
|
|
|
) -> str:
|
2016-03-07 22:20:48 +00:00
|
|
|
"""Return a string that is not present in current_strings.
|
|
|
|
|
|
|
|
If preferred string exists will append _2, _3, ..
|
|
|
|
"""
|
2015-01-18 05:55:33 +00:00
|
|
|
test_string = preferred_string
|
2016-07-28 03:33:49 +00:00
|
|
|
current_strings_set = set(current_strings)
|
2014-03-23 19:31:24 +00:00
|
|
|
|
|
|
|
tries = 1
|
|
|
|
|
2016-07-28 03:33:49 +00:00
|
|
|
while test_string in current_strings_set:
|
2014-03-23 19:31:24 +00:00
|
|
|
tries += 1
|
2019-08-23 16:53:33 +00:00
|
|
|
test_string = f"{preferred_string}_{tries}"
|
2014-03-23 19:31:24 +00:00
|
|
|
|
2015-01-18 05:55:33 +00:00
|
|
|
return test_string
|
2014-03-23 19:31:24 +00:00
|
|
|
|
|
|
|
|
2014-05-02 06:03:14 +00:00
|
|
|
# Taken from: http://stackoverflow.com/a/11735897
|
2018-07-23 08:24:39 +00:00
|
|
|
def get_local_ip() -> str:
|
2016-03-07 22:20:48 +00:00
|
|
|
"""Try to determine the local IP address of the machine."""
|
2014-05-02 06:03:14 +00:00
|
|
|
try:
|
|
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
|
|
|
|
|
|
# Use Google Public DNS server to determine own IP
|
2019-07-30 23:59:12 +00:00
|
|
|
sock.connect(("8.8.8.8", 80))
|
2014-05-02 06:03:14 +00:00
|
|
|
|
2018-07-23 08:24:39 +00:00
|
|
|
return sock.getsockname()[0] # type: ignore
|
2020-04-04 20:09:11 +00:00
|
|
|
except OSError:
|
2017-08-06 16:15:17 +00:00
|
|
|
try:
|
|
|
|
return socket.gethostbyname(socket.gethostname())
|
|
|
|
except socket.gaierror:
|
2019-07-30 23:59:12 +00:00
|
|
|
return "127.0.0.1"
|
2015-08-03 15:05:33 +00:00
|
|
|
finally:
|
|
|
|
sock.close()
|
2014-05-02 06:03:14 +00:00
|
|
|
|
|
|
|
|
2015-01-18 05:55:33 +00:00
|
|
|
# Taken from http://stackoverflow.com/a/23728630
|
2018-07-23 08:24:39 +00:00
|
|
|
def get_random_string(length: int = 10) -> str:
|
2016-03-07 22:20:48 +00:00
|
|
|
"""Return a random string with letters and digits."""
|
2015-01-18 05:55:33 +00:00
|
|
|
generator = random.SystemRandom()
|
|
|
|
source_chars = string.ascii_letters + string.digits
|
|
|
|
|
2019-07-30 23:59:12 +00:00
|
|
|
return "".join(generator.choice(source_chars) for _ in range(length))
|
2015-01-18 05:55:33 +00:00
|
|
|
|
|
|
|
|
2014-04-15 06:48:00 +00:00
|
|
|
class OrderedEnum(enum.Enum):
|
2016-03-07 22:20:48 +00:00
|
|
|
"""Taken from Python 3.4.0 docs."""
|
2014-04-15 06:48:00 +00:00
|
|
|
|
2018-07-26 06:55:42 +00:00
|
|
|
# https://github.com/PyCQA/pylint/issues/2306
|
|
|
|
# pylint: disable=comparison-with-callable
|
|
|
|
|
2018-07-30 11:44:31 +00:00
|
|
|
def __ge__(self, other: ENUM_T) -> bool:
|
2016-03-07 22:20:48 +00:00
|
|
|
"""Return the greater than element."""
|
2014-04-15 06:48:00 +00:00
|
|
|
if self.__class__ is other.__class__:
|
2018-07-23 08:24:39 +00:00
|
|
|
return bool(self.value >= other.value)
|
2014-04-15 06:48:00 +00:00
|
|
|
return NotImplemented
|
|
|
|
|
2018-07-30 11:44:31 +00:00
|
|
|
def __gt__(self, other: ENUM_T) -> bool:
|
2016-03-07 22:20:48 +00:00
|
|
|
"""Return the greater element."""
|
2014-04-15 06:48:00 +00:00
|
|
|
if self.__class__ is other.__class__:
|
2018-07-23 08:24:39 +00:00
|
|
|
return bool(self.value > other.value)
|
2014-04-15 06:48:00 +00:00
|
|
|
return NotImplemented
|
|
|
|
|
2018-07-30 11:44:31 +00:00
|
|
|
def __le__(self, other: ENUM_T) -> bool:
|
2016-03-07 22:20:48 +00:00
|
|
|
"""Return the lower than element."""
|
2014-04-15 06:48:00 +00:00
|
|
|
if self.__class__ is other.__class__:
|
2018-07-23 08:24:39 +00:00
|
|
|
return bool(self.value <= other.value)
|
2014-04-15 06:48:00 +00:00
|
|
|
return NotImplemented
|
|
|
|
|
2018-07-30 11:44:31 +00:00
|
|
|
def __lt__(self, other: ENUM_T) -> bool:
|
2016-03-07 22:20:48 +00:00
|
|
|
"""Return the lower element."""
|
2014-04-15 06:48:00 +00:00
|
|
|
if self.__class__ is other.__class__:
|
2018-07-23 08:24:39 +00:00
|
|
|
return bool(self.value < other.value)
|
2014-04-15 06:48:00 +00:00
|
|
|
return NotImplemented
|
|
|
|
|
|
|
|
|
2018-07-20 08:45:20 +00:00
|
|
|
class Throttle:
|
2016-03-07 22:20:48 +00:00
|
|
|
"""A class for throttling the execution of tasks.
|
|
|
|
|
|
|
|
This method decorator adds a cooldown to a method to prevent it from being
|
2014-12-05 05:06:45 +00:00
|
|
|
called more then 1 time within the timedelta interval `min_time` after it
|
|
|
|
returned its result.
|
2014-12-04 09:14:27 +00:00
|
|
|
|
2014-12-05 05:06:45 +00:00
|
|
|
Calling a method a second time during the interval will return None.
|
2014-12-04 09:14:27 +00:00
|
|
|
|
2014-12-05 05:06:45 +00:00
|
|
|
Pass keyword argument `no_throttle=True` to the wrapped method to make
|
|
|
|
the call not throttled.
|
|
|
|
|
|
|
|
Decorator takes in an optional second timedelta interval to throttle the
|
|
|
|
'no_throttle' calls.
|
|
|
|
|
|
|
|
Adds a datetime attribute `last_call` to the method.
|
2014-12-04 09:14:27 +00:00
|
|
|
"""
|
|
|
|
|
2019-07-30 23:59:12 +00:00
|
|
|
def __init__(
|
|
|
|
self, min_time: timedelta, limit_no_throttle: Optional[timedelta] = None
|
|
|
|
) -> None:
|
2016-03-07 22:20:48 +00:00
|
|
|
"""Initialize the throttle."""
|
2014-12-04 09:14:27 +00:00
|
|
|
self.min_time = min_time
|
2014-12-05 05:06:45 +00:00
|
|
|
self.limit_no_throttle = limit_no_throttle
|
2014-12-04 09:14:27 +00:00
|
|
|
|
2018-07-23 08:24:39 +00:00
|
|
|
def __call__(self, method: Callable) -> Callable:
|
2016-03-07 22:20:48 +00:00
|
|
|
"""Caller for the throttle."""
|
2018-03-17 03:27:05 +00:00
|
|
|
# Make sure we return a coroutine if the method is async.
|
|
|
|
if asyncio.iscoroutinefunction(method):
|
2019-07-30 23:59:12 +00:00
|
|
|
|
2018-07-23 08:24:39 +00:00
|
|
|
async def throttled_value() -> None:
|
2018-03-17 03:27:05 +00:00
|
|
|
"""Stand-in function for when real func is being throttled."""
|
|
|
|
return None
|
2019-07-30 23:59:12 +00:00
|
|
|
|
2018-03-17 03:27:05 +00:00
|
|
|
else:
|
2019-07-30 23:59:12 +00:00
|
|
|
|
2018-07-23 08:24:39 +00:00
|
|
|
def throttled_value() -> None: # type: ignore
|
2018-03-17 03:27:05 +00:00
|
|
|
"""Stand-in function for when real func is being throttled."""
|
|
|
|
return None
|
|
|
|
|
2014-12-05 05:06:45 +00:00
|
|
|
if self.limit_no_throttle is not None:
|
|
|
|
method = Throttle(self.limit_no_throttle)(method)
|
|
|
|
|
2015-10-11 17:42:42 +00:00
|
|
|
# Different methods that can be passed in:
|
|
|
|
# - a function
|
|
|
|
# - an unbound function on a class
|
|
|
|
# - a method (bound function on a class)
|
|
|
|
|
|
|
|
# We want to be able to differentiate between function and unbound
|
|
|
|
# methods (which are considered functions).
|
2017-09-23 15:15:46 +00:00
|
|
|
# All methods have the classname in their qualname separated by a '.'
|
2015-10-09 06:49:55 +00:00
|
|
|
# Functions have a '.' in their qualname if defined inline, but will
|
|
|
|
# be prefixed by '.<locals>.' so we strip that out.
|
2019-07-30 23:59:12 +00:00
|
|
|
is_func = (
|
|
|
|
not hasattr(method, "__self__")
|
|
|
|
and "." not in method.__qualname__.split(".<locals>.")[-1]
|
|
|
|
)
|
2015-10-09 06:49:55 +00:00
|
|
|
|
2014-12-04 09:14:27 +00:00
|
|
|
@wraps(method)
|
2018-07-23 08:24:39 +00:00
|
|
|
def wrapper(*args: Any, **kwargs: Any) -> Union[Callable, Coroutine]:
|
2017-05-02 16:18:47 +00:00
|
|
|
"""Wrap that allows wrapped to be called only once per min_time.
|
2016-03-07 22:20:48 +00:00
|
|
|
|
2015-01-06 04:50:34 +00:00
|
|
|
If we cannot acquire the lock, it is running so return None.
|
2014-12-04 09:14:27 +00:00
|
|
|
"""
|
2019-07-30 23:59:12 +00:00
|
|
|
if hasattr(method, "__self__"):
|
|
|
|
host = getattr(method, "__self__")
|
2015-10-11 17:42:42 +00:00
|
|
|
elif is_func:
|
|
|
|
host = wrapper
|
|
|
|
else:
|
|
|
|
host = args[0] if args else wrapper
|
|
|
|
|
2020-05-09 11:08:40 +00:00
|
|
|
# pylint: disable=protected-access # to _throttle
|
2019-07-30 23:59:12 +00:00
|
|
|
if not hasattr(host, "_throttle"):
|
2016-02-27 22:18:56 +00:00
|
|
|
host._throttle = {}
|
2015-10-09 06:49:55 +00:00
|
|
|
|
2016-02-27 22:18:56 +00:00
|
|
|
if id(self) not in host._throttle:
|
|
|
|
host._throttle[id(self)] = [threading.Lock(), None]
|
|
|
|
throttle = host._throttle[id(self)]
|
2020-05-09 11:08:40 +00:00
|
|
|
# pylint: enable=protected-access
|
2016-02-27 22:18:56 +00:00
|
|
|
|
|
|
|
if not throttle[0].acquire(False):
|
2018-03-10 03:38:51 +00:00
|
|
|
return throttled_value()
|
2015-09-13 05:56:49 +00:00
|
|
|
|
2015-10-09 06:49:55 +00:00
|
|
|
# Check if method is never called or no_throttle is given
|
2019-07-30 23:59:12 +00:00
|
|
|
force = kwargs.pop("no_throttle", False) or not throttle[1]
|
2015-09-13 05:56:49 +00:00
|
|
|
|
2015-10-09 06:49:55 +00:00
|
|
|
try:
|
2016-02-27 22:18:56 +00:00
|
|
|
if force or utcnow() - throttle[1] > self.min_time:
|
2015-09-13 05:56:49 +00:00
|
|
|
result = method(*args, **kwargs)
|
2016-02-27 22:18:56 +00:00
|
|
|
throttle[1] = utcnow()
|
2018-07-23 08:24:39 +00:00
|
|
|
return result # type: ignore
|
2017-07-06 03:02:16 +00:00
|
|
|
|
2018-03-10 03:38:51 +00:00
|
|
|
return throttled_value()
|
2015-09-13 05:56:49 +00:00
|
|
|
finally:
|
2016-02-27 22:18:56 +00:00
|
|
|
throttle[0].release()
|
2014-12-04 09:14:27 +00:00
|
|
|
|
|
|
|
return wrapper
|