diff --git a/deploy.sh b/deploy.sh index b77e9c0..72bb07d 100644 --- a/deploy.sh +++ b/deploy.sh @@ -1,14 +1,18 @@ #!/usr/bin/env sh set -eu -# If venv doesnt exists create it -[ -d venv/ ] || python3 -m venv venv +# If venv exists delete it +[ -d venv/ ] && rm venv/ + +# Create a new venv +python3 -m venv venv + # Activate venv . venv/bin/activate + # Install all requirements pip install -r requirements.txt -r requirements-dev.txt - # Collect static files ./manage.py collectstatic --noinput diff --git a/fetcher/musicbrainz.py b/fetcher/musicbrainz.py index 79893fe..45a731c 100644 --- a/fetcher/musicbrainz.py +++ b/fetcher/musicbrainz.py @@ -6,8 +6,7 @@ musicbrainz entrega exceptuando los errores. from urllib.parse import quote, urlencode import logging import requests -from ratelimit import limits, sleep_and_retry -from utils import sanitize_keys +from utils import sanitize_keys, ratelimit HEADERS = {'accept': 'application/json', 'user-agent': 'MusicList/1.0 (danielcortes.xyz)'} MB_HOST = 'https://musicbrainz.org/ws/2' @@ -37,8 +36,6 @@ def _do_request(url): return response -@sleep_and_retry -@limits(calls=1, period=1) def _do_request_mb(url): """Does a request to a path of musicbrainz and returns the dictionary representing the json response @@ -52,6 +49,7 @@ def _do_request_mb(url): :return: The dictionary with the response or the status and his an error message """ + ratelimit() response = _do_request(url) if response.status_code == 200: diff --git a/musiclist/settings/__init__.py b/musiclist/settings/__init__.py index fb98c7a..1854fa7 100644 --- a/musiclist/settings/__init__.py +++ b/musiclist/settings/__init__.py @@ -149,7 +149,7 @@ LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'formatters': { - 'base': {'format': '[{levelname}]\t({name}): {message}', 'style': '{'} + 'base': {'format': '[{levelname}]({asctime})({name}): {message}', 'style': '{'} }, 'handlers': { diff --git a/requirements.txt b/requirements.txt index ec22fec..b17f7fb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ django-cors-middleware # To handle cors request django-oauth-toolkit # To handle user authentication djangorestframework # The framework to create the api pygments # To make cute pretty prints XD -ratelimit # To rate limit request to musicbrainz redis # To comunicate with redis django-redis # Redis backend for django django-rq #Job queue diff --git a/utils/__init__.py b/utils/__init__.py index e4e092c..ef90b5f 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -10,7 +10,7 @@ from django.http import JsonResponse from pygments import highlight from pygments.lexers import JsonLexer # pylint: disable=no-name-in-module from pygments.formatters import TerminalTrueColorFormatter # pylint: disable=no-name-in-module - +from utils.ratelimit import ratelimit _log = logging.getLogger('utils') _log.addHandler(logging.NullHandler()) diff --git a/utils/ratelimit.py b/utils/ratelimit.py new file mode 100644 index 0000000..6b8581c --- /dev/null +++ b/utils/ratelimit.py @@ -0,0 +1,33 @@ +"""Modulo encargado de la función de ratelimit""" + +import time +import logging + +from django.core.cache import cache +from django.utils import timezone + +_log = logging.getLogger('utils_ratelimit') +_log.addHandler(logging.NullHandler()) + + +def ratelimit(): + """Rate limit básico, espera un segundo entre ejecuciones + + Esto lo logra a través del cache, utilizando la key musicbrainz_ratelimit_SS en donde SS es el + segundo actual, si es que ya esta seteada la key, se esperaran 0.001 segundos, y se volverá a + intentar + """ + while not cache.set( + f'musicbrainz_ratelimit_{timezone.now().second}', + 'locked <3', + nx=True, + timeout=10): + # Le tengo algo de respeto a este sleep, si es demasiado corto igual y es un busy + # loop y eso es horrible para el rendimiento, pero si es muy largo igual pierdo demasiadas + # oportunidades de obtener un lock. + # Ademas la precision de sleep depende del sistema operativo, supuestamente linux es mejor + # para llevar esto, pero no estoy nada seguro~ + # https://stackoverflow.com/questions/1133857/how-accurate-is-pythons-time-sleep + time.sleep(0.001) + + _log.debug('Ratelimit allowing request on second %s', timezone.now().second)