skrd/musiclist-server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Coverart archive no tiene limite de requests!

Browse Source
This commit is contained in:
Daniel Cortes
2020-06-04 06:18:13 -04:00
parent 50b5694865
commit 8b90a876dc
3 changed files with 73 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fetcher/medium.py
View File

@@ -34,7 +34,6 @@ def map_artist(mb_artist):
def map_artist_credit(mb_artist_credit):
pretty_print_json(mb_artist_credit)
return {
'id': mb_artist_credit.get('artist').get('id'),
'name': mb_artist_credit.get('artist').get('name'),

104
fetcher/musicbrainz.py
View File

@@ -19,51 +19,83 @@ _log = logging.getLogger(__name__)
_log.addHandler(logging.NullHandler())
@sleep_and_retry
@limits(calls=1, period=1)
def _do_request(url, host=_mb_host):
"""Does a request to a path and returns the dictionary representing the json response
If the response is 200 it will return the full dictionary of the json that the response
contains, and if is a 307 code, it will create a dictionary containing the response code
and the link where is redirecting, any other response code will be just appended to a
dictionary and returning.
def _do_request(url):
"""Does a request to an url
:param str url: URL where to do the request
:return: The whole response object
:raises ValueError when user-agent isn't set
:raises ValueError when url isn't set
:returns A dictionary with the request response
it always has a status key with the status code by its value,
the rest of the response will be contained in the 'response' key
if the request didn't fail
:return the request response
"""
if not _headers['user-agent']:
raise ValueError('User Agent isn\'t set')
if not url:
raise ValueError('URL cant be empty')
_log.info(f'Doing request to "{url}" with headers {_headers}')
r = requests.get(url, headers=_headers)
response = requests.get(url, headers=_headers)
_log.info(f'Request returned with status code {r.status_code}')
_log.info(f'Request returned with status code {response.status_code}')
return response
@sleep_and_retry
@limits(calls=1, period=1)
def _do_request_mb(url):
"""Does a request to a path of musicbrainz and returns the dictionary representing the json
response
If the response is 200 it will return the full dictionary of the json that the response
contains, and if is a code 500 it will inform of the server error, any other response
code will be just appended to a dictionary and returned.
:param str url: URL where to do the request
:raises ValueError when user-agent isn't set
:return: The dictionary with the response or the status and his an error message
"""
if not _headers['user-agent']:
raise ValueError('User Agent isn\'t set')
r = _do_request(url)
if r.status_code == 200:
response = r.json(object_hook=sanitize_keys)
elif r.status_code == 500:
response = {'status': r.status_code, 'error': f'Error del servidor'}
elif host == _ca_host:
if r.status_code == 400:
response = {'status': r.status_code, 'error': f'El mbid es invalido'}
elif r.status_code == 404:
response = {'status': r.status_code, 'error': f'No encontrado'}
elif r.status_code == 405:
response = {'status': r.status_code, 'error': f'Metodo erroneo'}
elif r.status_code == 503:
response = {'status': r.status_code, 'error': f'Rate limit exedido'}
else:
response = {'status': r.status_code, 'error': r.json()['error']}
return response
def _do_request_ca(url):
"""Does a request to a path of cover art archive and returns the dictionary representing
the json response
If the response is 200 it will return the full dictionary of the json that the response
contains, and if is a code 500 it will inform of the server error, any other response
code will be just appended to a dictionary and returned.
:param str url: URL where to do the request
:raises ValueError when user-agent isn't set
:return: The dictionary with the response or the status and his an error message
"""
if not _headers['user-agent']:
raise ValueError('User Agent isn\'t set')
r = _do_request(url)
if r.status_code == 200:
response = r.json(object_hook=sanitize_keys)
elif r.status_code == 500:
response = {'status': r.status_code, 'error': f'Error del servidor'}
elif r.status_code == 400:
response = {'status': r.status_code, 'error': f'El mbid es invalido'}
elif r.status_code == 404:
response = {'status': r.status_code, 'error': f'No encontrado'}
elif r.status_code == 405:
response = {'status': r.status_code, 'error': f'Metodo erroneo'}
elif r.status_code == 503:
response = {'status': r.status_code, 'error': f'Rate limit exedido'}
else:
response = {'status': r.status_code, 'error': r.json()['error']}
@@ -84,12 +116,12 @@ def _get(entity_type, mbid, includes=None):
if includes is None:
_log.info(f'Getting {entity_type} of mbid {mbid}')
return _do_request(f'{_mb_host}/{entity_type}/{mbid}')
return _do_request_mb(f'{_mb_host}/{entity_type}/{mbid}')
else:
_log.info(f'Getting {entity_type} of mbid {mbid} including {includes}')
_includes = quote('+'.join(includes))
return _do_request(f'{_mb_host}/{entity_type}/{mbid}?inc={_includes}')
return _do_request_mb(f'{_mb_host}/{entity_type}/{mbid}?inc={_includes}')
def _search(entity_type, query, includes=None, limit=25, offset=0):
@@ -114,7 +146,7 @@ def _search(entity_type, query, includes=None, limit=25, offset=0):
if includes is not None:
_query['inc'] = '+'.join(includes)
return _do_request(f'{_mb_host}/{entity_type}/?{urlencode(_query)}')
return _do_request_mb(f'{_mb_host}/{entity_type}/?{urlencode(_query)}')
def _browse(entity_type, params, includes=None, limit=25, offset=0):
@@ -142,7 +174,7 @@ def _browse(entity_type, params, includes=None, limit=25, offset=0):
_log.info(f'Browsing {entity_type} with parameters {params} including {includes} at offset {offset} with limit of {limit}')
_query = urlencode({**params, 'inc': '+'.join(includes), 'limit': limit, 'offset': offset})
return _do_request(f'{_mb_host}/{entity_type}?{_query}')
return _do_request_mb(f'{_mb_host}/{entity_type}?{_query}')
def _ca(entity_type, mbid):
@@ -158,7 +190,7 @@ def _ca(entity_type, mbid):
_url = f'{_ca_host}/{entity_type}/{mbid}'
return _do_request(_url, host=_ca_host)
return _do_request_ca(_url)
@cache

10
fetcher/tests.py
View File

@@ -11,21 +11,21 @@ class MusicBrainzTestCase(TestCase):
def test_can_do_basic_request(self):
artist_url = 'https://musicbrainz.org/ws/2/artist/65f4f0c5-ef9e-490c-aee3-909e7ae6b2ab'
musicbrainz._do_request(artist_url)
musicbrainz._do_request_mb(artist_url)
def test_do_request_response_format(self):
artist_url = 'https://musicbrainz.org/ws/2/artist/65f4f0c5-ef9e-490c-aee3-909e7ae6b2ab'
response = musicbrainz._do_request(artist_url)
response = musicbrainz._do_request_mb(artist_url)
self.assertIsInstance(response, dict)
self.assertEquals(response['status'], 200)
self.assertIsInstance(response['response'], dict)
def test_do_request_input_validation(self):
artist_url = 'https://musicbrainz.org/ws/2/artist/65f4f0c5-ef9e-490c-aee3-909e7ae6b2ab'
self.assertRaises(ValueError, musicbrainz._do_request, None)
self.assertRaises(ValueError, musicbrainz._do_request, '')
self.assertRaises(ValueError, musicbrainz._do_request_mb, None)
self.assertRaises(ValueError, musicbrainz._do_request_mb, '')
musicbrainz._headers['user-agent'] = None
self.assertRaises(ValueError, musicbrainz._do_request, artist_url)
self.assertRaises(ValueError, musicbrainz._do_request_mb, artist_url)
def test_cache(self):
def fun():