diff --git a/fetcher/cache.py b/fetcher/cache.py index 8255f55..4c4ccf3 100644 --- a/fetcher/cache.py +++ b/fetcher/cache.py @@ -54,15 +54,15 @@ def get_discs_of_artist(mbid, limit, offset): jobs.load_artist_on_cache.delay(mbid) return None, 0 - release_ids = redis.zrange(key_releases, offset, limit) + release_ids = redis.zrange(key_releases, offset, offset + limit) keys = [f'release_group:{mbid}' for mbid in release_ids] if redis.exists(*keys) != len(keys): _log.debug('Aun no se cargan todas las release_groups del artista %s', mbid) jobs.load_artist_on_cache.delay(mbid) - return None, None + return None, 0 _log.info('Se encontraron los discos en redis') - return [json.loads(disc) for disc in redis.mget(keys)] + return [json.loads(disc) for disc in redis.mget(keys)], count def get_artist_of_disc(mbid): @@ -77,3 +77,168 @@ def get_artist_of_disc(mbid): _log.debug('Se encontro el artista') return json.loads(redis.get(f'artist:{artist_id}')) + + +def get_release(mbid): + _log.info('Intentado obtener la release %s en redis', mbid) + + with get_redis_connection() as redis: + release = redis.get(f'release:{mbid}') + if not release: + _log.info('La release no estaba en redis') + jobs.load_entities_of_release.delay(mbid) + return None + + return json.loads(release) + + +def get_releases_of_disc(mbid, limit, offset): + _log.info('Intentando obtener las releases del disco %s', mbid) + + with get_redis_connection() as redis: + key_releases = f'release_group:{mbid}:releases' + + if key_releases not in redis: + _log.debug('%s no existe en redis', key_releases) + jobs.load_entities_of_release_group.delay(mbid) + return None, 0 + + count = int(redis.get(f'{key_releases}:count')) + if count != redis.zcard(key_releases): + _log.debug('La cantidad de releases del disco %s no coinciden con el total', mbid) + jobs.load_entities_of_release_group.delay(mbid) + return None, 0 + + release_ids = redis.zrange(key_releases, offset, offset + limit) + keys = [f'release:{mbid}' for mbid in release_ids] + if redis.exists(*keys) != len(keys): + _log.debug('Aun no se cargan todas las releases del disco %s', mbid) + jobs.load_entities_of_release_group.delay(mbid) + return None, 0 + + _log.info('Se encontraron las releases en redis') + return [json.loads(disc) for disc in redis.mget(keys)], count + + +def get_artist_of_release(mbid): + _log.info('Intentando obtener al artista de la release %s en redis', mbid) + + with get_redis_connection() as redis: + artist_id = redis.get(f'release:{mbid}:artist') + if not artist_id: + _log.debug('No se encontro el artista') + jobs.load_entities_of_release.delay(mbid) + return None + + artist = redis.get(f'artist:{artist_id}') + if not artist: + _log.debug('El artista aun no se carga en redis') + return None + + _log.debug('Se encontro el artista') + return json.loads(artist) + + +def get_recording(mbid): + _log.info('Intentado obtener la grabacion %s en redis', mbid) + + with get_redis_connection() as redis: + recording = redis.get(f'recording:{mbid}') + if not recording: + _log.info('La grabacion no estaba en redis') + jobs.load_entities_of_recording.delay(mbid) + return None + + return json.loads(recording) + + +def get_recordings_of_release(mbid): + _log.info('Intentando obtener las grabaciones de la release %s', mbid) + + with get_redis_connection() as redis: + medias_key = f'release:{mbid}:media' + + if medias_key not in redis: + _log.debug('%s no existe en redis', medias_key) + jobs.load_entities_of_release.delay(mbid) + return None + + media_count = int(redis.get(f'{medias_key}:count')) + if redis.zcard(medias_key) != media_count: + _log.debug('La cantidad de medias de la release no coinciden con el total', mbid) + jobs.load_entities_of_release.delay(mbid) + return None + + medias = [json.loads(media) for media in redis.zrange(medias_key, 0, -1)] + for media in medias: + recordings_key = f'{medias_key}:{media.get("position")}:recordings' + if recordings_key not in redis: + _log.debug('%s no existe en redis', medias_key) + jobs.load_entities_of_release.delay(mbid) + return None + + recordings_count = int(redis.get(f'{recordings_key}:count')) + if redis.zcard(recordings_key) != recordings_count: + _log.debug('La cantidad de recordings de la media no coinciden con el total', mbid) + jobs.load_entities_of_release.delay(mbid) + return None + + recording_ids = redis.zrange(recordings_key, 0, -1) + keys = [f'recording:{mbid}' for mbid in recording_ids] + if redis.exists(*keys) != len(keys): + _log.debug('No estan todos los recordings almacenados') + jobs.load_entities_of_release.delay(mbid) + return None + + media['recordings'] = [] + for key in keys: + recording = json.loads(redis.get(key)) + media['recordings'].append(recording) + + return medias + + +def get_releases_of_recording(mbid, limit, offset): + _log.info('Intentando obtener la release de la recording %s', mbid) + + with get_redis_connection() as redis: + releases_key = f'recording:{mbid}:release' + if releases_key not in redis or f'{releases_key}:count' not in redis: + _log.debug('No existe %s en redis', releases_key) + jobs.load_entities_of_recording.delay(mbid) + return None, 0 + + count = int(redis.get(f'{releases_key}:count')) + if redis.zcard(releases_key) != count: + _log.debug('No estan almacenadas todas las keys de las releases') + jobs.load_entities_of_recording.delay(mbid) + return None, 0 + + release_ids = redis.zrange(releases_key, offset, offset + limit) + keys = [f'release:{mbid}' for mbid in release_ids] + if redis.exists(*keys) != len(keys): + _log.debug('No estan todas las releses') + jobs.load_entities_of_recording.delay(mbid) + return None, 0 + + _log.debug('Se encontraron las releases en redis') + return [json.loads(release) for release in redis.mget(keys)], count + + +def get_artist_of_recording(mbid): + _log.info('Intentando obtener el artista de la grabacion %s desde redis', mbid) + + with get_redis_connection() as redis: + artist_id = redis.get(f'recording:{mbid}:artist') + if not artist_id: + _log.debug('El artista no se encuentra en redis') + jobs.load_entities_of_recording.delay(mbid) + return None + + artist = redis.get(f'artist:{artist_id}') + if not artist: + _log.debug('El artista aun no se carga en redis') + return None + + _log.debug('Se encontro el artista') + return json.loads(artist) diff --git a/fetcher/medium.py b/fetcher/medium.py index c41d11b..bdc048e 100644 --- a/fetcher/medium.py +++ b/fetcher/medium.py @@ -257,66 +257,42 @@ def get_release(mbid): """Obtiene una release desde musicbrainz incluyendo sus artistas""" _log.info('Obteniendo release %s', mbid) - with get_redis_connection() as redis: - _log.debug('Intentando obtener release %s desde redis', mbid) - mb_release = redis.get(f'release:{mbid}') - if mb_release is None: - _log.debug('La release %s no estaba en redis, cargando desde musicbrainz', mbid) - mb_release = mb.get_release_by_mbid(mbid, includes=['artists']) - else: - _log.debug('La release %s se encontró en redis', mbid) - mb_release = json.loads(mb_release) - - if 'error' in mb_release: - _log.error('Error en la release %s', mbid) - return mb_release - - jobs.load_entities_of_release.delay(mbid) + mb_release = cache.get_release(mbid) + if mb_release: return map_release(mb_release) + mb_release = mb.get_release_by_mbid(mbid, includes=['artists']) + + if 'error' in mb_release: + _log.error('Error al buscar', mb_release) + return mb_release + + return map_release(mb_release) + def get_releases_of_disc(mbid, limit, page): """Obtiene las releases de un disco desde musicbrainz""" _log.info('Obteniendo releases del disco %s', mbid) - mb_releases = [] offset = limit * (page - 1) - total = 0 - with get_redis_connection() as redis: - _log.debug('Intentando obtener las releases del disco %s desde redis', mbid) + mb_releases, total = cache.get_releases_of_disc(mbid, limit, offset) + if mb_releases and total: + return { + 'paginate': paginate(total, limit, page), + 'releases': [map_release(release) for release in mb_releases] + } - key_releases = f'release_group:{mbid}:releases' - if key_releases in redis: - # TODO, Misma race condition que en get disc of artist - if int(redis.get(f'{key_releases}:count')) == redis.zcard(key_releases): - _log.debug('Releases del disco %s encontradas en redis', mbid) - mb_releases = [get_release(mbid) for mbid in redis.zrange(key_releases, - offset, limit)] - total = redis.zcard(key_releases) - else: - _log.debug('La cantidad de releases que hay almacenadas para el disco %s no ' - 'coinciden con las totales') - else: - _log.debug('%s no se encuentra en redis', key_releases) + mb_releases = mb.browse_releases(params={'release-group': mbid}, + includes=['artist-credits'], + limit=limit, offset=limit * (page - 1)) - if len(mb_releases) == 0: - _log.debug('Cargar desde musicbrainz las releases del disco %s', mbid) + if 'error' in mb_releases: + _log.error('Error al buscar %s', mb_releases) + return mb_releases - # Si es que no se encontraron releases antes es probable que nunca se cargo en cache el - # release group - jobs.load_entities_of_release_group.delay(mbid) - - mb_releases = mb.browse_releases(params={'release-group': mbid}, - includes=['artist-credits'], - limit=limit, offset=limit * (page - 1)) - - if 'error' in mb_releases: - _log.error('Error en las releases %s', mb_releases) - return mb_releases - - total = mb_releases.get('release_count') - mb_releases = mb_releases.get('releases') + total = mb_releases.get('release_count') + mb_releases = mb_releases.get('releases') return { 'paginate': paginate(total, limit, page), @@ -324,40 +300,25 @@ def get_releases_of_disc(mbid, limit, page): } -def get_artist_of_release(mbid, limit, page): +def get_artist_of_release(mbid): """Obtiene el artista de una release""" _log.info('Obteniendo el artista de la release %s', mbid) - mb_artist = None + mb_artist = cache.get_artist_of_release(mbid) + if mb_artist: + return map_artist(mb_artist) - with get_redis_connection() as redis: - _log.debug('Intentando obtener al artista de la release %s desde redis', mbid) + mb_artist_browse = mb.browse_artists(params={'release': mbid}, + includes=['tags'], + limit=1, offset=0) - key = f'release:{mbid}:artist' - if key in redis: - _log.debug('El artista de la release %s se encontro en redis', mbid) - mb_artist = get_artist(redis.get(key)) - # TODO parece que tengo un error aqui, el mb_artist lo cargo con get_artist - # eso significa que ya lo mapee, pero al final de la funcion lo estoy mapeando de nuevo - # eso tiene que traer errores si o tambien + if 'error' in mb_artist_browse: + _log.error('Error al buscar %s', mb_artist_browse) + return mb_artist_browse - if mb_artist is None: - _log.debug('El artista de la release %s se no se encontraba en redis', mbid) - mb_artist_browse = mb.browse_artists(params={'release': mbid}, - includes=['tags'], - limit=limit, - offset=limit * (page - 1)) + mb_artist = mb_artist_browse.get('artists')[0] - if 'error' in mb_artist_browse: - _log.error('El browse de artista retorno con error %s', mb_artist_browse) - return mb_artist_browse - - mb_artist = mb_artist_browse.get('artists')[0] - jobs.load_artist_on_cache.delay(mb_artist) - - return { - 'artist': map_artist(mb_artist) - } + return map_artist(mb_artist) ## @@ -367,23 +328,17 @@ def get_artist_of_release(mbid, limit, page): def get_recording(mbid): """Obtiene una grabación""" - _log.info('Obteniendo el recording %s', mbid) + _log.info('Obteniendo la grabacion %s', mbid) - with get_redis_connection() as redis: - _log.debug('Intentando obtener el recording %s desde redis', mbid) - mb_recording = redis.get(f'recording:{mbid}') - if mb_recording is None: - _log.debug('El recording %s no estaba en redis, cargando desde musicbrainz', mbid) - mb_recording = mb.get_recording_by_mbid(mbid) - else: - _log.debug('El recording %s se enontro en redis', mbid) - mb_recording = json.loads(mb_recording) + mb_recording = cache.get_recording(mbid) + if mb_recording: + return map_recording(mb_recording) - if 'error' in mb_recording: - _log.error('Error al cargar recording') - return mb_recording + mb_recording = mb.get_recording_by_mbid(mbid) - jobs.load_entities_of_recording.delay(mbid) + if 'error' in mb_recording: + _log.error('Error al buscar %s', mb_recording) + return mb_recording return map_recording(mb_recording) @@ -397,89 +352,44 @@ def get_recordings_of_release(mbid): """ _log.info('Obteniendo recordings de la release %s', mbid) - medias = [] + mb_medias = cache.get_recordings_of_release(mbid) + if mb_medias: + return {'medias': mb_medias} - with get_redis_connection() as redis: - _log.debug('Intentando obtener los recordings de la release %s desde redis', mbid) - medias_key = f'release:{mbid}:media' - if medias_key in redis: - if redis.zcard(medias_key) == int(redis.get(f'{medias_key}:count')): - medias = [json.loads(media) for media in redis.zrange(medias_key, 0, -1)] - for media in medias: - recordings_key = f'{medias_key}:{media.get("position")}:recordings' - if redis.zcard(recordings_key) == int(redis.get(f'{recordings_key}:count')): - recordings_id = redis.zrange(recordings_key, 0, -1) - media['recordings'] = [] - for recording_id in recordings_id: - recording = json.loads(redis.get(f'recording:{recording_id}')) - media['recordings'].append(recording) - else: - _log.debug('No estan todas las recordings de la release %s que deberian ' - 'estar', mbid) - else: - _log.debug('La cantidad de medias de la release %s encontradas no corresponden al ' - 'total registrado', mbid) - else: - _log.debug('No existe %s en redis', medias_key) + mb_release = mb.get_release_by_mbid(mbid, ['recordings', 'artist-credits']) - if len(medias) == 0: - _log.debug('No se encontraron recordings de la release %s, se cargara con musicbrainz', - mbid) + if 'error' in mb_release: + _log.debug('Error al cargar recordings %s', mb_release) + return mb_release - # Si es que no habían medias cargadas en cache, puede ser que la release nunca se alla - # cargado. - jobs.load_entities_of_release.delay(mbid) - - mb_release = mb.get_release_by_mbid(mbid, ['recordings']) - - if 'error' in mb_release: - _log.debug('Error al cargar recordings %s', mb_release) - return mb_release - - medias = [map_media(media) for media in mb_release.get('media', [])] + medias = [map_media(media) for media in mb_release.get('media', [])] return {'medias': medias} -def get_release_of_recording(mbid, limit, page): +def get_releases_of_recording(mbid, limit, page): """Obtiene la release de una grabacion incluyendo los creditos a su artista""" _log.info('Obteniendo las releases de la recording %s', mbid) - releases = [] offset = limit * (page - 1) - total = 0 - with get_redis_connection() as redis: - _log.debug('Intentando obtener las releases de la recording %s desde redis', mbid) - releases_key = f'recording:{mbid}:release' - if releases_key in redis or f'{releases_key}:count' in redis: - if redis.zcard(releases_key) == int(redis.get(f'{releases_key}:count')): - release_mbids = redis.zrange(releases_key, offset, offset + limit) - if redis.exists(*[f'release:{mbid}' for mbid in release_mbids]): - total = redis.zcard(releases_key) - releases = [get_release(mbid) for mbid in release_mbids] - else: - _log.debug('No estan almacenadas todas las releases de la recording %s', mbid) - else: - _log.debug('No hay almacenadas tantas que de release de la recording %s como ' - 'deberian haber', mbid) - else: - _log.debug('No esta %s ni %s:count en redis', releases_key, releases_key) + releases, total = cache.get_releases_of_recording(mbid, limit, offset) + if releases and total: + return { + 'paginate': paginate(total, limit, page), + 'releases': releases + } - if len(releases) == 0: - _log.debug('Las releases de la recording %s no estaban cargadas en redis, se va a utilizar ' - 'musicbrainz', mbid) - mb_releases_browse = mb.browse_releases(params={'recording': mbid}, - includes=['artist-credits'], - limit=limit, offset=limit * (page - 1)) + mb_releases_browse = mb.browse_releases(params={'recording': mbid}, + includes=['artist-credits', 'recordings'], + limit=limit, offset=offset) - if 'error' in mb_releases_browse: - _log.debug('Error al hacer browse de las releases de la recording %s %s', - mbid, mb_releases_browse) - return mb_releases_browse + if 'error' in mb_releases_browse: + _log.debug('Error al buscar %s', mb_releases_browse) + return mb_releases_browse + + releases = [map_release(release) for release in mb_releases_browse.get('releases')] - releases = [map_release(release) for release in mb_releases_browse.get('releases')] - jobs.load_entities_of_recording.delay(mbid) return { 'paginate': paginate(total, limit, page), 'releases': releases @@ -490,33 +400,21 @@ def get_artist_of_recording(mbid): """Obtiene el artista de una grabacion""" _log.info('Obteniendo el artista de la recording %s', mbid) - artist = None + mb_artist = cache.get_artist_of_recording(mbid) + if mb_artist: + return map_artist(mb_artist) - with get_redis_connection() as redis: - _log.debug('Intentando obtener el artista de la recording %s desde redis', mbid) - artist_key = f'recording:{mbid}:artist' - if artist_key in redis: - _log.debug('Se encontro el artista de la recording %s en redis', mbid) - artist = get_artist(redis.get(artist_key)) - else: - _log.debug('El artista de la recording %s no estaba en redis', mbid) + mb_artist_browse = mb.browse_artists(params={'recording': mbid}, + includes=['tags'], + limit=1, offset=0) - if artist is None: - _log.debug('Obteniendo el artista de la recording %s desde musicbrainz', mbid) - mb_artist_browse = mb.browse_artists(params={'recording': mbid}, - includes=['tags'], - limit=1, offset=0) + if 'error' in mb_artist_browse: + _log.debug('Error al buscar %s', mb_artist_browse) + return mb_artist_browse - if 'error' in mb_artist_browse: - _log.debug('Erro al obtener artista %s', mb_artist_browse) - return mb_artist_browse + mb_artist = mb_artist_browse.get('artists')[0] - artist = mb_artist_browse.get('artists')[0] - jobs.load_artist_on_cache.delay(artist.get('id')) - - return { - 'artist': artist - } + return map_artist(mb_artist) ## @@ -563,7 +461,7 @@ def get_cover_art_release(mbid): def get_cover_art_recording(mbid): """Obtiene el cover art de una grabacion""" - release = get_release_of_recording(mbid, limit=1, page=1) + release = get_releases_of_recording(mbid, limit=1, page=1)[0] if 'error' in release: return None diff --git a/fetcher/urls.py b/fetcher/urls.py index 2ffec6b..f5c61b3 100644 --- a/fetcher/urls.py +++ b/fetcher/urls.py @@ -23,7 +23,7 @@ urlpatterns = [ path('recording/', views.search_recording), path('recording//', views.get_recording), - path('recording//release/', views.get_release_of_recording), + path('recording//releases/', views.get_releases_of_recording), path('recording//artist/', views.get_artist_of_recording), path('recording//coverart/', views.get_cover_art_of_recording), ] diff --git a/fetcher/views.py b/fetcher/views.py index 925f72a..0e8f602 100644 --- a/fetcher/views.py +++ b/fetcher/views.py @@ -114,15 +114,9 @@ def get_releases_of_disc(request, mbid): @api_view(['GET']) def get_artist_of_release(request, mbid): - """ Obtiene el artista de una release dada su mbid + """ Obtiene el artista de una release dada su mbid""" - Como los datos son paginables la query puede contener per_page y page para definir cuantos - elementos mostrar por pagina y que pagina mostrar. - """ - limit = int(request.GET.get('per_page', 10)) - page = int(request.GET.get('page', 1)) - - return Response(medium.get_artist_of_release(mbid, limit, page)) + return Response(medium.get_artist_of_release(mbid)) @api_view(['GET']) @@ -157,7 +151,7 @@ def get_recordings_of_release(request, mbid): @api_view(['GET']) -def get_release_of_recording(request, mbid): +def get_releases_of_recording(request, mbid): """ Obtiene la release de una recording dada su mbid Como los datos son paginables la query puede contener per_page y page para definir cuantos @@ -166,7 +160,7 @@ def get_release_of_recording(request, mbid): limit = int(request.GET.get('per_page', 100)) page = int(request.GET.get('page', 1)) - return Response(medium.get_release_of_recording(mbid, limit, page)) + return Response(medium.get_releases_of_recording(mbid, limit, page)) @api_view(['GET'])