Continua la separacion de cache

fetcher/cache.py

@@ -54,15 +54,15 @@ def get_discs_of_artist(mbid, limit, offset):
             jobs.load_artist_on_cache.delay(mbid)
             return None, 0
 
-        release_ids = redis.zrange(key_releases, offset, limit)
+        release_ids = redis.zrange(key_releases, offset, offset + limit)
         keys = [f'release_group:{mbid}' for mbid in release_ids]
         if redis.exists(*keys) != len(keys):
             _log.debug('Aun no se cargan todas las release_groups del artista %s', mbid)
             jobs.load_artist_on_cache.delay(mbid)
-            return None, None
+            return None, 0
 
         _log.info('Se encontraron los discos en redis')
-        return [json.loads(disc) for disc in redis.mget(keys)]
+        return [json.loads(disc) for disc in redis.mget(keys)], count
 
 
 def get_artist_of_disc(mbid):
@@ -77,3 +77,168 @@ def get_artist_of_disc(mbid):
 
         _log.debug('Se encontro el artista')
         return json.loads(redis.get(f'artist:{artist_id}'))
+
+
+def get_release(mbid):
+    _log.info('Intentado obtener la release %s en redis', mbid)
+
+    with get_redis_connection() as redis:
+        release = redis.get(f'release:{mbid}')
+        if not release:
+            _log.info('La release no estaba en redis')
+            jobs.load_entities_of_release.delay(mbid)
+            return None
+
+        return json.loads(release)
+
+
+def get_releases_of_disc(mbid, limit, offset):
+    _log.info('Intentando obtener las releases del disco %s', mbid)
+
+    with get_redis_connection() as redis:
+        key_releases = f'release_group:{mbid}:releases'
+
+        if key_releases not in redis:
+            _log.debug('%s no existe en redis', key_releases)
+            jobs.load_entities_of_release_group.delay(mbid)
+            return None, 0
+
+        count = int(redis.get(f'{key_releases}:count'))
+        if count != redis.zcard(key_releases):
+            _log.debug('La cantidad de releases del disco %s no coinciden con el total', mbid)
+            jobs.load_entities_of_release_group.delay(mbid)
+            return None, 0
+
+        release_ids = redis.zrange(key_releases, offset, offset + limit)
+        keys = [f'release:{mbid}' for mbid in release_ids]
+        if redis.exists(*keys) != len(keys):
+            _log.debug('Aun no se cargan todas las releases del disco %s', mbid)
+            jobs.load_entities_of_release_group.delay(mbid)
+            return None, 0
+
+        _log.info('Se encontraron las releases en redis')
+        return [json.loads(disc) for disc in redis.mget(keys)], count
+
+
+def get_artist_of_release(mbid):
+    _log.info('Intentando obtener al artista de la release %s en redis', mbid)
+
+    with get_redis_connection() as redis:
+        artist_id = redis.get(f'release:{mbid}:artist')
+        if not artist_id:
+            _log.debug('No se encontro el artista')
+            jobs.load_entities_of_release.delay(mbid)
+            return None
+
+        artist = redis.get(f'artist:{artist_id}')
+        if not artist:
+            _log.debug('El artista aun no se carga en redis')
+            return None
+
+        _log.debug('Se encontro el artista')
+        return json.loads(artist)
+
+
+def get_recording(mbid):
+    _log.info('Intentado obtener la grabacion %s en redis', mbid)
+
+    with get_redis_connection() as redis:
+        recording = redis.get(f'recording:{mbid}')
+        if not recording:
+            _log.info('La grabacion no estaba en redis')
+            jobs.load_entities_of_recording.delay(mbid)
+            return None
+
+        return json.loads(recording)
+
+
+def get_recordings_of_release(mbid):
+    _log.info('Intentando obtener las grabaciones de la release %s', mbid)
+
+    with get_redis_connection() as redis:
+        medias_key = f'release:{mbid}:media'
+
+        if medias_key not in redis:
+            _log.debug('%s no existe en redis', medias_key)
+            jobs.load_entities_of_release.delay(mbid)
+            return None
+
+        media_count = int(redis.get(f'{medias_key}:count'))
+        if redis.zcard(medias_key) != media_count:
+            _log.debug('La cantidad de medias de la release no coinciden con el total', mbid)
+            jobs.load_entities_of_release.delay(mbid)
+            return None
+
+        medias = [json.loads(media) for media in redis.zrange(medias_key, 0, -1)]
+        for media in medias:
+            recordings_key = f'{medias_key}:{media.get("position")}:recordings'
+            if recordings_key not in redis:
+                _log.debug('%s no existe en redis', medias_key)
+                jobs.load_entities_of_release.delay(mbid)
+                return None
+
+            recordings_count = int(redis.get(f'{recordings_key}:count'))
+            if redis.zcard(recordings_key) != recordings_count:
+                _log.debug('La cantidad de recordings de la media no coinciden con el total', mbid)
+                jobs.load_entities_of_release.delay(mbid)
+                return None
+
+            recording_ids = redis.zrange(recordings_key, 0, -1)
+            keys = [f'recording:{mbid}' for mbid in recording_ids]
+            if redis.exists(*keys) != len(keys):
+                _log.debug('No estan todos los recordings almacenados')
+                jobs.load_entities_of_release.delay(mbid)
+                return None
+
+            media['recordings'] = []
+            for key in keys:
+                recording = json.loads(redis.get(key))
+                media['recordings'].append(recording)
+
+        return medias
+
+
+def get_releases_of_recording(mbid, limit, offset):
+    _log.info('Intentando obtener la release de la recording %s', mbid)
+
+    with get_redis_connection() as redis:
+        releases_key = f'recording:{mbid}:release'
+        if releases_key not in redis or f'{releases_key}:count' not in redis:
+            _log.debug('No existe %s en redis', releases_key)
+            jobs.load_entities_of_recording.delay(mbid)
+            return None, 0
+
+        count = int(redis.get(f'{releases_key}:count'))
+        if redis.zcard(releases_key) != count:
+            _log.debug('No estan almacenadas todas las keys de las releases')
+            jobs.load_entities_of_recording.delay(mbid)
+            return None, 0
+
+        release_ids = redis.zrange(releases_key, offset, offset + limit)
+        keys = [f'release:{mbid}' for mbid in release_ids]
+        if redis.exists(*keys) != len(keys):
+            _log.debug('No estan todas las releses')
+            jobs.load_entities_of_recording.delay(mbid)
+            return None, 0
+
+        _log.debug('Se encontraron las releases en redis')
+        return [json.loads(release) for release in redis.mget(keys)], count
+
+
+def get_artist_of_recording(mbid):
+    _log.info('Intentando obtener el artista de la grabacion %s desde redis', mbid)
+
+    with get_redis_connection() as redis:
+        artist_id = redis.get(f'recording:{mbid}:artist')
+        if not artist_id:
+            _log.debug('El artista no se encuentra en redis')
+            jobs.load_entities_of_recording.delay(mbid)
+            return None
+
+        artist = redis.get(f'artist:{artist_id}')
+        if not artist:
+            _log.debug('El artista aun no se carga en redis')
+            return None
+
+        _log.debug('Se encontro el artista')
+        return json.loads(artist)

fetcher/medium.py

@@ -257,66 +257,42 @@ def get_release(mbid):
     """Obtiene una release desde musicbrainz incluyendo sus artistas"""
     _log.info('Obteniendo release %s', mbid)
 
-    with get_redis_connection() as redis:
+    mb_release = cache.get_release(mbid)
-        _log.debug('Intentando obtener release %s desde redis', mbid)
+    if mb_release:
-        mb_release = redis.get(f'release:{mbid}')
-        if mb_release is None:
-            _log.debug('La release %s no estaba en redis, cargando desde musicbrainz', mbid)
-            mb_release = mb.get_release_by_mbid(mbid, includes=['artists'])
-        else:
-            _log.debug('La release %s se encontró en redis', mbid)
-            mb_release = json.loads(mb_release)
-
-        if 'error' in mb_release:
-            _log.error('Error en la release %s', mbid)
-            return mb_release
-
-        jobs.load_entities_of_release.delay(mbid)
         return map_release(mb_release)
 
+    mb_release = mb.get_release_by_mbid(mbid, includes=['artists'])
+
+    if 'error' in mb_release:
+        _log.error('Error al buscar', mb_release)
+        return mb_release
+
+    return map_release(mb_release)
+
 
 def get_releases_of_disc(mbid, limit, page):
     """Obtiene las releases de un disco desde musicbrainz"""
     _log.info('Obteniendo releases del disco %s', mbid)
 
-    mb_releases = []
     offset = limit * (page - 1)
-    total = 0
 
-    with get_redis_connection() as redis:
+    mb_releases, total = cache.get_releases_of_disc(mbid, limit, offset)
-        _log.debug('Intentando obtener las releases del disco %s desde redis', mbid)
+    if mb_releases and total:
+        return {
+            'paginate': paginate(total, limit, page),
+            'releases': [map_release(release) for release in mb_releases]
+        }
 
-        key_releases = f'release_group:{mbid}:releases'
+    mb_releases = mb.browse_releases(params={'release-group': mbid},
-        if key_releases in redis:
+                                     includes=['artist-credits'],
-            # TODO, Misma race condition que en get disc of artist
+                                     limit=limit, offset=limit * (page - 1))
-            if int(redis.get(f'{key_releases}:count')) == redis.zcard(key_releases):
-                _log.debug('Releases del disco %s encontradas en redis', mbid)
-                mb_releases = [get_release(mbid) for mbid in redis.zrange(key_releases,
-                                                                          offset, limit)]
-                total = redis.zcard(key_releases)
-            else:
-                _log.debug('La cantidad de releases que hay almacenadas para el disco %s no '
-                           'coinciden con las totales')
-        else:
-            _log.debug('%s no se encuentra en redis', key_releases)
 
-    if len(mb_releases) == 0:
+    if 'error' in mb_releases:
-        _log.debug('Cargar desde musicbrainz las releases del disco %s', mbid)
+        _log.error('Error al buscar %s', mb_releases)
+        return mb_releases
 
-        # Si es que no se encontraron releases antes es probable que nunca se cargo en cache el
+    total = mb_releases.get('release_count')
-        # release group
+    mb_releases = mb_releases.get('releases')
-        jobs.load_entities_of_release_group.delay(mbid)
-
-        mb_releases = mb.browse_releases(params={'release-group': mbid},
-                                         includes=['artist-credits'],
-                                         limit=limit, offset=limit * (page - 1))
-
-        if 'error' in mb_releases:
-            _log.error('Error en las releases %s', mb_releases)
-            return mb_releases
-
-        total = mb_releases.get('release_count')
-        mb_releases = mb_releases.get('releases')
 
     return {
         'paginate': paginate(total, limit, page),
@@ -324,40 +300,25 @@ def get_releases_of_disc(mbid, limit, page):
     }
 
 
-def get_artist_of_release(mbid, limit, page):
+def get_artist_of_release(mbid):
     """Obtiene el artista de una release"""
     _log.info('Obteniendo el artista de la release %s', mbid)
 
-    mb_artist = None
+    mb_artist = cache.get_artist_of_release(mbid)
+    if mb_artist:
+        return map_artist(mb_artist)
 
-    with get_redis_connection() as redis:
+    mb_artist_browse = mb.browse_artists(params={'release': mbid},
-        _log.debug('Intentando obtener al artista de la release %s desde redis', mbid)
+                                         includes=['tags'],
+                                         limit=1, offset=0)
 
-        key = f'release:{mbid}:artist'
+    if 'error' in mb_artist_browse:
-        if key in redis:
+        _log.error('Error al buscar %s', mb_artist_browse)
-            _log.debug('El artista de la release %s se encontro en redis', mbid)
+        return mb_artist_browse
-            mb_artist = get_artist(redis.get(key))
-            # TODO parece que tengo un error aqui, el mb_artist lo cargo con get_artist
-            # eso significa que ya lo mapee, pero al final de la funcion lo estoy mapeando de nuevo
-            # eso tiene que traer errores si o tambien
 
-    if mb_artist is None:
+    mb_artist = mb_artist_browse.get('artists')[0]
-        _log.debug('El artista de la release %s se no se encontraba en redis', mbid)
-        mb_artist_browse = mb.browse_artists(params={'release': mbid},
-                                             includes=['tags'],
-                                             limit=limit,
-                                             offset=limit * (page - 1))
 
-        if 'error' in mb_artist_browse:
+    return map_artist(mb_artist)
-            _log.error('El browse de artista retorno con error %s', mb_artist_browse)
-            return mb_artist_browse
-
-        mb_artist = mb_artist_browse.get('artists')[0]
-        jobs.load_artist_on_cache.delay(mb_artist)
-
-    return {
-        'artist': map_artist(mb_artist)
-    }
 
 
 ##
@@ -367,23 +328,17 @@ def get_artist_of_release(mbid, limit, page):
 
 def get_recording(mbid):
     """Obtiene una grabación"""
-    _log.info('Obteniendo el recording %s', mbid)
+    _log.info('Obteniendo la grabacion %s', mbid)
 
-    with get_redis_connection() as redis:
+    mb_recording = cache.get_recording(mbid)
-        _log.debug('Intentando obtener el recording %s desde redis', mbid)
+    if mb_recording:
-        mb_recording = redis.get(f'recording:{mbid}')
+        return map_recording(mb_recording)
-        if mb_recording is None:
-            _log.debug('El recording %s no estaba en redis, cargando desde musicbrainz', mbid)
-            mb_recording = mb.get_recording_by_mbid(mbid)
-        else:
-            _log.debug('El recording %s se enontro en redis', mbid)
-            mb_recording = json.loads(mb_recording)
 
-        if 'error' in mb_recording:
+    mb_recording = mb.get_recording_by_mbid(mbid)
-            _log.error('Error al cargar recording')
-            return mb_recording
 
-        jobs.load_entities_of_recording.delay(mbid)
+    if 'error' in mb_recording:
+        _log.error('Error al buscar %s', mb_recording)
+        return mb_recording
 
     return map_recording(mb_recording)
 
@@ -397,89 +352,44 @@ def get_recordings_of_release(mbid):
     """
     _log.info('Obteniendo recordings de la release %s', mbid)
 
-    medias = []
+    mb_medias = cache.get_recordings_of_release(mbid)
+    if mb_medias:
+        return {'medias': mb_medias}
 
-    with get_redis_connection() as redis:
+    mb_release = mb.get_release_by_mbid(mbid, ['recordings', 'artist-credits'])
-        _log.debug('Intentando obtener los recordings de la release %s desde redis', mbid)
-        medias_key = f'release:{mbid}:media'
-        if medias_key in redis:
-            if redis.zcard(medias_key) == int(redis.get(f'{medias_key}:count')):
-                medias = [json.loads(media) for media in redis.zrange(medias_key, 0, -1)]
-                for media in medias:
-                    recordings_key = f'{medias_key}:{media.get("position")}:recordings'
-                    if redis.zcard(recordings_key) == int(redis.get(f'{recordings_key}:count')):
-                        recordings_id = redis.zrange(recordings_key, 0, -1)
-                        media['recordings'] = []
-                        for recording_id in recordings_id:
-                            recording = json.loads(redis.get(f'recording:{recording_id}'))
-                            media['recordings'].append(recording)
-                    else:
-                        _log.debug('No estan todas las recordings de la release %s que deberian '
-                                   'estar', mbid)
-            else:
-                _log.debug('La cantidad de medias de la release %s encontradas no corresponden al '
-                           'total registrado', mbid)
-        else:
-            _log.debug('No existe %s en redis', medias_key)
 
-    if len(medias) == 0:
+    if 'error' in mb_release:
-        _log.debug('No se encontraron recordings de la release %s, se cargara con musicbrainz',
+        _log.debug('Error al cargar recordings %s', mb_release)
-                   mbid)
+        return mb_release
 
-        # Si es que no habían medias cargadas en cache, puede ser que la release nunca se alla
+    medias = [map_media(media) for media in mb_release.get('media', [])]
-        # cargado.
-        jobs.load_entities_of_release.delay(mbid)
-
-        mb_release = mb.get_release_by_mbid(mbid, ['recordings'])
-
-        if 'error' in mb_release:
-            _log.debug('Error al cargar recordings %s', mb_release)
-            return mb_release
-
-        medias = [map_media(media) for media in mb_release.get('media', [])]
 
     return {'medias': medias}
 
 
-def get_release_of_recording(mbid, limit, page):
+def get_releases_of_recording(mbid, limit, page):
     """Obtiene la release de una grabacion incluyendo los creditos a su artista"""
     _log.info('Obteniendo las releases de la recording %s', mbid)
 
-    releases = []
     offset = limit * (page - 1)
-    total = 0
 
-    with get_redis_connection() as redis:
+    releases, total = cache.get_releases_of_recording(mbid, limit, offset)
-        _log.debug('Intentando obtener las releases de la recording %s desde redis', mbid)
+    if releases and total:
-        releases_key = f'recording:{mbid}:release'
+        return {
-        if releases_key in redis or f'{releases_key}:count' in redis:
+            'paginate': paginate(total, limit, page),
-            if redis.zcard(releases_key) == int(redis.get(f'{releases_key}:count')):
+            'releases': releases
-                release_mbids = redis.zrange(releases_key, offset, offset + limit)
+        }
-                if redis.exists(*[f'release:{mbid}' for mbid in release_mbids]):
-                    total = redis.zcard(releases_key)
-                    releases = [get_release(mbid) for mbid in release_mbids]
-                else:
-                    _log.debug('No estan almacenadas todas las releases de la recording %s', mbid)
-            else:
-                _log.debug('No hay almacenadas tantas que de release de la recording %s como '
-                           'deberian haber', mbid)
-        else:
-            _log.debug('No esta %s ni %s:count en redis', releases_key, releases_key)
 
-    if len(releases) == 0:
+    mb_releases_browse = mb.browse_releases(params={'recording': mbid},
-        _log.debug('Las releases de la recording %s no estaban cargadas en redis, se va a utilizar '
+                                            includes=['artist-credits', 'recordings'],
-                   'musicbrainz', mbid)
+                                            limit=limit, offset=offset)
-        mb_releases_browse = mb.browse_releases(params={'recording': mbid},
-                                                includes=['artist-credits'],
-                                                limit=limit, offset=limit * (page - 1))
 
-        if 'error' in mb_releases_browse:
+    if 'error' in mb_releases_browse:
-            _log.debug('Error al hacer browse de las releases de la recording %s %s',
+        _log.debug('Error al buscar %s', mb_releases_browse)
-                       mbid, mb_releases_browse)
+        return mb_releases_browse
-            return mb_releases_browse
+
+    releases = [map_release(release) for release in mb_releases_browse.get('releases')]
 
-        releases = [map_release(release) for release in mb_releases_browse.get('releases')]
-        jobs.load_entities_of_recording.delay(mbid)
     return {
         'paginate': paginate(total, limit, page),
         'releases': releases
@@ -490,33 +400,21 @@ def get_artist_of_recording(mbid):
     """Obtiene el artista de una grabacion"""
     _log.info('Obteniendo el artista de la recording %s', mbid)
 
-    artist = None
+    mb_artist = cache.get_artist_of_recording(mbid)
+    if mb_artist:
+        return map_artist(mb_artist)
 
-    with get_redis_connection() as redis:
+    mb_artist_browse = mb.browse_artists(params={'recording': mbid},
-        _log.debug('Intentando obtener el artista de la recording %s desde redis', mbid)
+                                         includes=['tags'],
-        artist_key = f'recording:{mbid}:artist'
+                                         limit=1, offset=0)
-        if artist_key in redis:
-            _log.debug('Se encontro el artista de la recording %s en redis', mbid)
-            artist = get_artist(redis.get(artist_key))
-        else:
-            _log.debug('El artista de la recording %s no estaba en redis', mbid)
 
-    if artist is None:
+    if 'error' in mb_artist_browse:
-        _log.debug('Obteniendo el artista de la recording %s desde musicbrainz', mbid)
+        _log.debug('Error al buscar %s', mb_artist_browse)
-        mb_artist_browse = mb.browse_artists(params={'recording': mbid},
+        return mb_artist_browse
-                                             includes=['tags'],
-                                             limit=1, offset=0)
 
-        if 'error' in mb_artist_browse:
+    mb_artist = mb_artist_browse.get('artists')[0]
-            _log.debug('Erro al obtener artista %s', mb_artist_browse)
-            return mb_artist_browse
 
-        artist = mb_artist_browse.get('artists')[0]
+    return map_artist(mb_artist)
-        jobs.load_artist_on_cache.delay(artist.get('id'))
-
-    return {
-        'artist': artist
-    }
 
 
 ##
@@ -563,7 +461,7 @@ def get_cover_art_release(mbid):
 
 def get_cover_art_recording(mbid):
     """Obtiene el cover art de una grabacion"""
-    release = get_release_of_recording(mbid, limit=1, page=1)
+    release = get_releases_of_recording(mbid, limit=1, page=1)[0]
 
     if 'error' in release:
         return None

fetcher/urls.py

@@ -23,7 +23,7 @@ urlpatterns = [
 
     path('recording/', views.search_recording),
     path('recording/<mbid>/', views.get_recording),
-    path('recording/<mbid>/release/', views.get_release_of_recording),
+    path('recording/<mbid>/releases/', views.get_releases_of_recording),
     path('recording/<mbid>/artist/', views.get_artist_of_recording),
     path('recording/<mbid>/coverart/', views.get_cover_art_of_recording),
 ]

fetcher/views.py

@@ -114,15 +114,9 @@ def get_releases_of_disc(request, mbid):
 
 @api_view(['GET'])
 def get_artist_of_release(request, mbid):
-    """ Obtiene el artista de una release dada su mbid
+    """ Obtiene el artista de una release dada su mbid"""
 
-    Como los datos son paginables la query puede contener per_page y page para definir cuantos
+    return Response(medium.get_artist_of_release(mbid))
-    elementos mostrar por pagina y que pagina mostrar.
-    """
-    limit = int(request.GET.get('per_page', 10))
-    page = int(request.GET.get('page', 1))
-
-    return Response(medium.get_artist_of_release(mbid, limit, page))
 
 
 @api_view(['GET'])
@@ -157,7 +151,7 @@ def get_recordings_of_release(request, mbid):
 
 
 @api_view(['GET'])
-def get_release_of_recording(request, mbid):
+def get_releases_of_recording(request, mbid):
     """ Obtiene la release de una recording dada su mbid
 
     Como los datos son paginables la query puede contener per_page y page para definir cuantos
@@ -166,7 +160,7 @@ def get_release_of_recording(request, mbid):
     limit = int(request.GET.get('per_page', 100))
     page = int(request.GET.get('page', 1))
 
-    return Response(medium.get_release_of_recording(mbid, limit, page))
+    return Response(medium.get_releases_of_recording(mbid, limit, page))
 
 
 @api_view(['GET'])