From 2edca8e25523cf6e09be4240a17252592de5b2cc Mon Sep 17 00:00:00 2001 From: Daniel Cortes Date: Mon, 8 Jun 2020 23:14:13 -0400 Subject: [PATCH] Se remplazo la conexion a cache de django a la nativa --- fetcher/jobs.py | 176 +++++++++++++++++++++++++----------------------- 1 file changed, 91 insertions(+), 85 deletions(-) diff --git a/fetcher/jobs.py b/fetcher/jobs.py index a93d198..210ce37 100644 --- a/fetcher/jobs.py +++ b/fetcher/jobs.py @@ -3,7 +3,7 @@ import logging import json import django_rq -from django.core.cache import cache +from django_redis import get_redis_connection from fetcher import musicbrainz as mb @@ -14,136 +14,142 @@ _log.addHandler(logging.NullHandler()) @django_rq.job('high') def load_release_cover_art(release): """Carga en cache el cover art de una release""" - mbid = release - if isinstance(release, dict): - mbid = release.get('id') + with get_redis_connection("default") as cache: + mbid = release + if isinstance(release, dict): + mbid = release.get('id') - if cache.ttl(f'release_cover_art_{mbid}') is None: - _log.info('El cover art ya estaba guardado') - return + if cache.ttl(f'release_cover_art_{mbid}') == -1: + _log.info('El cover art ya estaba guardado') + return - cover_art = mb.get_release_cover_art(mbid) - cache.set(f'release_cover_art_{mbid}', json.dumps(cover_art), timeout=None) - _log.info('Cover art de release %s almacenado en cache', mbid) + cover_art = mb.get_release_cover_art(mbid) + cache.set(f'release_cover_art_{mbid}', json.dumps(cover_art)) + _log.info('Cover art de release %s almacenado en cache', mbid) @django_rq.job('high') def load_release_group_cover_art(release_group): """Carga en cache el cover art de un release group""" - mbid = release_group - if isinstance(release_group, dict): - mbid = release_group.get('id') + with get_redis_connection("default") as cache: + mbid = release_group + if isinstance(release_group, dict): + mbid = release_group.get('id') - if cache.ttl(f'release_group_cover_art_{mbid}') is None: - _log.info('El cover art ya estaba guardado') - return + if cache.ttl(f'release_group_cover_art_{mbid}') == -1: + _log.info('El cover art ya estaba guardado') + return - cover_art = mb.get_release_group_cover_art(mbid) - cache.set(f'release_group_cover_art_{mbid}', json.dumps(cover_art), timeout=None) - _log.info('Cover art de release group %s almacenado en cache', mbid) + cover_art = mb.get_release_group_cover_art(mbid) + cache.set(f'release_group_cover_art_{mbid}', json.dumps(cover_art)) + _log.info('Cover art de release group %s almacenado en cache', mbid) @django_rq.job def load_entities_of_recording(recording): """Carga en cache una grabacion y sus entidades relacionadas""" - mbid = recording - if isinstance(recording, dict): - mbid = recording.get('id') + with get_redis_connection("default") as cache: + mbid = recording + if isinstance(recording, dict): + mbid = recording.get('id') - if cache.ttl(f'recording_{mbid}') is None: - _log.info('La recording ya se habia procesado anteriormente') - return + if cache.ttl(f'recording_{mbid}') == -1: + _log.info('La recording ya se habia procesado anteriormente') + return - if isinstance(recording, str): - recording = mb.get_recording_by_mbid(mbid) + if isinstance(recording, str): + recording = mb.get_recording_by_mbid(mbid) - cache.set(f'recording_{mbid}', json.dumps(recording), timeout=None) - _log.info('Recording %s fue almacenada correctamente', mbid) + cache.set(f'recording_{mbid}', json.dumps(recording)) + _log.info('Recording %s fue almacenada correctamente', mbid) @django_rq.job def load_entities_of_release(release): """Carga en cache una release y sus entidades relacionadas""" - mbid = release - if isinstance(release, dict): - mbid = release.get('id') + with get_redis_connection("default") as cache: + mbid = release + if isinstance(release, dict): + mbid = release.get('id') - if cache.ttl(f'release_{mbid}') is None: - _log.info('La release ya se habia procesado anteriormente') - return + if cache.ttl(f'release_{mbid}') == -1: + _log.info('La release ya se habia procesado anteriormente') + return - if isinstance(release, str): - release = mb.get_release_by_mbid(mbid) + if isinstance(release, str): + release = mb.get_release_by_mbid(mbid) - cache.set(f'release_{mbid}', json.dumps(release), timeout=None) - _log.info('Release %s fue almacenada en cache', mbid) + cache.set(f'release_{mbid}', json.dumps(release)) + _log.info('Release %s fue almacenada en cache', mbid) - load_release_cover_art.delay(release) + load_release_cover_art.delay(release) - offset = 0 - while True: - recordings = mb.browse_recordings({'release': mbid}, limit=100, offset=offset) - for recording in recordings.get('recordings'): - load_entities_of_recording(recording) + offset = 0 + while True: + recordings = mb.browse_recordings({'release': mbid}, limit=100, offset=offset) + for recording in recordings.get('recordings'): + load_entities_of_recording(recording) - offset += 100 - if offset > recordings.get('recording_count', 0): - break + offset += 100 + if offset > recordings.get('recording_count', 0): + break @django_rq.job def load_entities_of_release_group(release_group): """Carga en cache un release group y sus entidades relacionadas""" - mbid = release_group - if isinstance(release_group, dict): - mbid = release_group.get('id') + with get_redis_connection("default") as cache: + mbid = release_group + if isinstance(release_group, dict): + mbid = release_group.get('id') - if cache.ttl(f'release_group_{mbid}') is None: - _log.info('La release group ya se habia procesado anteriormente') - return + if cache.ttl(f'release_group_{mbid}') == -1: + _log.info('La release group ya se habia procesado anteriormente') + return - if isinstance(release_group, str): - release_group = mb.get_release_group_by_mbid(mbid) + if isinstance(release_group, str): + release_group = mb.get_release_group_by_mbid(mbid) - cache.set(f'release_group_{mbid}', json.dumps(release_group), timeout=None) - _log.info('Release Group %s almacenado en cache', mbid) + cache.set(f'release_group_{mbid}', json.dumps(release_group)) + _log.info('Release Group %s almacenado en cache', mbid) - load_release_group_cover_art.delay(release_group) + load_release_group_cover_art.delay(release_group) - offset = 0 - while True: - releases = mb.browse_releases({'release-group': mbid}, limit=100, offset=offset) - for release in releases.get('releases'): - load_entities_of_release(release) + offset = 0 + while True: + releases = mb.browse_releases({'release-group': mbid}, limit=100, offset=offset) + for release in releases.get('releases'): + load_entities_of_release(release) - offset += 100 - if offset > releases.get('release_count', 0): - break + offset += 100 + if offset > releases.get('release_count', 0): + break @django_rq.job def load_artist_on_cache(artist): """Carga en cache a un artista y todas sus entidades""" - mbid = artist - if isinstance(artist, dict): - mbid = artist.get('id') + with get_redis_connection("default") as cache: + mbid = artist + if isinstance(artist, dict): + mbid = artist.get('id') - if cache.ttl(f'artist_{mbid}') is None: - _log.info('El artista ya se había procesado anteriormente') - return + if cache.ttl(f'artist_{mbid}') == -1: + _log.info('El artista ya se había procesado anteriormente') + return - if isinstance(artist, str): - artist = mb.get_artist_by_mbid(mbid, includes=['tags']) + if isinstance(artist, str): + artist = mb.get_artist_by_mbid(mbid, includes=['tags']) - cache.set(f'artist_{mbid}', json.dumps(artist), timeout=None) - _log.info('Artista %s almacenado en cache', mbid) + cache.set(f'artist_{mbid}', json.dumps(artist)) + _log.info('Artista %s almacenado en cache', mbid) - offset = 0 - while True: - release_groups = mb.browse_release_groups({'artist': mbid}, limit=100, offset=offset) - for release_group in release_groups.get('release_groups'): - load_entities_of_release_group.delay(release_group) + offset = 0 + while True: + release_groups = mb.browse_release_groups({'artist': mbid}, limit=100, offset=offset) + for release_group in release_groups.get('release_groups'): + load_entities_of_release_group.delay(release_group) - offset += 100 - if offset > release_groups.get('release_group_count', 0): - break + offset += 100 + if offset > release_groups.get('release_group_count', 0): + break