skrd/musiclist-server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Volver a py-redis

Browse Source 
Resulta que django-redis es muy bonito pero esta pensado para usarse
como el cache por default de django, remplazando a memcached y por eso
no permite casi ninguna de las funcionalidades que redis permite, como
usar listas y sets.
Djang-redis permite usar el cliente directamente pero el codigo se
estaba haciendo largo y creando mas problemas de los que necesito
Asi que voy a usar el cliente regular, asi me evito dramas
This commit is contained in:
Daniel Cortes
2020-06-09 04:08:08 -04:00
parent 72bcf94d11
commit cabe607958
5 changed files with 97 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
fetcher/jobs.py
View File

@@ -3,10 +3,9 @@ import logging
import json
import django_rq
from django_redis import get_redis_connection
from fetcher import musicbrainz as mb
from utils import make_key
from utils import get_redis_connection
_log = logging.getLogger('fetcher_jobs')
_log.addHandler(logging.NullHandler())
@@ -15,72 +14,72 @@ _log.addHandler(logging.NullHandler())
@django_rq.job('high')
def load_release_cover_art(release):
"""Carga en cache el cover art de una release"""
with get_redis_connection("default") as cache:
with get_redis_connection() as redis:
mbid = release
if isinstance(release, dict):
mbid = release.get('id')
if cache.ttl(make_key(f'release_cover_art_{mbid}')) == -1:
if f'release_cover_art_{mbid}' in redis:
_log.info('El cover art ya estaba guardado')
return
cover_art = mb.get_release_cover_art(mbid)
cache.set(make_key(f'release_cover_art_{mbid}'), json.dumps(cover_art))
redis.set(f'release_cover_art_{mbid}', json.dumps(cover_art))
_log.info('Cover art de release %s almacenado en cache', mbid)
@django_rq.job('high')
def load_release_group_cover_art(release_group):
"""Carga en cache el cover art de un release group"""
with get_redis_connection("default") as cache:
with get_redis_connection() as redis:
mbid = release_group
if isinstance(release_group, dict):
mbid = release_group.get('id')
if cache.ttl(make_key(f'release_group_cover_art_{mbid}')) == -1:
if f'release_group_cover_art_{mbid}' in redis:
_log.info('El cover art ya estaba guardado')
return
cover_art = mb.get_release_group_cover_art(mbid)
cache.set(make_key(f'release_group_cover_art_{mbid}'), json.dumps(cover_art))
redis.set(f'release_group_cover_art_{mbid}', json.dumps(cover_art))
_log.info('Cover art de release group %s almacenado en cache', mbid)
@django_rq.job
def load_entities_of_recording(recording):
"""Carga en cache una grabacion y sus entidades relacionadas"""
with get_redis_connection("default") as cache:
with get_redis_connection() as redis:
mbid = recording
if isinstance(recording, dict):
mbid = recording.get('id')
if cache.ttl(make_key(f'recording_{mbid}')) == -1:
_log.info('La recording ya se habia procesado anteriormente')
if f'recording_{mbid}' in redis:
_log.info('La recording ya se había procesado anteriormente')
return
if isinstance(recording, str):
recording = mb.get_recording_by_mbid(mbid)
cache.set(make_key(f'recording_{mbid}'), json.dumps(recording))
redis.set(f'recording_{mbid}', json.dumps(recording))
_log.info('Recording %s fue almacenada correctamente', mbid)
@django_rq.job
def load_entities_of_release(release):
"""Carga en cache una release y sus entidades relacionadas"""
with get_redis_connection("default") as cache:
with get_redis_connection() as redis:
mbid = release
if isinstance(release, dict):
mbid = release.get('id')
if cache.ttl(make_key(f'release_{mbid}')) == -1:
_log.info('La release ya se habia procesado anteriormente')
if f'release_{mbid}' in redis:
_log.info('La release ya se había procesado anteriormente')
return
if isinstance(release, str):
release = mb.get_release_by_mbid(mbid)
cache.set(make_key(f'release_{mbid}'), json.dumps(release))
redis.set(f'release_{mbid}', json.dumps(release))
_log.info('Release %s fue almacenada en cache', mbid)
load_release_cover_art.delay(release)
@@ -99,19 +98,19 @@ def load_entities_of_release(release):
@django_rq.job
def load_entities_of_release_group(release_group):
"""Carga en cache un release group y sus entidades relacionadas"""
with get_redis_connection("default") as cache:
with get_redis_connection() as redis:
mbid = release_group
if isinstance(release_group, dict):
mbid = release_group.get('id')
if cache.ttl(make_key(f'release_group_{mbid}')) == -1:
if f'release_group_{mbid}' in redis:
_log.info('La release group ya se habia procesado anteriormente')
return
if isinstance(release_group, str):
release_group = mb.get_release_group_by_mbid(mbid)
cache.set(make_key(f'release_group_{mbid}'), json.dumps(release_group))
redis.set(f'release_group_{mbid}', json.dumps(release_group))
_log.info('Release Group %s almacenado en cache', mbid)
load_release_group_cover_art.delay(release_group)
@@ -130,29 +129,29 @@ def load_entities_of_release_group(release_group):
@django_rq.job
def load_artist_on_cache(artist):
"""Carga en cache a un artista y todas sus entidades"""
with get_redis_connection("default") as cache:
with get_redis_connection() as redis:
mbid = artist
if isinstance(artist, dict):
mbid = artist.get('id')
if cache.ttl(make_key(f'artist_{mbid}')) == -1:
if f'artist_{mbid}' in redis:
_log.info('El artista ya se había procesado anteriormente')
return
if isinstance(artist, str):
artist = mb.get_artist_by_mbid(mbid, includes=['tags'])
cache.set(make_key(f'artist_{mbid}'), json.dumps(artist))
redis.set(f'artist_{mbid}', json.dumps(artist))
_log.info('Artista %s almacenado en cache', mbid)
offset = 0
while True:
release_groups = mb.browse_release_groups({'artist': mbid}, limit=100, offset=offset)
if cache.ttl(make_key(f'artist_{mbid}:release_group_count')) != -1:
cache.set(make_key(f'artist_{mbid}:release_group_count'),
if f'artist_{mbid}:release_group_count' in redis:
redis.set(f'artist_{mbid}:release_group_count',
release_groups.get('release_group_count'))
for release_group in release_groups.get('release_groups'):
cache.rpush(make_key(f'artist_{mbid}:release_groups'), release_group.get('id'))
redis.rpush(f'artist_{mbid}:release_groups', release_group.get('id'))
load_entities_of_release_group.delay(release_group)
offset += 100

102
fetcher/medium.py
View File

@@ -7,13 +7,11 @@ traducción.
import json
from math import ceil
from django.core.cache import cache
from django_redis import get_redis_connection
from country_list import countries_for_language
import fetcher.musicbrainz as mb
from fetcher import jobs
from utils import make_key
from utils import get_redis_connection
###
@@ -155,33 +153,38 @@ def map_coverart(mb_cover):
def get_artist(mbid):
"""Obtiene un artista desde musicbrainz incluyendo sus tags"""
mb_artist = cache.get(f'artist_{mbid}')
if mb_artist is None:
mb_artist = mb.get_artist_by_mbid(mbid, includes=['tags'])
else:
mb_artist = json.loads(mb_artist)
if 'error' in mb_artist:
return mb_artist
with get_redis_connection() as redis:
mb_artist = redis.get(f'artist_{mbid}')
if mb_artist is None:
mb_artist = mb.get_artist_by_mbid(mbid, includes=['tags'])
else:
mb_artist = json.loads(mb_artist)
jobs.load_artist_on_cache.delay(mbid)
if 'error' in mb_artist:
return mb_artist
return map_artist(mb_artist)
jobs.load_artist_on_cache.delay(mbid)
return map_artist(mb_artist)
def get_disc(mbid):
"""Obtiene un disco desde musicbrainz"""
mb_disc = cache.get(f'release_group_{mbid}')
if mb_disc is None:
mb_disc = mb.get_release_group_by_mbid(mbid)
else:
mb_disc = json.loads(mb_disc)
if 'error' in mb_disc:
return mb_disc
jobs.load_entities_of_release_group.delay(mbid)
disc = map_disc(mb_disc)
with get_redis_connection() as redis:
mb_disc = redis.get(f'release_group_{mbid}')
if mb_disc is None:
mb_disc = mb.get_release_group_by_mbid(mbid)
else:
mb_disc = json.loads(mb_disc)
return disc
if 'error' in mb_disc:
return mb_disc
jobs.load_entities_of_release_group.delay(mbid)
disc = map_disc(mb_disc)
return disc
def get_discs_of_artist(mbid):
@@ -189,10 +192,10 @@ def get_discs_of_artist(mbid):
mb_discs = []
with get_redis_connection("default") as raw_cache:
if raw_cache.exists(make_key(f'artist_{mbid}:release_group_count')) == 1:
mb_discs_ids = raw_cache.lrange(make_key(f'artist_{mbid}:release_groups'), 0, -1)
mb_discs = [get_disc(str(mbid, 'utf-8')) for mbid in mb_discs_ids]
with get_redis_connection() as redis:
if f'artist_{mbid}:release_group_count' in redis:
mb_discs_ids = redis.lrange(f'artist_{mbid}:release_groups', 0, -1)
mb_discs = [get_disc(mbid) for mbid in mb_discs_ids]
if len(mb_discs) == 0:
jobs.load_artist_on_cache.delay(mbid)
@@ -204,7 +207,7 @@ def get_discs_of_artist(mbid):
if 'error' in mb_discs_browse:
return mb_discs_browse
mb_discs.extend(*mb_discs_browse.get('release_groups'))
mb_discs.extend(mb_discs_browse.get('release_groups'))
offset += 100
if offset > mb_discs_browse.get('release_group_count'):
@@ -217,7 +220,8 @@ def get_discs_of_artist(mbid):
def get_artist_of_disc(mbid, limit, page):
"""Obtiene el artista de un disco"""
mb_artists = mb.browse_artists(params={'disc': mbid}, limit=limit, offset=limit * (page - 1))
mb_artists = mb.browse_artists(params={'release-group': mbid},
limit=limit, offset=limit * (page - 1))
if 'error' in mb_artists:
return mb_artists
@@ -234,18 +238,19 @@ def get_artist_of_disc(mbid, limit, page):
def get_release(mbid):
"""Obtiene una release desde musicbrainz incluyendo sus artistas"""
mb_release = cache.get(f'release_{mbid}')
if mb_release is None:
mb_release = mb.get_release_by_mbid(mbid, includes=['artists'])
else:
mb_release = json.loads(mb_release)
if 'error' in mb_release:
return mb_release
with get_redis_connection() as redis:
mb_release = redis.get(f'release_{mbid}')
if mb_release is None:
mb_release = mb.get_release_by_mbid(mbid, includes=['artists'])
else:
mb_release = json.loads(mb_release)
if 'error' in mb_release:
return mb_release
jobs.load_entities_of_release.delay(mbid)
release = map_release(mb_release)
jobs.load_entities_of_release.delay(mbid)
release = map_release(mb_release)
return release
return release
def get_releases_of_disc(mbid, limit, page):
@@ -345,19 +350,20 @@ def get_artist_of_recording(mbid, limit, page):
def get_cover_art_disc(mbid):
"""Obtiene el cover art de un disco"""
mb_covers = cache.get(f'release_group_cover_art_{mbid}')
if mb_covers is None:
mb_covers = mb.get_release_group_cover_art(mbid)
else:
mb_covers = json.loads(mb_covers)
with get_redis_connection() as redis:
mb_covers = redis.get(f'release_group_cover_art_{mbid}')
if mb_covers is None:
mb_covers = mb.get_release_group_cover_art(mbid)
else:
mb_covers = json.loads(mb_covers)
if 'error' in mb_covers:
return None
if 'error' in mb_covers:
return None
jobs.load_entities_of_release_group.delay(mbid)
cover_art = map_coverart(find_best_cover(mb_covers))
jobs.load_entities_of_release_group.delay(mbid)
cover_art = map_coverart(find_best_cover(mb_covers))
return cover_art
return cover_art
def get_cover_art_release(mbid):

5
lint.sh
View File

@@ -1,3 +1,6 @@
#!/bin/sh
pylint --load-plugins pylint_django fetcher lists musiclist users utils welcome || pylint-exit --error-fail --warn-fail $?
plugins="pylint_django"
good_names="db"
modules="fetcher lists musiclist users utils welcome"
pylint --load-plugins $plugins --good-names=$good_names $modules || pylint-exit --error-fail --warn-fail $?
exit $?

7
musiclist/settings/__init__.py
View File

@@ -87,7 +87,6 @@ CACHES = {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
},
"KEY_PREFIX": "DJANGO_SERVER",
"KEY_FUNCTION": "utils.make_key"
}
}
@@ -103,6 +102,12 @@ RQ_QUEUES = {
},
}
DATA_CACHE = {
"host": "localhost",
"port": 6379,
"db": 0
}
SESSION_ENGINE = "django.contrib.sessions.backends.cache"
SESSION_CACHE_ALIAS = "default"

9
utils/__init__.py
View File

@@ -2,6 +2,7 @@
import json
import logging
import functools
import redis
from django.http import JsonResponse
from django.conf import settings
@@ -85,3 +86,11 @@ def make_key(key, key_prefix=None, version=1):
key_prefix = settings.CACHES.get('default').get('KEY_PREFIX')
return ':'.join([key_prefix, str(version), key])
def get_redis_connection():
"""Obtiene una conexion con el cliente de redis"""
host = settings.DATA_CACHE.get('host')
port = settings.DATA_CACHE.get('port')
db = settings.DATA_CACHE.get('db')
return redis.Redis(host=host, port=port, db=db, decode_responses=True)