Volver a py-redis

Resulta que django-redis es muy bonito pero esta pensado para usarse
como el cache por default de django, remplazando a memcached y por eso
no permite casi ninguna de las funcionalidades que redis permite, como
usar listas y sets.
Djang-redis permite usar el cliente directamente pero el codigo se
estaba haciendo largo y creando mas problemas de los que necesito
Asi que voy a usar el cliente regular, asi me evito dramas

fetcher/jobs.py

@@ -3,10 +3,9 @@ import logging
 import json
 
 import django_rq
-from django_redis import get_redis_connection
 
 from fetcher import musicbrainz as mb
-from utils import make_key
+from utils import get_redis_connection
 
 _log = logging.getLogger('fetcher_jobs')
 _log.addHandler(logging.NullHandler())
@@ -15,72 +14,72 @@ _log.addHandler(logging.NullHandler())
 @django_rq.job('high')
 def load_release_cover_art(release):
     """Carga en cache el cover art de una release"""
-    with get_redis_connection("default") as cache:
+    with get_redis_connection() as redis:
         mbid = release
         if isinstance(release, dict):
             mbid = release.get('id')
 
-        if cache.ttl(make_key(f'release_cover_art_{mbid}')) == -1:
+        if f'release_cover_art_{mbid}' in redis:
             _log.info('El cover art ya estaba guardado')
             return
 
         cover_art = mb.get_release_cover_art(mbid)
-        cache.set(make_key(f'release_cover_art_{mbid}'), json.dumps(cover_art))
+        redis.set(f'release_cover_art_{mbid}', json.dumps(cover_art))
         _log.info('Cover art de release %s almacenado en cache', mbid)
 
 
 @django_rq.job('high')
 def load_release_group_cover_art(release_group):
     """Carga en cache el cover art de un release group"""
-    with get_redis_connection("default") as cache:
+    with get_redis_connection() as redis:
         mbid = release_group
         if isinstance(release_group, dict):
             mbid = release_group.get('id')
 
-        if cache.ttl(make_key(f'release_group_cover_art_{mbid}')) == -1:
+        if f'release_group_cover_art_{mbid}' in redis:
             _log.info('El cover art ya estaba guardado')
             return
 
         cover_art = mb.get_release_group_cover_art(mbid)
-        cache.set(make_key(f'release_group_cover_art_{mbid}'), json.dumps(cover_art))
+        redis.set(f'release_group_cover_art_{mbid}', json.dumps(cover_art))
         _log.info('Cover art de release group %s almacenado en cache', mbid)
 
 
 @django_rq.job
 def load_entities_of_recording(recording):
     """Carga en cache una grabacion y sus entidades relacionadas"""
-    with get_redis_connection("default") as cache:
+    with get_redis_connection() as redis:
         mbid = recording
         if isinstance(recording, dict):
             mbid = recording.get('id')
 
-        if cache.ttl(make_key(f'recording_{mbid}')) == -1:
+        if f'recording_{mbid}' in redis:
-            _log.info('La recording ya se habia procesado anteriormente')
+            _log.info('La recording ya se había procesado anteriormente')
             return
 
         if isinstance(recording, str):
             recording = mb.get_recording_by_mbid(mbid)
 
-        cache.set(make_key(f'recording_{mbid}'), json.dumps(recording))
+        redis.set(f'recording_{mbid}', json.dumps(recording))
         _log.info('Recording %s fue almacenada correctamente', mbid)
 
 
 @django_rq.job
 def load_entities_of_release(release):
     """Carga en cache una release y sus entidades relacionadas"""
-    with get_redis_connection("default") as cache:
+    with get_redis_connection() as redis:
         mbid = release
         if isinstance(release, dict):
             mbid = release.get('id')
 
-        if cache.ttl(make_key(f'release_{mbid}')) == -1:
+        if f'release_{mbid}' in redis:
-            _log.info('La release ya se habia procesado anteriormente')
+            _log.info('La release ya se había procesado anteriormente')
             return
 
         if isinstance(release, str):
             release = mb.get_release_by_mbid(mbid)
 
-        cache.set(make_key(f'release_{mbid}'), json.dumps(release))
+        redis.set(f'release_{mbid}', json.dumps(release))
         _log.info('Release %s fue almacenada en cache', mbid)
 
         load_release_cover_art.delay(release)
@@ -99,19 +98,19 @@ def load_entities_of_release(release):
 @django_rq.job
 def load_entities_of_release_group(release_group):
     """Carga en cache un release group y sus entidades relacionadas"""
-    with get_redis_connection("default") as cache:
+    with get_redis_connection() as redis:
         mbid = release_group
         if isinstance(release_group, dict):
             mbid = release_group.get('id')
 
-        if cache.ttl(make_key(f'release_group_{mbid}')) == -1:
+        if f'release_group_{mbid}' in redis:
             _log.info('La release group ya se habia procesado anteriormente')
             return
 
         if isinstance(release_group, str):
             release_group = mb.get_release_group_by_mbid(mbid)
 
-        cache.set(make_key(f'release_group_{mbid}'), json.dumps(release_group))
+        redis.set(f'release_group_{mbid}', json.dumps(release_group))
         _log.info('Release Group %s almacenado en cache', mbid)
 
         load_release_group_cover_art.delay(release_group)
@@ -130,29 +129,29 @@ def load_entities_of_release_group(release_group):
 @django_rq.job
 def load_artist_on_cache(artist):
     """Carga en cache a un artista y todas sus entidades"""
-    with get_redis_connection("default") as cache:
+    with get_redis_connection() as redis:
         mbid = artist
         if isinstance(artist, dict):
             mbid = artist.get('id')
 
-        if cache.ttl(make_key(f'artist_{mbid}')) == -1:
+        if f'artist_{mbid}' in redis:
             _log.info('El artista ya se había procesado anteriormente')
             return
 
         if isinstance(artist, str):
             artist = mb.get_artist_by_mbid(mbid, includes=['tags'])
 
-        cache.set(make_key(f'artist_{mbid}'), json.dumps(artist))
+        redis.set(f'artist_{mbid}', json.dumps(artist))
         _log.info('Artista %s almacenado en cache', mbid)
 
         offset = 0
         while True:
             release_groups = mb.browse_release_groups({'artist': mbid}, limit=100, offset=offset)
-            if cache.ttl(make_key(f'artist_{mbid}:release_group_count')) != -1:
+            if f'artist_{mbid}:release_group_count' in redis:
-                cache.set(make_key(f'artist_{mbid}:release_group_count'),
+                redis.set(f'artist_{mbid}:release_group_count',
                           release_groups.get('release_group_count'))
             for release_group in release_groups.get('release_groups'):
-                cache.rpush(make_key(f'artist_{mbid}:release_groups'), release_group.get('id'))
+                redis.rpush(f'artist_{mbid}:release_groups', release_group.get('id'))
                 load_entities_of_release_group.delay(release_group)
 
             offset += 100

fetcher/medium.py

@@ -7,13 +7,11 @@ traducción.
 import json
 from math import ceil
 
-from django.core.cache import cache
-from django_redis import get_redis_connection
 from country_list import countries_for_language
 
 import fetcher.musicbrainz as mb
 from fetcher import jobs
-from utils import make_key
+from utils import get_redis_connection
 
 
 ###
@@ -155,11 +153,13 @@ def map_coverart(mb_cover):
 def get_artist(mbid):
     """Obtiene un artista desde musicbrainz incluyendo sus tags"""
 
-    mb_artist = cache.get(f'artist_{mbid}')
+    with get_redis_connection() as redis:
+        mb_artist = redis.get(f'artist_{mbid}')
         if mb_artist is None:
             mb_artist = mb.get_artist_by_mbid(mbid, includes=['tags'])
         else:
             mb_artist = json.loads(mb_artist)
+
         if 'error' in mb_artist:
             return mb_artist
 
@@ -170,11 +170,14 @@ def get_artist(mbid):
 
 def get_disc(mbid):
     """Obtiene un disco desde musicbrainz"""
-    mb_disc = cache.get(f'release_group_{mbid}')
+
+    with get_redis_connection() as redis:
+        mb_disc = redis.get(f'release_group_{mbid}')
         if mb_disc is None:
             mb_disc = mb.get_release_group_by_mbid(mbid)
         else:
             mb_disc = json.loads(mb_disc)
+
         if 'error' in mb_disc:
             return mb_disc
 
@@ -189,10 +192,10 @@ def get_discs_of_artist(mbid):
 
     mb_discs = []
 
-    with get_redis_connection("default") as raw_cache:
+    with get_redis_connection() as redis:
-        if raw_cache.exists(make_key(f'artist_{mbid}:release_group_count')) == 1:
+        if f'artist_{mbid}:release_group_count' in redis:
-            mb_discs_ids = raw_cache.lrange(make_key(f'artist_{mbid}:release_groups'), 0, -1)
+            mb_discs_ids = redis.lrange(f'artist_{mbid}:release_groups', 0, -1)
-            mb_discs = [get_disc(str(mbid, 'utf-8')) for mbid in mb_discs_ids]
+            mb_discs = [get_disc(mbid) for mbid in mb_discs_ids]
 
     if len(mb_discs) == 0:
         jobs.load_artist_on_cache.delay(mbid)
@@ -204,7 +207,7 @@ def get_discs_of_artist(mbid):
             if 'error' in mb_discs_browse:
                 return mb_discs_browse
 
-            mb_discs.extend(*mb_discs_browse.get('release_groups'))
+            mb_discs.extend(mb_discs_browse.get('release_groups'))
 
             offset += 100
             if offset > mb_discs_browse.get('release_group_count'):
@@ -217,7 +220,8 @@ def get_discs_of_artist(mbid):
 
 def get_artist_of_disc(mbid, limit, page):
     """Obtiene el artista de un disco"""
-    mb_artists = mb.browse_artists(params={'disc': mbid}, limit=limit, offset=limit * (page - 1))
+    mb_artists = mb.browse_artists(params={'release-group': mbid},
+                                   limit=limit, offset=limit * (page - 1))
 
     if 'error' in mb_artists:
         return mb_artists
@@ -234,7 +238,8 @@ def get_artist_of_disc(mbid, limit, page):
 
 def get_release(mbid):
     """Obtiene una release desde musicbrainz incluyendo sus artistas"""
-    mb_release = cache.get(f'release_{mbid}')
+    with get_redis_connection() as redis:
+        mb_release = redis.get(f'release_{mbid}')
         if mb_release is None:
             mb_release = mb.get_release_by_mbid(mbid, includes=['artists'])
         else:
@@ -345,7 +350,8 @@ def get_artist_of_recording(mbid, limit, page):
 
 def get_cover_art_disc(mbid):
     """Obtiene el cover art de un disco"""
-    mb_covers = cache.get(f'release_group_cover_art_{mbid}')
+    with get_redis_connection() as redis:
+        mb_covers = redis.get(f'release_group_cover_art_{mbid}')
         if mb_covers is None:
             mb_covers = mb.get_release_group_cover_art(mbid)
         else:

lint.sh

@@ -1,3 +1,6 @@
 #!/bin/sh
-pylint --load-plugins pylint_django fetcher lists musiclist users utils welcome || pylint-exit --error-fail --warn-fail $?
+plugins="pylint_django"
+good_names="db"
+modules="fetcher lists musiclist users utils welcome"
+pylint --load-plugins $plugins --good-names=$good_names $modules || pylint-exit --error-fail --warn-fail $?
 exit $?

musiclist/settings/__init__.py

@@ -87,7 +87,6 @@ CACHES = {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
         },
         "KEY_PREFIX": "DJANGO_SERVER",
-        "KEY_FUNCTION": "utils.make_key"
     }
 }
 
@@ -103,6 +102,12 @@ RQ_QUEUES = {
     },
 }
 
+DATA_CACHE = {
+    "host": "localhost",
+    "port": 6379,
+    "db": 0
+}
+
 SESSION_ENGINE = "django.contrib.sessions.backends.cache"
 SESSION_CACHE_ALIAS = "default"
 

utils/__init__.py

@@ -2,6 +2,7 @@
 import json
 import logging
 import functools
+import redis
 
 from django.http import JsonResponse
 from django.conf import settings
@@ -85,3 +86,11 @@ def make_key(key, key_prefix=None, version=1):
         key_prefix = settings.CACHES.get('default').get('KEY_PREFIX')
 
     return ':'.join([key_prefix, str(version), key])
+
+
+def get_redis_connection():
+    """Obtiene una conexion con el cliente de redis"""
+    host = settings.DATA_CACHE.get('host')
+    port = settings.DATA_CACHE.get('port')
+    db = settings.DATA_CACHE.get('db')
+    return redis.Redis(host=host, port=port, db=db, decode_responses=True)