skrd/musiclist-server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Continuando con el almacenamiento de datos en cache

Browse Source 
Va a mejor la cosa, tengo que hacer tests si o tambien porque no estoy
nada seguro si es que todo funciona como espero ya que toda llamada a la
api corresponde a una llamada a cache y descubrir si es que esta todo en
cache como se espera

Voy a terminar del modo que lo estoy haciendo y tal vez cambio esto a un
modulo de cache el cual se encargara de o obtener datos desde cache o
llamar a music brainz para suplir los datos que no puede responder
This commit is contained in:
Daniel Cortes
2020-06-10 00:02:45 -04:00
parent d62253051b
commit 65c77c679f
3 changed files with 255 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
fetcher/medium.py
View File

@@ -154,7 +154,7 @@ def get_artist(mbid):
"""Obtiene un artista desde musicbrainz incluyendo sus tags"""
with get_redis_connection() as redis:
mb_artist = redis.get(f'artist_{mbid}')
mb_artist = redis.get(f'artist:{mbid}')
if mb_artist is None:
mb_artist = mb.get_artist_by_mbid(mbid, includes=['tags'])
else:
@@ -172,9 +172,9 @@ def get_disc(mbid):
"""Obtiene un disco desde musicbrainz"""
with get_redis_connection() as redis:
mb_disc = redis.get(f'release_group_{mbid}')
mb_disc = redis.get(f'release_group:{mbid}')
if mb_disc is None:
mb_disc = mb.get_release_group_by_mbid(mbid)
mb_disc = mb.get_release_group_by_mbid(mbid, ['artists'])
else:
mb_disc = json.loads(mb_disc)
@@ -182,9 +182,7 @@ def get_disc(mbid):
return mb_disc
jobs.load_entities_of_release_group.delay(mbid)
disc = map_disc(mb_disc)
return disc
return map_disc(mb_disc)
def get_discs_of_artist(mbid, limit, page):
@@ -195,19 +193,23 @@ def get_discs_of_artist(mbid, limit, page):
mb_discs = []
total = 0
# Si es que tengo un set de release_groups en redis me fijo si es que sus counts coinciden
# Si es que coinciden significa que se cargaron todos los discos, pero si no, quizás aun no
# terminan de guardarse, por lo que salto el código de obtención y voy directo a musicbrainz
with get_redis_connection() as redis:
key_count = f'artist_{mbid}:release_group_count'
if key_count in redis:
key = f'artist_{mbid}:release_groups'
mb_disc_ids = redis.lrange(key, offset, limit)
if redis.exists(*[f'release_group_{mbid}' for mbid in mb_disc_ids]) == len(mb_disc_ids):
mb_discs = [get_disc(mbid) for mbid in mb_disc_ids]
total = redis.llen(key)
key_releases = f'artist:{mbid}:release_groups'
if key_releases in redis:
if int(redis.get(f'{key_releases}:count')) == redis.zcard(key_releases):
mb_discs = [get_disc(mbid) for mbid in redis.zrange(key_releases, offset, limit)]
total = redis.zcard(key_releases)
if len(mb_discs) == 0:
# Si es que no había ningún disco, enviar a cargar al artista, quizás nunca se a guardado
# en cache antes
jobs.load_artist_on_cache.delay(mbid)
mb_discs_browse = mb.browse_release_groups(params={'artist': mbid},
includes=['artist-credits'],
limit=limit, offset=offset)
if 'error' in mb_discs_browse:
return mb_discs_browse
@@ -227,12 +229,14 @@ def get_artist_of_disc(mbid):
mb_artist = None
with get_redis_connection() as redis:
key_id = f'release_group_{mbid}:artist'
key_id = f'release_group:{mbid}:artist'
if key_id in redis:
mb_artist = get_artist(redis.get(key_id))
if mb_artist is None:
mb_artist_browse = mb.browse_artists(params={'release-group': mbid}, limit=0, offset=0)
mb_artist_browse = mb.browse_artists(params={'release-group': mbid},
includes=['tags'],
limit=1, offset=0)
if 'error' in mb_artist_browse:
return mb_artist_browse
@@ -252,45 +256,82 @@ def get_artist_of_disc(mbid):
def get_release(mbid):
"""Obtiene una release desde musicbrainz incluyendo sus artistas"""
with get_redis_connection() as redis:
mb_release = redis.get(f'release_{mbid}')
mb_release = redis.get(f'release:{mbid}')
if mb_release is None:
mb_release = mb.get_release_by_mbid(mbid, includes=['artists'])
else:
mb_release = json.loads(mb_release)
if 'error' in mb_release:
return mb_release
jobs.load_entities_of_release.delay(mbid)
release = map_release(mb_release)
return release
return map_release(mb_release)
def get_releases_of_disc(mbid, limit, page):
"""Obtiene las releases de un disco desde musicbrainz incluyendo los creditos a su artistas"""
mb_releases = mb.browse_releases(params={'release-group': mbid}, includes=['artist-credits'],
limit=limit, offset=limit * (page - 1))
"""Obtiene las releases de un disco desde musicbrainz"""
if 'error' in mb_releases:
return mb_releases
mb_releases = []
offset = limit * (page - 1)
total = 0
with get_redis_connection() as redis:
key_releases = f'release_group:{mbid}:releases'
if key_releases in redis:
if int(redis.get(f'{key_releases}:count')) == redis.zcard(key_releases):
mb_releases = [get_release(mbid) for mbid in redis.zrange(key_releases,
offset,
limit)]
total = redis.zcard(key_releases)
if len(mb_releases) == 0:
# Si es que no se encontraron releases antes es probable que nunca se cargo en cache el
# release group
jobs.load_entities_of_release_group.delay(mbid)
mb_releases = mb.browse_releases(params={'release-group': mbid},
includes=['artist-credits'],
limit=limit, offset=limit * (page - 1))
if 'error' in mb_releases:
return mb_releases
total = mb_releases.get('release_count')
mb_releases = mb_releases.get('releases')
return {
'paginate': paginate(mb_releases.get('release_count', 0), limit, page),
'releases': [map_release(release) for release in mb_releases.get('releases')]
'paginate': paginate(total, limit, page),
'releases': [map_release(release) for release in mb_releases]
}
def get_artist_of_release(mbid, limit, page):
"""Obtiene el artista de una release"""
mb_artists = mb.browse_artists(params={'release': mbid}, limit=limit, offset=limit * (page - 1))
mb_artist = None
if 'error' in mb_artists:
return mb_artists
with get_redis_connection() as redis:
key = f'release:{mbid}:artist'
if key in redis:
mb_artist = get_artist(redis.get(key))
if mb_artist is None:
mb_artist_browse = mb.browse_artists(params={'release': mbid},
includes=['tags'],
limit=limit,
offset=limit * (page - 1))
if 'error' in mb_artist_browse:
return mb_artist_browse
mb_artist = mb_artist_browse.get('artists')[0]
jobs.load_artist_on_cache.delay(mb_artist)
return {
'paginate': paginate(mb_artists.get('artist_count', 0), limit, page),
'artists': [map_artist(artist) for artist in mb_artists.get('artists')]
'artist': map_artist(mb_artist)
}
@@ -309,7 +350,6 @@ def get_recording(mbid):
if 'error' in mb_recording:
return mb_recording
jobs.load_entities_of_recording.delay(mb_recording)
recording = map_recording(mb_recording)
return recording