greatbody
/
tubesync
mirror of https://github.com/meeb/tubesync
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Store title as column instead of just in metadata to speedup access

pull/515/head
Tim 2024-07-12 16:12:32 +08:00
parent e0a2187566
commit 9615aac8e9
4 changed files with 70 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
tubesync/sync/migrations/0023_media_duration_filter.py
View File

@ -1,14 +1,24 @@
# Generated by pac
from django.db import migrations, models
class Migration(migrations.Migration):
class Migration(migrations.Migration):
dependencies = [
('sync', '0022_add_delete_files_on_disk'),
]
operations = [
migrations.AddField(
model_name='media',
name='title',
field=models.CharField(
verbose_name='title',
max_length=100,
blank=True,
null=False,
default='',
help_text='Video title'
),
),
migrations.AddField(
model_name='media',
name='duration',
@ -16,7 +26,8 @@ class Migration(migrations.Migration):
verbose_name='duration',
blank=True,
null=True,
help_text='Duration of media in seconds'),
help_text='Duration of media in seconds'
),
),
migrations.AddField(
model_name='source',
@ -25,18 +36,18 @@ class Migration(migrations.Migration):
verbose_name='filter seconds',
blank=True,
null=True,
help_text='Filter Media based on Min/Max duration. Leave blank or 0 to disable filtering'),
help_text='Filter Media based on Min/Max duration. Leave blank or 0 to disable filtering'
),
),
migrations.AddField(
model_name='source',
name='filter_seconds_min',
field=models.BooleanField(
verbose_name='filter seconds min/max',
choices=[(True, 'Minimum Length'),(False, 'Maximum Length')],
choices=[(True, 'Minimum Length'), (False, 'Maximum Length')],
default=True,
help_text='When Filter Seconds is > 0, do we skip on minimum (video shorter than limit) or maximum ('
'video greater than maximum) video duration'
),
),
]

28
tubesync/sync/models.py
View File

@ -876,13 +876,20 @@ class Media(models.Model):
null=True,
help_text=_('Size of the downloaded media in bytes')
)
duration = models.PositiveIntegerField(
_('duration'),
blank=True,
null=True,
help_text=_('Duration of media in seconds')
)
title = models.CharField(
_('title'),
max_length=100,
blank=True,
null=False,
default='',
help_text=_('Video title')
)
def __str__(self):
return self.key
@ -894,6 +901,21 @@ class Media(models.Model):
('source', 'key'),
)
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
# Trigger an update of derived fields from metadata
if self.metadata:
self.title = self.metadata_title
self.duration = self.metadata_duration
if update_fields is not None and "metadata" in update_fields:
# If only some fields are being updated, make sure we update title and duration if metadata changes
update_fields = {"title", "duration"}.union(update_fields)
super().save(
force_insert=force_insert,
force_update=force_update,
using=using,
update_fields=update_fields,)
def get_metadata_field(self, field):
fields = self.METADATA_FIELDS.get(field, {})
return fields.get(self.source.source_type, '')
@ -1108,7 +1130,7 @@ class Media(models.Model):
return self.loaded_metadata.get(field, '').strip()
@property
def title(self):
def metadata_title(self):
field = self.get_metadata_field('title')
return self.loaded_metadata.get(field, '').strip()
@ -1152,7 +1174,7 @@ class Media(models.Model):
@property
def duration_formatted(self):
duration = self.duration
if duration > 0:
if duration and duration > 0:
return seconds_to_timestr(duration)
return '??:??:??'

4
tubesync/sync/tasks.py
View File

@ -294,6 +294,10 @@ def download_media_metadata(media_id):
if upload_date:
media.published = timezone.make_aware(upload_date)
# Store title in DB so it's fast to access
if media.metadata_title:
media.title = media.metadata_title
# Store duration in DB so it's fast to access
if media.metadata_duration:
media.duration = media.metadata_duration

27
tubesync/sync/tests.py
View File

@ -176,7 +176,8 @@ class FrontEndTestCase(TestCase):
'directory': 'testdirectory',
'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
'download_cap': 0,
'filter_text':'.*',
'filter_text': '.*',
'filter_seconds_min': True,
'index_schedule': 3600,
'delete_old_media': False,
'days_to_keep': 14,
@ -219,7 +220,8 @@ class FrontEndTestCase(TestCase):
'directory': 'testdirectory',
'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
'download_cap': 0,
'filter_text':'.*',
'filter_text': '.*',
'filter_seconds_min': True,
'index_schedule': Source.IndexSchedule.EVERY_HOUR,
'delete_old_media': False,
'days_to_keep': 14,
@ -250,7 +252,8 @@ class FrontEndTestCase(TestCase):
'directory': 'testdirectory',
'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
'download_cap': 0,
'filter_text':'.*',
'filter_text': '.*',
'filter_seconds_min': True,
'index_schedule': Source.IndexSchedule.EVERY_2_HOURS, # changed
'delete_old_media': False,
'days_to_keep': 14,
@ -819,6 +822,7 @@ class FormatMatchingTestCase(TestCase):
def test_combined_exact_format_matching(self):
self.source.fallback = Source.FALLBACK_FAIL
self.media.metadata = all_test_metadata['boring']
self.media.save()
expected_matches = {
# (format, vcodec, acodec, prefer_60fps, prefer_hdr): (match_type, code),
('360p', 'AVC1', 'MP4A', True, False): (False, False),
@ -948,6 +952,7 @@ class FormatMatchingTestCase(TestCase):
def test_audio_exact_format_matching(self):
self.source.fallback = Source.FALLBACK_FAIL
self.media.metadata = all_test_metadata['boring']
self.media.save()
expected_matches = {
# (format, vcodec, acodec, prefer_60fps, prefer_hdr): (match_type, code),
('360p', 'AVC1', 'MP4A', True, False): (True, '140'),
@ -1094,6 +1099,7 @@ class FormatMatchingTestCase(TestCase):
self.source.fallback = Source.FALLBACK_FAIL
# Test no 60fps, no HDR metadata
self.media.metadata = all_test_metadata['boring']
self.media.save()
expected_matches = {
# (format, vcodec, prefer_60fps, prefer_hdr): (match_type, code),
('360p', 'AVC1', False, True): (False, False),
@ -1141,6 +1147,7 @@ class FormatMatchingTestCase(TestCase):
self.assertEqual(match_type, expeceted_match_type)
# Test 60fps metadata
self.media.metadata = all_test_metadata['60fps']
self.media.save()
expected_matches = {
# (format, vcodec, prefer_60fps, prefer_hdr): (match_type, code),
('360p', 'AVC1', False, True): (False, False),
@ -1180,6 +1187,7 @@ class FormatMatchingTestCase(TestCase):
self.assertEqual(match_type, expeceted_match_type)
# Test hdr metadata
self.media.metadata = all_test_metadata['hdr']
self.media.save()
expected_matches = {
# (format, vcodec, prefer_60fps, prefer_hdr): (match_type, code),
('360p', 'AVC1', False, True): (False, False),
@ -1235,6 +1243,7 @@ class FormatMatchingTestCase(TestCase):
self.assertEqual(match_type, expeceted_match_type)
# Test 60fps+hdr metadata
self.media.metadata = all_test_metadata['60fps+hdr']
self.media.save()
expected_matches = {
# (format, vcodec, prefer_60fps, prefer_hdr): (match_type, code),
('360p', 'AVC1', False, True): (False, False),
@ -1300,6 +1309,7 @@ class FormatMatchingTestCase(TestCase):
self.source.fallback = Source.FALLBACK_NEXT_BEST
# Test no 60fps, no HDR metadata
self.media.metadata = all_test_metadata['boring']
self.media.save()
expected_matches = {
# (format, vcodec, prefer_60fps, prefer_hdr): (match_type, code),
('360p', 'AVC1', False, True): (False, '134'), # Fallback match, no hdr
@ -1347,6 +1357,7 @@ class FormatMatchingTestCase(TestCase):
self.assertEqual(match_type, expeceted_match_type)
# Test 60fps metadata
self.media.metadata = all_test_metadata['60fps']
self.media.save()
expected_matches = {
# (format, vcodec, prefer_60fps, prefer_hdr): (match_type, code),
('360p', 'AVC1', False, True): (False, '134'), # Fallback match, no hdr
@ -1386,6 +1397,7 @@ class FormatMatchingTestCase(TestCase):
self.assertEqual(match_type, expeceted_match_type)
# Test hdr metadata
self.media.metadata = all_test_metadata['hdr']
self.media.save()
expected_matches = {
# (format, vcodec, prefer_60fps, prefer_hdr): (match_type, code),
('360p', 'AVC1', False, True): (False, '332'), # Fallback match, hdr, switched to VP9
@ -1441,6 +1453,7 @@ class FormatMatchingTestCase(TestCase):
self.assertEqual(match_type, expeceted_match_type)
# Test 60fps+hdr metadata
self.media.metadata = all_test_metadata['60fps+hdr']
self.media.save()
expected_matches = {
# (format, vcodec, prefer_60fps, prefer_hdr): (match_type, code),
('360p', 'AVC1', False, True): (False, '134'), # Fallback match, no hdr
@ -1504,6 +1517,7 @@ class FormatMatchingTestCase(TestCase):
def test_metadata_20230629(self):
self.media.metadata = all_test_metadata['20230629']
self.media.save()
expected_matches = {
# (format, vcodec, prefer_60fps, prefer_hdr): (match_type, code),
('360p', 'AVC1', False, True): (False, '134'), # Fallback match, no hdr
@ -1568,6 +1582,7 @@ class FormatMatchingTestCase(TestCase):
def test_is_regex_match(self):
self.media.metadata = all_test_metadata['boring']
self.media.save()
expected_matches = {
('.*'): (True),
('no fancy stuff'): (True),
@ -1587,7 +1602,11 @@ class FormatMatchingTestCase(TestCase):
for params, expected in expected_matches.items():
self.source.filter_text = params
expected_match_result = expected
self.assertEqual(self.source.is_regex_match(self.media.title), expected_match_result)
self.assertEqual(
self.source.is_regex_match(self.media.title),
expected_match_result,
msg=f'Media title "{self.media.title}" checked against regex "{self.source.filter_text}" failed '
f'expected {expected_match_result}')
class TasksTestCase(TestCase):
def setUp(self):