From c30173cdc064e7b9c6950098216a93be398cc6a7 Mon Sep 17 00:00:00 2001 From: breakid Date: Sun, 14 May 2023 00:37:09 -0400 Subject: [PATCH] Reverted all formatting changes --- app/main.py | 6 +- app/ytdl.py | 229 ++++++++++++++++++++-------------------------------- 2 files changed, 92 insertions(+), 143 deletions(-) diff --git a/app/main.py b/app/main.py index fb55854..7aff1cf 100644 --- a/app/main.py +++ b/app/main.py @@ -139,13 +139,13 @@ def get_custom_dirs(): dirs = list(filter(None, map(convert, path.glob('**')))) return dirs - + download_dir = recursive_dirs(config.DOWNLOAD_DIR) audio_download_dir = download_dir if config.DOWNLOAD_DIR != config.AUDIO_DOWNLOAD_DIR: audio_download_dir = recursive_dirs(config.AUDIO_DOWNLOAD_DIR) - + return { "download_dir": download_dir, "audio_download_dir": audio_download_dir @@ -194,4 +194,4 @@ app.on_response_prepare.append(on_prepare) if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) log.info(f"Listening on {config.HOST}:{config.PORT}") - web.run_app(app, host=config.HOST, port=config.PORT, reuse_port=True) + web.run_app(app, host=config.HOST, port=config.PORT, reuse_port=True) \ No newline at end of file diff --git a/app/ytdl.py b/app/ytdl.py index 7632227..ed65e19 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -9,8 +9,7 @@ import logging import re from dl_formats import get_format, get_opts, AUDIO_FORMATS -log = logging.getLogger("ytdl") - +log = logging.getLogger('ytdl') class DownloadQueueNotifier: async def added(self, dl): @@ -28,11 +27,10 @@ class DownloadQueueNotifier: async def cleared(self, id): raise NotImplementedError - class DownloadInfo: def __init__(self, id, title, url, quality, format, folder, custom_name_prefix): - self.id = id if len(custom_name_prefix) == 0 else f"{custom_name_prefix}.{id}" - self.title = title if len(custom_name_prefix) == 0 else f"{custom_name_prefix}.{title}" + self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}' + self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}' self.url = url self.quality = quality self.format = format @@ -41,7 +39,6 @@ class DownloadInfo: self.status = self.msg = self.percent = self.speed = self.eta = None self.timestamp = time.time_ns() - class Download: manager = None @@ -62,7 +59,6 @@ class Download: def _download(self): try: - def put_status(st): self.status_queue.put({k: v for k, v in st.items() if k in ( 'tmpfilename', @@ -75,34 +71,28 @@ class Download: 'speed', 'eta', )}) - def put_status_postprocessor(d): - if d["postprocessor"] == "MoveFiles" and d["status"] == "finished": - if "__finaldir" in d["info_dict"]: - filename = os.path.join( - d["info_dict"]["__finaldir"], os.path.basename(d["info_dict"]["filepath"]) - ) + if d['postprocessor'] == 'MoveFiles' and d['status'] == 'finished': + if '__finaldir' in d['info_dict']: + filename = os.path.join(d['info_dict']['__finaldir'], os.path.basename(d['info_dict']['filepath'])) else: - filename = d["info_dict"]["filepath"] - self.status_queue.put({"status": "finished", "filename": filename}) - - ret = yt_dlp.YoutubeDL( - params={ - "quiet": True, - "no_color": True, - #'skip_download': True, - "paths": {"home": self.download_dir, "temp": self.temp_dir}, - "outtmpl": {"default": self.output_template, "chapter": self.output_template_chapter}, - "format": self.format, - "socket_timeout": 30, - "progress_hooks": [put_status], - "postprocessor_hooks": [put_status_postprocessor], - **self.ytdl_opts, - } - ).download([self.info.url]) - self.status_queue.put({"status": "finished" if ret == 0 else "error"}) + filename = d['info_dict']['filepath'] + self.status_queue.put({'status': 'finished', 'filename': filename}) + ret = yt_dlp.YoutubeDL(params={ + 'quiet': True, + 'no_color': True, + #'skip_download': True, + 'paths': {"home": self.download_dir, "temp": self.temp_dir}, + 'outtmpl': { "default": self.output_template, "chapter": self.output_template_chapter }, + 'format': self.format, + 'socket_timeout': 30, + 'progress_hooks': [put_status], + 'postprocessor_hooks': [put_status_postprocessor], + **self.ytdl_opts, + }).download([self.info.url]) + self.status_queue.put({'status': 'finished' if ret == 0 else 'error'}) except yt_dlp.utils.YoutubeDLError as exc: - self.status_queue.put({"status": "error", "msg": str(exc)}) + self.status_queue.put({'status': 'error', 'msg': str(exc)}) async def start(self, notifier): if Download.manager is None: @@ -112,7 +102,7 @@ class Download: self.proc.start() self.loop = asyncio.get_running_loop() self.notifier = notifier - self.info.status = "preparing" + self.info.status = 'preparing' await self.notifier.updated(self.info) asyncio.create_task(self.update_status()) return await self.loop.run_in_executor(None, self.proc.join) @@ -141,66 +131,65 @@ class Download: status = await self.loop.run_in_executor(None, self.status_queue.get) if status is None: return - self.tmpfilename = status.get("tmpfilename") - if "filename" in status: - self.info.filename = os.path.relpath(status.get("filename"), self.download_dir) + self.tmpfilename = status.get('tmpfilename') + if 'filename' in status: + self.info.filename = os.path.relpath(status.get('filename'), self.download_dir) # Set correct file extension for thumbnails - if self.info.format == "thumbnail": - self.info.filename = re.sub(r"\.webm$", ".jpg", self.info.filename) - self.info.status = status["status"] - self.info.msg = status.get("msg") - if "downloaded_bytes" in status: - total = status.get("total_bytes") or status.get("total_bytes_estimate") + if(self.info.format == 'thumbnail'): + self.info.filename = re.sub(r'\.webm$', '.jpg', self.info.filename) + self.info.status = status['status'] + self.info.msg = status.get('msg') + if 'downloaded_bytes' in status: + total = status.get('total_bytes') or status.get('total_bytes_estimate') if total: - self.info.percent = status["downloaded_bytes"] / total * 100 - self.info.speed = status.get("speed") - self.info.eta = status.get("eta") + self.info.percent = status['downloaded_bytes'] / total * 100 + self.info.speed = status.get('speed') + self.info.eta = status.get('eta') await self.notifier.updated(self.info) - class PersistentQueue: def __init__(self, path): pdir = os.path.dirname(path) if not os.path.isdir(pdir): os.mkdir(pdir) - with shelve.open(path, "c"): + with shelve.open(path, 'c'): pass self.path = path self.dict = OrderedDict() - + def load(self): for k, v in self.saved_items(): self.dict[k] = Download(None, None, None, None, None, None, {}, v) def exists(self, key): return key in self.dict - + def get(self, key): return self.dict[key] - + def items(self): return self.dict.items() def saved_items(self): - with shelve.open(self.path, "r") as shelf: + with shelve.open(self.path, 'r') as shelf: return sorted(shelf.items(), key=lambda item: item[1].timestamp) def put(self, value): key = value.info.url self.dict[key] = value - with shelve.open(self.path, "w") as shelf: + with shelve.open(self.path, 'w') as shelf: shelf[key] = value.info - + def delete(self, key): del self.dict[key] - with shelve.open(self.path, "w") as shelf: + with shelve.open(self.path, 'w') as shelf: shelf.pop(key) def next(self): k, v = next(iter(self.dict.items())) return k, v - + def empty(self): return not bool(self.dict) @@ -209,10 +198,10 @@ class DownloadQueue: def __init__(self, config, notifier): self.config = config self.notifier = notifier - self.queue = PersistentQueue(self.config.STATE_DIR + "/queue") - self.done = PersistentQueue(self.config.STATE_DIR + "/completed") + self.queue = PersistentQueue(self.config.STATE_DIR + '/queue') + self.done = PersistentQueue(self.config.STATE_DIR + '/completed') self.done.load() - + async def __import_queue(self): for k, v in self.queue.saved_items(): await self.add(v.url, v.quality, v.format, v.folder, v.custom_name_prefix) @@ -223,159 +212,119 @@ class DownloadQueue: asyncio.create_task(self.__import_queue()) def __extract_info(self, url): - return yt_dlp.YoutubeDL( - params={ - "quiet": True, - "no_color": True, - "extract_flat": True, - **self.config.YTDL_OPTIONS, - } - ).extract_info(url, download=False) + return yt_dlp.YoutubeDL(params={ + 'quiet': True, + 'no_color': True, + 'extract_flat': True, + **self.config.YTDL_OPTIONS, + }).extract_info(url, download=False) async def __add_entry(self, entry, quality, format, folder, custom_name_prefix, already): - etype = entry.get("_type") or "video" - if etype == "playlist": - entries = entry["entries"] - log.info(f"playlist detected with {len(entries)} entries") + etype = entry.get('_type') or 'video' + if etype == 'playlist': + entries = entry['entries'] + log.info(f'playlist detected with {len(entries)} entries') playlist_index_digits = len(str(len(entries))) results = [] for index, etr in enumerate(entries, start=1): etr["playlist"] = entry["id"] - etr["playlist_index"] = "{{0:0{0:d}d}}".format(playlist_index_digits).format(index) + etr["playlist_index"] = '{{0:0{0:d}d}}'.format(playlist_index_digits).format(index) for property in ("id", "title", "uploader", "uploader_id"): if property in entry: etr[f"playlist_{property}"] = entry[property] results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, already)) - if any(res["status"] == "error" for res in results): - return { - "status": "error", - "msg": ", ".join(res["msg"] for res in results if res["status"] == "error" and "msg" in res), - } - return {"status": "ok"} - elif etype == "video" or etype.startswith("url") and "id" in entry and "title" in entry: - if not self.queue.exists(entry["id"]): - dl = DownloadInfo( - entry["id"], - entry["title"], - entry.get("webpage_url") or entry["url"], - quality, - format, - folder, - custom_name_prefix, - ) + if any(res['status'] == 'error' for res in results): + return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)} + return {'status': 'ok'} + elif etype == 'video' or etype.startswith('url') and 'id' in entry and 'title' in entry: + if not self.queue.exists(entry['id']): + dl = DownloadInfo(entry['id'], entry['title'], entry.get('webpage_url') or entry['url'], quality, format, folder, custom_name_prefix) # Keep consistent with frontend - base_directory = ( - self.config.DOWNLOAD_DIR - if (quality != "audio" and format not in AUDIO_FORMATS) - else self.config.AUDIO_DOWNLOAD_DIR - ) + base_directory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format not in AUDIO_FORMATS) else self.config.AUDIO_DOWNLOAD_DIR if folder: if not self.config.CUSTOM_DIRS: - return { - "status": "error", - "msg": f"A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.", - } + return {'status': 'error', 'msg': f'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'} dldirectory = os.path.realpath(os.path.join(base_directory, folder)) real_base_directory = os.path.realpath(base_directory) if not dldirectory.startswith(real_base_directory): - return { - "status": "error", - "msg": f'Folder "{folder}" must resolve inside the base download directory "{real_base_directory}"', - } + return {'status': 'error', 'msg': f'Folder "{folder}" must resolve inside the base download directory "{real_base_directory}"'} if not os.path.isdir(dldirectory): if not self.config.CREATE_CUSTOM_DIRS: - return { - "status": "error", - "msg": f'Folder "{folder}" for download does not exist inside base directory "{real_base_directory}", and CREATE_CUSTOM_DIRS is not true in the configuration.', - } + return {'status': 'error', 'msg': f'Folder "{folder}" for download does not exist inside base directory "{real_base_directory}", and CREATE_CUSTOM_DIRS is not true in the configuration.'} os.makedirs(dldirectory, exist_ok=True) else: dldirectory = base_directory - output = ( - self.config.OUTPUT_TEMPLATE - if len(custom_name_prefix) == 0 - else f"{custom_name_prefix}.{self.config.OUTPUT_TEMPLATE}" - ) + output = self.config.OUTPUT_TEMPLATE if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{self.config.OUTPUT_TEMPLATE}' output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER for property, value in entry.items(): if property.startswith("playlist"): output = output.replace(f"%({property})s", str(value)) - self.queue.put( - Download( - dldirectory, - self.config.TEMP_DIR, - output, - output_chapter, - quality, - format, - self.config.YTDL_OPTIONS, - dl, - ) - ) + self.queue.put(Download(dldirectory, self.config.TEMP_DIR, output, output_chapter, quality, format, self.config.YTDL_OPTIONS, dl)) self.event.set() await self.notifier.added(dl) - return {"status": "ok"} - elif etype.startswith("url"): - return await self.add(entry["url"], quality, format, folder, custom_name_prefix, already) - return {"status": "error", "msg": f'Unsupported resource "{etype}"'} + return {'status': 'ok'} + elif etype.startswith('url'): + return await self.add(entry['url'], quality, format, folder, custom_name_prefix, already) + return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'} async def add(self, url, quality, format, folder, custom_name_prefix, already=None): - log.info(f"adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=}") + log.info(f'adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=}') already = set() if already is None else already if url in already: - log.info("recursion detected, skipping") - return {"status": "ok"} + log.info('recursion detected, skipping') + return {'status': 'ok'} else: already.add(url) try: entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url) except yt_dlp.utils.YoutubeDLError as exc: - return {"status": "error", "msg": str(exc)} + return {'status': 'error', 'msg': str(exc)} return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, already) async def cancel(self, ids): for id in ids: if not self.queue.exists(id): - log.warn(f"requested cancel for non-existent download {id}") + log.warn(f'requested cancel for non-existent download {id}') continue if self.queue.get(id).started(): self.queue.get(id).cancel() else: self.queue.delete(id) await self.notifier.canceled(id) - return {"status": "ok"} + return {'status': 'ok'} async def clear(self, ids): for id in ids: if not self.done.exists(id): - log.warn(f"requested delete for non-existent download {id}") + log.warn(f'requested delete for non-existent download {id}') continue if self.config.DELETE_FILE_ON_TRASHCAN: dl = self.done.get(id) os.remove(os.path.join(dl.download_dir, dl.info.filename)) self.done.delete(id) await self.notifier.cleared(id) - return {"status": "ok"} + return {'status': 'ok'} def get(self): - return (list((k, v.info) for k, v in self.queue.items()), list((k, v.info) for k, v in self.done.items())) + return(list((k, v.info) for k, v in self.queue.items()), + list((k, v.info) for k, v in self.done.items())) async def __download(self): while True: while self.queue.empty(): - log.info("waiting for item to download") + log.info('waiting for item to download') await self.event.wait() self.event.clear() id, entry = self.queue.next() - log.info(f"downloading {entry.info.title}") + log.info(f'downloading {entry.info.title}') await entry.start(self.notifier) - if entry.info.status != "finished": + if entry.info.status != 'finished': if entry.tmpfilename and os.path.isfile(entry.tmpfilename): try: os.remove(entry.tmpfilename) except: pass - entry.info.status = "error" + entry.info.status = 'error' entry.close() if self.queue.exists(id): self.queue.delete(id) @@ -383,4 +332,4 @@ class DownloadQueue: await self.notifier.canceled(id) else: self.done.put(entry) - await self.notifier.completed(entry.info) + await self.notifier.completed(entry.info) \ No newline at end of file