From 0329645c37b599976c0b4c4f92209cac1abf807c Mon Sep 17 00:00:00 2001 From: Tiberiu Chibici Date: Mon, 10 Dec 2018 01:15:10 +0200 Subject: [PATCH] Integrated django-dynamic-preferences library into project. Settings are now stored in the database, the ini file only contains a minimal number of settings. --- Dockerfile | 1 + app/YtManager/settings.py | 73 +++++------- app/YtManagerApp/appconfig.py | 112 ------------------ app/YtManagerApp/appmain.py | 37 +++++- app/YtManagerApp/apps.py | 1 - .../dynamic_preferences_registry.py | 57 +++++++-- app/YtManagerApp/management/appconfig.py | 3 + app/YtManagerApp/management/downloader.py | 17 +-- .../management/jobs/download_video.py | 47 +++++--- .../management/jobs/synchronize.py | 8 +- app/YtManagerApp/management/subscriptions.py | 0 app/YtManagerApp/utils/__init__.py | 9 ++ app/default/defaults.ini | 87 -------------- config/config.ini | 66 +---------- docker-compose.yml | 1 + requirements.txt | 3 +- 16 files changed, 169 insertions(+), 353 deletions(-) delete mode 100644 app/YtManagerApp/appconfig.py create mode 100644 app/YtManagerApp/management/appconfig.py delete mode 100644 app/YtManagerApp/management/subscriptions.py delete mode 100644 app/default/defaults.ini diff --git a/Dockerfile b/Dockerfile index e4630bd..b00c370 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ ENV YTSM_DATA_PATH='/usr/src/ytsm/data' VOLUME /usr/src/ytsm/config VOLUME /usr/src/ytsm/data +VOLUME /usr/src/ytsm/download COPY ./app/ ./ COPY ./docker/init.sh ./ diff --git a/app/YtManager/settings.py b/app/YtManager/settings.py index 8aca83c..7c384e3 100644 --- a/app/YtManager/settings.py +++ b/app/YtManager/settings.py @@ -23,17 +23,17 @@ SESSION_COOKIE_AGE = 3600 * 30 # one month # Application definition INSTALLED_APPS = [ + 'django.contrib.auth', + 'dynamic_preferences', + 'dynamic_preferences.users.apps.UserPreferencesConfig', 'YtManagerApp.apps.YtManagerAppConfig', 'crispy_forms', 'django.contrib.admin', - 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', 'django.contrib.humanize', - 'dynamic_preferences', - 'dynamic_preferences.users.apps.UserPreferencesConfig', ] MIDDLEWARE = [ @@ -125,16 +125,14 @@ LOG_FORMAT = '%(asctime)s|%(process)d|%(thread)d|%(name)s|%(filename)s|%(lineno) # Build paths inside the project like this: os.path.join(BASE_DIR, ...) PROJECT_ROOT = up(up(os.path.dirname(__file__))) # Project root BASE_DIR = up(os.path.dirname(__file__)) # Base dir of the application -CONFIG_DIR = os.path.join(PROJECT_ROOT, "config") -DATA_DIR = os.path.join(PROJECT_ROOT, "data") + +CONFIG_DIR = os.getenv("YTSM_CONFIG_DIR", os.path.join(PROJECT_ROOT, "config")) +DATA_DIR = os.getenv("YTSM_DATA_DIR", os.path.join(PROJECT_ROOT, "data")) +os.chdir(DATA_DIR) + STATIC_ROOT = os.path.join(PROJECT_ROOT, "static") +MEDIA_ROOT = os.path.join(DATA_DIR, 'media') -_DEFAULT_CONFIG_FILE = os.path.join(CONFIG_DIR, 'config.ini') -_DEFAULT_LOG_FILE = os.path.join(DATA_DIR, 'log.log') -_DEFAULT_MEDIA_ROOT = os.path.join(DATA_DIR, 'media') - -CONFIG_FILE = os.getenv('YTSM_CONFIG_FILE', _DEFAULT_CONFIG_FILE) -DATA_CONFIG_FILE = os.path.join(DATA_DIR, 'config.ini') # # Defaults @@ -142,7 +140,6 @@ DATA_CONFIG_FILE = os.path.join(DATA_DIR, 'config.ini') _DEFAULT_DEBUG = False _DEFAULT_SECRET_KEY = '^zv8@i2h!ko2lo=%ivq(9e#x=%q*i^^)6#4@(juzdx%&0c+9a0' -_DEFAULT_YOUTUBE_API_KEY = 'AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8' _DEFAULT_DATABASE = { 'ENGINE': 'django.db.backends.sqlite3', @@ -153,10 +150,6 @@ _DEFAULT_DATABASE = { 'PORT': None, } -_SCHEDULER_SYNC_SCHEDULE = '5 * * * *' -_DEFAULT_SCHEDULER_CONCURRENCY = 1 - - CONFIG_ERRORS = [] CONFIG_WARNINGS = [] @@ -171,6 +164,8 @@ def get_global_opt(name, cfgparser, env_variable=None, fallback=None, boolean=Fa 2. config parser 3. fallback + :param integer: + :param cfgparser: :param name: :param env_variable: :param fallback: @@ -216,25 +211,22 @@ def load_config_ini(): import dj_database_url cfg = ConfigParser(allow_no_value=True, interpolation=ExtendedInterpolatorWithEnv()) - read_ok = cfg.read([DEFAULTS_FILE, CONFIG_FILE, DATA_CONFIG_FILE]) - if DEFAULTS_FILE not in read_ok: - CONFIG_ERRORS.append(f'File {DEFAULTS_FILE} could not be read! Please make sure the file is in the right place,' - f' and it has read permissions.') + cfg_file = os.path.join(CONFIG_DIR, "config.ini") + read_ok = cfg.read([cfg_file]) + + if cfg_file not in read_ok: + CONFIG_ERRORS.append(f'Configuration file {cfg_file} could not be read! Please make sure the file is in the ' + 'right place, and it has read permissions.') # Debug global DEBUG DEBUG = get_global_opt('Debug', cfg, env_variable='YTSM_DEBUG', fallback=_DEFAULT_DEBUG, boolean=True) - # Media root, which is where thumbnails are stored - global MEDIA_ROOT - MEDIA_ROOT = get_global_opt('MediaRoot', cfg, env_variable='YTSM_MEDIA_ROOT', fallback=_DEFAULT_MEDIA_ROOT) - - # Keys - secret key, youtube API key + # Secret key # SECURITY WARNING: keep the secret key used in production secret! - global SECRET_KEY, YOUTUBE_API_KEY + global SECRET_KEY SECRET_KEY = get_global_opt('SecretKey', cfg, env_variable='YTSM_SECRET_KEY', fallback=_DEFAULT_SECRET_KEY) - YOUTUBE_API_KEY = get_global_opt('YoutubeApiKey', cfg, env_variable='YTSM_YTAPI_KEY', fallback=_DEFAULT_YOUTUBE_API_KEY) # Database global DATABASES @@ -247,16 +239,22 @@ def load_config_ini(): else: DATABASES['default'] = { - 'ENGINE': get_global_opt('DatabaseEngine', cfg, env_variable='YTSM_DB_ENGINE', fallback=_DEFAULT_DATABASE['ENGINE']), - 'NAME': get_global_opt('DatabaseName', cfg, env_variable='YTSM_DB_NAME', fallback=_DEFAULT_DATABASE['NAME']), - 'HOST': get_global_opt('DatabaseHost', cfg, env_variable='YTSM_DB_HOST', fallback=_DEFAULT_DATABASE['HOST']), - 'USER': get_global_opt('DatabaseUser', cfg, env_variable='YTSM_DB_USER', fallback=_DEFAULT_DATABASE['USER']), - 'PASSWORD': get_global_opt('DatabasePassword', cfg, env_variable='YTSM_DB_PASSWORD', fallback=_DEFAULT_DATABASE['PASSWORD']), - 'PORT': get_global_opt('DatabasePort', cfg, env_variable='YTSM_DB_PORT', fallback=_DEFAULT_DATABASE['PORT']), + 'ENGINE': get_global_opt('DatabaseEngine', cfg, + env_variable='YTSM_DB_ENGINE', fallback=_DEFAULT_DATABASE['ENGINE']), + 'NAME': get_global_opt('DatabaseName', cfg, + env_variable='YTSM_DB_NAME', fallback=_DEFAULT_DATABASE['NAME']), + 'HOST': get_global_opt('DatabaseHost', cfg, + env_variable='YTSM_DB_HOST', fallback=_DEFAULT_DATABASE['HOST']), + 'USER': get_global_opt('DatabaseUser', cfg, + env_variable='YTSM_DB_USER', fallback=_DEFAULT_DATABASE['USER']), + 'PASSWORD': get_global_opt('DatabasePassword', cfg, + env_variable='YTSM_DB_PASSWORD', fallback=_DEFAULT_DATABASE['PASSWORD']), + 'PORT': get_global_opt('DatabasePort', cfg, + env_variable='YTSM_DB_PORT', fallback=_DEFAULT_DATABASE['PORT']), } # Log settings - global LOG_LEVEL, LOG_FILE + global LOG_LEVEL log_level_str = get_global_opt('LogLevel', cfg, env_variable='YTSM_LOG_LEVEL', fallback='INFO') try: @@ -267,12 +265,5 @@ def load_config_ini(): print("Invalid log level " + LOG_LEVEL) LOG_LEVEL = logging.INFO - LOG_FILE = get_global_opt('LogFile', cfg, env_variable='YTSM_LOG_FILE', fallback=_DEFAULT_LOG_FILE) - - # Scheduler settings - global SCHEDULER_SYNC_SCHEDULE, SCHEDULER_CONCURRENCY - SCHEDULER_SYNC_SCHEDULE = get_global_opt('SynchronizationSchedule', cfg, fallback=_SCHEDULER_SYNC_SCHEDULE) - SCHEDULER_CONCURRENCY = get_global_opt('SchedulerConcurrency', cfg, fallback=_DEFAULT_SCHEDULER_CONCURRENCY, integer=True) - load_config_ini() diff --git a/app/YtManagerApp/appconfig.py b/app/YtManagerApp/appconfig.py deleted file mode 100644 index 9905558..0000000 --- a/app/YtManagerApp/appconfig.py +++ /dev/null @@ -1,112 +0,0 @@ -import logging -import os -import os.path -from collections import ChainMap -from configparser import ConfigParser -from shutil import copyfile -from typing import Optional, Any - -from django.conf import settings as dj_settings -from django.contrib.auth.models import User - -from .models import UserSettings, Subscription -from .utils.extended_interpolation_with_env import ExtendedInterpolatorWithEnv - - -class AppSettings(ConfigParser): - _DEFAULT_INTERPOLATION = ExtendedInterpolatorWithEnv() - - def __init__(self, *args, **kwargs): - super().__init__(allow_no_value=True, *args, **kwargs) - - def initialize(self): - self.read([dj_settings.DEFAULTS_FILE, dj_settings.CONFIG_FILE]) - - def save(self): - if os.path.exists(dj_settings.CONFIG_FILE): - # Create a backup - copyfile(dj_settings.CONFIG_FILE, dj_settings.CONFIG_FILE + ".backup") - else: - # Ensure directory exists - settings_dir = os.path.dirname(dj_settings.CONFIG_FILE) - os.makedirs(settings_dir, exist_ok=True) - - with open(dj_settings.CONFIG_FILE, 'w') as f: - self.write(f) - - def __get_combined_dict(self, vars: Optional[Any], sub: Optional[Subscription], user: Optional[User]) -> ChainMap: - vars_dict = {} - sub_overloads_dict = {} - user_settings_dict = {} - - if vars is not None: - vars_dict = vars - - if sub is not None: - sub_overloads_dict = sub.get_overloads_dict() - - if user is not None: - user_settings = UserSettings.find_by_user(user) - if user_settings is not None: - user_settings_dict = user_settings.to_dict() - - return ChainMap(vars_dict, sub_overloads_dict, user_settings_dict) - - def get_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> str: - return super().get(section, option, - fallback=fallback, - vars=self.__get_combined_dict(vars, None, user)) - - def getboolean_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> bool: - return super().getboolean(section, option, - fallback=fallback, - vars=self.__get_combined_dict(vars, None, user)) - - def getint_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> int: - return super().getint(section, option, - fallback=fallback, - vars=self.__get_combined_dict(vars, None, user)) - - def getfloat_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> float: - return super().getfloat(section, option, - fallback=fallback, - vars=self.__get_combined_dict(vars, None, user)) - - def get_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> str: - return super().get(section, option, - fallback=fallback, - vars=self.__get_combined_dict(vars, sub, sub.user)) - - def getboolean_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> bool: - return super().getboolean(section, option, - fallback=fallback, - vars=self.__get_combined_dict(vars, sub, sub.user)) - - def getint_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> int: - return super().getint(section, option, - fallback=fallback, - vars=self.__get_combined_dict(vars, sub, sub.user)) - - def getfloat_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> float: - return super().getfloat(section, option, - fallback=fallback, - vars=self.__get_combined_dict(vars, sub, sub.user)) - - -settings = AppSettings() - - -def initialize_app_config(): - settings.initialize() - __initialize_logger() - logging.info('Application started!') - - -def __initialize_logger(): - log_dir = os.path.dirname(dj_settings.LOG_FILE) - os.makedirs(log_dir, exist_ok=True) - - logging.basicConfig( - filename=dj_settings.LOG_FILE, - level=dj_settings.LOG_LEVEL, - format=dj_settings.LOG_FORMAT) diff --git a/app/YtManagerApp/appmain.py b/app/YtManagerApp/appmain.py index ad7b771..5b7c449 100644 --- a/app/YtManagerApp/appmain.py +++ b/app/YtManagerApp/appmain.py @@ -1,11 +1,36 @@ -from .appconfig import initialize_app_config -from .scheduler import initialize_scheduler -from .management.jobs.synchronize import schedule_synchronize_global import logging +import logging.handlers +import os + +from django.conf import settings as dj_settings + +from .management.appconfig import global_prefs +from .management.jobs.synchronize import schedule_synchronize_global +from .scheduler import initialize_scheduler + + +def __initialize_logger(): + log_dir = os.path.join(dj_settings.DATA_DIR, 'logs') + os.makedirs(log_dir, exist_ok=True) + + file_handler = logging.handlers.RotatingFileHandler( + os.path.join(log_dir, "log.log"), + maxBytes=1024 * 1024, + backupCount=5 + ) + + logging.basicConfig( + level=dj_settings.LOG_LEVEL, + format=dj_settings.LOG_FORMAT, + handlers=[file_handler] + ) def main(): - initialize_app_config() - initialize_scheduler() - schedule_synchronize_global() + __initialize_logger() + + if global_prefs['hidden__initialized']: + initialize_scheduler() + schedule_synchronize_global() + logging.info('Initialization complete.') diff --git a/app/YtManagerApp/apps.py b/app/YtManagerApp/apps.py index 3d7cd30..ba8c32b 100644 --- a/app/YtManagerApp/apps.py +++ b/app/YtManagerApp/apps.py @@ -1,5 +1,4 @@ from django.apps import AppConfig -import os class YtManagerAppConfig(AppConfig): diff --git a/app/YtManagerApp/dynamic_preferences_registry.py b/app/YtManagerApp/dynamic_preferences_registry.py index 9dcbb4d..13b0cdd 100644 --- a/app/YtManagerApp/dynamic_preferences_registry.py +++ b/app/YtManagerApp/dynamic_preferences_registry.py @@ -13,6 +13,7 @@ scheduler = Section('scheduler') manager = Section('manager') downloader = Section('downloader') + # Hidden settings @global_preferences_registry.register class Initialized(BooleanPreference): @@ -20,6 +21,7 @@ class Initialized(BooleanPreference): name = 'initialized' default = False + # General settings @global_preferences_registry.register class YouTubeAPIKey(StringPreference): @@ -28,13 +30,31 @@ class YouTubeAPIKey(StringPreference): default = 'AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8' required = True + +@global_preferences_registry.register +class AllowCDN(BooleanPreference): + section = general + name = 'allow_cdn' + default = True + required = True + + +@global_preferences_registry.register +class AllowRegistrations(BooleanPreference): + section = general + name = 'allow_registrations' + default = True + required = True + + @global_preferences_registry.register class SyncSchedule(StringPreference): section = scheduler name = 'synchronization_schedule' - default = '5 * * * *' # hourly + default = '5 * * * *' # hourly required = True + @global_preferences_registry.register class SchedulerConcurrency(IntegerPreference): section = scheduler @@ -42,8 +62,8 @@ class SchedulerConcurrency(IntegerPreference): default = 2 required = True -# User settings +# User settings @user_preferences_registry.register class MarkDeletedAsWatched(BooleanPreference): section = manager @@ -51,6 +71,7 @@ class MarkDeletedAsWatched(BooleanPreference): default = True required = True + @user_preferences_registry.register class AutoDeleteWatched(BooleanPreference): section = manager @@ -58,20 +79,23 @@ class AutoDeleteWatched(BooleanPreference): default = True required = True + @user_preferences_registry.register class AutoDownloadEnabled(BooleanPreference): section = downloader - name = 'enabled' + name = 'download_enabled' default = True required = True + @user_preferences_registry.register class DownloadGlobalLimit(IntegerPreference): section = downloader - name = 'global_limit' + name = 'download_global_limit' default = None required = False + @user_preferences_registry.register class DownloadGlobalSizeLimit(IntegerPreference): section = downloader @@ -79,49 +103,56 @@ class DownloadGlobalSizeLimit(IntegerPreference): default = None required = False + @user_preferences_registry.register class DownloadSubscriptionLimit(IntegerPreference): section = downloader - name = 'limit_per_subscription' + name = 'download_limit_per_subscription' default = 5 required = False + @user_preferences_registry.register class DownloadMaxAttempts(IntegerPreference): section = downloader - name = 'max_attempts' + name = 'download_max_attempts' default = 3 required = True + @user_preferences_registry.register class DownloadOrder(ChoicePreference): section = downloader - name = 'order' + name = 'download_order' choices = VIDEO_ORDER_CHOICES default = 'playlist' required = True + @user_preferences_registry.register class DownloadPath(StringPreference): section = downloader - name = 'path' + name = 'download_path' default = None required = False + @user_preferences_registry.register class DownloadFilePattern(StringPreference): section = downloader - name = 'file_pattern' + name = 'download_file_pattern' default = '${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]' required = True + @user_preferences_registry.register class DownloadFormat(StringPreference): section = downloader - name = 'format' + name = 'download_format' default = 'bestvideo+bestaudio' required = True + @user_preferences_registry.register class DownloadSubtitles(BooleanPreference): section = downloader @@ -129,6 +160,7 @@ class DownloadSubtitles(BooleanPreference): default = True required = True + @user_preferences_registry.register class DownloadAutogeneratedSubtitles(BooleanPreference): section = downloader @@ -136,6 +168,7 @@ class DownloadAutogeneratedSubtitles(BooleanPreference): default = False required = True + @user_preferences_registry.register class DownloadAllSubtitles(BooleanPreference): section = downloader @@ -143,6 +176,7 @@ class DownloadAllSubtitles(BooleanPreference): default = False required = False + @user_preferences_registry.register class DownloadSubtitlesLangs(StringPreference): section = downloader @@ -150,9 +184,10 @@ class DownloadSubtitlesLangs(StringPreference): default = 'en,ro' required = False + @user_preferences_registry.register class DownloadSubtitlesFormat(StringPreference): section = downloader name = 'subtitles_format' default = False - required = False \ No newline at end of file + required = False diff --git a/app/YtManagerApp/management/appconfig.py b/app/YtManagerApp/management/appconfig.py new file mode 100644 index 0000000..1743fd6 --- /dev/null +++ b/app/YtManagerApp/management/appconfig.py @@ -0,0 +1,3 @@ +from dynamic_preferences.registries import global_preferences_registry + +global_prefs = global_preferences_registry.manager() diff --git a/app/YtManagerApp/management/downloader.py b/app/YtManagerApp/management/downloader.py index edad07e..70c7f38 100644 --- a/app/YtManagerApp/management/downloader.py +++ b/app/YtManagerApp/management/downloader.py @@ -1,6 +1,6 @@ -from YtManagerApp.appconfig import settings from YtManagerApp.management.jobs.download_video import schedule_download_video from YtManagerApp.models import Video, Subscription, VIDEO_ORDER_MAPPING +from YtManagerApp.utils import first_non_null from django.conf import settings as srv_settings import logging import requests @@ -12,17 +12,12 @@ log = logging.getLogger('downloader') def __get_subscription_config(sub: Subscription): - enabled = settings.getboolean_sub(sub, 'user', 'AutoDownload') + user = sub.user - global_limit = -1 - if len(settings.get_sub(sub, 'user', 'DownloadGlobalLimit')) > 0: - global_limit = settings.getint_sub(sub, 'user', 'DownloadGlobalLimit') - - limit = -1 - if len(settings.get_sub(sub, 'user', 'DownloadSubscriptionLimit')) > 0: - limit = settings.getint_sub(sub, 'user', 'DownloadSubscriptionLimit') - - order = settings.get_sub(sub, 'user', 'DownloadOrder') + enabled = first_non_null(sub.auto_download, user.preferences['download_enabled']) + global_limit = user.preferences['download_global_limit'] + limit = first_non_null(sub.download_limit, user.preferences['download_limit_per_subscription']) + order = first_non_null(sub.download_order, user.preferences['download_order']) order = VIDEO_ORDER_MAPPING[order] return enabled, global_limit, limit, order diff --git a/app/YtManagerApp/management/jobs/download_video.py b/app/YtManagerApp/management/jobs/download_video.py index 76854e7..b35771b 100644 --- a/app/YtManagerApp/management/jobs/download_video.py +++ b/app/YtManagerApp/management/jobs/download_video.py @@ -1,12 +1,14 @@ -from YtManagerApp.models import Video -from YtManagerApp import scheduler -from YtManagerApp.appconfig import settings -import os -import youtube_dl import logging +import os import re +from string import Template from threading import Lock +import youtube_dl + +from YtManagerApp import scheduler +from YtManagerApp.models import Video + log = logging.getLogger('video_downloader') log_youtube_dl = log.getChild('youtube_dl') @@ -25,9 +27,8 @@ def __get_valid_path(path): return value -def __build_youtube_dl_params(video: Video): - # resolve path - pattern_dict = { +def __build_template_dict(video: Video): + return { 'channel': video.subscription.channel_name, 'channel_id': video.subscription.channel_id, 'playlist': video.subscription.name, @@ -37,22 +38,30 @@ def __build_youtube_dl_params(video: Video): 'id': video.video_id, } - download_path = settings.get_sub(video.subscription, 'user', 'DownloadPath') - output_pattern = __get_valid_path(settings.get_sub( - video.subscription, 'user', 'DownloadFilePattern', vars=pattern_dict)) + +def __build_youtube_dl_params(video: Video): + + sub = video.subscription + user = sub.user + + # resolve path + download_path = user.preferences['download_path'] + + template_dict = __build_template_dict(video) + output_pattern = Template(user.preferences['download_file_pattern']).safe_substitute(template_dict) output_path = os.path.join(download_path, output_pattern) output_path = os.path.normpath(output_path) youtube_dl_params = { 'logger': log_youtube_dl, - 'format': settings.get_sub(video.subscription, 'user', 'DownloadFormat'), + 'format': user.preferences['download_format'], 'outtmpl': output_path, 'writethumbnail': True, 'writedescription': True, - 'writesubtitles': settings.getboolean_sub(video.subscription, 'user', 'DownloadSubtitles'), - 'writeautomaticsub': settings.getboolean_sub(video.subscription, 'user', 'DownloadAutogeneratedSubtitles'), - 'allsubtitles': settings.getboolean_sub(video.subscription, 'user', 'DownloadSubtitlesAll'), + 'writesubtitles': user.preferences['subtitles_enabled'], + 'writeautomaticsub': user.preferences['autogenerated_subtitles'], + 'allsubtitles': user.preferences['all_subtitles'], 'postprocessors': [ { 'key': 'FFmpegMetadata' @@ -60,12 +69,12 @@ def __build_youtube_dl_params(video: Video): ] } - sub_langs = settings.get_sub(video.subscription, 'user', 'DownloadSubtitlesLangs').split(',') + sub_langs = user.preferences['subtitles_langs'].split(',') sub_langs = [i.strip() for i in sub_langs] if len(sub_langs) > 0: youtube_dl_params['subtitleslangs'] = sub_langs - sub_format = settings.get_sub(video.subscription, 'user', 'DownloadSubtitlesFormat') + sub_format = user.preferences['subtitles_format'] if len(sub_format) > 0: youtube_dl_params['subtitlesformat'] = sub_format @@ -74,6 +83,8 @@ def __build_youtube_dl_params(video: Video): def download_video(video: Video, attempt: int = 1): + user = video.subscription.user + log.info('Downloading video %d [%s %s]', video.id, video.video_id, video.name) # Issue: if multiple videos are downloaded at the same time, a race condition appears in the mkdirs() call that @@ -82,7 +93,7 @@ def download_video(video: Video, attempt: int = 1): _lock.acquire() try: - max_attempts = settings.getint_sub(video.subscription, 'user', 'DownloadMaxAttempts', fallback=3) + max_attempts = user.preferences['download_max_attempts'] youtube_dl_params, output_path = __build_youtube_dl_params(video) with youtube_dl.YoutubeDL(youtube_dl_params) as yt: diff --git a/app/YtManagerApp/management/jobs/synchronize.py b/app/YtManagerApp/management/jobs/synchronize.py index f6db0d8..04741b9 100644 --- a/app/YtManagerApp/management/jobs/synchronize.py +++ b/app/YtManagerApp/management/jobs/synchronize.py @@ -5,7 +5,7 @@ from threading import Lock from apscheduler.triggers.cron import CronTrigger from YtManagerApp import scheduler -from YtManagerApp.appconfig import settings +from YtManagerApp.management.appconfig import global_prefs from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_all, downloader_process_subscription from YtManagerApp.models import * from YtManagerApp.utils import youtube @@ -43,6 +43,8 @@ def __check_new_videos_sub(subscription: Subscription, yt_api: youtube.YoutubeAP def __detect_deleted(subscription: Subscription): + user = subscription.user + for video in Video.objects.filter(subscription=subscription, downloaded_path__isnull=False): found_video = False files = [] @@ -71,7 +73,7 @@ def __detect_deleted(subscription: Subscription): video.downloaded_path = None # Mark watched? - if settings.getboolean_sub(subscription, 'user', 'MarkDeletedAsWatched'): + if user.preferences['MarkDeletedAsWatched']: video.watched = True video.save() @@ -147,7 +149,7 @@ def synchronize_subscription(subscription: Subscription): def schedule_synchronize_global(): - trigger = CronTrigger.from_crontab(settings.get('global', 'SynchronizationSchedule')) + trigger = CronTrigger.from_crontab(global_prefs['synchronization_schedule']) job = scheduler.scheduler.add_job(synchronize, trigger, max_instances=1, coalesce=True) log.info('Scheduled synchronize job job=%s', job.id) diff --git a/app/YtManagerApp/management/subscriptions.py b/app/YtManagerApp/management/subscriptions.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/YtManagerApp/utils/__init__.py b/app/YtManagerApp/utils/__init__.py index e69de29..e198fb4 100644 --- a/app/YtManagerApp/utils/__init__.py +++ b/app/YtManagerApp/utils/__init__.py @@ -0,0 +1,9 @@ + +def first_non_null(*iterable): + ''' + Returns the first element from the iterable which is not None. + If all the elements are 'None', 'None' is returned. + :param iterable: Iterable containing list of elements. + :return: First non-null element, or None if all elements are 'None'. + ''' + return next((item for item in iterable if item is not None), None) diff --git a/app/default/defaults.ini b/app/default/defaults.ini deleted file mode 100644 index 2a6f045..0000000 --- a/app/default/defaults.ini +++ /dev/null @@ -1,87 +0,0 @@ -; Use ${env:environment_variable} to use the value of an environment variable. -; The global section contains settings that apply to the entire server -[global] - -; Controls whether django debug mode is enabled. Should be false in production. -Debug=False - -; This is the folder where thumbnails will be downloaded. By default project_root/data/media is used. -;MediaRoot= - -; Secret key - django secret key -SecretKey=^zv8@i2h!ko2lo=%ivq(9e#x=%q*i^^)6#4@(juzdx%&0c+9a0 - -; YouTube API key - get this from your user account -YoutubeApiKey=AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8 - -; Database settings -; You can use any database engine supported by Django, as long as you add the required dependencies. -; Built-in engines: https://docs.djangoproject.com/en/2.1/ref/settings/#std:setting-DATABASE-ENGINE -; Others databases might be supported by installing the corect pip package. - -;DatabaseEngine=django.db.backends.sqlite3 -;DatabaseName=data/ytmanager.db -;DatabaseHost= -;DatabaseUser= -;DatabasePassword= -;DatabasePort= - -; Database one-liner. If set, it will override any other Database* setting. -; Documentation: https://github.com/kennethreitz/dj-database-url -;DatabaseURL=sqlite:////full/path/to/your/database/file.sqlite - -; Log settings, sets the log file location and the log level -LogLevel=INFO -; LogFile=data/log.log - -; Specifies the synchronization schedule, in crontab format. -; Format: -SynchronizationSchedule=5 * * * * - -; Number of threads running the scheduler -; Since most of the jobs scheduled are downloads, there is no advantage to having -; a higher concurrency -SchedulerConcurrency=3 - - -; Default user settings -[user] -; When a video is deleted on the system, it will be marked as 'watched' -MarkDeletedAsWatched=True - -; Videos marked as watched are automatically deleted -DeleteWatched=True - -; Enable automatic downloading -AutoDownload=True - -; Limit the total number of videos downloaded (-1 or empty = no limit) -DownloadGlobalLimit= - -; Limit the numbers of videos per subscription (-1 or empty = no limit) -DownloadSubscriptionLimit=5 - -; Number of download attempts -DownloadMaxAttempts=3 - -; Download order -; Options: newest, oldest, playlist, playlist_reverse, popularity, rating -DownloadOrder=playlist - -; Path where downloaded videos are stored -DownloadPath=data/videos - -; A pattern which describes how downloaded files are organized. Extensions are automatically appended. -; Supported fields: channel, channel_id, playlist, playlist_id, playlist_index, title, id -; The default pattern should work pretty well with Plex -DownloadFilePattern=${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}] - -; Download format that will be passed to youtube-dl. See the youtube-dl documentation for more details. -DownloadFormat=bestvideo+bestaudio - -; Subtitles - these options match the youtube-dl options -DownloadSubtitles=True -DownloadAutogeneratedSubtitles=False -DownloadSubtitlesAll=False -DownloadSubtitlesLangs=en,ro -DownloadSubtitlesFormat= diff --git a/config/config.ini b/config/config.ini index 727f047..807f663 100644 --- a/config/config.ini +++ b/config/config.ini @@ -6,22 +6,16 @@ Debug=True -; This is the folder where thumbnails will be downloaded. By default project_root/data/media is used. -;MediaRoot= - ; Secret key - django secret key -;SecretKey=^zv8@i2h!ko2lo=%ivq(9e#x=%q*i^^)6#4@(juzdx%&0c+9a0 - -; YouTube API key - get this from your user account -;YoutubeApiKey=AIzaSyAonB6T-DrKjfGxBGuHyFMg0x_d0T9nlP8 +SecretKey=^zv8@i2h!ko2lo=%ivq(9e#x=%q*i^^)6#4@(juzdx%&0c+9a0 ; Database settings ; You can use any database engine supported by Django, as long as you add the required dependencies. ; Built-in engines: https://docs.djangoproject.com/en/2.1/ref/settings/#std:setting-DATABASE-ENGINE ; Others databases might be supported by installing the corect pip package. -;DatabaseEngine=django.db.backends.sqlite3 -;DatabaseName=data/ytmanager.db +DatabaseEngine=django.db.backends.sqlite3 +DatabaseName=ytmanager.db ;DatabaseHost= ;DatabaseUser= ;DatabasePassword= @@ -32,56 +26,4 @@ Debug=True ;DatabaseURL=sqlite:////full/path/to/your/database/file.sqlite ; Log settings, sets the log file location and the log level -;LogLevel=INFO -;LogFile=data/log.log - -; Specifies the synchronization schedule, in crontab format. -; Format: -;SynchronizationSchedule=5 * * * * - -; Number of threads running the scheduler -; Since most of the jobs scheduled are downloads, there is no advantage to having -; a higher concurrency -;SchedulerConcurrency=3 - -; Default user settings -[user] -; When a video is deleted on the system, it will be marked as 'watched' -;MarkDeletedAsWatched=True - -; Videos marked as watched are automatically deleted -;DeleteWatched=True - -; Enable automatic downloading -;AutoDownload=True - -; Limit the total number of videos downloaded (-1 or empty = no limit) -;DownloadGlobalLimit=10 - -; Limit the numbers of videos per subscription (-1 or empty = no limit) -;DownloadSubscriptionLimit=5 - -; Number of download attempts -;DownloadMaxAttempts=3 - -; Download order -; Options: newest, oldest, playlist, playlist_reverse, popularity, rating -;DownloadOrder=playlist - -; Path where downloaded videos are stored -DownloadPath=${env:YTSM_DATA_PATH}/videos - -; A pattern which describes how downloaded files are organized. Extensions are automatically appended. -; Supported fields: channel, channel_id, playlist, playlist_id, playlist_index, title, id -; The default pattern should work pretty well with Plex -;DownloadFilePattern=${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}] - -; Download format that will be passed to youtube-dl. See the youtube-dl documentation for more details. -;DownloadFormat=bestvideo+bestaudio - -; Subtitles - these options match the youtube-dl options -;DownloadSubtitles=True -;DownloadAutogeneratedSubtitles=False -;DownloadSubtitlesAll=False -;DownloadSubtitlesLangs=en,ro -;DownloadSubtitlesFormat= +LogLevel=INFO diff --git a/docker-compose.yml b/docker-compose.yml index d3069ab..54d4101 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,3 +20,4 @@ services: volumes: - ./config:/usr/src/ytsm/config - ./data:/usr/src/ytsm/data + - ./downloads:/usr/src/ytsm/downloads \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index d78e284..ef9a6d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -apscheduler +requests +apscheduler gunicorn django django-crispy-forms