Publish YouTubeDL 1.4.5 and website on Gitea
Check extension and helper / check (push) Canceled after 0s

This commit is contained in:
DepressedCat committed 2026-10-10 23:53:17 +03:00
commit 5f2aad00f4
97 files changed
+8451

No files matched your search

+1
View File
@@ -0,0 +1 @@
"""YouTubeDL native messaging helper."""
+37
View File
@@ -0,0 +1,37 @@
"""Verify and register an unpacked offline bundle. Never downloads anything."""
from __future__ import annotations
import json
import sys
from pathlib import Path
from host.bundle import ROOT, executable, manifest
from host.common import EXTENSION_ID, HOST_NAME
from host.dependencies import status
def verify():
if not manifest():
raise ValueError("Autonomous bundle metadata is missing.")
if Path(sys.executable).resolve() != Path(executable("python")):
raise ValueError("The helper must use its embedded Python.")
measured = status()
if any(item["status"] != "installed" for item in measured):
raise ValueError("Bundled components failed verification: " + json.dumps(measured))
print(json.dumps({"ready": True, "dependencies": measured}))
def configure():
verify()
runtime = ROOT / ".runtime"
runtime.mkdir(exist_ok=True)
python = executable("python")
launcher = runtime / "launch-host.cmd"
launcher.write_bytes((f'@echo off\r\nchcp 65001 >nul\r\nset PYTHONIOENCODING=utf-8\r\nset PYTHONUTF8=1\r\n'
f'set PYTHONDONTWRITEBYTECODE=1\r\n"{python}" -I -X utf8 -B -u "{ROOT / "host" / "main.py"}" %*\r\n').encode("utf-8"))
native = {"name": HOST_NAME, "description": "YouTubeDL autonomous helper", "path": str(launcher),
"type": "stdio", "allowed_origins": [f"chrome-extension://{EXTENSION_ID}/"]}
(runtime / "native-host.json").write_text(json.dumps(native, ensure_ascii=False, indent=2), encoding="utf-8")
if __name__ == "__main__":
configure() if sys.argv[1:] == ["configure"] else verify()
+49
View File
@@ -0,0 +1,49 @@
"""Application-local runtime; a bundle never falls back to tools on PATH."""
from __future__ import annotations
import json
import os
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
PRODUCT = "YouTubeDL.Helper"
def manifest():
path = ROOT / "bundle.json"
if not path.is_file():
return None
value = json.loads(path.read_text(encoding="utf-8"))
if value.get("product") != PRODUCT or value.get("mode") != "autonomous":
raise ValueError("Повреждён комплект помощника. Повторите установку.")
return value
def executable(name):
bundle = manifest()
if not bundle:
return None
relative = bundle.get("tools", {}).get(name)
if not isinstance(relative, str):
raise ValueError(f"В комплекте отсутствует {name}. Повторите установку помощника.")
candidate = (ROOT / relative).resolve()
if ROOT not in candidate.parents or not candidate.is_file():
raise ValueError(f"Встроенный {name} повреждён. Повторите установку помощника.")
return str(candidate)
def installation_root():
# The installer puts each complete release under <application>/releases/<id>.
if manifest() and ROOT.parent.name == "releases":
return ROOT.parent.parent
return None
def history_path():
installed = installation_root()
return (installed / "data" if installed else ROOT / ".runtime") / "completed-jobs.json"
def information():
bundle = manifest()
return {"mode": "autonomous" if bundle else "source", "version": bundle.get("version") if bundle else None,
"updateTogether": bool(bundle)}
+157
View File
@@ -0,0 +1,157 @@
from __future__ import annotations
import math
import re
from pathlib import Path
from urllib.parse import parse_qs, urlsplit
HOST_NAME = "com.youtubedl.native"
VERSION = "1.3.2"
EXTENSION_ID = "ndnoeiaecgmjnjneekaeapeecmmjheli"
VIDEO_CONTAINERS = {"mp4", "mkv", "webm"}
AUDIO_CONTAINERS = {"m4a", "mp3", "opus", "flac", "wav"}
VIDEO_CODECS = {"auto", "copy", "h264", "hevc", "av1", "vp9"}
ENGINES = {"auto", "nvenc", "amf", "cpu"}
MODES = {"video", "video_only", "audio"}
FORMAT_ID = re.compile(r"^[A-Za-z0-9_.-]{1,80}$")
VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
def youtube_url(value: object) -> str:
if not isinstance(value, str) or len(value) > 2048:
raise ValueError("Нужна ссылка на один ролик YouTube.")
try:
parsed = urlsplit(value.strip())
if parsed.scheme != "https" or parsed.username or parsed.password or parsed.port not in (None, 443):
raise ValueError()
host = (parsed.hostname or "").lower()
parts = parsed.path.strip("/").split("/")
if host == "youtu.be" and len(parts) == 1:
video_id = parts[0]
elif host in {"youtube.com", "www.youtube.com", "m.youtube.com", "music.youtube.com"}:
if parsed.path == "/watch":
video_id = parse_qs(parsed.query).get("v", [""])[0]
elif len(parts) == 2 and parts[0] in {"shorts", "live", "embed"}:
video_id = parts[1]
else:
raise ValueError()
else:
raise ValueError()
if not VIDEO_ID.fullmatch(video_id):
raise ValueError()
except (ValueError, TypeError):
raise ValueError("Поддерживаются HTTPS-ссылки на видео и Shorts с YouTube.") from None
return f"https://www.youtube.com/watch?v={video_id}"
def format_id(value: object) -> str:
if not isinstance(value, str) or not FORMAT_ID.fullmatch(value):
raise ValueError("Некорректный идентификатор дорожки.")
return value
def number(value, default=0):
return value if isinstance(value, (int, float)) and math.isfinite(value) else default
def video_formats(info: dict) -> list[dict]:
return [f for f in info.get("formats", []) if f.get("vcodec") not in (None, "none")
and number(f.get("height")) > 0 and not f.get("has_drm") and FORMAT_ID.fullmatch(str(f.get("format_id", "")))]
def audio_formats(info: dict) -> list[dict]:
return [f for f in info.get("formats", []) if f.get("vcodec") == "none"
and f.get("acodec") not in (None, "none") and not f.get("has_drm")
and FORMAT_ID.fullmatch(str(f.get("format_id", "")))]
def summarize(info: dict) -> dict:
if info.get("is_live"):
raise ValueError("Дождитесь окончания прямой трансляции. Сейчас поддерживаются готовые ролики.")
videos = []
for f in reversed(video_formats(info)):
videos.append({"id": str(f["format_id"]), "height": number(f.get("height")),
"width": number(f.get("width")), "fps": number(f.get("fps")),
"codec": f.get("vcodec", ""), "extension": f.get("ext", ""),
"hasAudio": f.get("acodec") not in (None, "none"),
"size": number(f.get("filesize") or f.get("filesize_approx")),
"hdr": f.get("dynamic_range", "SDR")})
audios = [{"id": str(f["format_id"]), "codec": f.get("acodec", ""),
"extension": f.get("ext", ""), "bitrate": number(f.get("abr") or f.get("tbr")),
"language": f.get("language") or "", "note": f.get("format_note") or "",
"size": number(f.get("filesize") or f.get("filesize_approx"))}
for f in reversed(audio_formats(info))]
if not videos and not audios:
raise ValueError("YouTube не вернул доступных дорожек для этого ролика.")
return {"id": str(info.get("id", "")), "url": youtube_url(info.get("webpage_url", "")),
"title": str(info.get("title", "Видео"))[:300],
"channel": str(info.get("uploader") or info.get("channel") or "")[:200],
"duration": number(info.get("duration")), "videos": videos[:180], "audios": audios[:100]}
def validate_download(payload: dict) -> dict:
if not isinstance(payload, dict):
raise ValueError("Некорректная заявка.")
mode = payload.get("mode", "video")
container = payload.get("container", "mp4")
codec = payload.get("codec", "auto")
engine = payload.get("engine", "auto")
if mode not in MODES or container not in (AUDIO_CONTAINERS if mode == "audio" else VIDEO_CONTAINERS):
raise ValueError("Выберите корректный тип загрузки и формат файла.")
if codec not in VIDEO_CODECS or engine not in ENGINES or payload.get("preset", "balanced") not in {"fast", "balanced", "quality"}:
raise ValueError("Некорректные настройки кодирования.")
bitrate = payload.get("bitrate", 256)
if isinstance(bitrate, bool) or bitrate not in {128, 192, 256, 320}:
raise ValueError("Некорректный битрейт аудио.")
folder = payload.get("folder")
if not isinstance(folder, str) or not folder or len(folder) > 4096:
raise ValueError("Выберите папку сохранения.")
path = Path(folder).expanduser()
if not path.is_absolute() or not path.is_dir():
raise ValueError("Папка сохранения должна существовать и иметь абсолютный путь.")
work_folder = payload.get("workFolder", "")
if not isinstance(work_folder, str) or len(work_folder) > 4096:
raise ValueError("Выберите корректную рабочую папку.")
work_folder = work_folder or folder
work_path = Path(work_folder).expanduser()
if not work_path.is_absolute() or not work_path.is_dir():
raise ValueError("Рабочая папка должна существовать и иметь абсолютный путь.")
name = payload.get("filename", "")
if not isinstance(name, str) or len(name) > 180:
raise ValueError("Имя файла слишком длинное.")
return {"url": youtube_url(payload.get("url")), "mode": mode, "container": container,
"codec": codec, "engine": engine, "preset": payload.get("preset", "balanced"),
"bitrate": bitrate, "folder": str(path.resolve()), "workFolder": str(work_path.resolve()), "filename": name,
"videoId": format_id(payload.get("videoId", "best")),
"audioId": format_id(payload.get("audioId", "best"))}
def select_streams(info: dict, options: dict) -> str:
videos, audios = video_formats(info), audio_formats(info)
def choose(items, wanted, label):
if not items:
raise ValueError(f"Недоступна дорожка: {label}.")
if wanted == "best":
return items[-1]
for item in items:
if str(item["format_id"]) == wanted:
return item
raise ValueError("Выбранная дорожка исчезла. Обновите список качества.")
if options["mode"] == "audio":
# A progressive video is a valid fallback when separate audio is absent.
return str(choose(audios or [f for f in videos if f.get("acodec") not in (None, "none")],
options["audioId"], "аудио")["format_id"])
video = choose(videos, options["videoId"], "видео")
if options["mode"] == "video_only" or video.get("acodec") not in (None, "none"):
return str(video["format_id"])
audio = choose(audios, options["audioId"], "аудио")
return f"{video['format_id']}+{audio['format_id']}"
def safe_filename(value: str, fallback: str) -> str:
value = re.sub(r'[<>:"/\\|?*\x00-\x1f]', "_", value).strip(" .")[:150].rstrip(" .")
if not value:
value = fallback
if value.split(".", 1)[0].upper() in {"CON", "PRN", "AUX", "NUL", *(f"COM{i}" for i in range(1, 10)), *(f"LPT{i}" for i in range(1, 10))}:
value = "_" + value
return value
+116
View File
@@ -0,0 +1,116 @@
"""Measured dependency status and isolated installers for the helper."""
from __future__ import annotations
import importlib.metadata
import importlib.util
import json
import re
import subprocess
import sys
import tempfile
import urllib.request
import zipfile
from pathlib import Path
from host.media import ROOT, NO_WINDOW, config, run_small, tool
from host.bundle import manifest
LABELS = {"python": "Python", "yt_dlp": "yt-dlp", "ejs": "YouTube JS (EJS)", "node": "Node.js", "ffmpeg": "FFmpeg + FFprobe"}
def status():
python_ok = sys.version_info >= (3, 10)
result = [{"id": "python", "label": LABELS["python"], "status": "installed" if python_ok else "missing", "version": sys.version.split()[0]}]
for key, package in (("yt_dlp", "yt-dlp"), ("ejs", "yt-dlp-ejs")):
try:
version = importlib.metadata.version(package)
result.append({"id": key, "label": LABELS[key], "status": "installed", "version": version})
except importlib.metadata.PackageNotFoundError:
result.append({"id": key, "label": LABELS[key], "status": "missing", "version": ""})
for key in ("node", "ffmpeg"):
try:
measured = run_small([tool(key), "--version" if key == "node" else "-version"], timeout=8)
first = measured.stdout.splitlines()[0] if measured.stdout else ""
if measured.returncode != 0 or not first:
raise ValueError(f"Version check failed ({measured.returncode}): {measured.stderr[-200:]}")
version = first if key == "node" else first.split("version ", 1)[-1].split(" ", 1)[0]
installed = int(version.lstrip("v").split(".", 1)[0]) >= 22 if key == "node" else run_small([tool("ffprobe"), "-version"], timeout=8).returncode == 0
result.append({"id": key, "label": LABELS[key], "status": "installed" if installed else "outdated", "version": version})
except (OSError, ValueError, subprocess.TimeoutExpired) as error:
result.append({"id": key, "label": LABELS[key], "status": "missing", "version": "", "message": str(error)[:300]})
return result
def fetch(url, destination, progress):
request = urllib.request.Request(url, headers={"User-Agent": "YouTubeDL-Helper/1.1"})
with urllib.request.urlopen(request, timeout=30) as response, destination.open("wb") as output:
total = int(response.headers.get("Content-Length", 0)); received = 0
while chunk := response.read(1024 * 512):
output.write(chunk); received += len(chunk)
progress(min(75, received / total * 75) if total else None, "Скачиваю компонент")
def extract(archive, folder):
root = folder.resolve()
with zipfile.ZipFile(archive) as bundle:
for entry in bundle.infolist():
target = (root / entry.filename).resolve()
if target != root and root not in target.parents:
raise ValueError("Некорректный путь в архиве компонента.")
bundle.extract(entry, root)
def install(key, progress, workspace=None):
if manifest():
raise ValueError("Нажмите «Обновить компоненты» для установки новых версий из интернета или «Восстановить» для переустановки встроенного комплекта.")
if key not in {"node", "ffmpeg", "yt_dlp", "ejs"}:
raise ValueError("Неизвестный компонент.")
runtime = ROOT / ".runtime"
runtime.mkdir(exist_ok=True)
cfg = config()
with tempfile.TemporaryDirectory(prefix=".ytdl-install-", dir=runtime) as temporary:
scratch = Path(temporary)
if workspace:
workspace(str(scratch))
if key in {"yt_dlp", "ejs"}:
progress(None, "Устанавливаю Python-пакет")
package = "yt-dlp[default]>=2026.8.19" if key == "yt_dlp" else "yt-dlp-ejs"
measured = subprocess.run([sys.executable, "-m", "pip", "install", "--disable-pip-version-check", "--upgrade", package],
stdin=subprocess.DEVNULL, capture_output=True, text=True, encoding="utf-8", errors="replace", creationflags=NO_WINDOW, timeout=600)
if measured.returncode:
raise ValueError("Не удалось установить компонент. " + measured.stderr[-1200:])
else:
archive = scratch / "component.zip"
if key == "node":
with urllib.request.urlopen("https://nodejs.org/dist/index.json", timeout=30) as response:
versions = json.load(response)
release = next(item["version"] for item in versions if item.get("lts") and "win-x64-zip" in item.get("files", []) and re.fullmatch(r"v\d+\.\d+\.\d+", item["version"]) and int(item["version"].split(".")[0][1:]) >= 22)
fetch(f"https://nodejs.org/dist/{release}/node-{release}-win-x64.zip", archive, progress)
destination = runtime / "tools" / "node"
else:
fetch("https://www.gyan.dev/ffmpeg/builds/ffmpeg-release-essentials.zip", archive, progress)
destination = runtime / "tools" / "ffmpeg"
progress(80, "Распаковываю компонент")
extract(archive, destination)
# Newer versions may coexist; choose the executable from this archive.
with zipfile.ZipFile(archive) as bundle:
relative = next(name for name in bundle.namelist() if name.endswith("/node.exe" if key == "node" else "/bin/ffmpeg.exe"))
executable = destination / relative
cfg[key] = str(executable.resolve())
if key == "ffmpeg":
cfg["ffprobe"] = str(executable.with_name("ffprobe.exe").resolve())
progress(95, "Проверяю компонент")
runtime.joinpath("config.json").write_text(json.dumps(cfg, ensure_ascii=False, indent=2), encoding="utf-8")
measured = next(item for item in status() if item["id"] == key)
if measured["status"] != "installed":
raise ValueError("Компонент установлен, но проверка запуска не прошла.")
progress(100, "Установка завершена")
return {"title": LABELS[key], "dependency": key, "version": measured["version"]}
def main():
key = sys.argv[1]
result = install(key, lambda percent, message: print(f"{message}: {percent if percent is not None else '…'}%", flush=True))
print(json.dumps(result, ensure_ascii=False))
if __name__ == "__main__": main()
+71
View File
@@ -0,0 +1,71 @@
"""Read FFmpeg's real timeline and output size without blocking stderr."""
from __future__ import annotations
import math
import subprocess
import threading
from host.media import NO_WINDOW
def run_ffmpeg(command, duration, stage, engine, report):
duration = float(duration or 0)
if not math.isfinite(duration) or duration < 0:
duration = 0
report(stage=stage, percent=0 if duration else None, engine=engine,
speed="", eta=None, processedSeconds=0, durationSeconds=duration, written=0, message="")
process = subprocess.Popen(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, encoding="utf-8", errors="replace", creationflags=NO_WINDOW)
errors, finalizing = [], threading.Event()
progress_lock = threading.Lock()
def read_errors():
for line in process.stderr:
errors.append(line)
if len(errors) > 80:
del errors[:40]
if "Starting second pass" in line:
with progress_lock:
finalizing.set()
report(stage="finalizing", percent=99, engine="copy", speed="", eta=None,
message="Финализация файла")
reader = threading.Thread(target=read_errors, daemon=True)
reader.start()
stats = {}
elapsed = 0.0
try:
for raw in process.stdout:
key, _, value = raw.strip().partition("=")
stats[key] = value
if key != "progress":
continue
try:
elapsed = max(elapsed, max(0, int(stats.get("out_time_us", "0")) / 1_000_000))
except ValueError:
pass
speed = stats.get("speed", "")
try:
rate = float(speed.rstrip("x"))
except ValueError:
rate = 0
eta = max(0, (duration - elapsed) / rate) if duration and rate > 0 and math.isfinite(rate) and not finalizing.is_set() else None
try:
written = max(0, int(stats.get("total_size", "0")))
except ValueError:
written = None
with progress_lock:
is_finalizing = finalizing.is_set()
report(stage="finalizing" if is_finalizing else stage, percent=99 if is_finalizing else min(99, elapsed / duration * 100) if duration else None,
engine="copy" if is_finalizing else engine, speed=speed if rate > 0 and math.isfinite(rate) and not is_finalizing else "", eta=None if is_finalizing else eta,
processedSeconds=min(elapsed, duration) if duration else elapsed,
durationSeconds=duration, written=written,
message="Финализация файла" if is_finalizing else "")
process.wait()
reader.join(timeout=3)
return process.returncode, "".join(errors)[-2000:]
finally:
if process.poll() is None:
process.kill()
process.wait(timeout=10)
for stream in (process.stdout, process.stderr):
stream.close()
+56
View File
@@ -0,0 +1,56 @@
"""Use the Windows folder picker without shipping Tcl/Tk."""
from __future__ import annotations
import base64
import json
import os
import subprocess
def pick_folder(current):
from host.bundle import installation_root
installed = installation_root()
if installed:
encoded = base64.b64encode(json.dumps(current or "").encode("utf-8")).decode("ascii")
result = subprocess.run([str(installed / "YouTubeDL-Manager.exe"), "--pick-folder", "--initial-folder", encoded],
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=600,
creationflags=subprocess.CREATE_NO_WINDOW)
if result.returncode:
raise ValueError("Не удалось открыть окно выбора папки Windows.")
return json.loads(result.stdout.strip().lstrip("\ufeff"))
if os.name != "nt":
import tkinter
from tkinter import filedialog
root = tkinter.Tk()
try:
root.withdraw(); root.attributes("-topmost", True)
return filedialog.askdirectory(initialdir=current or "", mustexist=True) or None
finally:
root.destroy()
# Only base64 data is substituted; paths are never interpolated into shell code.
encoded = base64.b64encode(json.dumps(current or "").encode("utf-8")).decode("ascii")
script = """
$ErrorActionPreference='Stop'
[Console]::OutputEncoding=[Text.UTF8Encoding]::new()
Add-Type -AssemblyName System.Windows.Forms
$taskDialog=[Windows.Forms.FolderBrowserDialog]::new()
$taskOwner=[Windows.Forms.Form]::new()
try {
$taskOwner.TopMost=$true; $taskOwner.ShowInTaskbar=$false; $taskOwner.Opacity=0
$taskDialog.Description='YouTubeDL — папка сохранения'
$taskDialog.ShowNewFolderButton=$true
$taskDialog.SelectedPath=([Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('ENCODED')) | ConvertFrom-Json)
$taskOwner.Show()
$taskChosen=$null
if($taskDialog.ShowDialog($taskOwner) -eq [Windows.Forms.DialogResult]::OK){$taskChosen=$taskDialog.SelectedPath}
ConvertTo-Json -InputObject $taskChosen -Compress
} finally {$taskDialog.Dispose();$taskOwner.Dispose()}
""".replace("ENCODED", encoded)
argument = base64.b64encode(script.encode("utf-16le")).decode("ascii")
powershell = os.path.join(os.environ.get("SystemRoot", r"C:\Windows"), "System32", "WindowsPowerShell", "v1.0", "powershell.exe")
result = subprocess.run([powershell, "-NoProfile", "-ExecutionPolicy", "Bypass", "-STA", "-EncodedCommand", argument],
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=600,
creationflags=subprocess.CREATE_NO_WINDOW,
env={**os.environ, "PSModulePath": os.path.join(os.path.dirname(powershell), "Modules")})
if result.returncode:
raise ValueError("Не удалось открыть окно выбора папки Windows.")
return json.loads(result.stdout.strip().lstrip("\ufeff"))
+20
View File
@@ -0,0 +1,20 @@
"""Manage only this helper's per-user browser registrations."""
from pathlib import Path
from host.common import HOST_NAME
def unregister(root: Path):
import winreg
expected = str((root / ".runtime" / "native-host.json").resolve()).casefold()
removed = 0
for browser in (r"Google\Chrome", r"Microsoft\Edge", "Chromium"):
key = rf"Software\{browser}\NativeMessagingHosts\{HOST_NAME}"
try:
with winreg.OpenKey(winreg.HKEY_CURRENT_USER, key) as handle:
current = winreg.QueryValueEx(handle, "")[0]
if isinstance(current, str) and str(Path(current).resolve()).casefold() == expected:
winreg.DeleteKey(winreg.HKEY_CURRENT_USER, key)
removed += 1
except FileNotFoundError:
continue
return {"removed": True, "registrations": removed}
+399
View File
@@ -0,0 +1,399 @@
"""Chrome/Edge native messaging host; no listening network port."""
from __future__ import annotations
import concurrent.futures
import json
import os
import shutil
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from host.common import EXTENSION_ID, VERSION, validate_download, youtube_url
from host.media import NO_WINDOW, ROOT, capabilities
from host.protocol import read_message, write_message
from host.session import validate_cookies
from host.bundle import history_path, installation_root
from host.processes import ProcessTree
class NativeHost:
def __init__(self, output):
self.output = output
self.output_lock = threading.Lock()
self.state_lock = threading.RLock()
self.capability_lock = threading.Lock()
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=4)
self.history_path = history_path() if installation_root() else ROOT / ".runtime" / "completed-jobs.json"
self.jobs = self.read_history()
self.processes = {}
self.closed = False
self._capabilities = None
self.picker_lock = threading.Lock()
self.auto_timer = None
self.auto_candidates = None
def read_history(self):
try:
history = json.loads(self.history_path.read_text(encoding="utf-8"))
if not isinstance(history, list):
return {}
return {job["id"]: job for job in history[:30] if isinstance(job, dict)
and isinstance(job.get("id"), str) and job.get("status") == "complete"
and isinstance(job.get("result"), dict) and isinstance(job["result"].get("path"), str)
and Path(job["result"]["path"]).is_absolute()}
except (OSError, ValueError):
return {}
def save_history(self):
# Keep completed file references so Reveal still works after a browser restart.
history = sorted((dict(job) for job in self.jobs.values() if job["status"] == "complete" and job.get("result", {}).get("path")),
key=lambda job: job.get("createdAt", 0), reverse=True)[:30]
temporary = self.history_path.with_suffix(".tmp")
try:
self.history_path.parent.mkdir(parents=True, exist_ok=True)
temporary.write_text(json.dumps(history, ensure_ascii=False), encoding="utf-8")
os.replace(temporary, self.history_path)
except OSError:
pass # Saving a history entry must not fail an otherwise completed download.
def send(self, message):
with self.output_lock:
if not self.closed:
try:
write_message(self.output, message)
except (BrokenPipeError, OSError):
self.closed = True
def reply(self, request_id, result=None, error=None):
self.send({"id": request_id, "ok": error is None, "result": result, "error": error})
def hardware(self):
with self.capability_lock:
if self._capabilities is None:
self._capabilities = capabilities()
return self._capabilities
def event(self, job_id, **changes):
with self.state_lock:
state = self.jobs[job_id]
if state.get("status") in {"cancelled", "complete", "error"} and changes.get("status") != state.get("status"):
return
if state.get("cancelling") and changes.get("status") == "complete":
return
if state.get("cancelling") and changes.get("status") is None and changes.get("stage") != "cancelling":
return
state.update(changes)
state["updatedAt"] = time.time()
if changes.get("status") == "complete":
self.save_history()
snapshot = dict(state)
self.send({"event": "job", "job": snapshot})
def kill(self, process):
tree = getattr(process, "ytdl_process_tree", None)
if isinstance(tree, ProcessTree):
tree.terminate()
return
if process.poll() is not None:
return
if os.name == "nt":
subprocess.run(["taskkill", "/PID", str(process.pid), "/T", "/F"], stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, creationflags=NO_WINDOW, timeout=15)
else:
import signal
os.killpg(process.pid, signal.SIGTERM)
def worker(self, action, payload, job_id=None):
hardware = self.hardware() if action == "download" else {}
encoders = {key: hardware.get(key, []) for key in ("nvenc", "amf", "amf10bit")}
# CREATE_SUSPENDED + job assignment happen before even the Python
# launcher can create children. stdin is a second gate for the task.
process = ProcessTree.launch([sys.executable, "-X", "utf8", "-B", "-u", str(ROOT / "host" / "worker.py")],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, encoding="utf-8", errors="replace", creationflags=NO_WINDOW,
start_new_session=os.name != "nt")
token = job_id or str(uuid.uuid4())
with self.state_lock:
self.processes[token] = process
cancelled = self.closed or (job_id and (self.jobs[job_id]["status"] == "cancelled" or self.jobs[job_id].get("cancelling")))
if cancelled:
self.kill(process)
process.ytdl_process_tree.close()
with self.state_lock:
self.processes.pop(token, None)
return None
try:
process.stdin.write(json.dumps({"action": action, "payload": payload, "encoders": encoders}) + "\n")
process.stdin.close()
except BaseException:
self.kill(process)
process.ytdl_process_tree.close()
with self.state_lock:
self.processes.pop(token, None)
raise
errors = []
def stderr_reader():
for line in process.stderr:
errors.append(line)
if len(errors) > 40:
del errors[:20]
reader = threading.Thread(target=stderr_reader, daemon=True)
reader.start()
result, error, workspaces = None, None, []
workspace_parent = Path(payload.get("workFolder") or payload["folder"]).resolve() if action == "download" else (ROOT / ".runtime").resolve()
if action == "helper_update": workspace_parent = (installation_root() / "releases").resolve()
workspace_parents = {workspace_parent}
if action == "download": workspace_parents.add(Path(payload["folder"]).resolve())
try:
for line in process.stdout:
message = json.loads(line)
if message["type"] == "progress" and job_id:
self.event(job_id, **{k: v for k, v in message.items() if k != "type"})
elif message["type"] == "result":
result = message["result"]
elif message["type"] == "error":
error = message["message"]
elif message["type"] == "workspace" and action in {"download", "dependency_install", "helper_update"}:
candidate = Path(message["path"])
if candidate.parent.resolve() in workspace_parents and candidate.name.startswith(".ytdl-") and not candidate.is_symlink():
if candidate not in workspaces: workspaces.append(candidate)
process.wait()
reader.join(timeout=2)
if job_id and (self.jobs[job_id]["status"] == "cancelled" or self.jobs[job_id].get("cancelling")):
return None
if error or process.returncode or result is None:
raise ValueError(error or "Локальный помощник завершился с ошибкой. " + "".join(errors)[-1000:])
return result
finally:
if process.poll() is None:
self.kill(process)
process.ytdl_process_tree.close()
for stream in (process.stdin, process.stdout, process.stderr):
stream.close()
with self.state_lock:
self.processes.pop(token, None)
for workspace in workspaces:
if not workspace.exists() or workspace.is_symlink() or workspace.resolve().parent not in workspace_parents:
continue
retained = False
if action == "helper_update":
marker = json.loads((installation_root() / "installation.json").read_text(encoding="utf-8-sig"))
retained = Path(marker["current"]).resolve() == workspace.resolve()
if not retained: shutil.rmtree(workspace)
def run_job(self, job_id, payload, action="download"):
try:
try:
result = self.worker(action, payload, job_id)
finally:
if action in {"dependency_install", "helper_update"}:
with self.capability_lock:
self._capabilities = None
if result is not None:
self.event(job_id, status="complete", stage="complete", percent=100, result=result)
except Exception as error:
with self.state_lock:
cancelled = self.jobs[job_id]["status"] == "cancelled" or self.jobs[job_id].get("cancelling")
if not cancelled:
self.event(job_id, status="error", stage="error", error=str(error)[:2000])
def start_update(self, candidates=None):
if not installation_root(): raise ValueError("Для обновления нужен автономный помощник.")
with self.state_lock:
if self.processes or any(job["status"] == "running" for job in self.jobs.values()):
raise ValueError("Дождитесь завершения текущих операций перед обновлением.")
job_id = str(uuid.uuid4())
self.jobs[job_id] = {"id": job_id, "status": "running", "stage": "updating", "kind": "update", "title": "Обновление компонентов",
"percent": None, "createdAt": time.time(), "canCancel": False}
self.event(job_id)
self.executor.submit(self.run_job, job_id, {"candidates": candidates}, "helper_update")
return {"jobId": job_id}
def schedule_updates(self, delay=30):
if self.closed or not installation_root() or self.auto_timer is not None: return
self.auto_timer = threading.Timer(delay, self.auto_check)
self.auto_timer.daemon = True; self.auto_timer.start()
def auto_check(self):
from host.updates import check_updates, due, settings
try:
with self.state_lock:
if self.closed or self.processes or self._capabilities and self._capabilities.get("ready") is False or any(job["status"] == "running" for job in self.jobs.values()): return
if not settings()["enabled"]: self.auto_candidates = None; return
if self.auto_candidates is None:
if not due(): return
self.auto_candidates = check_updates()
settings(lastCheck=time.time())
with self.state_lock:
if self.closed or self.processes or any(job["status"] == "running" for job in self.jobs.values()): return
if self.auto_candidates: self.start_update(self.auto_candidates)
self.auto_candidates = None
except Exception:
# Quietly retry a failed automatic metadata check in an hour.
try: settings(lastCheck=time.time() - 86400 + 3600)
except OSError: pass
finally:
self.auto_timer = None
self.schedule_updates(60)
def pick_folder(self, current):
from host.folders import pick_folder
if not self.picker_lock.acquire(blocking=False):
raise ValueError("Диалог выбора папки уже открыт.")
try:
initial = current if isinstance(current, str) and Path(current).is_dir() else str(Path.home())
folder = pick_folder(initial)
return {"folder": folder or None}
finally:
self.picker_lock.release()
def handle(self, request):
request_id = request.get("id")
if not isinstance(request_id, str) or len(request_id) > 100:
return
action, payload = request.get("action"), request.get("payload", {})
try:
if not isinstance(payload, dict):
raise ValueError("Некорректное сообщение.")
if action == "hello":
from host.updates import settings
result = {**self.hardware(), "supportsBrowserCookies": True, "supportsComponentUpdates": bool(installation_root()), "updateSettings": settings()}
self.schedule_updates()
elif action == "ping":
result = {"alive": True, "pid": os.getpid(), "version": VERSION}
elif action == "update_settings":
from host.updates import settings
if not isinstance(payload.get("enabled"), bool): raise ValueError("Некорректная настройка обновлений.")
result = settings(payload["enabled"])
elif action == "helper_update":
result = self.start_update()
elif action == "helper_uninstall":
with self.state_lock:
if any(job["status"] == "running" for job in self.jobs.values()):
raise ValueError("Дождитесь завершения загрузок перед удалением помощника.")
from host.installation import unregister
result = unregister(ROOT)
installed = installation_root()
if installed and (installed / "YouTubeDL-Manager.exe").is_file():
# The uninstaller outlives the native port. Its output must
# never inherit the browser's native-messaging pipes.
log = installed / "data" / "uninstall.log"
log.parent.mkdir(parents=True, exist_ok=True)
with log.open("wb") as output:
subprocess.Popen([str(installed / "YouTubeDL-Manager.exe"), "--uninstall", "--silent", "--wait-pid", str(os.getpid())],
stdin=subprocess.DEVNULL, stdout=output, stderr=subprocess.STDOUT,
close_fds=True, creationflags=NO_WINDOW)
result["uninstallStarted"] = True
elif action == "inspect":
result = self.worker("inspect", {"url": youtube_url(payload.get("url")), "browserCookies": validate_cookies(payload.get("browserCookies"))})
elif action == "pick_folder":
result = self.pick_folder(payload.get("current"))
elif action == "download":
payload = {**validate_download(payload), "browserCookies": validate_cookies(payload.get("browserCookies"))}
with self.state_lock:
if any(j["status"] == "running" and j.get("kind") in {"dependency", "update"} for j in self.jobs.values()):
raise ValueError("Дождитесь завершения установки компонентов.")
if sum(j["status"] == "running" for j in self.jobs.values()) >= 2:
raise ValueError("Уже выполняются две загрузки. Дождитесь завершения одной.")
job_id = str(uuid.uuid4())
self.jobs[job_id] = {"id": job_id, "url": payload["url"], "status": "running",
"stage": "preparing", "percent": None, "createdAt": time.time()}
self.reply(request_id, {"jobId": job_id})
self.executor.submit(self.run_job, job_id, payload)
return
elif action == "dependency_install":
from host.dependencies import LABELS
key = payload.get("dependency")
if key not in {"node", "ffmpeg", "yt_dlp", "ejs"}:
raise ValueError("Неизвестный компонент.")
with self.state_lock:
if any(job["status"] == "running" for job in self.jobs.values()):
raise ValueError("Дождитесь завершения текущих операций перед установкой компонентов.")
job_id = str(uuid.uuid4())
self.jobs[job_id] = {"id": job_id, "status": "running", "stage": "installing", "kind": "dependency",
"title": LABELS[key], "percent": None, "createdAt": time.time()}
self.reply(request_id, {"jobId": job_id})
self.executor.submit(self.run_job, job_id, {"dependency": key}, action)
return
elif action == "jobs":
with self.state_lock:
result = [dict(job) for job in self.jobs.values()]
elif action == "cancel":
job_id = payload.get("jobId")
with self.state_lock:
job = self.jobs.get(job_id)
if not job or job["status"] != "running":
raise ValueError("Эта загрузка уже завершена.")
if job.get("canCancel") is False:
raise ValueError("Дождитесь завершения обновления компонентов.")
self.event(job_id, stage="cancelling", percent=None, cancelling=True, canCancel=False,
speed="", eta=None, message="Останавливаю процессы загрузки и обработки")
process = self.processes.get(job_id)
if process:
try:
self.kill(process)
except Exception as error:
self.event(job_id, status="error", stage="error", cancelling=False,
error="Не удалось остановить процессы: " + str(error))
raise
self.event(job_id, status="cancelled", stage="cancelled", percent=None, cancelling=False)
result = {"cancelled": True}
elif action == "reveal":
with self.state_lock:
job = self.jobs.get(payload.get("jobId"))
if not job or job["status"] != "complete":
raise ValueError("Файл ещё не готов.")
path = Path(job["result"]["path"])
if not path.is_file():
raise ValueError("Файл был перемещён или удалён.")
if os.name == "nt":
subprocess.Popen(["explorer.exe", "/select,", str(path)], creationflags=NO_WINDOW)
result = {"shown": True}
else:
raise ValueError("Неизвестная операция.")
self.reply(request_id, result)
except Exception as error:
self.reply(request_id, error=str(error)[:2000])
def close(self):
self.closed = True
if self.auto_timer: self.auto_timer.cancel()
with self.state_lock:
processes = list(self.processes.values())
for process in processes:
try:
self.kill(process)
except (OSError, subprocess.TimeoutExpired):
pass
self.executor.shutdown(wait=True, cancel_futures=True)
def main():
if len(sys.argv) > 1 and sys.argv[1] != f"chrome-extension://{EXTENSION_ID}/":
print("Недопустимый источник native messaging.", file=sys.stderr)
return 1
if os.name == "nt":
import msvcrt
msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
host = NativeHost(sys.stdout.buffer)
try:
while (request := read_message(sys.stdin.buffer)) is not None:
if request.get("action") in {"cancel", "ping"}:
host.handle(request)
else:
host.executor.submit(host.handle, request)
except (EOFError, ValueError, OSError) as error:
print(str(error), file=sys.stderr)
finally:
host.close()
return 0
if __name__ == "__main__":
raise SystemExit(main())
+192
View File
@@ -0,0 +1,192 @@
from __future__ import annotations
import json
import os
import shutil
import subprocess
from pathlib import Path
from host.common import VERSION
from host.bundle import executable, information
ROOT = Path(__file__).resolve().parent.parent
NO_WINDOW = subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0
def config():
path = ROOT / ".runtime" / "config.json"
return json.loads(path.read_text(encoding="utf-8-sig")) if path.is_file() else {}
def tool(name):
bundled = executable(name)
if bundled:
return bundled
configured = config().get(name)
result = configured if configured and Path(configured).is_file() else shutil.which(name)
if not result or not Path(result).is_file():
raise ValueError(f"Не найден {name}. Запустите install.ps1.")
return str(Path(result).resolve())
def run_small(command, timeout=15):
return subprocess.run(command, capture_output=True, text=True, encoding="utf-8", errors="replace",
timeout=timeout, creationflags=NO_WINDOW, stdin=subprocess.DEVNULL)
def capabilities():
from host.dependencies import status
dependencies = status()
ready = all(item["status"] == "installed" for item in dependencies)
ffmpeg = next((item for item in dependencies if item["id"] == "ffmpeg" and item["status"] == "installed"), None)
checked = {"nvenc": [], "amf": [], "amf10bit": []}
for engine in ("nvenc", "amf"):
for codec in ("h264", "hevc", "av1"):
if not ffmpeg:
break
base = [tool("ffmpeg"), "-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i"]
preset = ["-preset", "p4"] if engine == "nvenc" else ["-quality", "balanced", "-rc", "cqp", "-qp_i", "20", "-qp_p", "20"]
try:
result = run_small([*base, "color=c=black:s=640x360:r=30", "-frames:v", "2", "-c:v", f"{codec}_{engine}", *preset, "-f", "null", "-"], timeout=20)
if result.returncode == 0:
checked[engine].append(codec)
if engine == "amf" and codec in {"hevc", "av1"}:
extra = ["-profile:v", "main10"] if codec == "hevc" else []
tenbit = run_small([*base, "color=c=black:s=640x360:r=30,format=p010le", "-frames:v", "2", "-c:v", f"{codec}_amf",
*preset, "-bitdepth", "10", *extra, "-pix_fmt", "+p010le", "-f", "null", "-"], timeout=20)
if tenbit.returncode == 0:
checked["amf10bit"].append(codec)
except (OSError, subprocess.TimeoutExpired):
continue
gpu = ""
smi = shutil.which("nvidia-smi")
if smi:
try:
result = run_small([smi, "--query-gpu=name", "--format=csv,noheader"], timeout=8)
gpu = result.stdout.strip()[:200] if result.returncode == 0 else ""
except (OSError, subprocess.TimeoutExpired):
pass
if os.name == "nt":
try:
detected = run_small(["powershell.exe", "-NoProfile", "-Command",
"[Console]::OutputEncoding=[Text.UTF8Encoding]::new(); "
"Get-CimInstance Win32_VideoController | Select-Object -ExpandProperty Name"], timeout=10)
if detected.returncode == 0:
gpu = detected.stdout.strip()[:200] or gpu
except (OSError, subprocess.TimeoutExpired):
pass
return {"version": VERSION, "minimumExtensionVersion": "1.4.5", "supportsWorkFolder": True, "ytDlp": next(item["version"] for item in dependencies if item["id"] == "yt_dlp"), "ffmpeg": ffmpeg["version"] if ffmpeg else "",
"gpu": gpu, **checked, "defaultFolder": str(Path.home() / "Downloads"),
"dependencies": dependencies, "ready": ready, "bundle": information()}
def probe(path):
result = run_small([tool("ffprobe"), "-v", "error", "-show_streams", "-show_format", "-of", "json", str(path)], timeout=30)
if result.returncode:
raise ValueError("FFprobe не смог прочитать загруженный файл.")
return json.loads(result.stdout)
def encoding_plan(options, media, encoders):
# Older development callers supply the NVENC codec list directly.
encoders = {"nvenc": encoders} if isinstance(encoders, (list, tuple)) else encoders
streams = media.get("streams", [])
video = next((s for s in streams if s.get("codec_type") == "video"), None)
audio = next((s for s in streams if s.get("codec_type") == "audio"), None)
mode, container = options["mode"], options["container"]
if mode != "audio" and not video:
raise ValueError("В загруженном файле нет видео.")
if mode in {"audio", "video"} and not audio:
raise ValueError("В загруженном файле нет звуковой дорожки.")
if mode == "audio":
target = {"m4a": "aac", "mp3": "libmp3lame", "opus": "libopus", "flac": "flac", "wav": "pcm_s16le"}[container]
source_audio = audio.get("codec_name")
if (container, source_audio) in {("m4a", "aac"), ("mp3", "mp3"), ("opus", "opus"), ("flac", "flac")}:
target = "copy"
return {"video": None, "audio": target, "engine": "copy" if target == "copy" else "cpu", "hdr": False}
source = video.get("codec_name", "")
compatible = container == "mkv" or (container == "mp4" and source in {"h264", "hevc", "av1"}) or (container == "webm" and source in {"vp8", "vp9", "av1"})
codec = options["codec"]
hdr = video.get("color_transfer") in {"smpte2084", "arib-std-b67"}
ten_bit = "10" in video.get("pix_fmt", "") or video.get("pix_fmt", "").startswith("p010")
if codec == "copy" and not compatible:
raise ValueError("Исходный видеокодек несовместим с контейнером. Выберите «Авто» или MKV.")
if codec == "auto":
codec = "copy" if compatible else ("av1" if container == "webm" else ("hevc" if hdr else "h264"))
if container == "webm" and codec not in {"copy", "vp9", "av1"}:
raise ValueError("WebM поддерживает AV1 и VP9. Для H.264/HEVC выберите MP4 или MKV.")
if hdr and codec == "h264":
raise ValueError("Для HDR выберите HEVC, AV1 или сохранение исходного кодека.")
engine = "copy" if codec == "copy" else "cpu"
if codec != "copy" and options["engine"] != "cpu":
for candidate in ("nvenc", "amf"):
supported = encoders.get("amf10bit" if candidate == "amf" and ten_bit and codec in {"hevc", "av1"} else candidate, [])
if options["engine"] in {"auto", candidate} and codec in supported:
engine = candidate
break
if options["engine"] in {"nvenc", "amf"} and engine != options["engine"]:
name = "AMD AMF" if options["engine"] == "amf" else "NVIDIA NVENC"
raise ValueError(f"Выбранный кодек или глубина цвета недоступны в {name}. Выберите поддерживаемый кодек или «Авто».")
audio_codec = None
if mode == "video":
ac = audio.get("codec_name", "")
if container == "mkv" or (container == "mp4" and ac in {"aac", "mp3", "alac"}) or (container == "webm" and ac in {"opus", "vorbis"}):
audio_codec = "copy"
else:
audio_codec = "libopus" if container == "webm" else "aac"
if engine == "copy" and audio_codec not in (None, "copy"):
engine = "cpu"
return {"video": codec, "audio": audio_codec, "engine": engine, "hdr": hdr, "tenBit": ten_bit}
def ffmpeg_command(source, target, options, plan, cuda_decode=True):
command = [tool("ffmpeg"), "-hide_banner", "-nostdin", "-loglevel", "info", "-y", "-progress", "pipe:1", "-stats_period", "0.4"]
if plan["engine"] == "nvenc" and cuda_decode:
command += ["-hwaccel", "cuda"]
elif plan["engine"] == "amf" and cuda_decode:
command += ["-hwaccel", "d3d11va"]
command += ["-i", str(source)]
if options["mode"] == "audio":
command += ["-map", "0:a:0", "-vn"]
else:
command += ["-map", "0:v:0"]
if options["mode"] == "video":
command += ["-map", "0:a:0"]
else:
command += ["-an"]
codec = plan["video"]
if codec:
if codec == "copy":
command += ["-c:v", "copy"]
elif plan["engine"] == "nvenc":
preset = {"fast": "p2", "balanced": "p4", "quality": "p6"}[options["preset"]]
command += ["-c:v", codec + "_nvenc", "-preset", preset, "-rc", "vbr", "-cq", "20", "-b:v", "0"]
command += ["-pix_fmt", "p010le" if plan.get("tenBit") and codec in {"hevc", "av1"} else "yuv420p"]
elif plan["engine"] == "amf":
quality = {"fast": "speed", "balanced": "balanced", "quality": "quality"}[options["preset"]]
command += ["-c:v", codec + "_amf", "-quality", quality, "-rc", "cqp", "-qp_i", "20", "-qp_p", "20"]
if plan.get("tenBit") and codec in {"hevc", "av1"}:
command += ["-bitdepth", "10", "-pix_fmt", "p010le"]
if codec == "hevc": command += ["-profile:v", "main10"]
else:
command += ["-pix_fmt", "yuv420p"]
else:
encoder = {"h264": "libx264", "hevc": "libx265", "av1": "libsvtav1", "vp9": "libvpx-vp9"}[codec]
command += ["-c:v", encoder]
if codec in {"h264", "hevc"}:
command += ["-preset", {"fast": "veryfast", "balanced": "fast", "quality": "medium"}[options["preset"]], "-crf", "20"]
elif codec == "av1":
command += ["-preset", "8", "-crf", "24"]
else:
command += ["-crf", "28", "-b:v", "0", "-row-mt", "1", "-cpu-used", "4"]
command += ["-pix_fmt", "yuv420p10le" if plan.get("tenBit") and codec != "h264" else "yuv420p"]
if plan["audio"]:
command += ["-c:a", plan["audio"]]
if plan["audio"] in {"aac", "libmp3lame", "libopus"}:
command += ["-b:a", f"{options['bitrate']}k"]
command += ["-map_metadata", "0"]
if options["container"] in {"mp4", "m4a"}:
command += ["-movflags", "+faststart"]
if codec == "hevc" and options["container"] == "mp4":
command += ["-tag:v", "hvc1"]
command += [str(target)]
return command
+172
View File
@@ -0,0 +1,172 @@
"""Own one worker's process tree, including children left after its exit."""
from __future__ import annotations
import os
import subprocess
import threading
import time
class ProcessTree:
@classmethod
def launch(cls, command, **kwargs):
if os.name == "nt":
kwargs["creationflags"] = kwargs.get("creationflags", 0) | 0x4 # CREATE_SUSPENDED
else:
kwargs["start_new_session"] = True
process = subprocess.Popen(command, **kwargs)
try:
tree = cls(process)
process.ytdl_process_tree = tree
if os.name == "nt": tree.resume()
return process
except BaseException:
tree = getattr(process, "ytdl_process_tree", None)
if tree: tree.close()
if process.poll() is None: process.kill()
process.wait(timeout=10)
for stream in (process.stdin, process.stdout, process.stderr):
if stream: stream.close()
raise
def __init__(self, process):
self.process = process
self.lock = threading.Lock()
self.handle = None
if os.name != "nt":
return
import ctypes
from ctypes import wintypes
class BasicLimits(ctypes.Structure):
_fields_ = [("ProcessTime", ctypes.c_longlong), ("JobTime", ctypes.c_longlong),
("Flags", wintypes.DWORD), ("MinWorkingSet", ctypes.c_size_t),
("MaxWorkingSet", ctypes.c_size_t), ("ActiveLimit", wintypes.DWORD),
("Affinity", ctypes.c_size_t), ("Priority", wintypes.DWORD), ("Scheduling", wintypes.DWORD)]
class Limits(ctypes.Structure):
_fields_ = [("Basic", BasicLimits), ("Io", ctypes.c_ulonglong * 6),
("ProcessMemory", ctypes.c_size_t), ("JobMemory", ctypes.c_size_t),
("PeakProcessMemory", ctypes.c_size_t), ("PeakJobMemory", ctypes.c_size_t)]
class Accounting(ctypes.Structure):
_fields_ = [("Times", ctypes.c_longlong * 4), ("PageFaults", wintypes.DWORD),
("TotalProcesses", wintypes.DWORD), ("ActiveProcesses", wintypes.DWORD),
("TerminatedProcesses", wintypes.DWORD)]
self.ctypes, self.Accounting = ctypes, Accounting
self.api = ctypes.WinDLL("kernel32", use_last_error=True)
for name, args, result in [
("CreateJobObjectW", [ctypes.c_void_p, wintypes.LPCWSTR], wintypes.HANDLE),
("SetInformationJobObject", [wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD], wintypes.BOOL),
("QueryInformationJobObject", [wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD, ctypes.c_void_p], wintypes.BOOL),
("AssignProcessToJobObject", [wintypes.HANDLE, wintypes.HANDLE], wintypes.BOOL),
("TerminateJobObject", [wintypes.HANDLE, wintypes.UINT], wintypes.BOOL),
("OpenProcess", [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD], wintypes.HANDLE),
("CloseHandle", [wintypes.HANDLE], wintypes.BOOL),
("CreateToolhelp32Snapshot", [wintypes.DWORD, wintypes.DWORD], wintypes.HANDLE),
("Thread32First", [wintypes.HANDLE, ctypes.c_void_p], wintypes.BOOL),
("Thread32Next", [wintypes.HANDLE, ctypes.c_void_p], wintypes.BOOL),
("OpenThread", [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD], wintypes.HANDLE),
("ResumeThread", [wintypes.HANDLE], wintypes.DWORD),
("WaitForSingleObject", [wintypes.HANDLE, wintypes.DWORD], wintypes.DWORD),
]:
function = getattr(self.api, name)
function.argtypes, function.restype = args, result
handle = self.api.CreateJobObjectW(None, None)
if not handle:
raise ctypes.WinError(ctypes.get_last_error())
try:
limits = Limits()
limits.Basic.Flags = 0x2000 # JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
if not self.api.SetInformationJobObject(handle, 9, ctypes.byref(limits), ctypes.sizeof(limits)):
raise ctypes.WinError(ctypes.get_last_error())
process_handle = self.api.OpenProcess(0x100 | 0x1, False, process.pid)
if not process_handle:
raise ctypes.WinError(ctypes.get_last_error())
try:
if not self.api.AssignProcessToJobObject(handle, process_handle):
raise ctypes.WinError(ctypes.get_last_error())
finally:
self.api.CloseHandle(process_handle)
self.handle = handle
except BaseException:
self.api.CloseHandle(handle)
raise
def resume(self):
from ctypes import wintypes
ctypes = self.ctypes
class ThreadEntry(ctypes.Structure):
_fields_ = [("Size", wintypes.DWORD), ("Usage", wintypes.DWORD), ("Id", wintypes.DWORD),
("Owner", wintypes.DWORD), ("BasePriority", wintypes.LONG),
("DeltaPriority", wintypes.LONG), ("Flags", wintypes.DWORD)]
snapshot = self.api.CreateToolhelp32Snapshot(0x4, 0) # TH32CS_SNAPTHREAD
if snapshot == ctypes.c_void_p(-1).value:
raise ctypes.WinError(ctypes.get_last_error())
try:
entry = ThreadEntry()
entry.Size = ctypes.sizeof(entry)
found = self.api.Thread32First(snapshot, ctypes.byref(entry))
while found:
if entry.Owner == self.process.pid:
thread = self.api.OpenThread(0x2, False, entry.Id)
if not thread: raise ctypes.WinError(ctypes.get_last_error())
try:
if self.api.ResumeThread(thread) == 0xffffffff:
raise ctypes.WinError(ctypes.get_last_error())
return
finally:
self.api.CloseHandle(thread)
found = self.api.Thread32Next(snapshot, ctypes.byref(entry))
raise RuntimeError("Не найден поток запуска помощника.")
finally:
self.api.CloseHandle(snapshot)
def terminate(self):
with self.lock:
if os.name == "nt":
if not self.handle:
return
from ctypes import wintypes
capacity = 16
while True:
class ProcessIds(self.ctypes.Structure):
_fields_ = [("Assigned", wintypes.DWORD), ("Count", wintypes.DWORD), ("Ids", self.ctypes.c_size_t * capacity)]
members = ProcessIds()
if self.api.QueryInformationJobObject(self.handle, 3, self.ctypes.byref(members), self.ctypes.sizeof(members), None):
break
if self.ctypes.get_last_error() != 234: # ERROR_MORE_DATA
raise self.ctypes.WinError(self.ctypes.get_last_error())
capacity = max(capacity * 2, members.Assigned)
handles = []
deadline = time.monotonic() + 10
try:
for pid in members.Ids[:members.Count]:
handle = self.api.OpenProcess(0x100000, False, pid) # SYNCHRONIZE
if handle: handles.append(handle)
if not self.api.TerminateJobObject(self.handle, 1):
raise self.ctypes.WinError(self.ctypes.get_last_error())
# Job accounting can reach zero before individual process
# handles signal. Wait for FFmpeg to release its files too.
for handle in handles:
timeout = max(0, int((deadline - time.monotonic()) * 1000))
result = self.api.WaitForSingleObject(handle, timeout)
if result == 258: raise TimeoutError("Не удалось дождаться остановки процессов загрузки.")
if result != 0: raise self.ctypes.WinError(self.ctypes.get_last_error())
finally:
for handle in handles: self.api.CloseHandle(handle)
else:
import signal
try:
os.killpg(self.process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
self.process.wait(timeout=10)
def close(self):
self.terminate()
with self.lock:
if self.handle:
self.api.CloseHandle(self.handle)
self.handle = None
+40
View File
@@ -0,0 +1,40 @@
import json
import struct
MAX_MESSAGE = 1024 * 1024
def read_exact(stream, length):
data = bytearray()
while len(data) < length:
chunk = stream.read(length - len(data))
if not chunk:
if not data:
return None
raise EOFError("Оборвано сообщение native messaging.")
data.extend(chunk)
return bytes(data)
def read_message(stream):
header = read_exact(stream, 4)
if header is None:
return None
size = struct.unpack("<I", header)[0]
if not 0 < size <= MAX_MESSAGE:
raise ValueError("Превышен размер сообщения.")
raw = read_exact(stream, size)
if raw is None:
raise EOFError("Отсутствует тело сообщения.")
message = json.loads(raw.decode("utf-8"))
if not isinstance(message, dict):
raise ValueError("Сообщение должно быть объектом.")
return message
def write_message(stream, message):
raw = json.dumps(message, ensure_ascii=False, allow_nan=False, separators=(",", ":")).encode("utf-8")
if len(raw) > MAX_MESSAGE:
raise ValueError("Ответ слишком большой.")
stream.write(struct.pack("<I", len(raw)) + raw)
stream.flush()
+60
View File
@@ -0,0 +1,60 @@
"""Validate YouTube-only cookies and attach them to an in-memory yt-dlp jar."""
from __future__ import annotations
import math
import re
from http.cookiejar import Cookie
DOMAIN = re.compile(r"^(?:[a-z0-9-]+\.)*youtube\.com$", re.I)
NAME = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]{1,256}$")
def validate_cookies(value):
if value is None:
return []
if not isinstance(value, list) or len(value) > 300:
raise ValueError("Некорректные данные входа YouTube.")
result, size = [], 0
for entry in value:
if not isinstance(entry, dict):
raise ValueError("Некорректные данные входа YouTube.")
name, content, domain, path = (entry.get(key) for key in ("name", "value", "domain", "path"))
if not isinstance(domain, str) or len(domain) > 253 or not DOMAIN.fullmatch(domain.lstrip(".")) or domain.startswith(".."):
raise ValueError("Данные входа разрешены только для YouTube.")
if (not isinstance(name, str) or not NAME.fullmatch(name) or not isinstance(content, str) or len(content) > 16384
or any(char in content for char in "\r\n\x00") or not isinstance(path, str) or not path.startswith("/")
or len(path) > 2048 or any(char in path for char in "\r\n\x00")):
raise ValueError("Некорректные данные входа YouTube.")
expires = entry.get("expirationDate")
if expires is not None and (isinstance(expires, bool) or not isinstance(expires, (int, float)) or not math.isfinite(expires) or not 0 <= expires < 2**63):
raise ValueError("Некорректный срок действия входа YouTube.")
size += len((name + content + domain + path).encode("utf-8"))
if size > 512 * 1024:
raise ValueError("Превышен размер данных входа YouTube.")
result.append({"name": name, "value": content, "domain": domain.lower(), "path": path,
"secure": bool(entry.get("secure", True)), "httpOnly": bool(entry.get("httpOnly")),
"hostOnly": bool(entry.get("hostOnly", not domain.startswith("."))), "expirationDate": expires})
return result
def apply_cookies(ydl, cookies):
for entry in validate_cookies(cookies):
expires = entry["expirationDate"]
ydl.cookiejar.set_cookie(Cookie(
version=0, name=entry["name"], value=entry["value"], port=None, port_specified=False,
domain=entry["domain"], domain_specified=not entry["hostOnly"], domain_initial_dot=entry["domain"].startswith("."),
path=entry["path"], path_specified=True, secure=entry["secure"],
expires=int(expires) if expires is not None else None, discard=expires is None,
comment=None, comment_url=None, rest={"HttpOnly": None} if entry["httpOnly"] else {}, rfc2109=False))
def is_access_error(error):
return bool(re.search(r"sign in to confirm|not a bot|cookies.*authentication|login required|log in to|HTTP Error (?:403|410)", str(error), re.I))
def public_error(error, authenticated=False):
message = str(error)
if re.search(r"sign in to confirm|not a bot|cookies.*authentication|login required|log in to", message, re.I):
if authenticated:
return "YouTube отклонил вход браузера или просит проверку. Откройте этот ролик на YouTube, пройдите проверку, затем нажмите «Проверить вход» и повторите загрузку."
return "YouTube отклонил запрос загрузчика и просит проверку на бота. Повторите запрос или обновите yt-dlp в компонентах. Дополнительно можно подключить вход кнопкой «Использовать вход в YouTube». Помощник работает; переустановка не нужна."
return message[:2000]
+289
View File
@@ -0,0 +1,289 @@
"""Verified vendor downloads, staged runtimes and publication through the manager."""
from __future__ import annotations
import concurrent.futures
import hashlib
import importlib.metadata
import json
import os
import re
import shutil
import subprocess
import sys
import time
import threading
import uuid
import zipfile
from pathlib import Path
from urllib.parse import urlsplit
import requests
from host.bundle import ROOT, installation_root, manifest
from host.media import NO_WINDOW, run_small
ALLOWED_HOSTS = {"www.python.org", "nodejs.org", "api.github.com", "github.com", "release-assets.githubusercontent.com", "objects.githubusercontent.com", "pypi.org", "files.pythonhosted.org"}
FFMPEG_ASSET = "ffmpeg-master-latest-win64-gpl-shared.zip"
SETTINGS_LOCK = threading.Lock()
def settings(enabled=None, **changes):
with SETTINGS_LOCK:
return _settings(enabled, **changes)
def _settings(enabled=None, **changes):
installed = installation_root()
if not installed:
return {"enabled": False, "supported": False}
path = installed / "data/update-settings.json"
try: value = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError): value = {}
value = {"enabled": value.get("enabled", True) is True, "lastCheck": value.get("lastCheck", 0), **value}
if enabled is not None or changes:
if enabled is not None: value["enabled"] = enabled is True
value.update(changes)
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(path.name + "." + uuid.uuid4().hex + ".tmp")
temporary.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
temporary.replace(path)
return value
def due():
value = settings()
return value["enabled"] and time.time() - value.get("lastCheck", 0) >= 86400
def trusted_url(url):
parsed = urlsplit(url)
if parsed.scheme != "https" or parsed.hostname not in ALLOWED_HOSTS or parsed.username or parsed.password or parsed.port not in {None, 443}:
raise ValueError("Неизвестный источник обновления.")
return url
def get(url, **kwargs):
trusted_url(url)
try:
headers = {"User-Agent": "YouTubeDL/1.3"}
if urlsplit(url).hostname == "api.github.com": headers["Accept"] = "application/vnd.github+json"
response = requests.get(url, headers=headers, timeout=(15, 40), **kwargs)
for item in [*response.history, response]: trusted_url(item.url)
response.raise_for_status()
return response
except requests.RequestException as error:
raise ValueError(f"Не удалось связаться с {urlsplit(url).hostname}. Проверьте подключение и повторите обновление.") from error
def digest(path):
result = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""): result.update(chunk)
return result.hexdigest()
def fetch(item, target, progress):
expected = item.get("sha256", "")
if not re.fullmatch("[a-f0-9]{64}", expected): raise ValueError("Источник не предоставил SHA256 обновления.")
with get(item["url"], stream=True) as response, target.open("wb") as output:
total = int(response.headers.get("Content-Length", 0)); received = 0
for chunk in response.iter_content(512 * 1024):
output.write(chunk); received += len(chunk)
if received > 512 * 1024 * 1024: raise ValueError("Превышен размер компонента.")
progress(received / total if total else None)
if digest(target) != expected: raise ValueError("Контрольная сумма обновления не совпала. Прежний комплект сохранён.")
def safe_members(package):
if sum(item.file_size for item in package.infolist()) > 1024 * 1024 * 1024:
raise ValueError("Превышен размер распакованного компонента.")
for item in package.infolist():
name = item.filename.replace("\\", "/")
parts = name.split("/")
if name.startswith("/") or any(part in {"..", "."} or ":" in part for part in parts) or (item.external_attr >> 16) & 0xF000 == 0xA000:
raise ValueError("Недопустимый путь в обновлении.")
if not item.is_dir(): yield item
def extract_zip(archive, target):
root = target.resolve()
with zipfile.ZipFile(archive) as package:
for item in safe_members(package):
destination = (root / item.filename).resolve()
if root not in destination.parents: raise ValueError("Путь вне папки компонента.")
destination.parent.mkdir(parents=True, exist_ok=True)
with package.open(item) as source, destination.open("wb") as output: shutil.copyfileobj(source, output)
def version_tuple(value):
return tuple(int(part) for part in value.lstrip("v").split(".")) if re.fullmatch(r"v?\d+(?:\.\d+)*", value) else ()
def python_release():
branch = f"{sys.version_info.major}.{sys.version_info.minor}"
current = version_tuple(sys.version.split()[0])
index = get("https://www.python.org/ftp/python/").text
versions = sorted(set(re.findall(r'href="(' + re.escape(branch) + r'\.\d+)/"', index)), key=version_tuple, reverse=True)
for version in versions:
if version_tuple(version) <= current: break
page = get(f"https://www.python.org/downloads/release/python-{version.replace('.', '')}/").text
name = f"python-{version}-embed-amd64.zip"
for row in re.findall(r"<tr\b[^>]*>(.*?)</tr>", page, re.S):
if name not in row: continue
match = re.search(r"\b[a-f0-9]{64}\b", re.sub(r"<[^>]+>", " ", row))
if match:
return {"id": "python", "version": version, "url": f"https://www.python.org/ftp/python/{version}/{name}", "sha256": match[0]}
return None
def node_release():
current = run_small([str(ROOT / "runtime/bin/node.exe"), "--version"]).stdout.strip()
releases = get("https://nodejs.org/dist/index.json").json()
compatible = [item for item in releases if item.get("lts") and "win-x64-zip" in item.get("files", [])
and version_tuple(item["version"])[:1] == version_tuple(current)[:1] and version_tuple(item["version"]) > version_tuple(current)]
if not compatible: return None
version = max(compatible, key=lambda item: version_tuple(item["version"]))["version"]
name = f"node-{version}-win-x64.zip"
checksum = next((line.split()[0] for line in get(f"https://nodejs.org/dist/{version}/SHASUMS256.txt").text.splitlines() if line.split() and line.split()[-1] == name), None)
return {"id": "node", "version": version, "url": f"https://nodejs.org/dist/{version}/{name}", "sha256": checksum}
def ffmpeg_release():
release = get("https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest").json()
asset = next(item for item in release["assets"] if item["name"] == FFMPEG_ASSET)
checksum = asset.get("digest", "").removeprefix("sha256:")
if not re.fullmatch("[a-f0-9]{64}", checksum): raise ValueError("FFmpeg не предоставил SHA256. Обновление отложено.")
previous = next((item.get("sha256") for item in manifest()["sources"] if FFMPEG_ASSET in item["url"]), None)
if previous == checksum: return None
return {"id": "ffmpeg", "version": asset["updated_at"], "url": asset["browser_download_url"], "sha256": checksum}
def packages_release():
installed = {item.metadata["Name"]: item.version for item in importlib.metadata.distributions(path=[str(ROOT / "runtime/python/Lib/site-packages")])}
def newer(item):
name, version = item
release = get(f"https://pypi.org/pypi/{name}/json").json()["info"]["version"]
return version_tuple(release) > version_tuple(version)
with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor:
changed = any(list(executor.map(newer, installed.items())))
return {"id": "packages", "version": "latest-compatible"} if changed else None
def check_updates():
if not installation_root(): raise ValueError("Для обновления установите автономный помощник.")
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
candidates = list(executor.map(lambda task: task(), [python_release, node_release, ffmpeg_release, packages_release]))
return [item for item in candidates if item]
def resolve_packages(stage, work, progress):
progress(None, "Подбираю совместимые версии Python-пакетов…")
release = get("https://pypi.org/pypi/pip/json").json()
wheel = next(item for item in release["urls"] if item["filename"].endswith("-py3-none-any.whl") and not item.get("yanked"))
pip = work / "pip.whl"
fetch({"url": wheel["url"], "sha256": wheel["digests"]["sha256"]}, pip, lambda _: None)
report = work / "python-packages.json"
script = "import sys;sys.path.insert(0,sys.argv.pop(1));from pip._internal.cli.main import main;raise SystemExit(main(sys.argv[1:]))"
command = [str(stage / "runtime/python/python.exe"), "-I", "-X", "utf8", "-B", "-c", script, str(pip), "--isolated", "--disable-pip-version-check", "install",
"--dry-run", "--ignore-installed", "--only-binary=:all:", "--no-cache-dir", "--quiet", "--report", str(report), "--index-url", "https://pypi.org/simple", "yt-dlp[default]", "yt-dlp-ejs"]
result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=600, creationflags=NO_WINDOW,
env={**os.environ, "TEMP": str(work), "TMP": str(work)}, stdin=subprocess.DEVNULL)
if result.returncode: raise ValueError("Не удалось подобрать совместимые Python-пакеты. " + result.stderr[-1200:])
records = json.loads(report.read_text(encoding="utf-8"))["install"]
packages = stage / "runtime/python/Lib/site-packages"
if packages.exists(): shutil.rmtree(packages)
packages.mkdir(parents=True)
for index, record in enumerate(records):
info = record["download_info"]; name = record["metadata"]["name"]
archive = work / f"wheel-{index}.whl"
fetch({"url": info["url"], "sha256": info["archive_info"]["hashes"]["sha256"]}, archive,
lambda fraction: progress(None, f"Скачиваю {name}…"))
with zipfile.ZipFile(archive) as wheel:
for member in safe_members(wheel):
relative = member.filename.split("/")
if relative[0].endswith(".data"):
if len(relative) < 3 or relative[1] not in {"purelib", "platlib"}: continue
relative = relative[2:]
destination = packages.joinpath(*relative).resolve()
if packages.resolve() not in destination.parents: raise ValueError("Некорректный путь Python-пакета.")
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(wheel.read(member))
return {item["metadata"]["name"]: item["metadata"]["version"] for item in records}
def publish(stage):
installed = installation_root()
manager = installed / "YouTubeDL-Manager.exe"
result = run_small([str(manager), "--activate-release", str(stage), "--expected-current", str(ROOT), "--target-root", str(installed), "--silent"], timeout=180)
if result.returncode: raise ValueError("Новый комплект не прошёл проверку. Прежняя версия сохранена. " + result.stderr[-1000:])
def update(progress, workspace, candidates=None):
installed = installation_root()
if not installed: raise ValueError("Обновление доступно в автономном комплекте.")
progress(None, "Проверяю новые версии компонентов…")
candidates = check_updates() if candidates is None else candidates
settings(lastCheck=time.time())
if not candidates: return {"title": "Все компоненты актуальны", "changed": False}
releases = installed / "releases"
stage = releases / (".ytdl-update-" + uuid.uuid4().hex)
if stage.resolve().parent != releases.resolve() or releases.is_symlink(): raise ValueError("Недопустимая папка обновления.")
if shutil.disk_usage(releases).free < 1024 ** 3: raise ValueError("Для обновления нужен 1 ГБ свободного места.")
stage.mkdir(); workspace(str(stage))
published = False
try:
progress(5, "Готовлю новую версию комплекта…")
owned = json.loads((ROOT / "bundle-files.json").read_text(encoding="utf-8"))
for relative in owned:
source, target = (ROOT / relative).resolve(), (stage / relative).resolve()
if ROOT not in source.parents or stage.resolve() not in target.parents or source.is_symlink():
raise ValueError("Некорректный файл установленного комплекта.")
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(source, target)
work = stage / ".update-work"; work.mkdir()
metadata = json.loads((stage / "bundle.json").read_text(encoding="utf-8"))
for index, item in enumerate(candidates):
if item["id"] == "packages":
metadata["pythonPackages"] = resolve_packages(stage, work, progress)
continue
archive = work / f"{item['id']}.zip"
label = {"python": "Python", "node": "Node.js", "ffmpeg": "FFmpeg"}[item["id"]]
fetch(item, archive, lambda fraction: progress(10 + (index + (fraction or 0)) / len(candidates) * 60 if fraction is not None else None, f"Скачиваю {label} {item['version']}…"))
if item["id"] == "python":
python = stage / "runtime/python"
saved = work / "site-packages"
shutil.copytree(python / "Lib/site-packages", saved)
shutil.rmtree(python); python.mkdir()
extract_zip(archive, python)
shutil.copytree(saved, python / "Lib/site-packages")
pth = next(python.glob("python*._pth"))
pth.write_text(f"{pth.stem}.zip\n.\nLib/site-packages\n../../\nimport site\n", encoding="ascii")
else:
with zipfile.ZipFile(archive) as package:
for member in safe_members(package):
name = Path(member.filename).name
binary = name == "node.exe" if item["id"] == "node" else "/bin/" in member.filename and (name in {"ffmpeg.exe", "ffprobe.exe"} or name.endswith(".dll"))
if binary: (stage / "runtime/bin" / name).write_bytes(package.read(member))
elif name in {"LICENSE", "LICENSE.txt"}: (stage / "licenses" / f"{label}-LICENSE.txt").write_bytes(package.read(member))
source = {"url": item["url"], "file": Path(urlsplit(item["url"]).path).name, "sha256": item["sha256"]}
metadata["sources"] = [old for old in metadata["sources"] if not (item["id"] == "python" and "python.org" in old["url"] or item["id"] == "node" and "nodejs.org" in old["url"] or item["id"] == "ffmpeg" and FFMPEG_ASSET in old["url"])] + [source]
shutil.rmtree(work)
for item in candidates:
if item["id"] in {"python", "node"}:
executable = stage / ("runtime/python/python.exe" if item["id"] == "python" else "runtime/bin/node.exe")
actual = run_small([str(executable), "--version"]).stdout.strip().removeprefix("Python ")
if actual != item["version"]: raise ValueError("Версия скачанного компонента не совпала с релизом.")
third_party = stage / "licenses/THIRD-PARTY.txt"
if third_party.exists():
third_party.write_text(re.sub(r"yt-dlp source:.*", "yt-dlp source: https://github.com/yt-dlp/yt-dlp (installed version in Python metadata)", third_party.read_text(encoding="utf-8")), encoding="utf-8")
metadata["updatedAt"] = time.time()
(stage / "bundle.json").write_text(json.dumps(metadata, indent=2), encoding="utf-8")
records = {str(path.relative_to(stage)).replace("\\", "/"): digest(path) for path in stage.rglob("*") if path.is_file() and path.name != "bundle-files.json"}
(stage / "bundle-files.json").write_text(json.dumps(records, indent=2), encoding="utf-8")
progress(85, "Проверяю новый комплект и подключаю помощник…")
publish(stage); published = True
settings(lastUpdate=time.time())
return {"title": "Компоненты обновлены", "changed": True, "restartRequired": True, "components": [item["id"] for item in candidates]}
finally:
# Only our generated, unpublished staging directory can be removed.
if not published and stage.exists() and stage.resolve().parent == releases.resolve() and not stage.is_symlink():
marker = json.loads((installed / "installation.json").read_text(encoding="utf-8-sig"))
if Path(marker["current"]).resolve() != stage.resolve(): shutil.rmtree(stage)
+293
View File
@@ -0,0 +1,293 @@
"""Isolated, cancellable download worker. Stdout contains JSON lines only."""
from __future__ import annotations
import json
import errno
import os
import subprocess
import sys
import tempfile
import threading
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from host.common import safe_filename, select_streams, summarize, validate_download, youtube_url
from host.media import NO_WINDOW, config, encoding_plan, ffmpeg_command, probe, tool
from host.session import apply_cookies, public_error, is_access_error
from host.ffmpeg_progress import run_ffmpeg
def emit(kind, **data):
print(json.dumps({"type": kind, **data}, ensure_ascii=False, allow_nan=False), flush=True)
class QuietLogger:
def debug(self, message):
pass
def info(self, message):
pass
def warning(self, message):
pass
def error(self, message):
# The raised DownloadError is reported through the structured channel.
pass
def ydl_options():
try:
node = tool("node")
except ValueError:
node = None
options = {"quiet": True, "no_warnings": True, "logger": QuietLogger(), "noplaylist": True,
"socket_timeout": 20, "retries": 3, "fragment_retries": 5,
# Parallel HLS requests can trigger YouTube's fragment throttling;
# never skip failed fragments and silently publish an incomplete clip.
"concurrent_fragment_downloads": 1, "skip_unavailable_fragments": False,
"ffmpeg_location": str(Path(tool("ffmpeg")).parent),
"windowsfilenames": True, "cachedir": False, "noprogress": True}
if node:
options["js_runtimes"] = {"node": {"path": node}}
return options
def inspect(url, cookies=None):
from yt_dlp import YoutubeDL
from yt_dlp.utils import DownloadError
for attempt in range(2):
try:
with YoutubeDL(ydl_options()) as ydl:
apply_cookies(ydl, cookies)
return summarize(ydl.extract_info(youtube_url(url), download=False))
except DownloadError as error:
if attempt or not is_access_error(error):
raise
time.sleep(1.5)
def convert(source, target, options, media, encoders):
plan = encoding_plan(options, media, encoders)
duration = float(media.get("format", {}).get("duration") or 0)
attempts = [(plan, True)]
if plan["engine"] in {"nvenc", "amf"}:
attempts.append((plan, False))
if options["engine"] == "auto":
attempts.append(({**plan, "engine": "cpu"}, False))
last_error = ""
for index, (attempt, cuda_decode) in enumerate(attempts):
stage = "encoding" if attempt["video"] not in (None, "copy") else ("audio" if attempt["audio"] not in (None, "copy") else "remuxing")
emit("progress", stage=stage, percent=0, engine=attempt["engine"], codec=attempt["video"] or attempt["audio"],
message="", speed="", eta=None)
command = ffmpeg_command(source, target, options, attempt, cuda_decode)
returncode, errors = run_ffmpeg(command, duration, stage, attempt["engine"], lambda **data: emit("progress", **data))
if returncode == 0 and target.is_file() and target.stat().st_size:
# Verify the result really has the requested streams before publishing it.
output = probe(target)
types = {s.get("codec_type") for s in output.get("streams", [])}
required = {"audio"} if options["mode"] == "audio" else ({"video", "audio"} if options["mode"] == "video" else {"video"})
if not required <= types or (options["mode"] == "video_only" and "audio" in types):
raise ValueError("Проверка итогового файла не прошла: отсутствует выбранная дорожка.")
return attempt
last_error = errors.strip()[-1600:]
if index + 1 < len(attempts):
next_plan, next_decode = attempts[index + 1]
emit("progress", stage="encoding", percent=0, engine=attempt["engine"],
message="Повтор с обычным декодированием" if next_plan["engine"] == attempt["engine"] else "Аппаратное кодирование недоступно: переход на CPU", speed="", eta=None)
raise ValueError("FFmpeg не смог обработать видео. " + last_error)
def publish_file(source: Path, folder: Path, name: str, extension: str) -> Path:
if source.stat().st_dev != folder.stat().st_dev:
# Copy to an unpublished file on the destination disk, then publish it
# atomically. Cancellation cannot leave a partial final filename.
with tempfile.TemporaryDirectory(prefix=".ytdl-", dir=folder) as staging:
staging_path = Path(staging).resolve()
emit("workspace", path=str(staging_path), purpose="saving")
target = staging_path / ("ready." + extension)
total, copied, last_emit = source.stat().st_size, 0, 0.0
emit("progress", stage="saving", percent=0, copied=0, total=total, speed=None, eta=None)
with source.open("rb") as incoming, target.open("xb") as outgoing:
while chunk := incoming.read(4 * 1024 * 1024):
outgoing.write(chunk)
copied += len(chunk)
if time.monotonic() - last_emit >= 0.2:
last_emit = time.monotonic()
emit("progress", stage="saving", percent=min(99, copied / total * 100) if total else 0,
copied=copied, total=total, speed=None, eta=None)
outgoing.flush()
os.fsync(outgoing.fileno())
destination = publish_file(target, folder, name, extension)
source.unlink()
return destination
# Same-volume hard-link creation is atomic and never overwrites an existing file.
for index in range(10000):
suffix = "" if index == 0 else f" ({index})"
destination = folder / f"{name}{suffix}.{extension}"
if destination.exists():
continue
try:
os.link(source, destination)
source.unlink()
return destination
except FileExistsError:
continue
except OSError as error:
if destination.exists():
continue
if error.errno == errno.EXDEV or getattr(error, "winerror", None) == 17:
raise ValueError("Не удалось сохранить файл на выбранный диск.") from error
# Windows rename also refuses to replace existing destinations, including on FAT/exFAT.
if os.name != "nt":
raise
try:
os.rename(source, destination)
return destination
except FileExistsError:
continue
except OSError:
if destination.exists():
continue
raise
raise ValueError("Не удалось выбрать свободное имя файла.")
def download(payload, encoders):
from yt_dlp import YoutubeDL
from yt_dlp.postprocessor.ffmpeg import FFmpegMergerPP, FFmpegPostProcessorError
from yt_dlp.utils import DownloadError
options = validate_download(payload)
emit("progress", stage="preparing", percent=None, message="Получаю свежие ссылки на дорожки")
folder = Path(options["folder"])
work_folder = Path(options["workFolder"])
# Only this job's generated directory is removed on exit; final files live outside it.
# A failed downloader can retain an open .part handle until this worker exits.
# Preserve its actual error; the native host cleans up again after process exit.
with tempfile.TemporaryDirectory(prefix=".ytdl-", dir=work_folder, ignore_cleanup_errors=True) as scratch:
scratch_path = Path(scratch).resolve()
if scratch_path.parent != work_folder.resolve():
raise ValueError("Некорректная временная папка.")
emit("workspace", path=str(scratch_path))
last_emit = 0.0
finished_tracks = set()
track_count = 1
def format_selector(context):
nonlocal track_count
selector = select_streams(context, options)
track_count = 2 if "+" in selector else 1
# Keep extraction and downloading in the same YoutubeDL instance so
# its visitor cookies and request state remain available.
return ydl.build_format_selector(selector)(context)
def reject_live(info, *, incomplete=False):
if info.get("is_live"):
return "Дождитесь окончания прямой трансляции."
def progress(event):
nonlocal last_emit
if event.get("status") == "finished":
finished_tracks.add(event.get("filename", ""))
return
if event.get("status") != "downloading" or time.monotonic() - last_emit < 0.2:
return
last_emit = time.monotonic()
downloaded = event.get("downloaded_bytes", 0)
total = event.get("total_bytes") or event.get("total_bytes_estimate") or 0
track_progress = min(1, downloaded / total) if total else 0
percent = min(99, (len(finished_tracks) + track_progress) / track_count * 100) if total else None
stream = event.get("info_dict", {})
track = "audio" if stream.get("vcodec") == "none" else "video"
emit("progress", stage="downloading", track=track, percent=percent,
downloaded=downloaded, total=total, speed=event.get("speed"), eta=event.get("eta"))
def postprocess(event):
if event.get("status") == "started" and event.get("postprocessor") == "Merger":
emit("progress", stage="merging", percent=0, engine="copy", speed="", eta=None,
processedSeconds=0, durationSeconds=0, written=0, message="Подготовка дорожек к объединению")
class ProgressMerger(FFmpegMergerPP):
def run(self, info):
self.duration = info.get("duration") or 0
return super().run(info)
def run_ffmpeg_multiple_files(self, inputs, output, opts, **kwargs):
self.check_version()
oldest_mtime = min(os.stat(path).st_mtime for path in inputs)
command = [self.executable, "-hide_banner", "-nostdin", "-loglevel", "info", "-y",
"-progress", "pipe:1", "-stats_period", "0.4"]
for path in inputs:
command += ["-i", str(path)]
# Preserve yt-dlp's maps and AAC fixup, including HLS audio.
command += list(opts) + [str(output)]
code, errors = run_ffmpeg(command, self.duration, "merging", "copy", lambda **data: emit("progress", **data))
if code:
raise FFmpegPostProcessorError(errors.strip()[-1600:] or "Не удалось объединить дорожки.")
self.try_utime(output, oldest_mtime, oldest_mtime)
def enable_merge_progress(ydl):
original_run_pp = ydl.run_pp
def run_pp(pp, info):
if isinstance(pp, FFmpegMergerPP) and not isinstance(pp, ProgressMerger):
pp = ProgressMerger(ydl)
return original_run_pp(pp, info)
ydl.run_pp = run_pp
download_options = {**ydl_options(), "format": format_selector, "match_filter": reject_live,
"outtmpl": str(scratch_path / "source.%(ext)s"),
"merge_output_format": "mkv", "progress_hooks": [progress],
"postprocessor_hooks": [postprocess], "overwrites": False}
for attempt in range(2):
try:
with YoutubeDL(download_options) as ydl:
enable_merge_progress(ydl)
apply_cookies(ydl, payload.get("browserCookies"))
downloaded_info = ydl.extract_info(options["url"], download=True)
break
except DownloadError as error:
if attempt == 0 and (is_access_error(error) or "403" in str(error) or "410" in str(error)):
finished_tracks.clear()
emit("progress", stage="preparing", percent=None, message="Обновляю ссылки YouTube и повторяю загрузку")
time.sleep(1.5)
else:
raise
if not downloaded_info:
raise ValueError("Ролик недоступен для загрузки.")
title = str(downloaded_info.get("title", "Видео"))
name = safe_filename(options["filename"] or title, str(downloaded_info.get("id", "video")))
candidates = [p for p in scratch_path.glob("source.*") if p.is_file() and p.suffix.lower() in {".mkv", ".mp4", ".webm", ".m4a", ".mp3", ".ogg", ".opus", ".flac", ".wav", ".ts"}]
if len(candidates) != 1:
raise ValueError("Не удалось определить загруженный медиафайл.")
source = candidates[0]
media = probe(source)
output = scratch_path / f"final.{options['container']}"
plan = convert(source, output, options, media, encoders)
destination = publish_file(output, folder, name, options["container"])
return {"path": str(destination), "filename": destination.name, "title": title,
"size": destination.stat().st_size, "engine": plan["engine"],
"codec": plan["video"] or plan["audio"], "mode": options["mode"]}
def main():
task = {}
try:
task = json.loads(sys.stdin.readline())
if task["action"] == "inspect":
result = inspect(task["payload"]["url"], task["payload"].get("browserCookies"))
elif task["action"] == "download":
result = download(task["payload"], task.get("encoders", task.get("nvenc", [])))
elif task["action"] == "dependency_install":
from host.dependencies import install
result = install(task["payload"]["dependency"],
lambda percent, message: emit("progress", stage="installing", percent=percent, message=message),
lambda path: emit("workspace", path=path))
elif task["action"] == "helper_update":
from host.updates import update
result = update(lambda percent, message: emit("progress", stage="updating", percent=percent, message=message),
lambda path: emit("workspace", path=path), task["payload"].get("candidates"))
else:
raise ValueError("Неизвестная операция.")
emit("result", result=result)
except Exception as error:
emit("error", message=public_error(error, bool(task.get("payload", {}).get("browserCookies"))))
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())