290 lines
17 KiB
Python
290 lines
17 KiB
Python
"""Verified vendor downloads, staged runtimes and publication through the manager."""
|
|
from __future__ import annotations
|
|
import concurrent.futures
|
|
import hashlib
|
|
import importlib.metadata
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import threading
|
|
import uuid
|
|
import zipfile
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit
|
|
|
|
import requests
|
|
from host.bundle import ROOT, installation_root, manifest
|
|
from host.media import NO_WINDOW, run_small
|
|
|
|
ALLOWED_HOSTS = {"www.python.org", "nodejs.org", "api.github.com", "github.com", "release-assets.githubusercontent.com", "objects.githubusercontent.com", "pypi.org", "files.pythonhosted.org"}
|
|
FFMPEG_ASSET = "ffmpeg-master-latest-win64-gpl-shared.zip"
|
|
SETTINGS_LOCK = threading.Lock()
|
|
|
|
|
|
def settings(enabled=None, **changes):
|
|
with SETTINGS_LOCK:
|
|
return _settings(enabled, **changes)
|
|
|
|
|
|
def _settings(enabled=None, **changes):
|
|
installed = installation_root()
|
|
if not installed:
|
|
return {"enabled": False, "supported": False}
|
|
path = installed / "data/update-settings.json"
|
|
try: value = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, ValueError): value = {}
|
|
value = {"enabled": value.get("enabled", True) is True, "lastCheck": value.get("lastCheck", 0), **value}
|
|
if enabled is not None or changes:
|
|
if enabled is not None: value["enabled"] = enabled is True
|
|
value.update(changes)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = path.with_name(path.name + "." + uuid.uuid4().hex + ".tmp")
|
|
temporary.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
|
|
temporary.replace(path)
|
|
return value
|
|
|
|
|
|
def due():
|
|
value = settings()
|
|
return value["enabled"] and time.time() - value.get("lastCheck", 0) >= 86400
|
|
|
|
|
|
def trusted_url(url):
|
|
parsed = urlsplit(url)
|
|
if parsed.scheme != "https" or parsed.hostname not in ALLOWED_HOSTS or parsed.username or parsed.password or parsed.port not in {None, 443}:
|
|
raise ValueError("Неизвестный источник обновления.")
|
|
return url
|
|
|
|
|
|
def get(url, **kwargs):
|
|
trusted_url(url)
|
|
try:
|
|
headers = {"User-Agent": "YouTubeDL/1.3"}
|
|
if urlsplit(url).hostname == "api.github.com": headers["Accept"] = "application/vnd.github+json"
|
|
response = requests.get(url, headers=headers, timeout=(15, 40), **kwargs)
|
|
for item in [*response.history, response]: trusted_url(item.url)
|
|
response.raise_for_status()
|
|
return response
|
|
except requests.RequestException as error:
|
|
raise ValueError(f"Не удалось связаться с {urlsplit(url).hostname}. Проверьте подключение и повторите обновление.") from error
|
|
|
|
|
|
def digest(path):
|
|
result = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""): result.update(chunk)
|
|
return result.hexdigest()
|
|
|
|
|
|
def fetch(item, target, progress):
|
|
expected = item.get("sha256", "")
|
|
if not re.fullmatch("[a-f0-9]{64}", expected): raise ValueError("Источник не предоставил SHA256 обновления.")
|
|
with get(item["url"], stream=True) as response, target.open("wb") as output:
|
|
total = int(response.headers.get("Content-Length", 0)); received = 0
|
|
for chunk in response.iter_content(512 * 1024):
|
|
output.write(chunk); received += len(chunk)
|
|
if received > 512 * 1024 * 1024: raise ValueError("Превышен размер компонента.")
|
|
progress(received / total if total else None)
|
|
if digest(target) != expected: raise ValueError("Контрольная сумма обновления не совпала. Прежний комплект сохранён.")
|
|
|
|
|
|
def safe_members(package):
|
|
if sum(item.file_size for item in package.infolist()) > 1024 * 1024 * 1024:
|
|
raise ValueError("Превышен размер распакованного компонента.")
|
|
for item in package.infolist():
|
|
name = item.filename.replace("\\", "/")
|
|
parts = name.split("/")
|
|
if name.startswith("/") or any(part in {"..", "."} or ":" in part for part in parts) or (item.external_attr >> 16) & 0xF000 == 0xA000:
|
|
raise ValueError("Недопустимый путь в обновлении.")
|
|
if not item.is_dir(): yield item
|
|
|
|
|
|
def extract_zip(archive, target):
|
|
root = target.resolve()
|
|
with zipfile.ZipFile(archive) as package:
|
|
for item in safe_members(package):
|
|
destination = (root / item.filename).resolve()
|
|
if root not in destination.parents: raise ValueError("Путь вне папки компонента.")
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
with package.open(item) as source, destination.open("wb") as output: shutil.copyfileobj(source, output)
|
|
|
|
|
|
def version_tuple(value):
|
|
return tuple(int(part) for part in value.lstrip("v").split(".")) if re.fullmatch(r"v?\d+(?:\.\d+)*", value) else ()
|
|
|
|
|
|
def python_release():
|
|
branch = f"{sys.version_info.major}.{sys.version_info.minor}"
|
|
current = version_tuple(sys.version.split()[0])
|
|
index = get("https://www.python.org/ftp/python/").text
|
|
versions = sorted(set(re.findall(r'href="(' + re.escape(branch) + r'\.\d+)/"', index)), key=version_tuple, reverse=True)
|
|
for version in versions:
|
|
if version_tuple(version) <= current: break
|
|
page = get(f"https://www.python.org/downloads/release/python-{version.replace('.', '')}/").text
|
|
name = f"python-{version}-embed-amd64.zip"
|
|
for row in re.findall(r"<tr\b[^>]*>(.*?)</tr>", page, re.S):
|
|
if name not in row: continue
|
|
match = re.search(r"\b[a-f0-9]{64}\b", re.sub(r"<[^>]+>", " ", row))
|
|
if match:
|
|
return {"id": "python", "version": version, "url": f"https://www.python.org/ftp/python/{version}/{name}", "sha256": match[0]}
|
|
return None
|
|
|
|
|
|
def node_release():
|
|
current = run_small([str(ROOT / "runtime/bin/node.exe"), "--version"]).stdout.strip()
|
|
releases = get("https://nodejs.org/dist/index.json").json()
|
|
compatible = [item for item in releases if item.get("lts") and "win-x64-zip" in item.get("files", [])
|
|
and version_tuple(item["version"])[:1] == version_tuple(current)[:1] and version_tuple(item["version"]) > version_tuple(current)]
|
|
if not compatible: return None
|
|
version = max(compatible, key=lambda item: version_tuple(item["version"]))["version"]
|
|
name = f"node-{version}-win-x64.zip"
|
|
checksum = next((line.split()[0] for line in get(f"https://nodejs.org/dist/{version}/SHASUMS256.txt").text.splitlines() if line.split() and line.split()[-1] == name), None)
|
|
return {"id": "node", "version": version, "url": f"https://nodejs.org/dist/{version}/{name}", "sha256": checksum}
|
|
|
|
|
|
def ffmpeg_release():
|
|
release = get("https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest").json()
|
|
asset = next(item for item in release["assets"] if item["name"] == FFMPEG_ASSET)
|
|
checksum = asset.get("digest", "").removeprefix("sha256:")
|
|
if not re.fullmatch("[a-f0-9]{64}", checksum): raise ValueError("FFmpeg не предоставил SHA256. Обновление отложено.")
|
|
previous = next((item.get("sha256") for item in manifest()["sources"] if FFMPEG_ASSET in item["url"]), None)
|
|
if previous == checksum: return None
|
|
return {"id": "ffmpeg", "version": asset["updated_at"], "url": asset["browser_download_url"], "sha256": checksum}
|
|
|
|
|
|
def packages_release():
|
|
installed = {item.metadata["Name"]: item.version for item in importlib.metadata.distributions(path=[str(ROOT / "runtime/python/Lib/site-packages")])}
|
|
def newer(item):
|
|
name, version = item
|
|
release = get(f"https://pypi.org/pypi/{name}/json").json()["info"]["version"]
|
|
return version_tuple(release) > version_tuple(version)
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor:
|
|
changed = any(list(executor.map(newer, installed.items())))
|
|
return {"id": "packages", "version": "latest-compatible"} if changed else None
|
|
|
|
|
|
def check_updates():
|
|
if not installation_root(): raise ValueError("Для обновления установите автономный помощник.")
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
|
|
candidates = list(executor.map(lambda task: task(), [python_release, node_release, ffmpeg_release, packages_release]))
|
|
return [item for item in candidates if item]
|
|
|
|
|
|
def resolve_packages(stage, work, progress):
|
|
progress(None, "Подбираю совместимые версии Python-пакетов…")
|
|
release = get("https://pypi.org/pypi/pip/json").json()
|
|
wheel = next(item for item in release["urls"] if item["filename"].endswith("-py3-none-any.whl") and not item.get("yanked"))
|
|
pip = work / "pip.whl"
|
|
fetch({"url": wheel["url"], "sha256": wheel["digests"]["sha256"]}, pip, lambda _: None)
|
|
report = work / "python-packages.json"
|
|
script = "import sys;sys.path.insert(0,sys.argv.pop(1));from pip._internal.cli.main import main;raise SystemExit(main(sys.argv[1:]))"
|
|
command = [str(stage / "runtime/python/python.exe"), "-I", "-X", "utf8", "-B", "-c", script, str(pip), "--isolated", "--disable-pip-version-check", "install",
|
|
"--dry-run", "--ignore-installed", "--only-binary=:all:", "--no-cache-dir", "--quiet", "--report", str(report), "--index-url", "https://pypi.org/simple", "yt-dlp[default]", "yt-dlp-ejs"]
|
|
result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=600, creationflags=NO_WINDOW,
|
|
env={**os.environ, "TEMP": str(work), "TMP": str(work)}, stdin=subprocess.DEVNULL)
|
|
if result.returncode: raise ValueError("Не удалось подобрать совместимые Python-пакеты. " + result.stderr[-1200:])
|
|
records = json.loads(report.read_text(encoding="utf-8"))["install"]
|
|
packages = stage / "runtime/python/Lib/site-packages"
|
|
if packages.exists(): shutil.rmtree(packages)
|
|
packages.mkdir(parents=True)
|
|
for index, record in enumerate(records):
|
|
info = record["download_info"]; name = record["metadata"]["name"]
|
|
archive = work / f"wheel-{index}.whl"
|
|
fetch({"url": info["url"], "sha256": info["archive_info"]["hashes"]["sha256"]}, archive,
|
|
lambda fraction: progress(None, f"Скачиваю {name}…"))
|
|
with zipfile.ZipFile(archive) as wheel:
|
|
for member in safe_members(wheel):
|
|
relative = member.filename.split("/")
|
|
if relative[0].endswith(".data"):
|
|
if len(relative) < 3 or relative[1] not in {"purelib", "platlib"}: continue
|
|
relative = relative[2:]
|
|
destination = packages.joinpath(*relative).resolve()
|
|
if packages.resolve() not in destination.parents: raise ValueError("Некорректный путь Python-пакета.")
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(wheel.read(member))
|
|
return {item["metadata"]["name"]: item["metadata"]["version"] for item in records}
|
|
|
|
|
|
def publish(stage):
|
|
installed = installation_root()
|
|
manager = installed / "YouTubeDL-Manager.exe"
|
|
result = run_small([str(manager), "--activate-release", str(stage), "--expected-current", str(ROOT), "--target-root", str(installed), "--silent"], timeout=180)
|
|
if result.returncode: raise ValueError("Новый комплект не прошёл проверку. Прежняя версия сохранена. " + result.stderr[-1000:])
|
|
|
|
|
|
def update(progress, workspace, candidates=None):
|
|
installed = installation_root()
|
|
if not installed: raise ValueError("Обновление доступно в автономном комплекте.")
|
|
progress(None, "Проверяю новые версии компонентов…")
|
|
candidates = check_updates() if candidates is None else candidates
|
|
settings(lastCheck=time.time())
|
|
if not candidates: return {"title": "Все компоненты актуальны", "changed": False}
|
|
releases = installed / "releases"
|
|
stage = releases / (".ytdl-update-" + uuid.uuid4().hex)
|
|
if stage.resolve().parent != releases.resolve() or releases.is_symlink(): raise ValueError("Недопустимая папка обновления.")
|
|
if shutil.disk_usage(releases).free < 1024 ** 3: raise ValueError("Для обновления нужен 1 ГБ свободного места.")
|
|
stage.mkdir(); workspace(str(stage))
|
|
published = False
|
|
try:
|
|
progress(5, "Готовлю новую версию комплекта…")
|
|
owned = json.loads((ROOT / "bundle-files.json").read_text(encoding="utf-8"))
|
|
for relative in owned:
|
|
source, target = (ROOT / relative).resolve(), (stage / relative).resolve()
|
|
if ROOT not in source.parents or stage.resolve() not in target.parents or source.is_symlink():
|
|
raise ValueError("Некорректный файл установленного комплекта.")
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copyfile(source, target)
|
|
work = stage / ".update-work"; work.mkdir()
|
|
metadata = json.loads((stage / "bundle.json").read_text(encoding="utf-8"))
|
|
for index, item in enumerate(candidates):
|
|
if item["id"] == "packages":
|
|
metadata["pythonPackages"] = resolve_packages(stage, work, progress)
|
|
continue
|
|
archive = work / f"{item['id']}.zip"
|
|
label = {"python": "Python", "node": "Node.js", "ffmpeg": "FFmpeg"}[item["id"]]
|
|
fetch(item, archive, lambda fraction: progress(10 + (index + (fraction or 0)) / len(candidates) * 60 if fraction is not None else None, f"Скачиваю {label} {item['version']}…"))
|
|
if item["id"] == "python":
|
|
python = stage / "runtime/python"
|
|
saved = work / "site-packages"
|
|
shutil.copytree(python / "Lib/site-packages", saved)
|
|
shutil.rmtree(python); python.mkdir()
|
|
extract_zip(archive, python)
|
|
shutil.copytree(saved, python / "Lib/site-packages")
|
|
pth = next(python.glob("python*._pth"))
|
|
pth.write_text(f"{pth.stem}.zip\n.\nLib/site-packages\n../../\nimport site\n", encoding="ascii")
|
|
else:
|
|
with zipfile.ZipFile(archive) as package:
|
|
for member in safe_members(package):
|
|
name = Path(member.filename).name
|
|
binary = name == "node.exe" if item["id"] == "node" else "/bin/" in member.filename and (name in {"ffmpeg.exe", "ffprobe.exe"} or name.endswith(".dll"))
|
|
if binary: (stage / "runtime/bin" / name).write_bytes(package.read(member))
|
|
elif name in {"LICENSE", "LICENSE.txt"}: (stage / "licenses" / f"{label}-LICENSE.txt").write_bytes(package.read(member))
|
|
source = {"url": item["url"], "file": Path(urlsplit(item["url"]).path).name, "sha256": item["sha256"]}
|
|
metadata["sources"] = [old for old in metadata["sources"] if not (item["id"] == "python" and "python.org" in old["url"] or item["id"] == "node" and "nodejs.org" in old["url"] or item["id"] == "ffmpeg" and FFMPEG_ASSET in old["url"])] + [source]
|
|
shutil.rmtree(work)
|
|
for item in candidates:
|
|
if item["id"] in {"python", "node"}:
|
|
executable = stage / ("runtime/python/python.exe" if item["id"] == "python" else "runtime/bin/node.exe")
|
|
actual = run_small([str(executable), "--version"]).stdout.strip().removeprefix("Python ")
|
|
if actual != item["version"]: raise ValueError("Версия скачанного компонента не совпала с релизом.")
|
|
third_party = stage / "licenses/THIRD-PARTY.txt"
|
|
if third_party.exists():
|
|
third_party.write_text(re.sub(r"yt-dlp source:.*", "yt-dlp source: https://github.com/yt-dlp/yt-dlp (installed version in Python metadata)", third_party.read_text(encoding="utf-8")), encoding="utf-8")
|
|
metadata["updatedAt"] = time.time()
|
|
(stage / "bundle.json").write_text(json.dumps(metadata, indent=2), encoding="utf-8")
|
|
records = {str(path.relative_to(stage)).replace("\\", "/"): digest(path) for path in stage.rglob("*") if path.is_file() and path.name != "bundle-files.json"}
|
|
(stage / "bundle-files.json").write_text(json.dumps(records, indent=2), encoding="utf-8")
|
|
progress(85, "Проверяю новый комплект и подключаю помощник…")
|
|
publish(stage); published = True
|
|
settings(lastUpdate=time.time())
|
|
return {"title": "Компоненты обновлены", "changed": True, "restartRequired": True, "components": [item["id"] for item in candidates]}
|
|
finally:
|
|
# Only our generated, unpublished staging directory can be removed.
|
|
if not published and stage.exists() and stage.resolve().parent == releases.resolve() and not stage.is_symlink():
|
|
marker = json.loads((installed / "installation.json").read_text(encoding="utf-8-sig"))
|
|
if Path(marker["current"]).resolve() != stage.resolve(): shutil.rmtree(stage)
|