cookies.py 54 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351
  1. import base64
  2. import collections
  3. import contextlib
  4. import datetime as dt
  5. import glob
  6. import http.cookiejar
  7. import http.cookies
  8. import io
  9. import json
  10. import os
  11. import re
  12. import shutil
  13. import struct
  14. import subprocess
  15. import sys
  16. import tempfile
  17. import time
  18. import urllib.request
  19. from enum import Enum, auto
  20. from hashlib import pbkdf2_hmac
  21. from .aes import (
  22. aes_cbc_decrypt_bytes,
  23. aes_gcm_decrypt_and_verify_bytes,
  24. unpad_pkcs7,
  25. )
  26. from .compat import functools # isort: split
  27. from .compat import compat_os_name
  28. from .dependencies import (
  29. _SECRETSTORAGE_UNAVAILABLE_REASON,
  30. secretstorage,
  31. sqlite3,
  32. )
  33. from .minicurses import MultilinePrinter, QuietMultilinePrinter
  34. from .utils import (
  35. DownloadError,
  36. Popen,
  37. error_to_str,
  38. expand_path,
  39. is_path_like,
  40. sanitize_url,
  41. str_or_none,
  42. try_call,
  43. write_string,
  44. )
  45. from .utils._utils import _YDLLogger
  46. from .utils.networking import normalize_url
  47. CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'}
  48. SUPPORTED_BROWSERS = CHROMIUM_BASED_BROWSERS | {'firefox', 'safari'}
  49. class YDLLogger(_YDLLogger):
  50. def warning(self, message, only_once=False): # compat
  51. return super().warning(message, once=only_once)
  52. class ProgressBar(MultilinePrinter):
  53. _DELAY, _timer = 0.1, 0
  54. def print(self, message):
  55. if time.time() - self._timer > self._DELAY:
  56. self.print_at_line(f'[Cookies] {message}', 0)
  57. self._timer = time.time()
  58. def progress_bar(self):
  59. """Return a context manager with a print method. (Optional)"""
  60. # Do not print to files/pipes, loggers, or when --no-progress is used
  61. if not self._ydl or self._ydl.params.get('noprogress') or self._ydl.params.get('logger'):
  62. return
  63. file = self._ydl._out_files.error
  64. try:
  65. if not file.isatty():
  66. return
  67. except BaseException:
  68. return
  69. return self.ProgressBar(file, preserve_output=False)
  70. def _create_progress_bar(logger):
  71. if hasattr(logger, 'progress_bar'):
  72. printer = logger.progress_bar()
  73. if printer:
  74. return printer
  75. printer = QuietMultilinePrinter()
  76. printer.print = lambda _: None
  77. return printer
  78. def load_cookies(cookie_file, browser_specification, ydl):
  79. cookie_jars = []
  80. if browser_specification is not None:
  81. browser_name, profile, keyring, container = _parse_browser_specification(*browser_specification)
  82. cookie_jars.append(
  83. extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl), keyring=keyring, container=container))
  84. if cookie_file is not None:
  85. is_filename = is_path_like(cookie_file)
  86. if is_filename:
  87. cookie_file = expand_path(cookie_file)
  88. jar = YoutubeDLCookieJar(cookie_file)
  89. if not is_filename or os.access(cookie_file, os.R_OK):
  90. jar.load()
  91. cookie_jars.append(jar)
  92. return _merge_cookie_jars(cookie_jars)
  93. def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(), *, keyring=None, container=None):
  94. if browser_name == 'firefox':
  95. return _extract_firefox_cookies(profile, container, logger)
  96. elif browser_name == 'safari':
  97. return _extract_safari_cookies(profile, logger)
  98. elif browser_name in CHROMIUM_BASED_BROWSERS:
  99. return _extract_chrome_cookies(browser_name, profile, keyring, logger)
  100. else:
  101. raise ValueError(f'unknown browser: {browser_name}')
  102. def _extract_firefox_cookies(profile, container, logger):
  103. logger.info('Extracting cookies from firefox')
  104. if not sqlite3:
  105. logger.warning('Cannot extract cookies from firefox without sqlite3 support. '
  106. 'Please use a Python interpreter compiled with sqlite3 support')
  107. return YoutubeDLCookieJar()
  108. if profile is None:
  109. search_roots = list(_firefox_browser_dirs())
  110. elif _is_path(profile):
  111. search_roots = [profile]
  112. else:
  113. search_roots = [os.path.join(path, profile) for path in _firefox_browser_dirs()]
  114. search_root = ', '.join(map(repr, search_roots))
  115. cookie_database_path = _newest(_firefox_cookie_dbs(search_roots))
  116. if cookie_database_path is None:
  117. raise FileNotFoundError(f'could not find firefox cookies database in {search_root}')
  118. logger.debug(f'Extracting cookies from: "{cookie_database_path}"')
  119. container_id = None
  120. if container not in (None, 'none'):
  121. containers_path = os.path.join(os.path.dirname(cookie_database_path), 'containers.json')
  122. if not os.path.isfile(containers_path) or not os.access(containers_path, os.R_OK):
  123. raise FileNotFoundError(f'could not read containers.json in {search_root}')
  124. with open(containers_path, encoding='utf8') as containers:
  125. identities = json.load(containers).get('identities', [])
  126. container_id = next((context.get('userContextId') for context in identities if container in (
  127. context.get('name'),
  128. try_call(lambda: re.fullmatch(r'userContext([^\.]+)\.label', context['l10nID']).group())
  129. )), None)
  130. if not isinstance(container_id, int):
  131. raise ValueError(f'could not find firefox container "{container}" in containers.json')
  132. with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
  133. cursor = None
  134. try:
  135. cursor = _open_database_copy(cookie_database_path, tmpdir)
  136. if isinstance(container_id, int):
  137. logger.debug(
  138. f'Only loading cookies from firefox container "{container}", ID {container_id}')
  139. cursor.execute(
  140. 'SELECT host, name, value, path, expiry, isSecure FROM moz_cookies WHERE originAttributes LIKE ? OR originAttributes LIKE ?',
  141. (f'%userContextId={container_id}', f'%userContextId={container_id}&%'))
  142. elif container == 'none':
  143. logger.debug('Only loading cookies not belonging to any container')
  144. cursor.execute(
  145. 'SELECT host, name, value, path, expiry, isSecure FROM moz_cookies WHERE NOT INSTR(originAttributes,"userContextId=")')
  146. else:
  147. cursor.execute('SELECT host, name, value, path, expiry, isSecure FROM moz_cookies')
  148. jar = YoutubeDLCookieJar()
  149. with _create_progress_bar(logger) as progress_bar:
  150. table = cursor.fetchall()
  151. total_cookie_count = len(table)
  152. for i, (host, name, value, path, expiry, is_secure) in enumerate(table):
  153. progress_bar.print(f'Loading cookie {i: 6d}/{total_cookie_count: 6d}')
  154. cookie = http.cookiejar.Cookie(
  155. version=0, name=name, value=value, port=None, port_specified=False,
  156. domain=host, domain_specified=bool(host), domain_initial_dot=host.startswith('.'),
  157. path=path, path_specified=bool(path), secure=is_secure, expires=expiry, discard=False,
  158. comment=None, comment_url=None, rest={})
  159. jar.set_cookie(cookie)
  160. logger.info(f'Extracted {len(jar)} cookies from firefox')
  161. return jar
  162. finally:
  163. if cursor is not None:
  164. cursor.connection.close()
  165. def _firefox_browser_dirs():
  166. if sys.platform in ('cygwin', 'win32'):
  167. yield os.path.expandvars(R'%APPDATA%\Mozilla\Firefox\Profiles')
  168. elif sys.platform == 'darwin':
  169. yield os.path.expanduser('~/Library/Application Support/Firefox/Profiles')
  170. else:
  171. yield from map(os.path.expanduser, (
  172. '~/.mozilla/firefox',
  173. '~/snap/firefox/common/.mozilla/firefox',
  174. '~/.var/app/org.mozilla.firefox/.mozilla/firefox',
  175. ))
  176. def _firefox_cookie_dbs(roots):
  177. for root in map(os.path.abspath, roots):
  178. for pattern in ('', '*/', 'Profiles/*/'):
  179. yield from glob.iglob(os.path.join(root, pattern, 'cookies.sqlite'))
  180. def _get_chromium_based_browser_settings(browser_name):
  181. # https://chromium.googlesource.com/chromium/src/+/HEAD/docs/user_data_dir.md
  182. if sys.platform in ('cygwin', 'win32'):
  183. appdata_local = os.path.expandvars('%LOCALAPPDATA%')
  184. appdata_roaming = os.path.expandvars('%APPDATA%')
  185. browser_dir = {
  186. 'brave': os.path.join(appdata_local, R'BraveSoftware\Brave-Browser\User Data'),
  187. 'chrome': os.path.join(appdata_local, R'Google\Chrome\User Data'),
  188. 'chromium': os.path.join(appdata_local, R'Chromium\User Data'),
  189. 'edge': os.path.join(appdata_local, R'Microsoft\Edge\User Data'),
  190. 'opera': os.path.join(appdata_roaming, R'Opera Software\Opera Stable'),
  191. 'vivaldi': os.path.join(appdata_local, R'Vivaldi\User Data'),
  192. }[browser_name]
  193. elif sys.platform == 'darwin':
  194. appdata = os.path.expanduser('~/Library/Application Support')
  195. browser_dir = {
  196. 'brave': os.path.join(appdata, 'BraveSoftware/Brave-Browser'),
  197. 'chrome': os.path.join(appdata, 'Google/Chrome'),
  198. 'chromium': os.path.join(appdata, 'Chromium'),
  199. 'edge': os.path.join(appdata, 'Microsoft Edge'),
  200. 'opera': os.path.join(appdata, 'com.operasoftware.Opera'),
  201. 'vivaldi': os.path.join(appdata, 'Vivaldi'),
  202. }[browser_name]
  203. else:
  204. config = _config_home()
  205. browser_dir = {
  206. 'brave': os.path.join(config, 'BraveSoftware/Brave-Browser'),
  207. 'chrome': os.path.join(config, 'google-chrome'),
  208. 'chromium': os.path.join(config, 'chromium'),
  209. 'edge': os.path.join(config, 'microsoft-edge'),
  210. 'opera': os.path.join(config, 'opera'),
  211. 'vivaldi': os.path.join(config, 'vivaldi'),
  212. }[browser_name]
  213. # Linux keyring names can be determined by snooping on dbus while opening the browser in KDE:
  214. # dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
  215. keyring_name = {
  216. 'brave': 'Brave',
  217. 'chrome': 'Chrome',
  218. 'chromium': 'Chromium',
  219. 'edge': 'Microsoft Edge' if sys.platform == 'darwin' else 'Chromium',
  220. 'opera': 'Opera' if sys.platform == 'darwin' else 'Chromium',
  221. 'vivaldi': 'Vivaldi' if sys.platform == 'darwin' else 'Chrome',
  222. }[browser_name]
  223. browsers_without_profiles = {'opera'}
  224. return {
  225. 'browser_dir': browser_dir,
  226. 'keyring_name': keyring_name,
  227. 'supports_profiles': browser_name not in browsers_without_profiles
  228. }
  229. def _extract_chrome_cookies(browser_name, profile, keyring, logger):
  230. logger.info(f'Extracting cookies from {browser_name}')
  231. if not sqlite3:
  232. logger.warning(f'Cannot extract cookies from {browser_name} without sqlite3 support. '
  233. 'Please use a Python interpreter compiled with sqlite3 support')
  234. return YoutubeDLCookieJar()
  235. config = _get_chromium_based_browser_settings(browser_name)
  236. if profile is None:
  237. search_root = config['browser_dir']
  238. elif _is_path(profile):
  239. search_root = profile
  240. config['browser_dir'] = os.path.dirname(profile) if config['supports_profiles'] else profile
  241. else:
  242. if config['supports_profiles']:
  243. search_root = os.path.join(config['browser_dir'], profile)
  244. else:
  245. logger.error(f'{browser_name} does not support profiles')
  246. search_root = config['browser_dir']
  247. cookie_database_path = _newest(_find_files(search_root, 'Cookies', logger))
  248. if cookie_database_path is None:
  249. raise FileNotFoundError(f'could not find {browser_name} cookies database in "{search_root}"')
  250. logger.debug(f'Extracting cookies from: "{cookie_database_path}"')
  251. decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name'], logger, keyring=keyring)
  252. with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
  253. cursor = None
  254. try:
  255. cursor = _open_database_copy(cookie_database_path, tmpdir)
  256. cursor.connection.text_factory = bytes
  257. column_names = _get_column_names(cursor, 'cookies')
  258. secure_column = 'is_secure' if 'is_secure' in column_names else 'secure'
  259. cursor.execute(f'SELECT host_key, name, value, encrypted_value, path, expires_utc, {secure_column} FROM cookies')
  260. jar = YoutubeDLCookieJar()
  261. failed_cookies = 0
  262. unencrypted_cookies = 0
  263. with _create_progress_bar(logger) as progress_bar:
  264. table = cursor.fetchall()
  265. total_cookie_count = len(table)
  266. for i, line in enumerate(table):
  267. progress_bar.print(f'Loading cookie {i: 6d}/{total_cookie_count: 6d}')
  268. is_encrypted, cookie = _process_chrome_cookie(decryptor, *line)
  269. if not cookie:
  270. failed_cookies += 1
  271. continue
  272. elif not is_encrypted:
  273. unencrypted_cookies += 1
  274. jar.set_cookie(cookie)
  275. if failed_cookies > 0:
  276. failed_message = f' ({failed_cookies} could not be decrypted)'
  277. else:
  278. failed_message = ''
  279. logger.info(f'Extracted {len(jar)} cookies from {browser_name}{failed_message}')
  280. counts = decryptor._cookie_counts.copy()
  281. counts['unencrypted'] = unencrypted_cookies
  282. logger.debug(f'cookie version breakdown: {counts}')
  283. return jar
  284. except PermissionError as error:
  285. if compat_os_name == 'nt' and error.errno == 13:
  286. message = 'Could not copy Chrome cookie database. See https://github.com/yt-dlp/yt-dlp/issues/7271 for more info'
  287. logger.error(message)
  288. raise DownloadError(message) # force exit
  289. raise
  290. finally:
  291. if cursor is not None:
  292. cursor.connection.close()
  293. def _process_chrome_cookie(decryptor, host_key, name, value, encrypted_value, path, expires_utc, is_secure):
  294. host_key = host_key.decode()
  295. name = name.decode()
  296. value = value.decode()
  297. path = path.decode()
  298. is_encrypted = not value and encrypted_value
  299. if is_encrypted:
  300. value = decryptor.decrypt(encrypted_value)
  301. if value is None:
  302. return is_encrypted, None
  303. return is_encrypted, http.cookiejar.Cookie(
  304. version=0, name=name, value=value, port=None, port_specified=False,
  305. domain=host_key, domain_specified=bool(host_key), domain_initial_dot=host_key.startswith('.'),
  306. path=path, path_specified=bool(path), secure=is_secure, expires=expires_utc, discard=False,
  307. comment=None, comment_url=None, rest={})
  308. class ChromeCookieDecryptor:
  309. """
  310. Overview:
  311. Linux:
  312. - cookies are either v10 or v11
  313. - v10: AES-CBC encrypted with a fixed key
  314. - also attempts empty password if decryption fails
  315. - v11: AES-CBC encrypted with an OS protected key (keyring)
  316. - also attempts empty password if decryption fails
  317. - v11 keys can be stored in various places depending on the activate desktop environment [2]
  318. Mac:
  319. - cookies are either v10 or not v10
  320. - v10: AES-CBC encrypted with an OS protected key (keyring) and more key derivation iterations than linux
  321. - not v10: 'old data' stored as plaintext
  322. Windows:
  323. - cookies are either v10 or not v10
  324. - v10: AES-GCM encrypted with a key which is encrypted with DPAPI
  325. - not v10: encrypted with DPAPI
  326. Sources:
  327. - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/
  328. - [2] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/key_storage_linux.cc
  329. - KeyStorageLinux::CreateService
  330. """
  331. _cookie_counts = {}
  332. def decrypt(self, encrypted_value):
  333. raise NotImplementedError('Must be implemented by sub classes')
  334. def get_cookie_decryptor(browser_root, browser_keyring_name, logger, *, keyring=None):
  335. if sys.platform == 'darwin':
  336. return MacChromeCookieDecryptor(browser_keyring_name, logger)
  337. elif sys.platform in ('win32', 'cygwin'):
  338. return WindowsChromeCookieDecryptor(browser_root, logger)
  339. return LinuxChromeCookieDecryptor(browser_keyring_name, logger, keyring=keyring)
  340. class LinuxChromeCookieDecryptor(ChromeCookieDecryptor):
  341. def __init__(self, browser_keyring_name, logger, *, keyring=None):
  342. self._logger = logger
  343. self._v10_key = self.derive_key(b'peanuts')
  344. self._empty_key = self.derive_key(b'')
  345. self._cookie_counts = {'v10': 0, 'v11': 0, 'other': 0}
  346. self._browser_keyring_name = browser_keyring_name
  347. self._keyring = keyring
  348. @functools.cached_property
  349. def _v11_key(self):
  350. password = _get_linux_keyring_password(self._browser_keyring_name, self._keyring, self._logger)
  351. return None if password is None else self.derive_key(password)
  352. @staticmethod
  353. def derive_key(password):
  354. # values from
  355. # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_linux.cc
  356. return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1, key_length=16)
  357. def decrypt(self, encrypted_value):
  358. """
  359. following the same approach as the fix in [1]: if cookies fail to decrypt then attempt to decrypt
  360. with an empty password. The failure detection is not the same as what chromium uses so the
  361. results won't be perfect
  362. References:
  363. - [1] https://chromium.googlesource.com/chromium/src/+/bbd54702284caca1f92d656fdcadf2ccca6f4165%5E%21/
  364. - a bugfix to try an empty password as a fallback
  365. """
  366. version = encrypted_value[:3]
  367. ciphertext = encrypted_value[3:]
  368. if version == b'v10':
  369. self._cookie_counts['v10'] += 1
  370. return _decrypt_aes_cbc_multi(ciphertext, (self._v10_key, self._empty_key), self._logger)
  371. elif version == b'v11':
  372. self._cookie_counts['v11'] += 1
  373. if self._v11_key is None:
  374. self._logger.warning('cannot decrypt v11 cookies: no key found', only_once=True)
  375. return None
  376. return _decrypt_aes_cbc_multi(ciphertext, (self._v11_key, self._empty_key), self._logger)
  377. else:
  378. self._logger.warning(f'unknown cookie version: "{version}"', only_once=True)
  379. self._cookie_counts['other'] += 1
  380. return None
  381. class MacChromeCookieDecryptor(ChromeCookieDecryptor):
  382. def __init__(self, browser_keyring_name, logger):
  383. self._logger = logger
  384. password = _get_mac_keyring_password(browser_keyring_name, logger)
  385. self._v10_key = None if password is None else self.derive_key(password)
  386. self._cookie_counts = {'v10': 0, 'other': 0}
  387. @staticmethod
  388. def derive_key(password):
  389. # values from
  390. # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_mac.mm
  391. return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1003, key_length=16)
  392. def decrypt(self, encrypted_value):
  393. version = encrypted_value[:3]
  394. ciphertext = encrypted_value[3:]
  395. if version == b'v10':
  396. self._cookie_counts['v10'] += 1
  397. if self._v10_key is None:
  398. self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
  399. return None
  400. return _decrypt_aes_cbc_multi(ciphertext, (self._v10_key,), self._logger)
  401. else:
  402. self._cookie_counts['other'] += 1
  403. # other prefixes are considered 'old data' which were stored as plaintext
  404. # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_mac.mm
  405. return encrypted_value
  406. class WindowsChromeCookieDecryptor(ChromeCookieDecryptor):
  407. def __init__(self, browser_root, logger):
  408. self._logger = logger
  409. self._v10_key = _get_windows_v10_key(browser_root, logger)
  410. self._cookie_counts = {'v10': 0, 'other': 0}
  411. def decrypt(self, encrypted_value):
  412. version = encrypted_value[:3]
  413. ciphertext = encrypted_value[3:]
  414. if version == b'v10':
  415. self._cookie_counts['v10'] += 1
  416. if self._v10_key is None:
  417. self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
  418. return None
  419. # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_win.cc
  420. # kNonceLength
  421. nonce_length = 96 // 8
  422. # boringssl
  423. # EVP_AEAD_AES_GCM_TAG_LEN
  424. authentication_tag_length = 16
  425. raw_ciphertext = ciphertext
  426. nonce = raw_ciphertext[:nonce_length]
  427. ciphertext = raw_ciphertext[nonce_length:-authentication_tag_length]
  428. authentication_tag = raw_ciphertext[-authentication_tag_length:]
  429. return _decrypt_aes_gcm(ciphertext, self._v10_key, nonce, authentication_tag, self._logger)
  430. else:
  431. self._cookie_counts['other'] += 1
  432. # any other prefix means the data is DPAPI encrypted
  433. # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_win.cc
  434. return _decrypt_windows_dpapi(encrypted_value, self._logger).decode()
  435. def _extract_safari_cookies(profile, logger):
  436. if sys.platform != 'darwin':
  437. raise ValueError(f'unsupported platform: {sys.platform}')
  438. if profile:
  439. cookies_path = os.path.expanduser(profile)
  440. if not os.path.isfile(cookies_path):
  441. raise FileNotFoundError('custom safari cookies database not found')
  442. else:
  443. cookies_path = os.path.expanduser('~/Library/Cookies/Cookies.binarycookies')
  444. if not os.path.isfile(cookies_path):
  445. logger.debug('Trying secondary cookie location')
  446. cookies_path = os.path.expanduser('~/Library/Containers/com.apple.Safari/Data/Library/Cookies/Cookies.binarycookies')
  447. if not os.path.isfile(cookies_path):
  448. raise FileNotFoundError('could not find safari cookies database')
  449. with open(cookies_path, 'rb') as f:
  450. cookies_data = f.read()
  451. jar = parse_safari_cookies(cookies_data, logger=logger)
  452. logger.info(f'Extracted {len(jar)} cookies from safari')
  453. return jar
  454. class ParserError(Exception):
  455. pass
  456. class DataParser:
  457. def __init__(self, data, logger):
  458. self._data = data
  459. self.cursor = 0
  460. self._logger = logger
  461. def read_bytes(self, num_bytes):
  462. if num_bytes < 0:
  463. raise ParserError(f'invalid read of {num_bytes} bytes')
  464. end = self.cursor + num_bytes
  465. if end > len(self._data):
  466. raise ParserError('reached end of input')
  467. data = self._data[self.cursor:end]
  468. self.cursor = end
  469. return data
  470. def expect_bytes(self, expected_value, message):
  471. value = self.read_bytes(len(expected_value))
  472. if value != expected_value:
  473. raise ParserError(f'unexpected value: {value} != {expected_value} ({message})')
  474. def read_uint(self, big_endian=False):
  475. data_format = '>I' if big_endian else '<I'
  476. return struct.unpack(data_format, self.read_bytes(4))[0]
  477. def read_double(self, big_endian=False):
  478. data_format = '>d' if big_endian else '<d'
  479. return struct.unpack(data_format, self.read_bytes(8))[0]
  480. def read_cstring(self):
  481. buffer = []
  482. while True:
  483. c = self.read_bytes(1)
  484. if c == b'\x00':
  485. return b''.join(buffer).decode()
  486. else:
  487. buffer.append(c)
  488. def skip(self, num_bytes, description='unknown'):
  489. if num_bytes > 0:
  490. self._logger.debug(f'skipping {num_bytes} bytes ({description}): {self.read_bytes(num_bytes)!r}')
  491. elif num_bytes < 0:
  492. raise ParserError(f'invalid skip of {num_bytes} bytes')
  493. def skip_to(self, offset, description='unknown'):
  494. self.skip(offset - self.cursor, description)
  495. def skip_to_end(self, description='unknown'):
  496. self.skip_to(len(self._data), description)
  497. def _mac_absolute_time_to_posix(timestamp):
  498. return int((dt.datetime(2001, 1, 1, 0, 0, tzinfo=dt.timezone.utc) + dt.timedelta(seconds=timestamp)).timestamp())
  499. def _parse_safari_cookies_header(data, logger):
  500. p = DataParser(data, logger)
  501. p.expect_bytes(b'cook', 'database signature')
  502. number_of_pages = p.read_uint(big_endian=True)
  503. page_sizes = [p.read_uint(big_endian=True) for _ in range(number_of_pages)]
  504. return page_sizes, p.cursor
  505. def _parse_safari_cookies_page(data, jar, logger):
  506. p = DataParser(data, logger)
  507. p.expect_bytes(b'\x00\x00\x01\x00', 'page signature')
  508. number_of_cookies = p.read_uint()
  509. record_offsets = [p.read_uint() for _ in range(number_of_cookies)]
  510. if number_of_cookies == 0:
  511. logger.debug(f'a cookies page of size {len(data)} has no cookies')
  512. return
  513. p.skip_to(record_offsets[0], 'unknown page header field')
  514. with _create_progress_bar(logger) as progress_bar:
  515. for i, record_offset in enumerate(record_offsets):
  516. progress_bar.print(f'Loading cookie {i: 6d}/{number_of_cookies: 6d}')
  517. p.skip_to(record_offset, 'space between records')
  518. record_length = _parse_safari_cookies_record(data[record_offset:], jar, logger)
  519. p.read_bytes(record_length)
  520. p.skip_to_end('space in between pages')
  521. def _parse_safari_cookies_record(data, jar, logger):
  522. p = DataParser(data, logger)
  523. record_size = p.read_uint()
  524. p.skip(4, 'unknown record field 1')
  525. flags = p.read_uint()
  526. is_secure = bool(flags & 0x0001)
  527. p.skip(4, 'unknown record field 2')
  528. domain_offset = p.read_uint()
  529. name_offset = p.read_uint()
  530. path_offset = p.read_uint()
  531. value_offset = p.read_uint()
  532. p.skip(8, 'unknown record field 3')
  533. expiration_date = _mac_absolute_time_to_posix(p.read_double())
  534. _creation_date = _mac_absolute_time_to_posix(p.read_double()) # noqa: F841
  535. try:
  536. p.skip_to(domain_offset)
  537. domain = p.read_cstring()
  538. p.skip_to(name_offset)
  539. name = p.read_cstring()
  540. p.skip_to(path_offset)
  541. path = p.read_cstring()
  542. p.skip_to(value_offset)
  543. value = p.read_cstring()
  544. except UnicodeDecodeError:
  545. logger.warning('failed to parse Safari cookie because UTF-8 decoding failed', only_once=True)
  546. return record_size
  547. p.skip_to(record_size, 'space at the end of the record')
  548. cookie = http.cookiejar.Cookie(
  549. version=0, name=name, value=value, port=None, port_specified=False,
  550. domain=domain, domain_specified=bool(domain), domain_initial_dot=domain.startswith('.'),
  551. path=path, path_specified=bool(path), secure=is_secure, expires=expiration_date, discard=False,
  552. comment=None, comment_url=None, rest={})
  553. jar.set_cookie(cookie)
  554. return record_size
  555. def parse_safari_cookies(data, jar=None, logger=YDLLogger()):
  556. """
  557. References:
  558. - https://github.com/libyal/dtformats/blob/main/documentation/Safari%20Cookies.asciidoc
  559. - this data appears to be out of date but the important parts of the database structure is the same
  560. - there are a few bytes here and there which are skipped during parsing
  561. """
  562. if jar is None:
  563. jar = YoutubeDLCookieJar()
  564. page_sizes, body_start = _parse_safari_cookies_header(data, logger)
  565. p = DataParser(data[body_start:], logger)
  566. for page_size in page_sizes:
  567. _parse_safari_cookies_page(p.read_bytes(page_size), jar, logger)
  568. p.skip_to_end('footer')
  569. return jar
  570. class _LinuxDesktopEnvironment(Enum):
  571. """
  572. https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util.h
  573. DesktopEnvironment
  574. """
  575. OTHER = auto()
  576. CINNAMON = auto()
  577. DEEPIN = auto()
  578. GNOME = auto()
  579. KDE3 = auto()
  580. KDE4 = auto()
  581. KDE5 = auto()
  582. KDE6 = auto()
  583. PANTHEON = auto()
  584. UKUI = auto()
  585. UNITY = auto()
  586. XFCE = auto()
  587. LXQT = auto()
  588. class _LinuxKeyring(Enum):
  589. """
  590. https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/key_storage_util_linux.h
  591. SelectedLinuxBackend
  592. """
  593. KWALLET = auto() # KDE4
  594. KWALLET5 = auto()
  595. KWALLET6 = auto()
  596. GNOMEKEYRING = auto()
  597. BASICTEXT = auto()
  598. SUPPORTED_KEYRINGS = _LinuxKeyring.__members__.keys()
  599. def _get_linux_desktop_environment(env, logger):
  600. """
  601. https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util.cc
  602. GetDesktopEnvironment
  603. """
  604. xdg_current_desktop = env.get('XDG_CURRENT_DESKTOP', None)
  605. desktop_session = env.get('DESKTOP_SESSION', None)
  606. if xdg_current_desktop is not None:
  607. xdg_current_desktop = xdg_current_desktop.split(':')[0].strip()
  608. if xdg_current_desktop == 'Unity':
  609. if desktop_session is not None and 'gnome-fallback' in desktop_session:
  610. return _LinuxDesktopEnvironment.GNOME
  611. else:
  612. return _LinuxDesktopEnvironment.UNITY
  613. elif xdg_current_desktop == 'Deepin':
  614. return _LinuxDesktopEnvironment.DEEPIN
  615. elif xdg_current_desktop == 'GNOME':
  616. return _LinuxDesktopEnvironment.GNOME
  617. elif xdg_current_desktop == 'X-Cinnamon':
  618. return _LinuxDesktopEnvironment.CINNAMON
  619. elif xdg_current_desktop == 'KDE':
  620. kde_version = env.get('KDE_SESSION_VERSION', None)
  621. if kde_version == '5':
  622. return _LinuxDesktopEnvironment.KDE5
  623. elif kde_version == '6':
  624. return _LinuxDesktopEnvironment.KDE6
  625. elif kde_version == '4':
  626. return _LinuxDesktopEnvironment.KDE4
  627. else:
  628. logger.info(f'unknown KDE version: "{kde_version}". Assuming KDE4')
  629. return _LinuxDesktopEnvironment.KDE4
  630. elif xdg_current_desktop == 'Pantheon':
  631. return _LinuxDesktopEnvironment.PANTHEON
  632. elif xdg_current_desktop == 'XFCE':
  633. return _LinuxDesktopEnvironment.XFCE
  634. elif xdg_current_desktop == 'UKUI':
  635. return _LinuxDesktopEnvironment.UKUI
  636. elif xdg_current_desktop == 'LXQt':
  637. return _LinuxDesktopEnvironment.LXQT
  638. else:
  639. logger.info(f'XDG_CURRENT_DESKTOP is set to an unknown value: "{xdg_current_desktop}"')
  640. elif desktop_session is not None:
  641. if desktop_session == 'deepin':
  642. return _LinuxDesktopEnvironment.DEEPIN
  643. elif desktop_session in ('mate', 'gnome'):
  644. return _LinuxDesktopEnvironment.GNOME
  645. elif desktop_session in ('kde4', 'kde-plasma'):
  646. return _LinuxDesktopEnvironment.KDE4
  647. elif desktop_session == 'kde':
  648. if 'KDE_SESSION_VERSION' in env:
  649. return _LinuxDesktopEnvironment.KDE4
  650. else:
  651. return _LinuxDesktopEnvironment.KDE3
  652. elif 'xfce' in desktop_session or desktop_session == 'xubuntu':
  653. return _LinuxDesktopEnvironment.XFCE
  654. elif desktop_session == 'ukui':
  655. return _LinuxDesktopEnvironment.UKUI
  656. else:
  657. logger.info(f'DESKTOP_SESSION is set to an unknown value: "{desktop_session}"')
  658. else:
  659. if 'GNOME_DESKTOP_SESSION_ID' in env:
  660. return _LinuxDesktopEnvironment.GNOME
  661. elif 'KDE_FULL_SESSION' in env:
  662. if 'KDE_SESSION_VERSION' in env:
  663. return _LinuxDesktopEnvironment.KDE4
  664. else:
  665. return _LinuxDesktopEnvironment.KDE3
  666. return _LinuxDesktopEnvironment.OTHER
  667. def _choose_linux_keyring(logger):
  668. """
  669. SelectBackend in [1]
  670. There is currently support for forcing chromium to use BASIC_TEXT by creating a file called
  671. `Disable Local Encryption` [1] in the user data dir. The function to write this file (`WriteBackendUse()` [1])
  672. does not appear to be called anywhere other than in tests, so the user would have to create this file manually
  673. and so would be aware enough to tell yt-dlp to use the BASIC_TEXT keyring.
  674. References:
  675. - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/key_storage_util_linux.cc
  676. """
  677. desktop_environment = _get_linux_desktop_environment(os.environ, logger)
  678. logger.debug(f'detected desktop environment: {desktop_environment.name}')
  679. if desktop_environment == _LinuxDesktopEnvironment.KDE4:
  680. linux_keyring = _LinuxKeyring.KWALLET
  681. elif desktop_environment == _LinuxDesktopEnvironment.KDE5:
  682. linux_keyring = _LinuxKeyring.KWALLET5
  683. elif desktop_environment == _LinuxDesktopEnvironment.KDE6:
  684. linux_keyring = _LinuxKeyring.KWALLET6
  685. elif desktop_environment in (
  686. _LinuxDesktopEnvironment.KDE3, _LinuxDesktopEnvironment.LXQT, _LinuxDesktopEnvironment.OTHER
  687. ):
  688. linux_keyring = _LinuxKeyring.BASICTEXT
  689. else:
  690. linux_keyring = _LinuxKeyring.GNOMEKEYRING
  691. return linux_keyring
  692. def _get_kwallet_network_wallet(keyring, logger):
  693. """ The name of the wallet used to store network passwords.
  694. https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/kwallet_dbus.cc
  695. KWalletDBus::NetworkWallet
  696. which does a dbus call to the following function:
  697. https://api.kde.org/frameworks/kwallet/html/classKWallet_1_1Wallet.html
  698. Wallet::NetworkWallet
  699. """
  700. default_wallet = 'kdewallet'
  701. try:
  702. if keyring == _LinuxKeyring.KWALLET:
  703. service_name = 'org.kde.kwalletd'
  704. wallet_path = '/modules/kwalletd'
  705. elif keyring == _LinuxKeyring.KWALLET5:
  706. service_name = 'org.kde.kwalletd5'
  707. wallet_path = '/modules/kwalletd5'
  708. elif keyring == _LinuxKeyring.KWALLET6:
  709. service_name = 'org.kde.kwalletd6'
  710. wallet_path = '/modules/kwalletd6'
  711. else:
  712. raise ValueError(keyring)
  713. stdout, _, returncode = Popen.run([
  714. 'dbus-send', '--session', '--print-reply=literal',
  715. f'--dest={service_name}',
  716. wallet_path,
  717. 'org.kde.KWallet.networkWallet'
  718. ], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
  719. if returncode:
  720. logger.warning('failed to read NetworkWallet')
  721. return default_wallet
  722. else:
  723. logger.debug(f'NetworkWallet = "{stdout.strip()}"')
  724. return stdout.strip()
  725. except Exception as e:
  726. logger.warning(f'exception while obtaining NetworkWallet: {e}')
  727. return default_wallet
  728. def _get_kwallet_password(browser_keyring_name, keyring, logger):
  729. logger.debug(f'using kwallet-query to obtain password from {keyring.name}')
  730. if shutil.which('kwallet-query') is None:
  731. logger.error('kwallet-query command not found. KWallet and kwallet-query '
  732. 'must be installed to read from KWallet. kwallet-query should be'
  733. 'included in the kwallet package for your distribution')
  734. return b''
  735. network_wallet = _get_kwallet_network_wallet(keyring, logger)
  736. try:
  737. stdout, _, returncode = Popen.run([
  738. 'kwallet-query',
  739. '--read-password', f'{browser_keyring_name} Safe Storage',
  740. '--folder', f'{browser_keyring_name} Keys',
  741. network_wallet
  742. ], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
  743. if returncode:
  744. logger.error(f'kwallet-query failed with return code {returncode}. '
  745. 'Please consult the kwallet-query man page for details')
  746. return b''
  747. else:
  748. if stdout.lower().startswith(b'failed to read'):
  749. logger.debug('failed to read password from kwallet. Using empty string instead')
  750. # this sometimes occurs in KDE because chrome does not check hasEntry and instead
  751. # just tries to read the value (which kwallet returns "") whereas kwallet-query
  752. # checks hasEntry. To verify this:
  753. # dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
  754. # while starting chrome.
  755. # this was identified as a bug later and fixed in
  756. # https://chromium.googlesource.com/chromium/src/+/bbd54702284caca1f92d656fdcadf2ccca6f4165%5E%21/#F0
  757. # https://chromium.googlesource.com/chromium/src/+/5463af3c39d7f5b6d11db7fbd51e38cc1974d764
  758. return b''
  759. else:
  760. logger.debug('password found')
  761. return stdout.rstrip(b'\n')
  762. except Exception as e:
  763. logger.warning(f'exception running kwallet-query: {error_to_str(e)}')
  764. return b''
  765. def _get_gnome_keyring_password(browser_keyring_name, logger):
  766. if not secretstorage:
  767. logger.error(f'secretstorage not available {_SECRETSTORAGE_UNAVAILABLE_REASON}')
  768. return b''
  769. # the Gnome keyring does not seem to organise keys in the same way as KWallet,
  770. # using `dbus-monitor` during startup, it can be observed that chromium lists all keys
  771. # and presumably searches for its key in the list. It appears that we must do the same.
  772. # https://github.com/jaraco/keyring/issues/556
  773. with contextlib.closing(secretstorage.dbus_init()) as con:
  774. col = secretstorage.get_default_collection(con)
  775. for item in col.get_all_items():
  776. if item.get_label() == f'{browser_keyring_name} Safe Storage':
  777. return item.get_secret()
  778. else:
  779. logger.error('failed to read from keyring')
  780. return b''
  781. def _get_linux_keyring_password(browser_keyring_name, keyring, logger):
  782. # note: chrome/chromium can be run with the following flags to determine which keyring backend
  783. # it has chosen to use
  784. # chromium --enable-logging=stderr --v=1 2>&1 | grep key_storage_
  785. # Chromium supports a flag: --password-store=<basic|gnome|kwallet> so the automatic detection
  786. # will not be sufficient in all cases.
  787. keyring = _LinuxKeyring[keyring] if keyring else _choose_linux_keyring(logger)
  788. logger.debug(f'Chosen keyring: {keyring.name}')
  789. if keyring in (_LinuxKeyring.KWALLET, _LinuxKeyring.KWALLET5, _LinuxKeyring.KWALLET6):
  790. return _get_kwallet_password(browser_keyring_name, keyring, logger)
  791. elif keyring == _LinuxKeyring.GNOMEKEYRING:
  792. return _get_gnome_keyring_password(browser_keyring_name, logger)
  793. elif keyring == _LinuxKeyring.BASICTEXT:
  794. # when basic text is chosen, all cookies are stored as v10 (so no keyring password is required)
  795. return None
  796. assert False, f'Unknown keyring {keyring}'
  797. def _get_mac_keyring_password(browser_keyring_name, logger):
  798. logger.debug('using find-generic-password to obtain password from OSX keychain')
  799. try:
  800. stdout, _, returncode = Popen.run(
  801. ['security', 'find-generic-password',
  802. '-w', # write password to stdout
  803. '-a', browser_keyring_name, # match 'account'
  804. '-s', f'{browser_keyring_name} Safe Storage'], # match 'service'
  805. stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
  806. if returncode:
  807. logger.warning('find-generic-password failed')
  808. return None
  809. return stdout.rstrip(b'\n')
  810. except Exception as e:
  811. logger.warning(f'exception running find-generic-password: {error_to_str(e)}')
  812. return None
  813. def _get_windows_v10_key(browser_root, logger):
  814. """
  815. References:
  816. - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_win.cc
  817. """
  818. path = _newest(_find_files(browser_root, 'Local State', logger))
  819. if path is None:
  820. logger.error('could not find local state file')
  821. return None
  822. logger.debug(f'Found local state file at "{path}"')
  823. with open(path, encoding='utf8') as f:
  824. data = json.load(f)
  825. try:
  826. # kOsCryptEncryptedKeyPrefName in [1]
  827. base64_key = data['os_crypt']['encrypted_key']
  828. except KeyError:
  829. logger.error('no encrypted key in Local State')
  830. return None
  831. encrypted_key = base64.b64decode(base64_key)
  832. # kDPAPIKeyPrefix in [1]
  833. prefix = b'DPAPI'
  834. if not encrypted_key.startswith(prefix):
  835. logger.error('invalid key')
  836. return None
  837. return _decrypt_windows_dpapi(encrypted_key[len(prefix):], logger)
  838. def pbkdf2_sha1(password, salt, iterations, key_length):
  839. return pbkdf2_hmac('sha1', password, salt, iterations, key_length)
  840. def _decrypt_aes_cbc_multi(ciphertext, keys, logger, initialization_vector=b' ' * 16):
  841. for key in keys:
  842. plaintext = unpad_pkcs7(aes_cbc_decrypt_bytes(ciphertext, key, initialization_vector))
  843. try:
  844. return plaintext.decode()
  845. except UnicodeDecodeError:
  846. pass
  847. logger.warning('failed to decrypt cookie (AES-CBC) because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
  848. return None
  849. def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, logger):
  850. try:
  851. plaintext = aes_gcm_decrypt_and_verify_bytes(ciphertext, key, authentication_tag, nonce)
  852. except ValueError:
  853. logger.warning('failed to decrypt cookie (AES-GCM) because the MAC check failed. Possibly the key is wrong?', only_once=True)
  854. return None
  855. try:
  856. return plaintext.decode()
  857. except UnicodeDecodeError:
  858. logger.warning('failed to decrypt cookie (AES-GCM) because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
  859. return None
  860. def _decrypt_windows_dpapi(ciphertext, logger):
  861. """
  862. References:
  863. - https://docs.microsoft.com/en-us/windows/win32/api/dpapi/nf-dpapi-cryptunprotectdata
  864. """
  865. import ctypes
  866. import ctypes.wintypes
  867. class DATA_BLOB(ctypes.Structure):
  868. _fields_ = [('cbData', ctypes.wintypes.DWORD),
  869. ('pbData', ctypes.POINTER(ctypes.c_char))]
  870. buffer = ctypes.create_string_buffer(ciphertext)
  871. blob_in = DATA_BLOB(ctypes.sizeof(buffer), buffer)
  872. blob_out = DATA_BLOB()
  873. ret = ctypes.windll.crypt32.CryptUnprotectData(
  874. ctypes.byref(blob_in), # pDataIn
  875. None, # ppszDataDescr: human readable description of pDataIn
  876. None, # pOptionalEntropy: salt?
  877. None, # pvReserved: must be NULL
  878. None, # pPromptStruct: information about prompts to display
  879. 0, # dwFlags
  880. ctypes.byref(blob_out) # pDataOut
  881. )
  882. if not ret:
  883. logger.warning('failed to decrypt with DPAPI', only_once=True)
  884. return None
  885. result = ctypes.string_at(blob_out.pbData, blob_out.cbData)
  886. ctypes.windll.kernel32.LocalFree(blob_out.pbData)
  887. return result
  888. def _config_home():
  889. return os.environ.get('XDG_CONFIG_HOME', os.path.expanduser('~/.config'))
  890. def _open_database_copy(database_path, tmpdir):
  891. # cannot open sqlite databases if they are already in use (e.g. by the browser)
  892. database_copy_path = os.path.join(tmpdir, 'temporary.sqlite')
  893. shutil.copy(database_path, database_copy_path)
  894. conn = sqlite3.connect(database_copy_path)
  895. return conn.cursor()
  896. def _get_column_names(cursor, table_name):
  897. table_info = cursor.execute(f'PRAGMA table_info({table_name})').fetchall()
  898. return [row[1].decode() for row in table_info]
  899. def _newest(files):
  900. return max(files, key=lambda path: os.lstat(path).st_mtime, default=None)
  901. def _find_files(root, filename, logger):
  902. # if there are multiple browser profiles, take the most recently used one
  903. i = 0
  904. with _create_progress_bar(logger) as progress_bar:
  905. for curr_root, _, files in os.walk(root):
  906. for file in files:
  907. i += 1
  908. progress_bar.print(f'Searching for "{filename}": {i: 6d} files searched')
  909. if file == filename:
  910. yield os.path.join(curr_root, file)
  911. def _merge_cookie_jars(jars):
  912. output_jar = YoutubeDLCookieJar()
  913. for jar in jars:
  914. for cookie in jar:
  915. output_jar.set_cookie(cookie)
  916. if jar.filename is not None:
  917. output_jar.filename = jar.filename
  918. return output_jar
  919. def _is_path(value):
  920. return any(sep in value for sep in (os.path.sep, os.path.altsep) if sep)
  921. def _parse_browser_specification(browser_name, profile=None, keyring=None, container=None):
  922. if browser_name not in SUPPORTED_BROWSERS:
  923. raise ValueError(f'unsupported browser: "{browser_name}"')
  924. if keyring not in (None, *SUPPORTED_KEYRINGS):
  925. raise ValueError(f'unsupported keyring: "{keyring}"')
  926. if profile is not None and _is_path(expand_path(profile)):
  927. profile = expand_path(profile)
  928. return browser_name, profile, keyring, container
  929. class LenientSimpleCookie(http.cookies.SimpleCookie):
  930. """More lenient version of http.cookies.SimpleCookie"""
  931. # From https://github.com/python/cpython/blob/v3.10.7/Lib/http/cookies.py
  932. # We use Morsel's legal key chars to avoid errors on setting values
  933. _LEGAL_KEY_CHARS = r'\w\d' + re.escape('!#$%&\'*+-.:^_`|~')
  934. _LEGAL_VALUE_CHARS = _LEGAL_KEY_CHARS + re.escape('(),/<=>?@[]{}')
  935. _RESERVED = {
  936. "expires",
  937. "path",
  938. "comment",
  939. "domain",
  940. "max-age",
  941. "secure",
  942. "httponly",
  943. "version",
  944. "samesite",
  945. }
  946. _FLAGS = {"secure", "httponly"}
  947. # Added 'bad' group to catch the remaining value
  948. _COOKIE_PATTERN = re.compile(r"""
  949. \s* # Optional whitespace at start of cookie
  950. (?P<key> # Start of group 'key'
  951. [""" + _LEGAL_KEY_CHARS + r"""]+?# Any word of at least one letter
  952. ) # End of group 'key'
  953. ( # Optional group: there may not be a value.
  954. \s*=\s* # Equal Sign
  955. ( # Start of potential value
  956. (?P<val> # Start of group 'val'
  957. "(?:[^\\"]|\\.)*" # Any doublequoted string
  958. | # or
  959. \w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr
  960. | # or
  961. [""" + _LEGAL_VALUE_CHARS + r"""]* # Any word or empty string
  962. ) # End of group 'val'
  963. | # or
  964. (?P<bad>(?:\\;|[^;])*?) # 'bad' group fallback for invalid values
  965. ) # End of potential value
  966. )? # End of optional value group
  967. \s* # Any number of spaces.
  968. (\s+|;|$) # Ending either at space, semicolon, or EOS.
  969. """, re.ASCII | re.VERBOSE)
  970. def load(self, data):
  971. # Workaround for https://github.com/yt-dlp/yt-dlp/issues/4776
  972. if not isinstance(data, str):
  973. return super().load(data)
  974. morsel = None
  975. for match in self._COOKIE_PATTERN.finditer(data):
  976. if match.group('bad'):
  977. morsel = None
  978. continue
  979. key, value = match.group('key', 'val')
  980. is_attribute = False
  981. if key.startswith('$'):
  982. key = key[1:]
  983. is_attribute = True
  984. lower_key = key.lower()
  985. if lower_key in self._RESERVED:
  986. if morsel is None:
  987. continue
  988. if value is None:
  989. if lower_key not in self._FLAGS:
  990. morsel = None
  991. continue
  992. value = True
  993. else:
  994. value, _ = self.value_decode(value)
  995. morsel[key] = value
  996. elif is_attribute:
  997. morsel = None
  998. elif value is not None:
  999. morsel = self.get(key, http.cookies.Morsel())
  1000. real_value, coded_value = self.value_decode(value)
  1001. morsel.set(key, real_value, coded_value)
  1002. self[key] = morsel
  1003. else:
  1004. morsel = None
  1005. class YoutubeDLCookieJar(http.cookiejar.MozillaCookieJar):
  1006. """
  1007. See [1] for cookie file format.
  1008. 1. https://curl.haxx.se/docs/http-cookies.html
  1009. """
  1010. _HTTPONLY_PREFIX = '#HttpOnly_'
  1011. _ENTRY_LEN = 7
  1012. _HEADER = '''# Netscape HTTP Cookie File
  1013. # This file is generated by yt-dlp. Do not edit.
  1014. '''
  1015. _CookieFileEntry = collections.namedtuple(
  1016. 'CookieFileEntry',
  1017. ('domain_name', 'include_subdomains', 'path', 'https_only', 'expires_at', 'name', 'value'))
  1018. def __init__(self, filename=None, *args, **kwargs):
  1019. super().__init__(None, *args, **kwargs)
  1020. if is_path_like(filename):
  1021. filename = os.fspath(filename)
  1022. self.filename = filename
  1023. @staticmethod
  1024. def _true_or_false(cndn):
  1025. return 'TRUE' if cndn else 'FALSE'
  1026. @contextlib.contextmanager
  1027. def open(self, file, *, write=False):
  1028. if is_path_like(file):
  1029. with open(file, 'w' if write else 'r', encoding='utf-8') as f:
  1030. yield f
  1031. else:
  1032. if write:
  1033. file.truncate(0)
  1034. yield file
  1035. def _really_save(self, f, ignore_discard, ignore_expires):
  1036. now = time.time()
  1037. for cookie in self:
  1038. if (not ignore_discard and cookie.discard
  1039. or not ignore_expires and cookie.is_expired(now)):
  1040. continue
  1041. name, value = cookie.name, cookie.value
  1042. if value is None:
  1043. # cookies.txt regards 'Set-Cookie: foo' as a cookie
  1044. # with no name, whereas http.cookiejar regards it as a
  1045. # cookie with no value.
  1046. name, value = '', name
  1047. f.write('%s\n' % '\t'.join((
  1048. cookie.domain,
  1049. self._true_or_false(cookie.domain.startswith('.')),
  1050. cookie.path,
  1051. self._true_or_false(cookie.secure),
  1052. str_or_none(cookie.expires, default=''),
  1053. name, value
  1054. )))
  1055. def save(self, filename=None, ignore_discard=True, ignore_expires=True):
  1056. """
  1057. Save cookies to a file.
  1058. Code is taken from CPython 3.6
  1059. https://github.com/python/cpython/blob/8d999cbf4adea053be6dbb612b9844635c4dfb8e/Lib/http/cookiejar.py#L2091-L2117 """
  1060. if filename is None:
  1061. if self.filename is not None:
  1062. filename = self.filename
  1063. else:
  1064. raise ValueError(http.cookiejar.MISSING_FILENAME_TEXT)
  1065. # Store session cookies with `expires` set to 0 instead of an empty string
  1066. for cookie in self:
  1067. if cookie.expires is None:
  1068. cookie.expires = 0
  1069. with self.open(filename, write=True) as f:
  1070. f.write(self._HEADER)
  1071. self._really_save(f, ignore_discard, ignore_expires)
  1072. def load(self, filename=None, ignore_discard=True, ignore_expires=True):
  1073. """Load cookies from a file."""
  1074. if filename is None:
  1075. if self.filename is not None:
  1076. filename = self.filename
  1077. else:
  1078. raise ValueError(http.cookiejar.MISSING_FILENAME_TEXT)
  1079. def prepare_line(line):
  1080. if line.startswith(self._HTTPONLY_PREFIX):
  1081. line = line[len(self._HTTPONLY_PREFIX):]
  1082. # comments and empty lines are fine
  1083. if line.startswith('#') or not line.strip():
  1084. return line
  1085. cookie_list = line.split('\t')
  1086. if len(cookie_list) != self._ENTRY_LEN:
  1087. raise http.cookiejar.LoadError('invalid length %d' % len(cookie_list))
  1088. cookie = self._CookieFileEntry(*cookie_list)
  1089. if cookie.expires_at and not cookie.expires_at.isdigit():
  1090. raise http.cookiejar.LoadError('invalid expires at %s' % cookie.expires_at)
  1091. return line
  1092. cf = io.StringIO()
  1093. with self.open(filename) as f:
  1094. for line in f:
  1095. try:
  1096. cf.write(prepare_line(line))
  1097. except http.cookiejar.LoadError as e:
  1098. if f'{line.strip()} '[0] in '[{"':
  1099. raise http.cookiejar.LoadError(
  1100. 'Cookies file must be Netscape formatted, not JSON. See '
  1101. 'https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp')
  1102. write_string(f'WARNING: skipping cookie file entry due to {e}: {line!r}\n')
  1103. continue
  1104. cf.seek(0)
  1105. self._really_load(cf, filename, ignore_discard, ignore_expires)
  1106. # Session cookies are denoted by either `expires` field set to
  1107. # an empty string or 0. MozillaCookieJar only recognizes the former
  1108. # (see [1]). So we need force the latter to be recognized as session
  1109. # cookies on our own.
  1110. # Session cookies may be important for cookies-based authentication,
  1111. # e.g. usually, when user does not check 'Remember me' check box while
  1112. # logging in on a site, some important cookies are stored as session
  1113. # cookies so that not recognizing them will result in failed login.
  1114. # 1. https://bugs.python.org/issue17164
  1115. for cookie in self:
  1116. # Treat `expires=0` cookies as session cookies
  1117. if cookie.expires == 0:
  1118. cookie.expires = None
  1119. cookie.discard = True
  1120. def get_cookie_header(self, url):
  1121. """Generate a Cookie HTTP header for a given url"""
  1122. cookie_req = urllib.request.Request(normalize_url(sanitize_url(url)))
  1123. self.add_cookie_header(cookie_req)
  1124. return cookie_req.get_header('Cookie')
  1125. def get_cookies_for_url(self, url):
  1126. """Generate a list of Cookie objects for a given url"""
  1127. # Policy `_now` attribute must be set before calling `_cookies_for_request`
  1128. # Ref: https://github.com/python/cpython/blob/3.7/Lib/http/cookiejar.py#L1360
  1129. self._policy._now = self._now = int(time.time())
  1130. return self._cookies_for_request(urllib.request.Request(normalize_url(sanitize_url(url))))
  1131. def clear(self, *args, **kwargs):
  1132. with contextlib.suppress(KeyError):
  1133. return super().clear(*args, **kwargs)