Coverage for /var/srv/projects/api.amasfac.comuna18.com/tmp/venv/lib/python3.9/site-packages/django/utils/autoreload.py: 17%

1import functools

2import itertools

3import logging

4import os

5import signal

6import subprocess

7import sys

8import threading

9import time

10import traceback

11import weakref

12from collections import defaultdict

13from pathlib import Path

14from types import ModuleType

15from zipimport import zipimporter

17import django

18from django.apps import apps

19from django.core.signals import request_finished

20from django.dispatch import Signal

21from django.utils.functional import cached_property

22from django.utils.version import get_version_tuple

24autoreload_started = Signal()

25file_changed = Signal()

27DJANGO_AUTORELOAD_ENV = "RUN_MAIN"

29logger = logging.getLogger("django.utils.autoreload")

31# If an error is raised while importing a file, it's not placed in sys.modules.

32# This means that any future modifications aren't caught. Keep a list of these

33# file paths to allow watching them in the future.

34_error_files = []

35_exception = None

37try:

38 import termios

39except ImportError:

40 termios = None

43try:

44 import pywatchman

45except ImportError:

46 pywatchman = None

49def is_django_module(module):

50 """Return True if the given module is nested under Django."""

51 return module.__name__.startswith("django.")

54def is_django_path(path):

55 """Return True if the given file path is nested under Django."""

56 return Path(django.__file__).parent in Path(path).parents

59def check_errors(fn):

60 @functools.wraps(fn)

61 def wrapper(*args, **kwargs):

62 global _exception

63 try:

64 fn(*args, **kwargs)

65 except Exception:

66 _exception = sys.exc_info()

68 et, ev, tb = _exception

70 if getattr(ev, "filename", None) is None:

71 # get the filename from the last item in the stack

72 filename = traceback.extract_tb(tb)[-1][0]

73 else:

74 filename = ev.filename

76 if filename not in _error_files:

77 _error_files.append(filename)

79 raise

81 return wrapper

84def raise_last_exception():

85 global _exception

86 if _exception is not None:

87 raise _exception[1]

90def ensure_echo_on():

91 """

92 Ensure that echo mode is enabled. Some tools such as PDB disable

93 it which causes usability issues after reload.

94 """

95 if not termios or not sys.stdin.isatty():

96 return

97 attr_list = termios.tcgetattr(sys.stdin)

98 if not attr_list[3] & termios.ECHO:

99 attr_list[3] |= termios.ECHO

100 if hasattr(signal, "SIGTTOU"):

101 old_handler = signal.signal(signal.SIGTTOU, signal.SIG_IGN)

102 else:

103 old_handler = None

104 termios.tcsetattr(sys.stdin, termios.TCSANOW, attr_list)

105 if old_handler is not None:

106 signal.signal(signal.SIGTTOU, old_handler)

107

108

109def iter_all_python_module_files():

110 # This is a hot path during reloading. Create a stable sorted list of

111 # modules based on the module name and pass it to iter_modules_and_files().

112 # This ensures cached results are returned in the usual case that modules

113 # aren't loaded on the fly.

114 keys = sorted(sys.modules)

115 modules = tuple(

116 m

117 for m in map(sys.modules.__getitem__, keys)

118 if not isinstance(m, weakref.ProxyTypes)

119 )

120 return iter_modules_and_files(modules, frozenset(_error_files))

121

122

123@functools.lru_cache(maxsize=1)

124def iter_modules_and_files(modules, extra_files):

125 """Iterate through all modules needed to be watched."""

126 sys_file_paths = []

127 for module in modules:

128 # During debugging (with PyDev) the 'typing.io' and 'typing.re' objects

129 # are added to sys.modules, however they are types not modules and so

130 # cause issues here.

131 if not isinstance(module, ModuleType):

132 continue

133 if module.__name__ == "__main__":

134 # __main__ (usually manage.py) doesn't always have a __spec__ set.

135 # Handle this by falling back to using __file__, resolved below.

136 # See https://docs.python.org/reference/import.html#main-spec

137 # __file__ may not exists, e.g. when running ipdb debugger.

138 if hasattr(module, "__file__"):

139 sys_file_paths.append(module.__file__)

140 continue

141 if getattr(module, "__spec__", None) is None:

142 continue

143 spec = module.__spec__

144 # Modules could be loaded from places without a concrete location. If

145 # this is the case, skip them.

146 if spec.has_location:

147 origin = (

148 spec.loader.archive

149 if isinstance(spec.loader, zipimporter)

150 else spec.origin

151 )

152 sys_file_paths.append(origin)

153

154 results = set()

155 for filename in itertools.chain(sys_file_paths, extra_files):

156 if not filename:

157 continue

158 path = Path(filename)

159 try:

160 if not path.exists():

161 # The module could have been removed, don't fail loudly if this

162 # is the case.

163 continue

164 except ValueError as e:

165 # Network filesystems may return null bytes in file paths.

166 logger.debug('"%s" raised when resolving path: "%s"', e, path)

167 continue

168 resolved_path = path.resolve().absolute()

169 results.add(resolved_path)

170 return frozenset(results)

171

172

173@functools.lru_cache(maxsize=1)

174def common_roots(paths):

175 """

176 Return a tuple of common roots that are shared between the given paths.

177 File system watchers operate on directories and aren't cheap to create.

178 Try to find the minimum set of directories to watch that encompass all of

179 the files that need to be watched.

180 """

181 # Inspired from Werkzeug:

182 # https://github.com/pallets/werkzeug/blob/7477be2853df70a022d9613e765581b9411c3c39/werkzeug/_reloader.py

183 # Create a sorted list of the path components, longest first.

184 path_parts = sorted([x.parts for x in paths], key=len, reverse=True)

185 tree = {}

186 for chunks in path_parts:

187 node = tree

188 # Add each part of the path to the tree.

189 for chunk in chunks:

190 node = node.setdefault(chunk, {})

191 # Clear the last leaf in the tree.

192 node.clear()

193

194 # Turn the tree into a list of Path instances.

195 def _walk(node, path):

196 for prefix, child in node.items():

197 yield from _walk(child, path + (prefix,))

198 if not node:

199 yield Path(*path)

200

201 return tuple(_walk(tree, ()))

202

203

204def sys_path_directories():

205 """

206 Yield absolute directories from sys.path, ignoring entries that don't

207 exist.

208 """

209 for path in sys.path:

210 path = Path(path)

211 if not path.exists():

212 continue

213 resolved_path = path.resolve().absolute()

214 # If the path is a file (like a zip file), watch the parent directory.

215 if resolved_path.is_file():

216 yield resolved_path.parent

217 else:

218 yield resolved_path

219

220

221def get_child_arguments():

222 """

223 Return the executable. This contains a workaround for Windows if the

224 executable is reported to not have the .exe extension which can cause bugs

225 on reloading.

226 """

227 import __main__

228

229 py_script = Path(sys.argv[0])

230

231 args = [sys.executable] + ["-W%s" % o for o in sys.warnoptions]

232 if sys.implementation.name == "cpython":

233 args.extend(

234 f"-X{key}" if value is True else f"-X{key}={value}"

235 for key, value in sys._xoptions.items()

236 )

237 # __spec__ is set when the server was started with the `-m` option,

238 # see https://docs.python.org/3/reference/import.html#main-spec

239 # __spec__ may not exist, e.g. when running in a Conda env.

240 if getattr(__main__, "__spec__", None) is not None:

241 spec = __main__.__spec__

242 if (spec.name == "__main__" or spec.name.endswith(".__main__")) and spec.parent:

243 name = spec.parent

244 else:

245 name = spec.name

246 args += ["-m", name]

247 args += sys.argv[1:]

248 elif not py_script.exists():

249 # sys.argv[0] may not exist for several reasons on Windows.

250 # It may exist with a .exe extension or have a -script.py suffix.

251 exe_entrypoint = py_script.with_suffix(".exe")

252 if exe_entrypoint.exists():

253 # Should be executed directly, ignoring sys.executable.

254 return [exe_entrypoint, *sys.argv[1:]]

255 script_entrypoint = py_script.with_name("%s-script.py" % py_script.name)

256 if script_entrypoint.exists():

257 # Should be executed as usual.

258 return [*args, script_entrypoint, *sys.argv[1:]]

259 raise RuntimeError("Script %s does not exist." % py_script)

260 else:

261 args += sys.argv

262 return args

263

264

265def trigger_reload(filename):

266 logger.info("%s changed, reloading.", filename)

267 sys.exit(3)

268

269

270def restart_with_reloader():

271 new_environ = {**os.environ, DJANGO_AUTORELOAD_ENV: "true"}

272 args = get_child_arguments()

273 while True:

274 p = subprocess.run(args, env=new_environ, close_fds=False)

275 if p.returncode != 3:

276 return p.returncode

277

278

279class BaseReloader:

280 def __init__(self):

281 self.extra_files = set()

282 self.directory_globs = defaultdict(set)

283 self._stop_condition = threading.Event()

284

285 def watch_dir(self, path, glob):

286 path = Path(path)

287 try:

288 path = path.absolute()

289 except FileNotFoundError:

290 logger.debug(

291 "Unable to watch directory %s as it cannot be resolved.",

292 path,

293 exc_info=True,

294 )

295 return

296 logger.debug("Watching dir %s with glob %s.", path, glob)

297 self.directory_globs[path].add(glob)

298

299 def watched_files(self, include_globs=True):

300 """

301 Yield all files that need to be watched, including module files and

302 files within globs.

303 """

304 yield from iter_all_python_module_files()

305 yield from self.extra_files

306 if include_globs:

307 for directory, patterns in self.directory_globs.items():

308 for pattern in patterns:

309 yield from directory.glob(pattern)

310

311 def wait_for_apps_ready(self, app_reg, django_main_thread):

312 """

313 Wait until Django reports that the apps have been loaded. If the given

314 thread has terminated before the apps are ready, then a SyntaxError or

315 other non-recoverable error has been raised. In that case, stop waiting

316 for the apps_ready event and continue processing.

317

318 Return True if the thread is alive and the ready event has been

319 triggered, or False if the thread is terminated while waiting for the

320 event.

321 """

322 while django_main_thread.is_alive():

323 if app_reg.ready_event.wait(timeout=0.1):

324 return True

325 else:

326 logger.debug("Main Django thread has terminated before apps are ready.")

327 return False

328

329 def run(self, django_main_thread):

330 logger.debug("Waiting for apps ready_event.")

331 self.wait_for_apps_ready(apps, django_main_thread)

332 from django.urls import get_resolver

333

334 # Prevent a race condition where URL modules aren't loaded when the

335 # reloader starts by accessing the urlconf_module property.

336 try:

337 get_resolver().urlconf_module

338 except Exception:

339 # Loading the urlconf can result in errors during development.

340 # If this occurs then swallow the error and continue.

341 pass

342 logger.debug("Apps ready_event triggered. Sending autoreload_started signal.")

343 autoreload_started.send(sender=self)

344 self.run_loop()

345

346 def run_loop(self):

347 ticker = self.tick()

348 while not self.should_stop:

349 try:

350 next(ticker)

351 except StopIteration:

352 break

353 self.stop()

354

355 def tick(self):

356 """

357 This generator is called in a loop from run_loop. It's important that

358 the method takes care of pausing or otherwise waiting for a period of

359 time. This split between run_loop() and tick() is to improve the

360 testability of the reloader implementations by decoupling the work they

361 do from the loop.

362 """

363 raise NotImplementedError("subclasses must implement tick().")

364

365 @classmethod

366 def check_availability(cls):

367 raise NotImplementedError("subclasses must implement check_availability().")

368

369 def notify_file_changed(self, path):

370 results = file_changed.send(sender=self, file_path=path)

371 logger.debug("%s notified as changed. Signal results: %s.", path, results)

372 if not any(res[1] for res in results):

373 trigger_reload(path)

374

375 # These are primarily used for testing.

376 @property

377 def should_stop(self):

378 return self._stop_condition.is_set()

379

380 def stop(self):

381 self._stop_condition.set()

382

383

384class StatReloader(BaseReloader):

385 SLEEP_TIME = 1 # Check for changes once per second.

386

387 def tick(self):

388 mtimes = {}

389 while True:

390 for filepath, mtime in self.snapshot_files():

391 old_time = mtimes.get(filepath)

392 mtimes[filepath] = mtime

393 if old_time is None:

394 logger.debug("File %s first seen with mtime %s", filepath, mtime)

395 continue

396 elif mtime > old_time:

397 logger.debug(

398 "File %s previous mtime: %s, current mtime: %s",

399 filepath,

400 old_time,

401 mtime,

402 )

403 self.notify_file_changed(filepath)

404

405 time.sleep(self.SLEEP_TIME)

406 yield

407

408 def snapshot_files(self):

409 # watched_files may produce duplicate paths if globs overlap.

410 seen_files = set()

411 for file in self.watched_files():

412 if file in seen_files:

413 continue

414 try:

415 mtime = file.stat().st_mtime

416 except OSError:

417 # This is thrown when the file does not exist.

418 continue

419 seen_files.add(file)

420 yield file, mtime

421

422 @classmethod

423 def check_availability(cls):

424 return True

425

426

427class WatchmanUnavailable(RuntimeError):

428 pass

429

430

431class WatchmanReloader(BaseReloader):

432 def __init__(self):

433 self.roots = defaultdict(set)

434 self.processed_request = threading.Event()

435 self.client_timeout = int(os.environ.get("DJANGO_WATCHMAN_TIMEOUT", 5))

436 super().__init__()

437

438 @cached_property

439 def client(self):

440 return pywatchman.client(timeout=self.client_timeout)

441

442 def _watch_root(self, root):

443 # In practice this shouldn't occur, however, it's possible that a

444 # directory that doesn't exist yet is being watched. If it's outside of

445 # sys.path then this will end up a new root. How to handle this isn't

446 # clear: Not adding the root will likely break when subscribing to the

447 # changes, however, as this is currently an internal API, no files

448 # will be being watched outside of sys.path. Fixing this by checking

449 # inside watch_glob() and watch_dir() is expensive, instead this could

450 # could fall back to the StatReloader if this case is detected? For

451 # now, watching its parent, if possible, is sufficient.

452 if not root.exists():

453 if not root.parent.exists():

454 logger.warning(

455 "Unable to watch root dir %s as neither it or its parent exist.",

456 root,

457 )

458 return

459 root = root.parent

460 result = self.client.query("watch-project", str(root.absolute()))

461 if "warning" in result:

462 logger.warning("Watchman warning: %s", result["warning"])

463 logger.debug("Watchman watch-project result: %s", result)

464 return result["watch"], result.get("relative_path")

465

466 @functools.lru_cache()

467 def _get_clock(self, root):

468 return self.client.query("clock", root)["clock"]

469

470 def _subscribe(self, directory, name, expression):

471 root, rel_path = self._watch_root(directory)

472 # Only receive notifications of files changing, filtering out other types

473 # like special files: https://facebook.github.io/watchman/docs/type

474 only_files_expression = [

475 "allof",

476 ["anyof", ["type", "f"], ["type", "l"]],

477 expression,

478 ]

479 query = {

480 "expression": only_files_expression,

481 "fields": ["name"],

482 "since": self._get_clock(root),

483 "dedup_results": True,

484 }

485 if rel_path:

486 query["relative_root"] = rel_path

487 logger.debug(

488 "Issuing watchman subscription %s, for root %s. Query: %s",

489 name,

490 root,

491 query,

492 )

493 self.client.query("subscribe", root, name, query)

494

495 def _subscribe_dir(self, directory, filenames):

496 if not directory.exists():

497 if not directory.parent.exists():

498 logger.warning(

499 "Unable to watch directory %s as neither it or its parent exist.",

500 directory,

501 )

502 return

503 prefix = "files-parent-%s" % directory.name

504 filenames = ["%s/%s" % (directory.name, filename) for filename in filenames]

505 directory = directory.parent

506 expression = ["name", filenames, "wholename"]

507 else:

508 prefix = "files"

509 expression = ["name", filenames]

510 self._subscribe(directory, "%s:%s" % (prefix, directory), expression)

511

512 def _watch_glob(self, directory, patterns):

513 """

514 Watch a directory with a specific glob. If the directory doesn't yet

515 exist, attempt to watch the parent directory and amend the patterns to

516 include this. It's important this method isn't called more than one per

517 directory when updating all subscriptions. Subsequent calls will

518 overwrite the named subscription, so it must include all possible glob

519 expressions.

520 """

521 prefix = "glob"

522 if not directory.exists():

523 if not directory.parent.exists():

524 logger.warning(

525 "Unable to watch directory %s as neither it or its parent exist.",

526 directory,

527 )

528 return

529 prefix = "glob-parent-%s" % directory.name

530 patterns = ["%s/%s" % (directory.name, pattern) for pattern in patterns]

531 directory = directory.parent

532

533 expression = ["anyof"]

534 for pattern in patterns:

535 expression.append(["match", pattern, "wholename"])

536 self._subscribe(directory, "%s:%s" % (prefix, directory), expression)

537

538 def watched_roots(self, watched_files):

539 extra_directories = self.directory_globs.keys()

540 watched_file_dirs = [f.parent for f in watched_files]

541 sys_paths = list(sys_path_directories())

542 return frozenset((*extra_directories, *watched_file_dirs, *sys_paths))

543

544 def _update_watches(self):

545 watched_files = list(self.watched_files(include_globs=False))

546 found_roots = common_roots(self.watched_roots(watched_files))

547 logger.debug("Watching %s files", len(watched_files))

548 logger.debug("Found common roots: %s", found_roots)

549 # Setup initial roots for performance, shortest roots first.

550 for root in sorted(found_roots):

551 self._watch_root(root)

552 for directory, patterns in self.directory_globs.items():

553 self._watch_glob(directory, patterns)

554 # Group sorted watched_files by their parent directory.

555 sorted_files = sorted(watched_files, key=lambda p: p.parent)

556 for directory, group in itertools.groupby(sorted_files, key=lambda p: p.parent):

557 # These paths need to be relative to the parent directory.

558 self._subscribe_dir(

559 directory, [str(p.relative_to(directory)) for p in group]

560 )

561

562 def update_watches(self):

563 try:

564 self._update_watches()

565 except Exception as ex:

566 # If the service is still available, raise the original exception.

567 if self.check_server_status(ex):

568 raise

569

570 def _check_subscription(self, sub):

571 subscription = self.client.getSubscription(sub)

572 if not subscription:

573 return

574 logger.debug("Watchman subscription %s has results.", sub)

575 for result in subscription:

576 # When using watch-project, it's not simple to get the relative

577 # directory without storing some specific state. Store the full

578 # path to the directory in the subscription name, prefixed by its

579 # type (glob, files).

580 root_directory = Path(result["subscription"].split(":", 1)[1])

581 logger.debug("Found root directory %s", root_directory)

582 for file in result.get("files", []):

583 self.notify_file_changed(root_directory / file)

584

585 def request_processed(self, **kwargs):

586 logger.debug("Request processed. Setting update_watches event.")

587 self.processed_request.set()

588

589 def tick(self):

590 request_finished.connect(self.request_processed)

591 self.update_watches()

592 while True:

593 if self.processed_request.is_set():

594 self.update_watches()

595 self.processed_request.clear()

596 try:

597 self.client.receive()

598 except pywatchman.SocketTimeout:

599 pass

600 except pywatchman.WatchmanError as ex:

601 logger.debug("Watchman error: %s, checking server status.", ex)

602 self.check_server_status(ex)

603 else:

604 for sub in list(self.client.subs.keys()):

605 self._check_subscription(sub)

606 yield

607 # Protect against busy loops.

608 time.sleep(0.1)

609

610 def stop(self):

611 self.client.close()

612 super().stop()

613

614 def check_server_status(self, inner_ex=None):

615 """Return True if the server is available."""

616 try:

617 self.client.query("version")

618 except Exception:

619 raise WatchmanUnavailable(str(inner_ex)) from inner_ex

620 return True

621

622 @classmethod

623 def check_availability(cls):

624 if not pywatchman:

625 raise WatchmanUnavailable("pywatchman not installed.")

626 client = pywatchman.client(timeout=0.1)

627 try:

628 result = client.capabilityCheck()

629 except Exception:

630 # The service is down?

631 raise WatchmanUnavailable("Cannot connect to the watchman service.")

632 version = get_version_tuple(result["version"])

633 # Watchman 4.9 includes multiple improvements to watching project

634 # directories as well as case insensitive filesystems.

635 logger.debug("Watchman version %s", version)

636 if version < (4, 9):

637 raise WatchmanUnavailable("Watchman 4.9 or later is required.")

638

639

640def get_reloader():

641 """Return the most suitable reloader for this environment."""

642 try:

643 WatchmanReloader.check_availability()

644 except WatchmanUnavailable:

645 return StatReloader()

646 return WatchmanReloader()

647

648

649def start_django(reloader, main_func, *args, **kwargs):

650 ensure_echo_on()

651

652 main_func = check_errors(main_func)

653 django_main_thread = threading.Thread(

654 target=main_func, args=args, kwargs=kwargs, name="django-main-thread"

655 )

656 django_main_thread.daemon = True

657 django_main_thread.start()

658

659 while not reloader.should_stop:

660 try:

661 reloader.run(django_main_thread)

662 except WatchmanUnavailable as ex:

663 # It's possible that the watchman service shuts down or otherwise

664 # becomes unavailable. In that case, use the StatReloader.

665 reloader = StatReloader()

666 logger.error("Error connecting to Watchman: %s", ex)

667 logger.info(

668 "Watching for file changes with %s", reloader.__class__.__name__

669 )

670

671

672def run_with_reloader(main_func, *args, **kwargs):

673 signal.signal(signal.SIGTERM, lambda *args: sys.exit(0))

674 try:

675 if os.environ.get(DJANGO_AUTORELOAD_ENV) == "true":

676 reloader = get_reloader()

677 logger.info(

678 "Watching for file changes with %s", reloader.__class__.__name__

679 )

680 start_django(reloader, main_func, *args, **kwargs)

681 else:

682 exit_code = restart_with_reloader()

683 sys.exit(exit_code)

684 except KeyboardInterrupt:

685 pass