""" Code concerned with waiting in different contexts (blocking, async, etc). These functions are designed to consume the generators returned by the `generators` module function and to return their final value. """ # Copyright (C) 2020 The Psycopg Team import os import sys import select import selectors from typing import Optional from asyncio import get_event_loop, wait_for, Event, TimeoutError from selectors import DefaultSelector from . import errors as e from .abc import RV, PQGen, PQGenConn, WaitFunc from ._enums import Wait as Wait, Ready as Ready # re-exported from ._cmodule import _psycopg WAIT_R = Wait.R WAIT_W = Wait.W WAIT_RW = Wait.RW READY_R = Ready.R READY_W = Ready.W READY_RW = Ready.RW def wait_selector(gen: PQGen[RV], fileno: int, timeout: Optional[float] = None) -> RV: """ Wait for a generator using the best strategy available. :param gen: a generator performing database operations and yielding `Ready` values when it would block. :param fileno: the file descriptor to wait on. :param timeout: timeout (in seconds) to check for other interrupt, e.g. to allow Ctrl-C. :type timeout: float :return: whatever `!gen` returns on completion. Consume `!gen`, scheduling `fileno` for completion when it is reported to block. Once ready again send the ready state back to `!gen`. """ try: s = next(gen) with DefaultSelector() as sel: while True: sel.register(fileno, s) rlist = None while not rlist: rlist = sel.select(timeout=timeout) sel.unregister(fileno) # note: this line should require a cast, but mypy doesn't complain ready: Ready = rlist[0][1] assert s & ready s = gen.send(ready) except StopIteration as ex: rv: RV = ex.args[0] if ex.args else None return rv def wait_conn(gen: PQGenConn[RV], timeout: Optional[float] = None) -> RV: """ Wait for a connection generator using the best strategy available. :param gen: a generator performing database operations and yielding (fd, `Ready`) pairs when it would block. :param timeout: timeout (in seconds) to check for other interrupt, e.g. to allow Ctrl-C. If zero or None, wait indefinitely. :type timeout: float :return: whatever `!gen` returns on completion. Behave like in `wait()`, but take the fileno to wait from the generator itself, which might change during processing. """ try: fileno, s = next(gen) if not timeout: timeout = None with DefaultSelector() as sel: while True: sel.register(fileno, s) rlist = sel.select(timeout=timeout) sel.unregister(fileno) if not rlist: raise e.ConnectionTimeout("connection timeout expired") ready: Ready = rlist[0][1] # type: ignore[assignment] fileno, s = gen.send(ready) except StopIteration as ex: rv: RV = ex.args[0] if ex.args else None return rv async def wait_async( gen: PQGen[RV], fileno: int, timeout: Optional[float] = None ) -> RV: """ Coroutine waiting for a generator to complete. :param gen: a generator performing database operations and yielding `Ready` values when it would block. :param fileno: the file descriptor to wait on. :return: whatever `!gen` returns on completion. Behave like in `wait()`, but exposing an `asyncio` interface. """ # Use an event to block and restart after the fd state changes. # Not sure this is the best implementation but it's a start. ev = Event() loop = get_event_loop() ready: Ready s: Wait def wakeup(state: Ready) -> None: nonlocal ready ready |= state # type: ignore[assignment] ev.set() try: s = next(gen) while True: reader = s & WAIT_R writer = s & WAIT_W if not reader and not writer: raise e.InternalError(f"bad poll status: {s}") ev.clear() ready = 0 # type: ignore[assignment] if reader: loop.add_reader(fileno, wakeup, READY_R) if writer: loop.add_writer(fileno, wakeup, READY_W) try: if timeout is None: await ev.wait() else: try: await wait_for(ev.wait(), timeout) except TimeoutError: pass finally: if reader: loop.remove_reader(fileno) if writer: loop.remove_writer(fileno) s = gen.send(ready) except StopIteration as ex: rv: RV = ex.args[0] if ex.args else None return rv async def wait_conn_async(gen: PQGenConn[RV], timeout: Optional[float] = None) -> RV: """ Coroutine waiting for a connection generator to complete. :param gen: a generator performing database operations and yielding (fd, `Ready`) pairs when it would block. :param timeout: timeout (in seconds) to check for other interrupt, e.g. to allow Ctrl-C. If zero or None, wait indefinitely. :return: whatever `!gen` returns on completion. Behave like in `wait()`, but take the fileno to wait from the generator itself, which might change during processing. """ # Use an event to block and restart after the fd state changes. # Not sure this is the best implementation but it's a start. ev = Event() loop = get_event_loop() ready: Ready s: Wait def wakeup(state: Ready) -> None: nonlocal ready ready = state ev.set() try: fileno, s = next(gen) if not timeout: timeout = None while True: reader = s & WAIT_R writer = s & WAIT_W if not reader and not writer: raise e.InternalError(f"bad poll status: {s}") ev.clear() ready = 0 # type: ignore[assignment] if reader: loop.add_reader(fileno, wakeup, READY_R) if writer: loop.add_writer(fileno, wakeup, READY_W) try: await wait_for(ev.wait(), timeout) finally: if reader: loop.remove_reader(fileno) if writer: loop.remove_writer(fileno) fileno, s = gen.send(ready) except TimeoutError: raise e.ConnectionTimeout("connection timeout expired") except StopIteration as ex: rv: RV = ex.args[0] if ex.args else None return rv # Specialised implementation of wait functions. def wait_select(gen: PQGen[RV], fileno: int, timeout: Optional[float] = None) -> RV: """ Wait for a generator using select where supported. BUG: on Linux, can't select on FD >= 1024. On Windows it's fine. """ try: s = next(gen) empty = () fnlist = (fileno,) while True: rl, wl, xl = select.select( fnlist if s & WAIT_R else empty, fnlist if s & WAIT_W else empty, fnlist, timeout, ) ready = 0 if rl: ready = READY_R if wl: ready |= READY_W if not ready: continue # assert s & ready s = gen.send(ready) # type: ignore except StopIteration as ex: rv: RV = ex.args[0] if ex.args else None return rv if hasattr(selectors, "EpollSelector"): _epoll_evmasks = { WAIT_R: select.EPOLLONESHOT | select.EPOLLIN | select.EPOLLERR, WAIT_W: select.EPOLLONESHOT | select.EPOLLOUT | select.EPOLLERR, WAIT_RW: select.EPOLLONESHOT | (select.EPOLLIN | select.EPOLLOUT | select.EPOLLERR), } else: _epoll_evmasks = {} def wait_epoll(gen: PQGen[RV], fileno: int, timeout: Optional[float] = None) -> RV: """ Wait for a generator using epoll where supported. Parameters are like for `wait()`. If it is detected that the best selector strategy is `epoll` then this function will be used instead of `wait`. See also: https://linux.die.net/man/2/epoll_ctl BUG: if the connection FD is closed, `epoll.poll()` hangs. Same for EpollSelector. For this reason, wait_poll() is currently preferable. To reproduce the bug: export PSYCOPG_WAIT_FUNC=wait_epoll pytest tests/test_concurrency.py::test_concurrent_close """ try: s = next(gen) if timeout is None or timeout < 0: timeout = 0 else: timeout = int(timeout * 1000.0) with select.epoll() as epoll: evmask = _epoll_evmasks[s] epoll.register(fileno, evmask) while True: fileevs = None while not fileevs: fileevs = epoll.poll(timeout) ev = fileevs[0][1] ready = 0 if ev & ~select.EPOLLOUT: ready = READY_R if ev & ~select.EPOLLIN: ready |= READY_W # assert s & ready s = gen.send(ready) evmask = _epoll_evmasks[s] epoll.modify(fileno, evmask) except StopIteration as ex: rv: RV = ex.args[0] if ex.args else None return rv if hasattr(selectors, "PollSelector"): _poll_evmasks = { WAIT_R: select.POLLIN, WAIT_W: select.POLLOUT, WAIT_RW: select.POLLIN | select.POLLOUT, } else: _poll_evmasks = {} def wait_poll(gen: PQGen[RV], fileno: int, timeout: Optional[float] = None) -> RV: """ Wait for a generator using poll where supported. Parameters are like for `wait()`. """ try: s = next(gen) if timeout is None or timeout < 0: timeout = 0 else: timeout = int(timeout * 1000.0) poll = select.poll() evmask = _poll_evmasks[s] poll.register(fileno, evmask) while True: fileevs = None while not fileevs: fileevs = poll.poll(timeout) ev = fileevs[0][1] ready = 0 if ev & ~select.POLLOUT: ready = READY_R if ev & ~select.POLLIN: ready |= READY_W # assert s & ready s = gen.send(ready) evmask = _poll_evmasks[s] poll.modify(fileno, evmask) except StopIteration as ex: rv: RV = ex.args[0] if ex.args else None return rv if _psycopg: wait_c = _psycopg.wait_c # Choose the best wait strategy for the platform. # # the selectors objects have a generic interface but come with some overhead, # so we also offer more finely tuned implementations. wait: WaitFunc # Allow the user to choose a specific function for testing if "PSYCOPG_WAIT_FUNC" in os.environ: fname = os.environ["PSYCOPG_WAIT_FUNC"] if not fname.startswith("wait_") or fname not in globals(): raise ImportError( "PSYCOPG_WAIT_FUNC should be the name of an available wait function;" f" got {fname!r}" ) wait = globals()[fname] # On Windows, for the moment, avoid using wait_c, because it was reported to # use excessive CPU (see #645). # TODO: investigate why. elif _psycopg and sys.platform != "win32": wait = wait_c elif selectors.DefaultSelector is getattr(selectors, "SelectSelector", None): # On Windows, SelectSelector should be the default. wait = wait_select elif hasattr(selectors, "PollSelector"): # On linux, EpollSelector is the default. However, it hangs if the fd is # closed while polling. wait = wait_poll else: wait = wait_selector