From 3fae5aa84f43dfcad1b13604f1ceadf58b96c6f6 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 5 Jun 2020 09:44:02 -0700 Subject: [PATCH] Redo docs --- docs/source/reference-io.rst | 99 +++++++++--------------- trio/_subprocess.py | 145 ++++++++++++++++++++++++----------- 2 files changed, 135 insertions(+), 109 deletions(-) diff --git a/docs/source/reference-io.rst b/docs/source/reference-io.rst index a0b8c77879..77ae4cbd88 100644 --- a/docs/source/reference-io.rst +++ b/docs/source/reference-io.rst @@ -664,22 +664,43 @@ Spawning subprocesses Trio provides support for spawning other programs as subprocesses, communicating with them via pipes, sending them signals, and waiting -for them to exit. The interface for doing so consists of two layers: +for them to exit. -* :func:`trio.run_process` runs a process from start to - finish and returns a :class:`~subprocess.CompletedProcess` object describing - its outputs and return value. This is what you should reach for if you - want to run a process to completion before continuing, while possibly - sending it some input or capturing its output. It is modelled after - the standard :func:`subprocess.run` with some additional features - and safer defaults. +Most of the time, this is done through our high-level interface, +`trio.run_process`. It lets you either run a process to completion +while optionally capturing the output, or else run it in a background +task and interact with it while it's running: -* `trio.open_process` starts a process in the background and returns a - `Process` object to let you interact with it. Using it requires a - bit more code than `run_process`, but exposes additional - capabilities: back-and-forth communication, processing output as - soon as it is generated, and so forth. It is modelled after the - standard library :class:`subprocess.Popen`. +.. autofunction:: trio.run_process + +.. autoclass:: trio.Process + + .. autoattribute:: returncode + + .. automethod:: wait + + .. automethod:: poll + + .. automethod:: kill + + .. automethod:: terminate + + .. automethod:: send_signal + + .. note:: :meth:`~subprocess.Popen.communicate` is not provided as a + method on :class:`~trio.Process` objects; call :func:`~trio.run_process` + normally for simple capturing, or write the loop yourself if you + have unusual needs. :meth:`~subprocess.Popen.communicate` has + quite unusual cancellation behavior in the standard library (on + some platforms it spawns a background thread which continues to + read from the child process even after the timeout has expired) + and we wanted to provide an interface with fewer surprises. + +If `trio.run_process` is too limiting, we also offer a low-level API, +`trio.open_process`. For example, use `open_process` if you want to +spawn a child process that outlives the parent process: + +.. autofunction:: trio.open_process .. _subprocess-options: @@ -705,56 +726,6 @@ with a process, so it does not support the ``encoding``, ``errors``, options. -Running a process and waiting for it to finish -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The basic interface for running a subprocess start-to-finish is -:func:`trio.run_process`. It always waits for the subprocess to exit -before returning, so there's no need to worry about leaving a process -running by mistake after you've gone on to do other things. -:func:`~trio.run_process` is similar to the standard library -:func:`subprocess.run` function, but tries to have safer defaults: -with no options, the subprocess's input is empty rather than coming -from the user's terminal, and a failure in the subprocess will be -propagated as a :exc:`subprocess.CalledProcessError` exception. Of -course, these defaults can be changed where necessary. - -.. autofunction:: trio.run_process - - -Interacting with a process as it runs -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you want more control than :func:`~trio.run_process` affords, you -can use `trio.open_process` to spawn a subprocess, and then interact -with it using the `Process` interface. - -.. autofunction:: trio.open_process - -.. autoclass:: trio.Process - - .. autoattribute:: returncode - - .. automethod:: wait - - .. automethod:: poll - - .. automethod:: kill - - .. automethod:: terminate - - .. automethod:: send_signal - - .. note:: :meth:`~subprocess.Popen.communicate` is not provided as a - method on :class:`~trio.Process` objects; use :func:`~trio.run_process` - instead, or write the loop yourself if you have unusual - needs. :meth:`~subprocess.Popen.communicate` has quite unusual - cancellation behavior in the standard library (on some platforms it - spawns a background thread which continues to read from the child - process even after the timeout has expired) and we wanted to - provide an interface with fewer surprises. - - .. _subprocess-quoting: Quoting: more than you wanted to know diff --git a/trio/_subprocess.py b/trio/_subprocess.py index a4c79a2b8f..aab9b84266 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -67,19 +67,20 @@ def pidfd_open(fd: int, flags: int) -> int: class Process(AsyncResource, metaclass=NoPublicConstructor): r"""A child process. Like :class:`subprocess.Popen`, but async. - This class has no public constructor. The most common way to create a - `Process` is to combine `Nursery.start` with `run_process`:: + This class has no public constructor. The most common way to get a + `Process` object is to combine `Nursery.start` with `run_process`:: - process = await nursery.start(run_process, ...) + process_object = await nursery.start(run_process, ...) - This way, `run_process` supervises the process, and makes sure that it is - cleaned up, while optionally checking the output, feeding it input, and so - on. + This way, `run_process` supervises the process and makes sure that it is + cleaned up properly, while optionally checking the return value, feeding + it input, and so on. If you need more control – for example, because you want to spawn a child - process that outlives your program – then you can use `open_process`:: + process that outlives your program – then another option is to use + `open_process`:: - process = await trio.open_process(...) + process_object = await trio.open_process(...) Attributes: args (str or list): The ``command`` passed at construction time, @@ -437,21 +438,39 @@ async def run_process( task_status=trio.TASK_STATUS_IGNORED, **options, ): - """Run ``command`` in a subprocess, wait for it to complete, and - return a :class:`subprocess.CompletedProcess` instance describing - the results. - - If cancelled, :func:`run_process` terminates the subprocess and - waits for it to exit before propagating the cancellation, like - :meth:`Process.aclose`. - - **Input:** The subprocess's standard input stream is set up to - receive the bytes provided as ``stdin``. Once the given input has - been fully delivered, or if none is provided, the subprocess will - receive end-of-file when reading from its standard input. - Alternatively, if you want the subprocess to read its - standard input from the same place as the parent Trio process, you - can pass ``stdin=None``. + """Run ``command`` in a subprocess and wait for it to complete. + + This function can be called in two different ways. + + One option is a direct call, like:: + + completed_process_info = await trio.run_process(...) + + In this case, it returns a :class:`subprocess.CompletedProcess` instance + describing the results. Use this if you want to treat a process like a + function call. + + The other option is to run it as a task using `Nursery.start` – the enhanced version + of `~Nursery.start_soon` that lets a task pass back a value during startup:: + + process = await nursery.start(trio.run_process, ...) + + In this case, `~Nursery.start` returns a `Process` object that you can use + to interact with the process while it's running. Use this if you want to + treat a process like a background task. + + Either way, `run_process` makes sure that the process has exited before + returning, handles cancellation, optionally checks for errors, and + provides some convenient shorthands for dealing with the child's + input/output. + + **Input:** `run_process` supports all the same ``stdin=`` arguments as + `subprocess.Popen`. In addition, if you simply want to pass in some fixed + data, you can pass a plain `bytes` object, and `run_process` will take + care of setting up a pipe, feeding in the data you gave, and then sending + end-of-file. The default is ``b""``, which means that the child will receive + an empty stdin. If you want the child to instead read from the parent's + stdin, use ``stdin=None``. **Output:** By default, any output produced by the subprocess is passed through to the standard output and error streams of the @@ -481,8 +500,28 @@ async def run_process( the :attr:`~subprocess.CalledProcessError.stdout` and :attr:`~subprocess.CalledProcessError.stderr` attributes of that exception. To disable this behavior, so that :func:`run_process` - returns normally even if the subprocess exits abnormally, pass - ``check=False``. + returns normally even if the subprocess exits abnormally, pass ``check=False``. + + Note that this can make the ``capture_stdout`` and ``capture_stderr`` + arguments useful even when starting `run_process` as a task: if you only + care about the output if the process fails, then you can enable capturing + and then read the output off of the `~subprocess.CalledProcessError`. + + **Cancellation:** If cancelled, `run_process` sends a termination + request to the subprocess, then waits for it to fully exit. The + ``deliver_cancel`` argument lets you control how the process is terminated. + + .. note:: `run_process` is intentionally similar to the standard library + `subprocess.run`, but some of the defaults are different. Specifically, we + default to: + + - ``check=True``, because `"errors should never pass silently / unless + explicitly silenced "`__. + + - ``stdin=b""``, because it produces less-confusing results if a subprocess + unexpectedly tries to read from stdin. + + To get the `subprocess.run` semantics, use ``check=False, stdin=None``. Args: command (list or str): The command to run. Typically this is a @@ -493,24 +532,26 @@ async def run_process( be a string, which will be parsed following platform-dependent :ref:`quoting rules `. - stdin (:obj:`bytes`, file descriptor, or None): The bytes to provide to - the subprocess on its standard input stream, or ``None`` if the - subprocess's standard input should come from the same place as - the parent Trio process's standard input. As is the case with - the :mod:`subprocess` module, you can also pass a - file descriptor or an object with a ``fileno()`` method, - in which case the subprocess's standard input will come from - that file. + stdin (:obj:`bytes`, subprocess.PIPE, file descriptor, or None): The + bytes to provide to the subprocess on its standard input stream, or + ``None`` if the subprocess's standard input should come from the + same place as the parent Trio process's standard input. As is the + case with the :mod:`subprocess` module, you can also pass a file + descriptor or an object with a ``fileno()`` method, in which case + the subprocess's standard input will come from that file. And when + starting `run_process` as a background task, you can use + ``stdin=subprocess.PIPE``, in which case `Process.stdin` will be a + `~trio.abc.SendStream` that you can use to send data to the child. capture_stdout (bool): If true, capture the bytes that the subprocess writes to its standard output stream and return them in the - :attr:`~subprocess.CompletedProcess.stdout` attribute - of the returned :class:`~subprocess.CompletedProcess` object. + `~subprocess.CompletedProcess.stdout` attribute of the returned + `subprocess.CompletedProcess` or `subprocess.CalledProcessError`. capture_stderr (bool): If true, capture the bytes that the subprocess writes to its standard error stream and return them in the - :attr:`~subprocess.CompletedProcess.stderr` attribute - of the returned :class:`~subprocess.CompletedProcess` object. + `~subprocess.CompletedProcess.stderr` attribute of the returned + `~subprocess.CompletedProcess` or `subprocess.CalledProcessError`. check (bool): If false, don't validate that the subprocess exits successfully. You should be sure to check the @@ -555,8 +596,11 @@ async def my_deliver_cancel(process): ``stdout=subprocess.DEVNULL``, or file descriptors. Returns: - A :class:`subprocess.CompletedProcess` instance describing the - return code and outputs. + + When called normally – a `subprocess.CompletedProcess` instance + describing the return code and outputs. + + When called via `Nursery.start` – a `trio.Process` instance. Raises: UnicodeError: if ``stdin`` is specified as a Unicode string, rather @@ -579,12 +623,23 @@ async def my_deliver_cancel(process): if isinstance(stdin, str): raise UnicodeError("process stdin must be bytes, not str") - if stdin == subprocess.PIPE and task_status is trio.TASK_STATUS_IGNORED: - raise ValueError( - "stdin=subprocess.PIPE doesn't make sense since the pipe " - "is internal to run_process(); pass the actual data you " - "want to send over that pipe instead" - ) + if task_status is trio.TASK_STATUS_IGNORED: + if stdin is subprocess.PIPE: + raise ValueError( + "stdin=subprocess.PIPE doesn't make sense without " + "nursery.start, since there's no way to access the " + "pipe; pass the data you want to send or use nursery.start" + ) + if options.get("stdout") is subprocess.PIPE: + raise ValueError( + "stdout=subprocess.PIPE doesn't make sense without " + "nursery.start, since there's no way to access the pipe" + ) + if options.get("stderr") is subprocess.PIPE: + raise ValueError( + "stderr=subprocess.PIPE doesn't make sense without " + "nursery.start, since there's no way to access the pipe" + ) if isinstance(stdin, (bytes, bytearray, memoryview)): input = stdin options["stdin"] = subprocess.PIPE