diff --git a/.ci/latest_deps_build_failed_issue_template.md b/.ci/latest_deps_build_failed_issue_template.md new file mode 100644 index 000000000000..0525402503fd --- /dev/null +++ b/.ci/latest_deps_build_failed_issue_template.md @@ -0,0 +1,4 @@ +--- +title: CI run against latest deps is failing +--- +See https://github.com/{{env.GITHUB_REPOSITORY}}/actions/runs/{{env.GITHUB_RUN_ID}} diff --git a/.ci/patch_for_twisted_trunk.sh b/.ci/patch_for_twisted_trunk.sh deleted file mode 100755 index f524581986a9..000000000000 --- a/.ci/patch_for_twisted_trunk.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -# replaces the dependency on Twisted in `python_dependencies` with trunk. - -set -e -cd "$(dirname "$0")"/.. - -sed -i -e 's#"Twisted.*"#"Twisted @ git+https://github.com/twisted/twisted"#' synapse/python_dependencies.py diff --git a/.dockerignore b/.dockerignore index a236760cf1fd..7809863ef328 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,8 +8,4 @@ !pyproject.toml !poetry.lock -# TODO: remove these once we have moved over to using poetry-core in pyproject.toml -!MANIFEST.in -!setup.py - **/__pycache__ diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 83ddd568c207..50d28c68eeb8 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -6,3 +6,6 @@ aff1eb7c671b0a3813407321d2702ec46c71fa56 # Update black to 20.8b1 (#9381). 0a00b7ff14890987f09112a2ae696c61001e6cf1 + +# Convert tests/rest/admin/test_room.py to unix file endings (#7953). +c4268e3da64f1abb5b31deaeb5769adb6510c0a7 \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 124b17458f45..d20d30c0353c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -34,32 +34,24 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - # TODO: consider using https://github.com/docker/metadata-action instead of this - # custom magic - name: Calculate docker image tag id: set-tag - run: | - case "${GITHUB_REF}" in - refs/heads/develop) - tag=develop - ;; - refs/heads/master|refs/heads/main) - tag=latest - ;; - refs/tags/*) - tag=${GITHUB_REF#refs/tags/} - ;; - *) - tag=${GITHUB_SHA} - ;; - esac - echo "::set-output name=tag::$tag" + uses: docker/metadata-action@master + with: + images: matrixdotorg/synapse + flavor: | + latest=false + tags: | + type=raw,value=develop,enable=${{ github.ref == 'refs/heads/develop' }} + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }} + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + type=pep440,pattern={{raw}} - name: Build and push all platforms uses: docker/build-push-action@v2 with: push: true labels: "gitsha1=${{ github.sha }}" - tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}" + tags: "${{ steps.set-tag.outputs.tags }}" file: "docker/Dockerfile" platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml new file mode 100644 index 000000000000..c537a5a60f9f --- /dev/null +++ b/.github/workflows/latest_deps.yml @@ -0,0 +1,159 @@ +# People who are freshly `pip install`ing from PyPI will pull in the latest versions of +# dependencies which match the broad requirements. Since most CI runs are against +# the locked poetry environment, run specifically against the latest dependencies to +# know if there's an upcoming breaking change. +# +# As an overview this workflow: +# - checks out develop, +# - installs from source, pulling in the dependencies like a fresh `pip install` would, and +# - runs mypy and test suites in that checkout. +# +# Based on the twisted trunk CI job. + +name: Latest dependencies + +on: + schedule: + - cron: 0 7 * * * + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + mypy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + # The dev dependencies aren't exposed in the wheel metadata (at least with current + # poetry-core versions), so we install with poetry. + - uses: matrix-org/setup-python-poetry@v1 + with: + python-version: "3.x" + poetry-version: "1.2.0b1" + extras: "all" + # Dump installed versions for debugging. + - run: poetry run pip list > before.txt + # Upgrade all runtime dependencies only. This is intended to mimic a fresh + # `pip install matrix-synapse[all]` as closely as possible. + - run: poetry update --no-dev + - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true) + - name: Remove warn_unused_ignores from mypy config + run: sed '/warn_unused_ignores = True/d' -i mypy.ini + - run: poetry run mypy + trial: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - database: "sqlite" + - database: "postgres" + postgres-version: "14" + + steps: + - uses: actions/checkout@v2 + - run: sudo apt-get -qq install xmlsec1 + - name: Set up PostgreSQL ${{ matrix.postgres-version }} + if: ${{ matrix.postgres-version }} + run: | + docker run -d -p 5432:5432 \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ + postgres:${{ matrix.postgres-version }} + - uses: actions/setup-python@v2 + with: + python-version: "3.x" + - run: pip install .[all,test] + - name: Await PostgreSQL + if: ${{ matrix.postgres-version }} + timeout-minutes: 2 + run: until pg_isready -h localhost; do sleep 1; done + - run: python -m twisted.trial --jobs=2 tests + env: + SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }} + SYNAPSE_POSTGRES_HOST: localhost + SYNAPSE_POSTGRES_USER: postgres + SYNAPSE_POSTGRES_PASSWORD: postgres + - name: Dump logs + # Logs are most useful when the command fails, always include them. + if: ${{ always() }} + # Note: Dumps to workflow logs instead of using actions/upload-artifact + # This keeps logs colocated with failing jobs + # It also ignores find's exit code; this is a best effort affair + run: >- + find _trial_temp -name '*.log' + -exec echo "::group::{}" \; + -exec cat {} \; + -exec echo "::endgroup::" \; + || true + + + sytest: + runs-on: ubuntu-latest + container: + image: matrixdotorg/sytest-synapse:testing + volumes: + - ${{ github.workspace }}:/src + strategy: + fail-fast: false + matrix: + include: + - sytest-tag: focal + + - sytest-tag: focal + postgres: postgres + workers: workers + redis: redis + env: + POSTGRES: ${{ matrix.postgres && 1}} + WORKERS: ${{ matrix.workers && 1 }} + REDIS: ${{ matrix.redis && 1 }} + BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }} + + steps: + - uses: actions/checkout@v2 + - name: Ensure sytest runs `pip install` + # Delete the lockfile so sytest will `pip install` rather than `poetry install` + run: rm /src/poetry.lock + working-directory: /src + - name: Prepare test blacklist + run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers + - name: Run SyTest + run: /bootstrap.sh synapse + working-directory: /src + - name: Summarise results.tap + if: ${{ always() }} + run: /sytest/scripts/tap_to_gha.pl /logs/results.tap + - name: Upload SyTest logs + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) + path: | + /logs/results.tap + /logs/**/*.log* + + + # TODO: run complement (as with twisted trunk, see #12473). + + # open an issue if the build fails, so we know about it. + open-issue: + if: failure() + needs: + # TODO: should mypy be included here? It feels more brittle than the other two. + - mypy + - trial + - sytest + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + update_existing: true + filename: .ci/latest_deps_build_failed_issue_template.md + diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5a98f619328f..efa35b71df18 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,24 +15,14 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - - run: pip install -e . + - run: pip install . - run: scripts-dev/generate_sample_config.sh --check - run: scripts-dev/config-lint.sh lint: - runs-on: ubuntu-latest - strategy: - matrix: - toxenv: - - "check_codestyle" - - "check_isort" - - "mypy" - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: pip install tox - - run: tox -e ${{ matrix.toxenv }} + uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1" + with: + typechecking-extras: "all" lint-crlf: runs-on: ubuntu-latest @@ -71,23 +61,23 @@ jobs: matrix: python-version: ["3.7", "3.8", "3.9", "3.10"] database: ["sqlite"] - toxenv: ["py"] + extras: ["all"] include: # Newest Python without optional deps - python-version: "3.10" - toxenv: "py-noextras" + extras: "" # Oldest Python with PostgreSQL - python-version: "3.7" database: "postgres" postgres-version: "10" - toxenv: "py" + extras: "all" # Newest Python with newest PostgreSQL - python-version: "3.10" database: "postgres" postgres-version: "14" - toxenv: "py" + extras: "all" steps: - uses: actions/checkout@v2 @@ -99,17 +89,16 @@ jobs: -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ postgres:${{ matrix.postgres-version }} - - uses: actions/setup-python@v2 + - uses: matrix-org/setup-python-poetry@v1 with: python-version: ${{ matrix.python-version }} - - run: pip install tox + extras: ${{ matrix.extras }} - name: Await PostgreSQL if: ${{ matrix.postgres-version }} timeout-minutes: 2 run: until pg_isready -h localhost; do sleep 1; done - - run: tox -e ${{ matrix.toxenv }} + - run: poetry run trial --jobs=2 tests env: - TRIAL_FLAGS: "--jobs=2" SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }} SYNAPSE_POSTGRES_HOST: localhost SYNAPSE_POSTGRES_USER: postgres @@ -156,23 +145,24 @@ jobs: trial-pypy: # Very slow; only run if the branch name includes 'pypy' + # Note: sqlite only; no postgres. Completely untested since poetry move. if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }} needs: linting-done runs-on: ubuntu-latest strategy: matrix: python-version: ["pypy-3.7"] + extras: ["all"] steps: - uses: actions/checkout@v2 + # Install libs necessary for PyPy to build binary wheels for dependencies - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - - uses: actions/setup-python@v2 + - uses: matrix-org/setup-python-poetry@v1 with: python-version: ${{ matrix.python-version }} - - run: pip install tox - - run: tox -e py - env: - TRIAL_FLAGS: "--jobs=2" + extras: ${{ matrix.extras }} + - run: poetry run trial --jobs=2 tests - name: Dump logs # Logs are most useful when the command fails, always include them. if: ${{ always() }} diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index fb9d46b7bfdd..5f0671f3503a 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -6,16 +6,27 @@ on: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: mypy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: .ci/patch_for_twisted_trunk.sh - - run: pip install tox - - run: tox -e mypy + - uses: matrix-org/setup-python-poetry@v1 + with: + python-version: "3.x" + extras: "all" + - run: | + poetry remove twisted + poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk + poetry install --no-interaction --extras "all test" + - name: Remove warn_unused_ignores from mypy config + run: sed '/warn_unused_ignores = True/d' -i mypy.ini + - run: poetry run mypy trial: runs-on: ubuntu-latest @@ -23,14 +34,15 @@ jobs: steps: - uses: actions/checkout@v2 - run: sudo apt-get -qq install xmlsec1 - - uses: actions/setup-python@v2 + - uses: matrix-org/setup-python-poetry@v1 with: - python-version: 3.7 - - run: .ci/patch_for_twisted_trunk.sh - - run: pip install tox - - run: tox -e py - env: - TRIAL_FLAGS: "--jobs=2" + python-version: "3.x" + extras: "all test" + - run: | + poetry remove twisted + poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk + poetry install --no-interaction --extras "all test" + - run: poetry run trial --jobs 2 tests - name: Dump logs # Logs are most useful when the command fails, always include them. @@ -55,11 +67,23 @@ jobs: steps: - uses: actions/checkout@v2 - name: Patch dependencies - run: .ci/patch_for_twisted_trunk.sh + # Note: The poetry commands want to create a virtualenv in /src/.venv/, + # but the sytest-synapse container expects it to be in /venv/. + # We symlink it before running poetry so that poetry actually + # ends up installing to `/venv`. + run: | + ln -s -T /venv /src/.venv + poetry remove twisted + poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk + poetry install --no-interaction --extras "all test" working-directory: /src - name: Run SyTest run: /bootstrap.sh synapse working-directory: /src + env: + # Use offline mode to avoid reinstalling the pinned version of + # twisted. + OFFLINE: 1 - name: Summarise results.tap if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap diff --git a/.gitignore b/.gitignore index c011cd27a4df..e58affb24125 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,7 @@ _trial_temp*/ .DS_Store __pycache__/ -# We do want the poetry lockfile. TODO: is there a good reason for ignoring -# '*.lock' above? If not, let's nuke it. +# We do want the poetry lockfile. !poetry.lock # stuff that is likely to exist when you run a server locally diff --git a/CHANGES.md b/CHANGES.md index 74a5dbf424b1..de6bf95e43d1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,403 @@ +Synapse 1.60.0 (2022-05-31) +=========================== + +This release of Synapse adds a unique index to the `state_group_edges` table, in +order to prevent accidentally introducing duplicate information (for example, +because a database backup was restored multiple times). If your Synapse database +already has duplicate rows in this table, this could fail with an error and +require manual remediation. + +Additionally, the signature of the `check_event_for_spam` module callback has changed. +The previous signature has been deprecated and remains working for now. Module authors +should update their modules to use the new signature where possible. + +See [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1600) +for more details. + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.60.0rc1 that would break some imports from `synapse.module_api`. ([\#12918](https://github.com/matrix-org/synapse/issues/12918)) + + +Synapse 1.60.0rc2 (2022-05-27) +============================== + +Features +-------- + +- Add an option allowing users to use their password to reauthenticate for privileged actions even though password login is disabled. ([\#12883](https://github.com/matrix-org/synapse/issues/12883)) + + +Bugfixes +-------- + +- Explicitly close `ijson` coroutines once we are done with them, instead of leaving the garbage collector to close them. ([\#12875](https://github.com/matrix-org/synapse/issues/12875)) + + +Internal Changes +---------------- + +- Improve URL previews by not including the content of media tags in the generated description. ([\#12887](https://github.com/matrix-org/synapse/issues/12887)) + + +Synapse 1.60.0rc1 (2022-05-24) +============================== + +Features +-------- + +- Measure the time taken in spam-checking callbacks and expose those measurements as metrics. ([\#12513](https://github.com/matrix-org/synapse/issues/12513)) +- Add a `default_power_level_content_override` config option to set default room power levels per room preset. ([\#12618](https://github.com/matrix-org/synapse/issues/12618)) +- Add support for [MSC3787: Allowing knocks to restricted rooms](https://github.com/matrix-org/matrix-spec-proposals/pull/3787). ([\#12623](https://github.com/matrix-org/synapse/issues/12623)) +- Send `USER_IP` commands on a different Redis channel, in order to reduce traffic to workers that do not process these commands. ([\#12672](https://github.com/matrix-org/synapse/issues/12672), [\#12809](https://github.com/matrix-org/synapse/issues/12809)) +- Synapse will now reload [cache config](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#caching) when it receives a [SIGHUP](https://en.wikipedia.org/wiki/SIGHUP) signal. ([\#12673](https://github.com/matrix-org/synapse/issues/12673)) +- Add a config options to allow for auto-tuning of caches. ([\#12701](https://github.com/matrix-org/synapse/issues/12701)) +- Update [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) implementation to process marker events from the current state to avoid markers being lost in timeline gaps for federated servers which would cause the imported history to be undiscovered. ([\#12718](https://github.com/matrix-org/synapse/issues/12718)) +- Add a `drop_federated_event` callback to `SpamChecker` to disregard inbound federated events before they take up much processing power, in an emergency. ([\#12744](https://github.com/matrix-org/synapse/issues/12744)) +- Implement [MSC3818: Copy room type on upgrade](https://github.com/matrix-org/matrix-spec-proposals/pull/3818). ([\#12786](https://github.com/matrix-org/synapse/issues/12786), [\#12792](https://github.com/matrix-org/synapse/issues/12792)) +- Update to the `check_event_for_spam` module callback. Deprecate the current callback signature, replace it with a new signature that is both less ambiguous (replacing booleans with explicit allow/block) and more powerful (ability to return explicit error codes). ([\#12808](https://github.com/matrix-org/synapse/issues/12808)) + + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.7.0 that would prevent events from being sent to clients if there's a retention policy in the room when the support for retention policies is disabled. ([\#12611](https://github.com/matrix-org/synapse/issues/12611)) +- Fix a bug introduced in Synapse 1.57.0 where `/messages` would throw a 500 error when querying for a non-existent room. ([\#12683](https://github.com/matrix-org/synapse/issues/12683)) +- Add a unique index to `state_group_edges` to prevent duplicates being accidentally introduced and the consequential impact to performance. ([\#12687](https://github.com/matrix-org/synapse/issues/12687)) +- Fix a long-standing bug where an empty room would be created when a user with an insufficient power level tried to upgrade a room. ([\#12696](https://github.com/matrix-org/synapse/issues/12696)) +- Fix a bug introduced in Synapse 1.30.0 where empty rooms could be automatically created if a monthly active users limit is set. ([\#12713](https://github.com/matrix-org/synapse/issues/12713)) +- Fix push to dismiss notifications when read on another client. Contributed by @SpiritCroc @ Beeper. ([\#12721](https://github.com/matrix-org/synapse/issues/12721)) +- Fix poor database performance when reading the cache invalidation stream for large servers with lots of workers. ([\#12747](https://github.com/matrix-org/synapse/issues/12747)) +- Delete events from the `federation_inbound_events_staging` table when a room is purged through the admin API. ([\#12770](https://github.com/matrix-org/synapse/issues/12770)) +- Give a meaningful error message when a client tries to create a room with an invalid alias localpart. ([\#12779](https://github.com/matrix-org/synapse/issues/12779)) +- Fix a bug introduced in 1.43.0 where a file (`providers.json`) was never closed. Contributed by @arkamar. ([\#12794](https://github.com/matrix-org/synapse/issues/12794)) +- Fix a long-standing bug where finished log contexts would be re-started when failing to contact remote homeservers. ([\#12803](https://github.com/matrix-org/synapse/issues/12803)) +- Fix a bug, introduced in Synapse 1.21.0, that led to media thumbnails being unusable before the index has been added in the background. ([\#12823](https://github.com/matrix-org/synapse/issues/12823)) + + +Updates to the Docker image +--------------------------- + +- Fix the docker file after a dependency update. ([\#12853](https://github.com/matrix-org/synapse/issues/12853)) + + +Improved Documentation +---------------------- + +- Fix a typo in the Media Admin API documentation. ([\#12715](https://github.com/matrix-org/synapse/issues/12715)) +- Update the OpenID Connect example for Keycloak to be compatible with newer versions of Keycloak. Contributed by @nhh. ([\#12727](https://github.com/matrix-org/synapse/issues/12727)) +- Fix typo in server listener documentation. ([\#12742](https://github.com/matrix-org/synapse/issues/12742)) +- Link to the configuration manual from the welcome page of the documentation. ([\#12748](https://github.com/matrix-org/synapse/issues/12748)) +- Fix typo in `run_background_tasks_on` option name in configuration manual documentation. ([\#12749](https://github.com/matrix-org/synapse/issues/12749)) +- Add information regarding the `rc_invites` ratelimiting option to the configuration docs. ([\#12759](https://github.com/matrix-org/synapse/issues/12759)) +- Add documentation for cancellation of request processing. ([\#12761](https://github.com/matrix-org/synapse/issues/12761)) +- Recommend using docker to run tests against postgres. ([\#12765](https://github.com/matrix-org/synapse/issues/12765)) +- Add missing user directory endpoint from the generic worker documentation. Contributed by @olmari. ([\#12773](https://github.com/matrix-org/synapse/issues/12773)) +- Add additional info to documentation of config option `cache_autotuning`. ([\#12776](https://github.com/matrix-org/synapse/issues/12776)) +- Update configuration manual documentation to document size-related suffixes. ([\#12777](https://github.com/matrix-org/synapse/issues/12777)) +- Fix invalid YAML syntax in the example documentation for the `url_preview_accept_language` config option. ([\#12785](https://github.com/matrix-org/synapse/issues/12785)) + + +Deprecations and Removals +------------------------- + +- Require a body in POST requests to `/rooms/{roomId}/receipt/{receiptType}/{eventId}`, as required by the [Matrix specification](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3roomsroomidreceiptreceipttypeeventid). This breaks compatibility with Element Android 1.2.0 and earlier: users of those clients will be unable to send read receipts. ([\#12709](https://github.com/matrix-org/synapse/issues/12709)) + + +Internal Changes +---------------- + +- Improve event caching mechanism to avoid having multiple copies of an event in memory at a time. ([\#10533](https://github.com/matrix-org/synapse/issues/10533)) +- Preparation for faster-room-join work: return subsets of room state which we already have, immediately. ([\#12498](https://github.com/matrix-org/synapse/issues/12498)) +- Add `@cancellable` decorator, for use on endpoint methods that can be cancelled when clients disconnect. ([\#12586](https://github.com/matrix-org/synapse/issues/12586), [\#12588](https://github.com/matrix-org/synapse/issues/12588), [\#12630](https://github.com/matrix-org/synapse/issues/12630), [\#12694](https://github.com/matrix-org/synapse/issues/12694), [\#12698](https://github.com/matrix-org/synapse/issues/12698), [\#12699](https://github.com/matrix-org/synapse/issues/12699), [\#12700](https://github.com/matrix-org/synapse/issues/12700), [\#12705](https://github.com/matrix-org/synapse/issues/12705)) +- Enable cancellation of `GET /rooms/$room_id/members`, `GET /rooms/$room_id/state` and `GET /rooms/$room_id/state/$event_type/*` requests. ([\#12708](https://github.com/matrix-org/synapse/issues/12708)) +- Improve documentation of the `synapse.push` module. ([\#12676](https://github.com/matrix-org/synapse/issues/12676)) +- Refactor functions to on `PushRuleEvaluatorForEvent`. ([\#12677](https://github.com/matrix-org/synapse/issues/12677)) +- Preparation for database schema simplifications: stop writing to `event_reference_hashes`. ([\#12679](https://github.com/matrix-org/synapse/issues/12679)) +- Remove code which updates unused database column `application_services_state.last_txn`. ([\#12680](https://github.com/matrix-org/synapse/issues/12680)) +- Refactor `EventContext` class. ([\#12689](https://github.com/matrix-org/synapse/issues/12689)) +- Remove an unneeded class in the push code. ([\#12691](https://github.com/matrix-org/synapse/issues/12691)) +- Consolidate parsing of relation information from events. ([\#12693](https://github.com/matrix-org/synapse/issues/12693)) +- Convert namespace class `Codes` into a string enum. ([\#12703](https://github.com/matrix-org/synapse/issues/12703)) +- Optimize private read receipt filtering. ([\#12711](https://github.com/matrix-org/synapse/issues/12711)) +- Drop the logging level of status messages for the URL preview cache expiry job from INFO to DEBUG. ([\#12720](https://github.com/matrix-org/synapse/issues/12720)) +- Downgrade some OIDC errors to warnings in the logs, to reduce the noise of Sentry reports. ([\#12723](https://github.com/matrix-org/synapse/issues/12723)) +- Update configs used by Complement to allow more invites/3PID validations during tests. ([\#12731](https://github.com/matrix-org/synapse/issues/12731)) +- Fix a long-standing bug where the user directory background process would fail to make forward progress if a user included a null codepoint in their display name or avatar. ([\#12762](https://github.com/matrix-org/synapse/issues/12762)) +- Tweak the mypy plugin so that `@cached` can accept `on_invalidate=None`. ([\#12769](https://github.com/matrix-org/synapse/issues/12769)) +- Move methods that call `add_push_rule` to the `PushRuleStore` class. ([\#12772](https://github.com/matrix-org/synapse/issues/12772)) +- Make handling of federation Authorization header (more) compliant with RFC7230. ([\#12774](https://github.com/matrix-org/synapse/issues/12774)) +- Refactor `resolve_state_groups_for_events` to not pull out full state when no state resolution happens. ([\#12775](https://github.com/matrix-org/synapse/issues/12775)) +- Do not keep going if there are 5 back-to-back background update failures. ([\#12781](https://github.com/matrix-org/synapse/issues/12781)) +- Fix federation when using the demo scripts. ([\#12783](https://github.com/matrix-org/synapse/issues/12783)) +- The `hash_password` script now fails when it is called without specifying a config file. Contributed by @jae1911. ([\#12789](https://github.com/matrix-org/synapse/issues/12789)) +- Improve and fix type hints. ([\#12567](https://github.com/matrix-org/synapse/issues/12567), [\#12477](https://github.com/matrix-org/synapse/issues/12477), [\#12717](https://github.com/matrix-org/synapse/issues/12717), [\#12753](https://github.com/matrix-org/synapse/issues/12753), [\#12695](https://github.com/matrix-org/synapse/issues/12695), [\#12734](https://github.com/matrix-org/synapse/issues/12734), [\#12716](https://github.com/matrix-org/synapse/issues/12716), [\#12726](https://github.com/matrix-org/synapse/issues/12726), [\#12790](https://github.com/matrix-org/synapse/issues/12790), [\#12833](https://github.com/matrix-org/synapse/issues/12833)) +- Update EventContext `get_current_event_ids` and `get_prev_event_ids` to accept state filters and update calls where possible. ([\#12791](https://github.com/matrix-org/synapse/issues/12791)) +- Remove Caddy from the Synapse workers image used in Complement. ([\#12818](https://github.com/matrix-org/synapse/issues/12818)) +- Add Complement's shared registration secret to the Complement worker image. This fixes tests that depend on it. ([\#12819](https://github.com/matrix-org/synapse/issues/12819)) +- Support registering Application Services when running with workers under Complement. ([\#12826](https://github.com/matrix-org/synapse/issues/12826)) +- Disable 'faster room join' Complement tests when testing against Synapse with workers. ([\#12842](https://github.com/matrix-org/synapse/issues/12842)) + + +Synapse 1.59.1 (2022-05-18) +=========================== + +This release fixes a long-standing issue which could prevent Synapse's user directory for updating properly. + +Bugfixes +---------------- + +- Fix a long-standing bug where the user directory background process would fail to make forward progress if a user included a null codepoint in their display name or avatar. Contributed by Nick @ Beeper. ([\#12762](https://github.com/matrix-org/synapse/issues/12762)) + + +Synapse 1.59.0 (2022-05-17) +=========================== + +Synapse 1.59 makes several changes that server administrators should be aware of: + +- Device name lookup over federation is now disabled by default. ([\#12616](https://github.com/matrix-org/synapse/issues/12616)) +- The `synapse.app.appservice` and `synapse.app.user_dir` worker application types are now deprecated. ([\#12452](https://github.com/matrix-org/synapse/issues/12452), [\#12654](https://github.com/matrix-org/synapse/issues/12654)) + +See [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1590) for more details. + +Additionally, this release removes the non-standard `m.login.jwt` login type from Synapse. It can be replaced with `org.matrix.login.jwt` for identical behaviour. This is only used if `jwt_config.enabled` is set to `true` in the configuration. ([\#12597](https://github.com/matrix-org/synapse/issues/12597)) + + +Bugfixes +-------- + +- Fix DB performance regression introduced in Synapse 1.59.0rc2. ([\#12745](https://github.com/matrix-org/synapse/issues/12745)) + + +Synapse 1.59.0rc2 (2022-05-16) +============================== + +Note: this release candidate includes a performance regression which can cause database disruption. Other release candidates in the v1.59.0 series are not affected, and a fix will be included in the v1.59.0 final release. + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.58.0 where `/sync` would fail if the most recent event in a room was rejected. ([\#12729](https://github.com/matrix-org/synapse/issues/12729)) + + +Synapse 1.59.0rc1 (2022-05-10) +============================== + +Features +-------- + +- Support [MSC3266](https://github.com/matrix-org/matrix-doc/pull/3266) room summaries over federation. ([\#11507](https://github.com/matrix-org/synapse/issues/11507)) +- Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. ([\#12168](https://github.com/matrix-org/synapse/issues/12168), [\#12635](https://github.com/matrix-org/synapse/issues/12635), [\#12636](https://github.com/matrix-org/synapse/issues/12636), [\#12670](https://github.com/matrix-org/synapse/issues/12670)) +- Extend the [module API](https://github.com/matrix-org/synapse/blob/release-v1.59/synapse/module_api/__init__.py) to allow modules to change actions for existing push rules of local users. ([\#12406](https://github.com/matrix-org/synapse/issues/12406)) +- Add the `notify_appservices_from_worker` configuration option (superseding `notify_appservices`) to allow a generic worker to be designated as the worker to send traffic to Application Services. ([\#12452](https://github.com/matrix-org/synapse/issues/12452)) +- Add the `update_user_directory_from_worker` configuration option (superseding `update_user_directory`) to allow a generic worker to be designated as the worker to update the user directory. ([\#12654](https://github.com/matrix-org/synapse/issues/12654)) +- Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid. ([\#12526](https://github.com/matrix-org/synapse/issues/12526)) +- Implement [MSC3786](https://github.com/matrix-org/matrix-spec-proposals/pull/3786): Add a default push rule to ignore `m.room.server_acl` events. ([\#12601](https://github.com/matrix-org/synapse/issues/12601)) +- Add new `mau_appservice_trial_days` configuration option to specify a different trial period for users registered via an appservice. ([\#12619](https://github.com/matrix-org/synapse/issues/12619)) + + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.48.0 where the latest thread reply provided failed to include the proper bundled aggregations. ([\#12273](https://github.com/matrix-org/synapse/issues/12273)) +- Fix a bug introduced in Synapse 1.22.0 where attempting to send a large amount of read receipts to an application service all at once would result in duplicate content and abnormally high memory usage. Contributed by Brad & Nick @ Beeper. ([\#12544](https://github.com/matrix-org/synapse/issues/12544)) +- Fix a bug introduced in Synapse 1.57.0 which could cause `Failed to calculate hosts in room` errors to be logged for outbound federation. ([\#12570](https://github.com/matrix-org/synapse/issues/12570)) +- Fix a long-standing bug where status codes would almost always get logged as `200!`, irrespective of the actual status code, when clients disconnect before a request has finished processing. ([\#12580](https://github.com/matrix-org/synapse/issues/12580)) +- Fix race when persisting an event and deleting a room that could lead to outbound federation breaking. ([\#12594](https://github.com/matrix-org/synapse/issues/12594)) +- Fix a bug introduced in Synapse 1.53.0 where bundled aggregations for annotations/edits were incorrectly calculated. ([\#12633](https://github.com/matrix-org/synapse/issues/12633)) +- Fix a long-standing bug where rooms containing power levels with string values could not be upgraded. ([\#12657](https://github.com/matrix-org/synapse/issues/12657)) +- Prevent memory leak from reoccurring when presence is disabled. ([\#12656](https://github.com/matrix-org/synapse/issues/12656)) + + +Updates to the Docker image +--------------------------- + +- Explicitly opt-in to using [BuildKit-specific features](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md) in the Dockerfile. This fixes issues with building images in some GitLab CI environments. ([\#12541](https://github.com/matrix-org/synapse/issues/12541)) +- Update the "Build docker images" GitHub Actions workflow to use `docker/metadata-action` to generate docker image tags, instead of a custom shell script. Contributed by @henryclw. ([\#12573](https://github.com/matrix-org/synapse/issues/12573)) + + +Improved Documentation +---------------------- + +- Update SQL statements and replace use of old table `user_stats_historical` in docs for Synapse Admins. ([\#12536](https://github.com/matrix-org/synapse/issues/12536)) +- Add missing linebreak to `pipx` install instructions. ([\#12579](https://github.com/matrix-org/synapse/issues/12579)) +- Add information about the TCP replication module to docs. ([\#12621](https://github.com/matrix-org/synapse/issues/12621)) +- Fixes to the formatting of `README.rst`. ([\#12627](https://github.com/matrix-org/synapse/issues/12627)) +- Fix docs on how to run specific Complement tests using the `complement.sh` test runner. ([\#12664](https://github.com/matrix-org/synapse/issues/12664)) + + +Deprecations and Removals +------------------------- + +- Remove unstable identifiers from [MSC3069](https://github.com/matrix-org/matrix-doc/pull/3069). ([\#12596](https://github.com/matrix-org/synapse/issues/12596)) +- Remove the unspecified `m.login.jwt` login type and the unstable `uk.half-shot.msc2778.login.application_service` from + [MSC2778](https://github.com/matrix-org/matrix-doc/pull/2778). ([\#12597](https://github.com/matrix-org/synapse/issues/12597)) +- Synapse now requires at least Python 3.7.1 (up from 3.7.0), for compatibility with the latest Twisted trunk. ([\#12613](https://github.com/matrix-org/synapse/issues/12613)) + + +Internal Changes +---------------- + +- Use supervisord to supervise Postgres and Caddy in the Complement image to reduce restart time. ([\#12480](https://github.com/matrix-org/synapse/issues/12480)) +- Immediately retry any requests that have backed off when a server comes back online. ([\#12500](https://github.com/matrix-org/synapse/issues/12500)) +- Use `make_awaitable` instead of `defer.succeed` for return values of mocks in tests. ([\#12505](https://github.com/matrix-org/synapse/issues/12505)) +- Consistently check if an object is a `frozendict`. ([\#12564](https://github.com/matrix-org/synapse/issues/12564)) +- Protect module callbacks with read semantics against cancellation. ([\#12568](https://github.com/matrix-org/synapse/issues/12568)) +- Improve comments and error messages around access tokens. ([\#12577](https://github.com/matrix-org/synapse/issues/12577)) +- Improve docstrings for the receipts store. ([\#12581](https://github.com/matrix-org/synapse/issues/12581)) +- Use constants for read-receipts in tests. ([\#12582](https://github.com/matrix-org/synapse/issues/12582)) +- Log status code of cancelled requests as 499 and avoid logging stack traces for them. ([\#12587](https://github.com/matrix-org/synapse/issues/12587), [\#12663](https://github.com/matrix-org/synapse/issues/12663)) +- Remove special-case for `twisted` logger from default log config. ([\#12589](https://github.com/matrix-org/synapse/issues/12589)) +- Use `getClientAddress` instead of the deprecated `getClientIP`. ([\#12599](https://github.com/matrix-org/synapse/issues/12599)) +- Add link to documentation in Grafana Dashboard. ([\#12602](https://github.com/matrix-org/synapse/issues/12602)) +- Reduce log spam when running multiple event persisters. ([\#12610](https://github.com/matrix-org/synapse/issues/12610)) +- Add extra debug logging to federation sender. ([\#12614](https://github.com/matrix-org/synapse/issues/12614)) +- Prevent remote homeservers from requesting local user device names by default. ([\#12616](https://github.com/matrix-org/synapse/issues/12616)) +- Add a consistency check on events which we read from the database. ([\#12620](https://github.com/matrix-org/synapse/issues/12620)) +- Remove use of the `constantly` library and switch to enums for `EventRedactBehaviour`. Contributed by @andrewdoh. ([\#12624](https://github.com/matrix-org/synapse/issues/12624)) +- Remove unused code related to receipts. ([\#12632](https://github.com/matrix-org/synapse/issues/12632)) +- Minor improvements to the scripts for running Synapse in worker mode under Complement. ([\#12637](https://github.com/matrix-org/synapse/issues/12637)) +- Move `pympler` back in to the `all` extras. ([\#12652](https://github.com/matrix-org/synapse/issues/12652)) +- Fix spelling of `M_UNRECOGNIZED` in comments. ([\#12665](https://github.com/matrix-org/synapse/issues/12665)) +- Release script: confirm the commit to be tagged before tagging. ([\#12556](https://github.com/matrix-org/synapse/issues/12556)) +- Fix a typo in the announcement text generated by the Synapse release development script. ([\#12612](https://github.com/matrix-org/synapse/issues/12612)) + +### Typechecking + +- Fix scripts-dev to pass typechecking. ([\#12356](https://github.com/matrix-org/synapse/issues/12356)) +- Add some type hints to datastore. ([\#12485](https://github.com/matrix-org/synapse/issues/12485)) +- Remove unused `# type: ignore`s. ([\#12531](https://github.com/matrix-org/synapse/issues/12531)) +- Allow unused `# type: ignore` comments in bleeding edge CI jobs. ([\#12576](https://github.com/matrix-org/synapse/issues/12576)) +- Remove redundant lines of config from `mypy.ini`. ([\#12608](https://github.com/matrix-org/synapse/issues/12608)) +- Update to mypy 0.950. ([\#12650](https://github.com/matrix-org/synapse/issues/12650)) +- Use `Concatenate` to better annotate `_do_execute`. ([\#12666](https://github.com/matrix-org/synapse/issues/12666)) +- Use `ParamSpec` to refine type hints. ([\#12667](https://github.com/matrix-org/synapse/issues/12667)) +- Fix mypy against latest pillow stubs. ([\#12671](https://github.com/matrix-org/synapse/issues/12671)) + +Synapse 1.58.1 (2022-05-05) +=========================== + +This patch release includes a fix to the Debian packages, installing the +`systemd` and `cache_memory` extra package groups, which were incorrectly +omitted in v1.58.0. This primarily prevented Synapse from starting +when the `systemd.journal.JournalHandler` log handler was configured. +See [#12631](https://github.com/matrix-org/synapse/issues/12631) for further information. + +Otherwise, no significant changes since 1.58.0. + + +Synapse 1.58.0 (2022-05-03) +=========================== + +As of this release, the groups/communities feature in Synapse is now disabled by default. See [\#11584](https://github.com/matrix-org/synapse/issues/11584) for details. As mentioned in [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1580), this feature will be removed in Synapse 1.61. + +No significant changes since 1.58.0rc2. + + +Synapse 1.58.0rc2 (2022-04-26) +============================== + +This release candidate fixes bugs related to Synapse 1.58.0rc1's logic for handling device list updates. + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.58.0rc1 where the main process could consume excessive amounts of CPU and memory while handling sentry logging failures. ([\#12554](https://github.com/matrix-org/synapse/issues/12554)) +- Fix a bug introduced in Synapse 1.58.0rc1 where opentracing contexts were not correctly sent to whitelisted remote servers with device lists updates. ([\#12555](https://github.com/matrix-org/synapse/issues/12555)) + + +Internal Changes +---------------- + +- Reduce unnecessary work when handling remote device list updates. ([\#12557](https://github.com/matrix-org/synapse/issues/12557)) + + +Synapse 1.58.0rc1 (2022-04-26) +============================== + +Features +-------- + +- Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly. ([\#11398](https://github.com/matrix-org/synapse/issues/11398)) +- Docker images and Debian packages from matrix.org now contain a locked set of Python dependencies, greatly improving build reproducibility. ([Board](https://github.com/orgs/matrix-org/projects/54), [\#11537](https://github.com/matrix-org/synapse/issues/11537)) +- Enable processing of device list updates asynchronously. ([\#12365](https://github.com/matrix-org/synapse/issues/12365), [\#12465](https://github.com/matrix-org/synapse/issues/12465)) +- Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir @ Beeper. ([\#12427](https://github.com/matrix-org/synapse/issues/12427)) +- Build Debian packages for Ubuntu 22.04 "Jammy Jellyfish". ([\#12543](https://github.com/matrix-org/synapse/issues/12543)) + + +Bugfixes +-------- + +- Prevent a sync request from removing a user's busy presence status. ([\#12213](https://github.com/matrix-org/synapse/issues/12213)) +- Fix bug with incremental sync missing events when rejoining/backfilling. Contributed by Nick @ Beeper. ([\#12319](https://github.com/matrix-org/synapse/issues/12319)) +- Fix a long-standing bug which incorrectly caused `GET /_matrix/client/v3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original. ([\#12476](https://github.com/matrix-org/synapse/issues/12476)) +- Fix a bug introduced in Synapse 1.27.0 where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1, no matter how many extremities were deleted. ([\#12496](https://github.com/matrix-org/synapse/issues/12496)) +- Fix a long-standing bug where the image thumbnails embedded into email notifications were broken. ([\#12510](https://github.com/matrix-org/synapse/issues/12510)) +- Fix a bug in the implementation of [MSC3202](https://github.com/matrix-org/matrix-spec-proposals/pull/3202) where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`. ([\#12520](https://github.com/matrix-org/synapse/issues/12520)) +- Fix a bug introduced in Synapse 0.99.3 which could cause Synapse to consume large amounts of RAM when back-paginating in a large room. ([\#12522](https://github.com/matrix-org/synapse/issues/12522)) + + +Improved Documentation +---------------------- + +- Fix rendering of the documentation site when using the 'print' feature. ([\#12340](https://github.com/matrix-org/synapse/issues/12340)) +- Add a manual documenting config file options. ([\#12368](https://github.com/matrix-org/synapse/issues/12368), [\#12527](https://github.com/matrix-org/synapse/issues/12527)) +- Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental. ([\#12451](https://github.com/matrix-org/synapse/issues/12451)) +- Update worker documentation and replace old `federation_reader` with `generic_worker`. ([\#12457](https://github.com/matrix-org/synapse/issues/12457)) +- Strongly recommend [Poetry](https://python-poetry.org/) for development. ([\#12475](https://github.com/matrix-org/synapse/issues/12475)) +- Add some example configurations for workers and update architectural diagram. ([\#12492](https://github.com/matrix-org/synapse/issues/12492)) +- Fix a broken link in `README.rst`. ([\#12495](https://github.com/matrix-org/synapse/issues/12495)) +- Add HAProxy delegation example with CORS headers to docs. ([\#12501](https://github.com/matrix-org/synapse/issues/12501)) +- Remove extraneous comma in User Admin API's device deletion section so that the example JSON is actually valid and works. Contributed by @olmari. ([\#12533](https://github.com/matrix-org/synapse/issues/12533)) + + +Deprecations and Removals +------------------------- + +- The groups/communities feature in Synapse is now disabled by default. ([\#12344](https://github.com/matrix-org/synapse/issues/12344)) +- Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#12382](https://github.com/matrix-org/synapse/issues/12382)) + + +Internal Changes +---------------- + +- Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join. ([\#12394](https://github.com/matrix-org/synapse/issues/12394)) +- Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive. ([\#12399](https://github.com/matrix-org/synapse/issues/12399)) +- Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#12395](https://github.com/matrix-org/synapse/issues/12395)) +- Run CI in the locked [Poetry](https://python-poetry.org/) environment, and remove corresponding `tox` jobs. ([\#12425](https://github.com/matrix-org/synapse/issues/12425), [\#12434](https://github.com/matrix-org/synapse/issues/12434), [\#12438](https://github.com/matrix-org/synapse/issues/12438), [\#12441](https://github.com/matrix-org/synapse/issues/12441), [\#12449](https://github.com/matrix-org/synapse/issues/12449), [\#12478](https://github.com/matrix-org/synapse/issues/12478), [\#12514](https://github.com/matrix-org/synapse/issues/12514), [\#12472](https://github.com/matrix-org/synapse/issues/12472)) +- Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current iteration of [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666). ([\#12445](https://github.com/matrix-org/synapse/issues/12445)) +- Fix typo in the release script help string. ([\#12450](https://github.com/matrix-org/synapse/issues/12450)) +- Fix a minor typo in the Debian changelogs generated by the release script. ([\#12497](https://github.com/matrix-org/synapse/issues/12497)) +- Reintroduce the list of targets to the linter script, to avoid linting unwanted local-only directories during development. ([\#12455](https://github.com/matrix-org/synapse/issues/12455)) +- Limit length of `device_id` to less than 512 characters. ([\#12454](https://github.com/matrix-org/synapse/issues/12454)) +- Dockerfile-workers: reduce the amount we install in the image. ([\#12464](https://github.com/matrix-org/synapse/issues/12464)) +- Dockerfile-workers: give the master its own log config. ([\#12466](https://github.com/matrix-org/synapse/issues/12466)) +- complement-synapse-workers: factor out separate entry point script. ([\#12467](https://github.com/matrix-org/synapse/issues/12467)) +- Back out experimental implementation of [MSC2314](https://github.com/matrix-org/matrix-spec-proposals/pull/2314). ([\#12474](https://github.com/matrix-org/synapse/issues/12474)) +- Fix grammatical error in federation error response when the room version of a room is unknown. ([\#12483](https://github.com/matrix-org/synapse/issues/12483)) +- Remove unnecessary configuration overrides in tests. ([\#12511](https://github.com/matrix-org/synapse/issues/12511)) +- Refactor the relations code for clarity. ([\#12519](https://github.com/matrix-org/synapse/issues/12519)) +- Add type hints so `docker` and `stubs` directories pass `mypy --disallow-untyped-defs`. ([\#12528](https://github.com/matrix-org/synapse/issues/12528)) +- Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s. ([\#12468](https://github.com/matrix-org/synapse/issues/12468)) +- Handle cancellation in `EventsWorkerStore._get_events_from_cache_or_db`. ([\#12529](https://github.com/matrix-org/synapse/issues/12529)) + + +Synapse 1.57.1 (2022-04-20) +=========================== + +This is a patch release that only affects the Docker image. It is only of interest to administrators using [the LDAP module][LDAPModule] to authenticate their users. +If you have already upgraded to Synapse 1.57.0 without problem, then you have no need to upgrade to this patch release. + +[LDAPModule]: https://github.com/matrix-org/matrix-synapse-ldap3 + + +Updates to the Docker image +--------------------------- + +- Include version 0.2.0 of the Synapse LDAP Auth Provider module in the Docker image. This matches the version that was present in the Docker image for Synapse v1.56.0. ([\#12512](https://github.com/matrix-org/synapse/issues/12512)) + + Synapse 1.57.0 (2022-04-19) =========================== diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index d744c090acde..000000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,54 +0,0 @@ -include LICENSE -include VERSION -include *.rst -include *.md -include demo/README -include demo/demo.tls.dh -include demo/*.py -include demo/*.sh - -include synapse/py.typed -recursive-include synapse/storage *.sql -recursive-include synapse/storage *.sql.postgres -recursive-include synapse/storage *.sql.sqlite -recursive-include synapse/storage *.py -recursive-include synapse/storage *.txt -recursive-include synapse/storage *.md - -recursive-include docs * -recursive-include scripts-dev * -recursive-include synapse *.pyi -recursive-include tests *.py -recursive-include tests *.pem -recursive-include tests *.p8 -recursive-include tests *.crt -recursive-include tests *.key - -recursive-include synapse/res * -recursive-include synapse/static *.css -recursive-include synapse/static *.gif -recursive-include synapse/static *.html -recursive-include synapse/static *.js - -exclude .codecov.yml -exclude .coveragerc -exclude .dockerignore -exclude .editorconfig -exclude Dockerfile -exclude mypy.ini -exclude sytest-blacklist -exclude test_postgresql.sh - -include book.toml -include pyproject.toml -recursive-include changelog.d * - -include .flake8 -prune .circleci -prune .github -prune .ci -prune contrib -prune debian -prune demo/etc -prune docker -prune stubs diff --git a/README.rst b/README.rst index 595fb5ff62a6..219e32de8ea4 100644 --- a/README.rst +++ b/README.rst @@ -55,7 +55,7 @@ solutions. The hope is for Matrix to act as the building blocks for a new generation of fully open and interoperable messaging and VoIP apps for the internet. -Synapse is a Matrix "homeserver" implementation developed by the matrix.org core +Synapse is a Matrix "homeserver" implementation developed by the matrix.org core team, written in Python 3/Twisted. In Matrix, every user runs one or more Matrix clients, which connect through to @@ -293,39 +293,42 @@ directory of your choice:: git clone https://github.com/matrix-org/synapse.git cd synapse -Synapse has a number of external dependencies, that are easiest -to install using pip and a virtualenv:: +Synapse has a number of external dependencies. We maintain a fixed development +environment using `Poetry `_. First, install poetry. We recommend:: - python3 -m venv ./env - source ./env/bin/activate - pip install -e ".[all,dev]" + pip install --user pipx + pipx install poetry -This will run a process of downloading and installing all the needed -dependencies into a virtual env. If any dependencies fail to install, -try installing the failing modules individually:: +as described `here `_. +(See `poetry's installation docs `_ +for other installation methods.) Then ask poetry to create a virtual environment +from the project and install Synapse's dependencies:: + + poetry install --extras "all test" - pip install -e "module-name" +This will run a process of downloading and installing all the needed +dependencies into a virtual env. -We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082` +We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`:: - ./demo/start.sh + poetry run ./demo/start.sh -(to stop, you can use `./demo/stop.sh`) +(to stop, you can use ``poetry run ./demo/stop.sh``) -See the [demo documentation](https://matrix-org.github.io/synapse/develop/development/demo.html) +See the `demo documentation `_ for more information. If you just want to start a single instance of the app and run it directly:: # Create the homeserver.yaml config once - python -m synapse.app.homeserver \ + poetry run synapse_homeserver \ --server-name my.domain.name \ --config-path homeserver.yaml \ --generate-config \ --report-stats=[yes|no] # Start the app - python -m synapse.app.homeserver --config-path homeserver.yaml + poetry run synapse_homeserver --config-path homeserver.yaml Running the unit tests @@ -334,7 +337,7 @@ Running the unit tests After getting up and running, you may wish to run Synapse's unit tests to check that everything is installed correctly:: - trial tests + poetry run trial tests This should end with a 'PASSED' result (note that exact numbers will differ):: diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json index 2c839c30d036..819426b8ea2b 100644 --- a/contrib/grafana/synapse.json +++ b/contrib/grafana/synapse.json @@ -66,6 +66,18 @@ ], "title": "Dashboards", "type": "dashboards" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Synapse Documentation", + "tooltip": "Open Documentation", + "type": "link", + "url": "https://matrix-org.github.io/synapse/latest/" } ], "panels": [ @@ -10889,4 +10901,4 @@ "title": "Synapse", "uid": "000000012", "version": 100 -} \ No newline at end of file +} diff --git a/debian/build_virtualenv b/debian/build_virtualenv index e6911636192c..f1ec60916325 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -30,9 +30,23 @@ case $(dpkg-architecture -q DEB_HOST_ARCH) in ;; esac -# Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather -# than the 2/3 compatible `virtualenv`. - +# Manually install Poetry and export a pip-compatible `requirements.txt` +# We need a Poetry pre-release as the export command is buggy in < 1.2 +TEMP_VENV="$(mktemp -d)" +python3 -m venv "$TEMP_VENV" +source "$TEMP_VENV/bin/activate" +pip install -U pip +pip install poetry==1.2.0b1 +poetry export \ + --extras all \ + --extras test \ + --extras systemd \ + -o exported_requirements.txt +deactivate +rm -rf "$TEMP_VENV" + +# Use --no-deps to only install pinned versions in exported_requirements.txt, +# and to avoid https://github.com/pypa/pip/issues/9644 dh_virtualenv \ --install-suffix "matrix-synapse" \ --builtin-venv \ @@ -41,9 +55,11 @@ dh_virtualenv \ --preinstall="lxml" \ --preinstall="mock" \ --preinstall="wheel" \ + --extra-pip-arg="--no-deps" \ --extra-pip-arg="--no-cache-dir" \ --extra-pip-arg="--compile" \ - --extras="all,systemd,test" + --extras="all,systemd,test" \ + --requirements="exported_requirements.txt" PACKAGE_BUILD_DIR="debian/matrix-synapse-py3" VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse" diff --git a/debian/changelog b/debian/changelog index 71dcf9de8e3e..5d332cedefd7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,81 @@ +matrix-synapse-py3 (1.60.0) stable; urgency=medium + + * New Synapse release 1.60.0. + + -- Synapse Packaging team Tue, 31 May 2022 13:41:22 +0100 + +matrix-synapse-py3 (1.60.0~rc2) stable; urgency=medium + + * New Synapse release 1.60.0rc2. + + -- Synapse Packaging team Fri, 27 May 2022 11:04:55 +0100 + +matrix-synapse-py3 (1.60.0~rc1) stable; urgency=medium + + * New Synapse release 1.60.0rc1. + + -- Synapse Packaging team Tue, 24 May 2022 12:05:01 +0100 + +matrix-synapse-py3 (1.59.1) stable; urgency=medium + + * New Synapse release 1.59.1. + + -- Synapse Packaging team Wed, 18 May 2022 11:41:46 +0100 + +matrix-synapse-py3 (1.59.0) stable; urgency=medium + + * New Synapse release 1.59.0. + + -- Synapse Packaging team Tue, 17 May 2022 10:26:50 +0100 + +matrix-synapse-py3 (1.59.0~rc2) stable; urgency=medium + + * New Synapse release 1.59.0rc2. + + -- Synapse Packaging team Mon, 16 May 2022 12:52:15 +0100 + +matrix-synapse-py3 (1.59.0~rc1) stable; urgency=medium + + * Adjust how the `exported-requirements.txt` file is generated as part of + the process of building these packages. This affects the package + maintainers only; end-users are unaffected. + * New Synapse release 1.59.0rc1. + + -- Synapse Packaging team Tue, 10 May 2022 10:45:08 +0100 + +matrix-synapse-py3 (1.58.1) stable; urgency=medium + + * Include python dependencies from the `systemd` and `cache_memory` extras package groups, which + were incorrectly omitted from the 1.58.0 package. + * New Synapse release 1.58.1. + + -- Synapse Packaging team Thu, 05 May 2022 14:58:23 +0100 + +matrix-synapse-py3 (1.58.0) stable; urgency=medium + + * New Synapse release 1.58.0. + + -- Synapse Packaging team Tue, 03 May 2022 10:52:58 +0100 + +matrix-synapse-py3 (1.58.0~rc2) stable; urgency=medium + + * New Synapse release 1.58.0rc2. + + -- Synapse Packaging team Tue, 26 Apr 2022 17:14:56 +0100 + +matrix-synapse-py3 (1.58.0~rc1) stable; urgency=medium + + * Use poetry to manage the bundled virtualenv included with this package. + * New Synapse release 1.58.0rc1. + + -- Synapse Packaging team Tue, 26 Apr 2022 11:15:20 +0100 + +matrix-synapse-py3 (1.57.1) stable; urgency=medium + + * New synapse release 1.57.1. + + -- Synapse Packaging team Wed, 20 Apr 2022 15:27:21 +0100 + matrix-synapse-py3 (1.57.0) stable; urgency=medium * New synapse release 1.57.0. diff --git a/debian/clean b/debian/clean new file mode 100644 index 000000000000..d488f298d587 --- /dev/null +++ b/debian/clean @@ -0,0 +1 @@ +exported_requirements.txt diff --git a/demo/start.sh b/demo/start.sh index 5a9972d24c2c..96b3a2ceab2f 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -12,6 +12,7 @@ export PYTHONPATH echo "$PYTHONPATH" +# Create servers which listen on HTTP at 808x and HTTPS at 848x. for port in 8080 8081 8082; do echo "Starting server on port $port... " @@ -19,10 +20,12 @@ for port in 8080 8081 8082; do mkdir -p demo/$port pushd demo/$port || exit - # Generate the configuration for the homeserver at localhost:848x. + # Generate the configuration for the homeserver at localhost:848x, note that + # the homeserver name needs to match the HTTPS listening port for federation + # to properly work.. python3 -m synapse.app.homeserver \ --generate-config \ - --server-name "localhost:$port" \ + --server-name "localhost:$https_port" \ --config-path "$port.config" \ --report-stats no diff --git a/docker/Dockerfile b/docker/Dockerfile index 56d01c06883d..1d313cb57a8a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 # Dockerfile to build the matrixdotorg/synapse docker images. # # Note that it uses features which are only available in BuildKit - see @@ -46,12 +47,13 @@ RUN apt-get update && apt-get install -y git \ # # NB: In poetry 1.2 `poetry export` will be moved into a plugin; we'll need to also # pip install poetry-plugin-export (https://github.com/python-poetry/poetry-plugin-export). -RUN pip install --user git+https://github.com/python-poetry/poetry.git@fb13b3a676f476177f7937ffa480ee5cff9a90a5 +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --user "poetry-core==1.1.0a7" "git+https://github.com/python-poetry/poetry.git@fb13b3a676f476177f7937ffa480ee5cff9a90a5" WORKDIR /synapse # Copy just what we need to run `poetry export`... -COPY pyproject.toml poetry.lock README.rst /synapse/ +COPY pyproject.toml poetry.lock /synapse/ RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt @@ -86,9 +88,7 @@ RUN pip install --prefix="/install" --no-deps --no-warn-script-location -r /syna # Copy over the rest of the synapse source code. COPY synapse /synapse/synapse/ # ... and what we need to `pip install`. -# TODO: once pyproject.toml declares poetry-core as its build system, we'll need to copy -# pyproject.toml here, ditching setup.py and MANIFEST.in. -COPY setup.py MANIFEST.in README.rst /synapse/ +COPY pyproject.toml README.rst /synapse/ # Install the synapse package itself. RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers index 6fb1cdbfb020..24b03585f9a2 100644 --- a/docker/Dockerfile-workers +++ b/docker/Dockerfile-workers @@ -2,15 +2,27 @@ FROM matrixdotorg/synapse # Install deps -RUN apt-get update -RUN apt-get install -y supervisor redis nginx +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + redis-server nginx-light -# Remove the default nginx sites +# Install supervisord with pip instead of apt, to avoid installing a second +# copy of python. +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install supervisor~=4.2 + +# Disable the default nginx sites RUN rm /etc/nginx/sites-enabled/default # Copy Synapse worker, nginx and supervisord configuration template files COPY ./docker/conf-workers/* /conf/ +# Copy a script to prefix log lines with the supervisor program name +COPY ./docker/prefix-log /usr/local/bin/ + # Expose nginx listener port EXPOSE 8080/tcp @@ -19,5 +31,7 @@ EXPOSE 8080/tcp COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py ENTRYPOINT ["/configure_workers_and_start.py"] +# Replace the healthcheck with one which checks *all* the workers. The script +# is generated by configure_workers_and_start.py. HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \ CMD /bin/sh /healthcheck.sh diff --git a/docker/complement/SynapseWorkers.Dockerfile b/docker/complement/SynapseWorkers.Dockerfile index 982219a91e7e..99a09cbc2bab 100644 --- a/docker/complement/SynapseWorkers.Dockerfile +++ b/docker/complement/SynapseWorkers.Dockerfile @@ -6,15 +6,9 @@ # https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse FROM matrixdotorg/synapse-workers -# Download a caddy server to stand in front of nginx and terminate TLS using Complement's -# custom CA. -# We include this near the top of the file in order to cache the result. -RUN curl -OL "https://github.com/caddyserver/caddy/releases/download/v2.3.0/caddy_2.3.0_linux_amd64.tar.gz" && \ - tar xzf caddy_2.3.0_linux_amd64.tar.gz && rm caddy_2.3.0_linux_amd64.tar.gz && mv caddy /root - # Install postgresql -RUN apt-get update -RUN apt-get install -y postgresql +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y postgresql-13 # Configure a user and create a database for Synapse RUN pg_ctlcluster 13 main start && su postgres -c "echo \ @@ -31,43 +25,16 @@ COPY conf-workers/workers-shared.yaml /conf/workers/shared.yaml WORKDIR /data -# Copy the caddy config -COPY conf-workers/caddy.complement.json /root/caddy.json +COPY conf-workers/postgres.supervisord.conf /etc/supervisor/conf.d/postgres.conf + +# Copy the entrypoint +COPY conf-workers/start-complement-synapse-workers.sh / -# Expose caddy's listener ports +# Expose nginx's listener ports EXPOSE 8008 8448 -ENTRYPOINT \ - # Replace the server name in the caddy config - sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json && \ - # Start postgres - pg_ctlcluster 13 main start 2>&1 && \ - # Start caddy - /root/caddy start --config /root/caddy.json 2>&1 && \ - # Set the server name of the homeserver - SYNAPSE_SERVER_NAME=${SERVER_NAME} \ - # No need to report stats here - SYNAPSE_REPORT_STATS=no \ - # Set postgres authentication details which will be placed in the homeserver config file - POSTGRES_PASSWORD=somesecret POSTGRES_USER=postgres POSTGRES_HOST=localhost \ - # Specify the workers to test with - SYNAPSE_WORKER_TYPES="\ - event_persister, \ - event_persister, \ - background_worker, \ - frontend_proxy, \ - event_creator, \ - user_dir, \ - media_repository, \ - federation_inbound, \ - federation_reader, \ - federation_sender, \ - synchrotron, \ - appservice, \ - pusher" \ - # Run the script that writes the necessary config files and starts supervisord, which in turn - # starts everything else - /configure_workers_and_start.py +ENTRYPOINT ["/start-complement-synapse-workers.sh"] +# Update the healthcheck to have a shorter check interval HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \ CMD /bin/sh /healthcheck.sh diff --git a/docker/complement/conf-workers/caddy.complement.json b/docker/complement/conf-workers/caddy.complement.json deleted file mode 100644 index 09e2136af2e2..000000000000 --- a/docker/complement/conf-workers/caddy.complement.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "apps": { - "http": { - "servers": { - "srv0": { - "listen": [ - ":8448" - ], - "routes": [ - { - "match": [ - { - "host": [ - "{{ server_name }}" - ] - } - ], - "handle": [ - { - "handler": "subroute", - "routes": [ - { - "handle": [ - { - "handler": "reverse_proxy", - "upstreams": [ - { - "dial": "localhost:8008" - } - ] - } - ] - } - ] - } - ], - "terminal": true - } - ] - } - } - }, - "tls": { - "automation": { - "policies": [ - { - "subjects": [ - "{{ server_name }}" - ], - "issuers": [ - { - "module": "internal" - } - ], - "on_demand": true - } - ] - } - }, - "pki": { - "certificate_authorities": { - "local": { - "name": "Complement CA", - "root": { - "certificate": "/complement/ca/ca.crt", - "private_key": "/complement/ca/ca.key" - } - } - } - } - } - } diff --git a/docker/complement/conf-workers/postgres.supervisord.conf b/docker/complement/conf-workers/postgres.supervisord.conf new file mode 100644 index 000000000000..5608342d1a9e --- /dev/null +++ b/docker/complement/conf-workers/postgres.supervisord.conf @@ -0,0 +1,16 @@ +[program:postgres] +command=/usr/local/bin/prefix-log /usr/bin/pg_ctlcluster 13 main start --foreground + +# Lower priority number = starts first +priority=1 + +autorestart=unexpected +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +# Use 'Fast Shutdown' mode which aborts current transactions and closes connections quickly. +# (Default (TERM) is 'Smart Shutdown' which stops accepting new connections but +# lets existing connections close gracefully.) +stopsignal=INT diff --git a/docker/complement/conf-workers/start-complement-synapse-workers.sh b/docker/complement/conf-workers/start-complement-synapse-workers.sh new file mode 100755 index 000000000000..b7e24440006f --- /dev/null +++ b/docker/complement/conf-workers/start-complement-synapse-workers.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# +# Default ENTRYPOINT for the docker image used for testing synapse with workers under complement + +set -e + +function log { + d=$(date +"%Y-%m-%d %H:%M:%S,%3N") + echo "$d $@" +} + +# Set the server name of the homeserver +export SYNAPSE_SERVER_NAME=${SERVER_NAME} + +# No need to report stats here +export SYNAPSE_REPORT_STATS=no + +# Set postgres authentication details which will be placed in the homeserver config file +export POSTGRES_PASSWORD=somesecret +export POSTGRES_USER=postgres +export POSTGRES_HOST=localhost + +# Specify the workers to test with +export SYNAPSE_WORKER_TYPES="\ + event_persister, \ + event_persister, \ + background_worker, \ + frontend_proxy, \ + event_creator, \ + user_dir, \ + media_repository, \ + federation_inbound, \ + federation_reader, \ + federation_sender, \ + synchrotron, \ + appservice, \ + pusher" + +# Add Complement's appservice registration directory, if there is one +# (It can be absent when there are no application services in this test!) +if [ -d /complement/appservice ]; then + export SYNAPSE_AS_REGISTRATION_DIR=/complement/appservice +fi + +# Generate a TLS key, then generate a certificate by having Complement's CA sign it +# Note that both the key and certificate are in PEM format (not DER). +openssl genrsa -out /conf/server.tls.key 2048 + +openssl req -new -key /conf/server.tls.key -out /conf/server.tls.csr \ + -subj "/CN=${SERVER_NAME}" + +openssl x509 -req -in /conf/server.tls.csr \ + -CA /complement/ca/ca.crt -CAkey /complement/ca/ca.key -set_serial 1 \ + -out /conf/server.tls.crt + +export SYNAPSE_TLS_CERT=/conf/server.tls.crt +export SYNAPSE_TLS_KEY=/conf/server.tls.key + +# Run the script that writes the necessary config files and starts supervisord, which in turn +# starts everything else +exec /configure_workers_and_start.py diff --git a/docker/complement/conf-workers/workers-shared.yaml b/docker/complement/conf-workers/workers-shared.yaml index 8b6987037715..cd7b50c65cc3 100644 --- a/docker/complement/conf-workers/workers-shared.yaml +++ b/docker/complement/conf-workers/workers-shared.yaml @@ -5,6 +5,12 @@ enable_registration: true enable_registration_without_verification: true bcrypt_rounds: 4 +## Registration ## + +# Needed by Complement to register admin users +# DO NOT USE in a production configuration! This should be a random secret. +registration_shared_secret: complement + ## Federation ## # trust certs signed by Complement's CA @@ -53,6 +59,18 @@ rc_joins: per_second: 9999 burst_count: 9999 +rc_3pid_validation: + per_second: 1000 + burst_count: 1000 + +rc_invites: + per_room: + per_second: 1000 + burst_count: 1000 + per_user: + per_second: 1000 + burst_count: 1000 + federation_rr_transactions_per_room_per_second: 9999 ## Experimental Features ## diff --git a/docker/complement/conf/homeserver.yaml b/docker/complement/conf/homeserver.yaml index c9d6a312401a..e2be540bbb9e 100644 --- a/docker/complement/conf/homeserver.yaml +++ b/docker/complement/conf/homeserver.yaml @@ -87,6 +87,18 @@ rc_joins: per_second: 9999 burst_count: 9999 +rc_3pid_validation: + per_second: 1000 + burst_count: 1000 + +rc_invites: + per_room: + per_second: 1000 + burst_count: 1000 + per_user: + per_second: 1000 + burst_count: 1000 + federation_rr_transactions_per_room_per_second: 9999 ## API Configuration ## @@ -103,8 +115,10 @@ experimental_features: spaces_enabled: true # Enable history backfilling support msc2716_enabled: true - # server-side support for partial state in /send_join + # server-side support for partial state in /send_join responses msc3706_enabled: true + # client-side support for partial state in /send_join responses + faster_joins: true # Enable jump to date endpoint msc3030_enabled: true diff --git a/docker/conf-workers/nginx.conf.j2 b/docker/conf-workers/nginx.conf.j2 index 1081979e06a0..967fc65e798c 100644 --- a/docker/conf-workers/nginx.conf.j2 +++ b/docker/conf-workers/nginx.conf.j2 @@ -9,6 +9,22 @@ server { listen 8008; listen [::]:8008; + {% if tls_cert_path is not none and tls_key_path is not none %} + listen 8448 ssl; + listen [::]:8448 ssl; + + ssl_certificate {{ tls_cert_path }}; + ssl_certificate_key {{ tls_key_path }}; + + # Some directives from cipherlist.eu (fka cipherli.st): + ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers on; + ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; + ssl_ecdh_curve secp384r1; # Requires nginx >= 1.1.0 + ssl_session_cache shared:SSL:10m; + ssl_session_tickets off; # Requires nginx >= 1.5.9 + {% endif %} + server_name localhost; # Nginx by default only allows file uploads up to 1M in size diff --git a/docker/conf-workers/shared.yaml.j2 b/docker/conf-workers/shared.yaml.j2 index f94b8c6aca0f..644ed788f3d5 100644 --- a/docker/conf-workers/shared.yaml.j2 +++ b/docker/conf-workers/shared.yaml.j2 @@ -6,4 +6,13 @@ redis: enabled: true -{{ shared_worker_config }} \ No newline at end of file +{% if appservice_registrations is not none %} +## Application Services ## +# A list of application service config files to use. +app_service_config_files: +{%- for path in appservice_registrations %} + - "{{ path }}" +{%- endfor %} +{%- endif %} + +{{ shared_worker_config }} diff --git a/docker/conf-workers/supervisord.conf.j2 b/docker/conf-workers/supervisord.conf.j2 index 0de2c6143b5a..ca1f7aef8e3f 100644 --- a/docker/conf-workers/supervisord.conf.j2 +++ b/docker/conf-workers/supervisord.conf.j2 @@ -5,8 +5,11 @@ nodaemon=true user=root +[include] +files = /etc/supervisor/conf.d/*.conf + [program:nginx] -command=/usr/sbin/nginx -g "daemon off;" +command=/usr/local/bin/prefix-log /usr/sbin/nginx -g "daemon off;" priority=500 stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 @@ -16,7 +19,7 @@ username=www-data autorestart=true [program:redis] -command=/usr/bin/redis-server /etc/redis/redis.conf --daemonize no +command=/usr/local/bin/prefix-log /usr/bin/redis-server /etc/redis/redis.conf --daemonize no priority=1 stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 @@ -26,7 +29,7 @@ username=redis autorestart=true [program:synapse_main] -command=/usr/local/bin/python -m synapse.app.homeserver --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml +command=/usr/local/bin/prefix-log /usr/local/bin/python -m synapse.app.homeserver --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml priority=10 # Log startup failures to supervisord's stdout/err # Regular synapse logs will still go in the configured data directory diff --git a/docker/conf/log.config b/docker/conf/log.config index 7a216a36a046..dc8c70befd40 100644 --- a/docker/conf/log.config +++ b/docker/conf/log.config @@ -2,11 +2,7 @@ version: 1 formatters: precise: -{% if worker_name %} - format: '%(asctime)s - worker:{{ worker_name }} - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' -{% else %} format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' -{% endif %} handlers: {% if LOG_FILE_PATH %} diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 3e91024e8c92..b6ad14117325 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -21,6 +21,11 @@ # * SYNAPSE_REPORT_STATS: Whether to report stats. # * SYNAPSE_WORKER_TYPES: A comma separated list of worker names as specified in WORKER_CONFIG # below. Leave empty for no workers, or set to '*' for all possible workers. +# * SYNAPSE_AS_REGISTRATION_DIR: If specified, a directory in which .yaml and .yml files +# will be treated as Application Service registration files. +# * SYNAPSE_TLS_CERT: Path to a TLS certificate in PEM format. +# * SYNAPSE_TLS_KEY: Path to a TLS key. If this and SYNAPSE_TLS_CERT are specified, +# Nginx will be configured to serve TLS on port 8448. # # NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined # in the project's README), this script may be run multiple times, and functionality should @@ -29,7 +34,8 @@ import os import subprocess import sys -from typing import Any, Dict, Set +from pathlib import Path +from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Set import jinja2 import yaml @@ -69,10 +75,10 @@ "worker_extra_conf": "enable_media_repo: true", }, "appservice": { - "app": "synapse.app.appservice", + "app": "synapse.app.generic_worker", "listener_resources": [], "endpoint_patterns": [], - "shared_extra_conf": {"notify_appservices": False}, + "shared_extra_conf": {"notify_appservices_from_worker": "appservice"}, "worker_extra_conf": "", }, "federation_sender": { @@ -171,7 +177,7 @@ # Templates for sections that may be inserted multiple times in config files SUPERVISORD_PROCESS_CONFIG_BLOCK = """ [program:synapse_{name}] -command=/usr/local/bin/python -m {app} \ +command=/usr/local/bin/prefix-log /usr/local/bin/python -m {app} \ --config-path="{config_path}" \ --config-path=/conf/workers/shared.yaml \ --config-path=/conf/workers/{name}.yaml @@ -201,7 +207,7 @@ # Utility functions -def log(txt: str): +def log(txt: str) -> None: """Log something to the stdout. Args: @@ -210,7 +216,7 @@ def log(txt: str): print(txt) -def error(txt: str): +def error(txt: str) -> NoReturn: """Log something and exit with an error code. Args: @@ -220,7 +226,7 @@ def error(txt: str): sys.exit(2) -def convert(src: str, dst: str, **template_vars): +def convert(src: str, dst: str, **template_vars: object) -> None: """Generate a file from a template Args: @@ -290,7 +296,7 @@ def add_sharding_to_shared_config( shared_config.setdefault("media_instance_running_background_jobs", worker_name) -def generate_base_homeserver_config(): +def generate_base_homeserver_config() -> None: """Starts Synapse and generates a basic homeserver config, which will later be modified for worker support. @@ -302,12 +308,14 @@ def generate_base_homeserver_config(): subprocess.check_output(["/usr/local/bin/python", "/start.py", "migrate_config"]) -def generate_worker_files(environ, config_path: str, data_dir: str): +def generate_worker_files( + environ: Mapping[str, str], config_path: str, data_dir: str +) -> None: """Read the desired list of workers from environment variables and generate shared homeserver, nginx and supervisord configs. Args: - environ: _Environ[str] + environ: os.environ instance. config_path: The location of the generated Synapse main worker config file. data_dir: The location of the synapse data directory. Where log and user-facing config files live. @@ -341,7 +349,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str): # base shared worker jinja2 template. # # This config file will be passed to all workers, included Synapse's main process. - shared_config = {"listeners": listeners} + shared_config: Dict[str, Any] = {"listeners": listeners} # The supervisord config. The contents of which will be inserted into the # base supervisord jinja2 template. @@ -369,13 +377,13 @@ def generate_worker_files(environ, config_path: str, data_dir: str): nginx_locations = {} # Read the desired worker configuration from the environment - worker_types = environ.get("SYNAPSE_WORKER_TYPES") - if worker_types is None: + worker_types_env = environ.get("SYNAPSE_WORKER_TYPES") + if worker_types_env is None: # No workers, just the main process worker_types = [] else: # Split type names by comma - worker_types = worker_types.split(",") + worker_types = worker_types_env.split(",") # Create the worker configuration directory if it doesn't already exist os.makedirs("/conf/workers", exist_ok=True) @@ -446,21 +454,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str): # Write out the worker's logging config file - # Check whether we should write worker logs to disk, in addition to the console - extra_log_template_args = {} - if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"): - extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format( - dir=data_dir, name=worker_name - ) - - # Render and write the file - log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name) - convert( - "/conf/log.config", - log_config_filepath, - worker_name=worker_name, - **extra_log_template_args, - ) + log_config_filepath = generate_worker_log_config(environ, worker_name, data_dir) # Then a worker config file convert( @@ -496,11 +490,27 @@ def generate_worker_files(environ, config_path: str, data_dir: str): # Finally, we'll write out the config files. + # log config for the master process + master_log_config = generate_worker_log_config(environ, "master", data_dir) + shared_config["log_config"] = master_log_config + + # Find application service registrations + appservice_registrations = None + appservice_registration_dir = os.environ.get("SYNAPSE_AS_REGISTRATION_DIR") + if appservice_registration_dir: + # Scan for all YAML files that should be application service registrations. + appservice_registrations = [ + str(reg_path.resolve()) + for reg_path in Path(appservice_registration_dir).iterdir() + if reg_path.suffix.lower() in (".yaml", ".yml") + ] + # Shared homeserver config convert( "/conf/shared.yaml.j2", "/conf/workers/shared.yaml", shared_worker_config=yaml.dump(shared_config), + appservice_registrations=appservice_registrations, ) # Nginx config @@ -509,12 +519,15 @@ def generate_worker_files(environ, config_path: str, data_dir: str): "/etc/nginx/conf.d/matrix-synapse.conf", worker_locations=nginx_location_config, upstream_directives=nginx_upstream_config, + tls_cert_path=os.environ.get("SYNAPSE_TLS_CERT"), + tls_key_path=os.environ.get("SYNAPSE_TLS_KEY"), ) # Supervisord config + os.makedirs("/etc/supervisor", exist_ok=True) convert( "/conf/supervisord.conf.j2", - "/etc/supervisor/conf.d/supervisord.conf", + "/etc/supervisor/supervisord.conf", main_config_path=config_path, worker_config=supervisord_config, ) @@ -532,15 +545,31 @@ def generate_worker_files(environ, config_path: str, data_dir: str): os.mkdir(log_dir) -def start_supervisord(): - """Starts up supervisord which then starts and monitors all other necessary processes +def generate_worker_log_config( + environ: Mapping[str, str], worker_name: str, data_dir: str +) -> str: + """Generate a log.config file for the given worker. - Raises: CalledProcessError if calling start.py return a non-zero exit code. + Returns: the path to the generated file """ - subprocess.run(["/usr/bin/supervisord"], stdin=subprocess.PIPE) + # Check whether we should write worker logs to disk, in addition to the console + extra_log_template_args = {} + if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"): + extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format( + dir=data_dir, name=worker_name + ) + # Render and write the file + log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name) + convert( + "/conf/log.config", + log_config_filepath, + worker_name=worker_name, + **extra_log_template_args, + ) + return log_config_filepath -def main(args, environ): +def main(args: List[str], environ: MutableMapping[str, str]) -> None: config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml") data_dir = environ.get("SYNAPSE_DATA_DIR", "/data") @@ -567,7 +596,13 @@ def main(args, environ): # Start supervisord, which will start Synapse, all of the configured worker # processes, redis, nginx etc. according to the config we created above. - start_supervisord() + log("Starting supervisord") + os.execl( + "/usr/local/bin/supervisord", + "supervisord", + "-c", + "/etc/supervisor/supervisord.conf", + ) if __name__ == "__main__": diff --git a/docker/prefix-log b/docker/prefix-log new file mode 100755 index 000000000000..0e26a4f19d33 --- /dev/null +++ b/docker/prefix-log @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Prefixes all lines on stdout and stderr with the process name (as determined by +# the SUPERVISOR_PROCESS_NAME env var, which is automatically set by Supervisor). +# +# Usage: +# prefix-log command [args...] +# + +exec 1> >(awk '{print "'"${SUPERVISOR_PROCESS_NAME}"' | "$0}' >&1) +exec 2> >(awk '{print "'"${SUPERVISOR_PROCESS_NAME}"' | "$0}' >&2) +exec "$@" diff --git a/docker/start.py b/docker/start.py index ac62bbc8baf9..4ac8f034777f 100755 --- a/docker/start.py +++ b/docker/start.py @@ -6,27 +6,28 @@ import platform import subprocess import sys +from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Optional import jinja2 # Utility functions -def log(txt): +def log(txt: str) -> None: print(txt, file=sys.stderr) -def error(txt): +def error(txt: str) -> NoReturn: log(txt) sys.exit(2) -def convert(src, dst, environ): +def convert(src: str, dst: str, environ: Mapping[str, object]) -> None: """Generate a file from a template Args: - src (str): path to input file - dst (str): path to file to write - environ (dict): environment dictionary, for replacement mappings. + src: path to input file + dst: path to file to write + environ: environment dictionary, for replacement mappings. """ with open(src) as infile: template = infile.read() @@ -35,25 +36,30 @@ def convert(src, dst, environ): outfile.write(rendered) -def generate_config_from_template(config_dir, config_path, environ, ownership): +def generate_config_from_template( + config_dir: str, + config_path: str, + os_environ: Mapping[str, str], + ownership: Optional[str], +) -> None: """Generate a homeserver.yaml from environment variables Args: - config_dir (str): where to put generated config files - config_path (str): where to put the main config file - environ (dict): environment dictionary - ownership (str|None): ":" string which will be used to set + config_dir: where to put generated config files + config_path: where to put the main config file + os_environ: environment mapping + ownership: ":" string which will be used to set ownership of the generated configs. If None, ownership will not change. """ for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"): - if v not in environ: + if v not in os_environ: error( "Environment variable '%s' is mandatory when generating a config file." % (v,) ) # populate some params from data files (if they exist, else create new ones) - environ = environ.copy() + environ: Dict[str, Any] = dict(os_environ) secrets = { "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET", "macaroon": "SYNAPSE_MACAROON_SECRET_KEY", @@ -127,12 +133,12 @@ def generate_config_from_template(config_dir, config_path, environ, ownership): subprocess.check_output(args) -def run_generate_config(environ, ownership): +def run_generate_config(environ: Mapping[str, str], ownership: Optional[str]) -> None: """Run synapse with a --generate-config param to generate a template config file Args: - environ (dict): env var dict - ownership (str|None): "userid:groupid" arg for chmod. If None, ownership will not change. + environ: env vars from `os.enrivon`. + ownership: "userid:groupid" arg for chmod. If None, ownership will not change. Never returns. """ @@ -178,7 +184,7 @@ def run_generate_config(environ, ownership): os.execv(sys.executable, args) -def main(args, environ): +def main(args: List[str], environ: MutableMapping[str, str]) -> None: mode = args[1] if len(args) > 1 else "run" # if we were given an explicit user to switch to, do so diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 6aa48e191999..8400a6539a4e 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -17,6 +17,7 @@ # Usage - [Federation](federate.md) - [Configuration](usage/configuration/README.md) + - [Configuration Manual](usage/configuration/config_documentation.md) - [Homeserver Sample Config File](usage/configuration/homeserver_sample_config.md) - [Logging Sample Config File](usage/configuration/logging_sample_config.md) - [Structured Logging](structured_logging.md) @@ -88,6 +89,7 @@ - [Database Schemas](development/database_schema.md) - [Experimental features](development/experimental_features.md) - [Synapse Architecture]() + - [Cancellation](development/synapse_architecture/cancellation.md) - [Log Contexts](log_contexts.md) - [Replication](replication.md) - [TCP Replication](tcp_replication.md) diff --git a/docs/admin_api/media_admin_api.md b/docs/admin_api/media_admin_api.md index 96b3668f2a08..d57c5aedae4c 100644 --- a/docs/admin_api/media_admin_api.md +++ b/docs/admin_api/media_admin_api.md @@ -289,7 +289,7 @@ POST /_synapse/admin/v1/purge_media_cache?before_ts= URL Parameters -* `unix_timestamp_in_ms`: string representing a positive integer - Unix timestamp in milliseconds. +* `before_ts`: string representing a positive integer - Unix timestamp in milliseconds. All cached media that was last accessed before this timestamp will be removed. Response: diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 4076fcab65f1..c8794299e790 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -804,7 +804,7 @@ POST /_synapse/admin/v2/users//delete_devices "devices": [ "QBUAZIFURK", "AUIECTSRND" - ], + ] } ``` diff --git a/docs/code_style.md b/docs/code_style.md index ebda6dcc85f4..db7edcd76b69 100644 --- a/docs/code_style.md +++ b/docs/code_style.md @@ -6,60 +6,36 @@ The Synapse codebase uses a number of code formatting tools in order to quickly and automatically check for formatting (and sometimes logical) errors in code. -The necessary tools are detailed below. +The necessary tools are: -First install them with: +- [black](https://black.readthedocs.io/en/stable/), a source code formatter; +- [isort](https://pycqa.github.io/isort/), which organises each file's imports; +- [flake8](https://flake8.pycqa.org/en/latest/), which can spot common errors; and +- [mypy](https://mypy.readthedocs.io/en/stable/), a type checker. + +Install them with: ```sh pip install -e ".[lint,mypy]" ``` -- **black** - - The Synapse codebase uses [black](https://pypi.org/project/black/) - as an opinionated code formatter, ensuring all comitted code is - properly formatted. - - Have `black` auto-format your code (it shouldn't change any - functionality) with: - - ```sh - black . - ``` - -- **flake8** - - `flake8` is a code checking tool. We require code to pass `flake8` - before being merged into the codebase. - - Check all application and test code with: +The easiest way to run the lints is to invoke the linter script as follows. - ```sh - flake8 . - ``` - -- **isort** - - `isort` ensures imports are nicely formatted, and can suggest and - auto-fix issues such as double-importing. - - Auto-fix imports with: - - ```sh - isort . - ``` +```sh +scripts-dev/lint.sh +``` It's worth noting that modern IDEs and text editors can run these tools automatically on save. It may be worth looking into whether this functionality is supported in your editor for a more convenient -development workflow. It is not, however, recommended to run `flake8` on -save as it takes a while and is very resource intensive. +development workflow. It is not, however, recommended to run `flake8` or `mypy` +on save as they take a while and can be very resource intensive. ## General rules - **Naming**: - - Use camel case for class and type names - - Use underscores for functions and variables. + - Use `CamelCase` for class and type names + - Use underscores for `function_names` and `variable_names`. - **Docstrings**: should follow the [google code style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings). See the diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index 0d9cf6019607..2b3714df66f9 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -48,19 +48,28 @@ can find many good git tutorials on the web. # 4. Install the dependencies -Once you have installed Python 3 and added the source, please open a terminal and -setup a *virtualenv*, as follows: +Synapse uses the [poetry](https://python-poetry.org/) project to manage its dependencies +and development environment. Once you have installed Python 3 and added the +source, you should install `poetry`. +Of their installation methods, we recommend +[installing `poetry` using `pipx`](https://python-poetry.org/docs/#installing-with-pipx), + +```shell +pip install --user pipx +pipx install poetry +``` + +but see poetry's [installation instructions](https://python-poetry.org/docs/#installation) +for other installation methods. + +Next, open a terminal and install dependencies as follows: ```sh cd path/where/you/have/cloned/the/repository -python3 -m venv ./env -source ./env/bin/activate -pip install wheel -pip install -e ".[all,dev]" -pip install tox +poetry install --extras all ``` -This will install the developer dependencies for the project. +This will install the runtime and developer dependencies for the project. # 5. Get in touch. @@ -117,11 +126,10 @@ The linters look at your code and do two things: - ensure that your code follows the coding style adopted by the project; - catch a number of errors in your code. -The linter takes no time at all to run as soon as you've [downloaded the dependencies into your python virtual environment](#4-install-the-dependencies). +The linter takes no time at all to run as soon as you've [downloaded the dependencies](#4-install-the-dependencies). ```sh -source ./env/bin/activate -./scripts-dev/lint.sh +poetry run ./scripts-dev/lint.sh ``` Note that this script *will modify your files* to fix styling errors. @@ -131,15 +139,13 @@ If you wish to restrict the linters to only the files changed since the last com (much faster!), you can instead run: ```sh -source ./env/bin/activate -./scripts-dev/lint.sh -d +poetry run ./scripts-dev/lint.sh -d ``` Or if you know exactly which files you wish to lint, you can instead run: ```sh -source ./env/bin/activate -./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder +poetry run ./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder ``` ## Run the unit tests (Twisted trial). @@ -148,16 +154,14 @@ The unit tests run parts of Synapse, including your changes, to see if anything was broken. They are slower than the linters but will typically catch more errors. ```sh -source ./env/bin/activate -trial tests +poetry run trial tests ``` If you wish to only run *some* unit tests, you may specify another module instead of `tests` - or a test class or a method: ```sh -source ./env/bin/activate -trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite +poetry run trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite ``` If your tests fail, you may wish to look at the logs (the default log level is `ERROR`): @@ -169,7 +173,7 @@ less _trial_temp/test.log To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`: ```sh -SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests +SYNAPSE_TEST_LOG_LEVEL=DEBUG poetry run trial tests ``` By default, tests will use an in-memory SQLite database for test data. For additional @@ -180,7 +184,7 @@ database state to be stored in a file named `test.db` under the trial process' working directory. Typically, this ends up being `_trial_temp/test.db`. For example: ```sh -SYNAPSE_TEST_PERSIST_SQLITE_DB=1 trial tests +SYNAPSE_TEST_PERSIST_SQLITE_DB=1 poetry run trial tests ``` The database file can then be inspected with: @@ -202,7 +206,32 @@ This means that we need to run our unit tests against PostgreSQL too. Our CI doe this automatically for pull requests and release candidates, but it's sometimes useful to reproduce this locally. -To do so, [configure Postgres](../postgres.md) and run `trial` with the +#### Using Docker + +The easiest way to do so is to run Postgres via a docker container. In one +terminal: + +```shell +docker run --rm -e POSTGRES_PASSWORD=mysecretpassword -e POSTGRES_USER=postgres -e POSTGRES_DB=postgress -p 5432:5432 postgres:14 +``` + +If you see an error like + +``` +docker: Error response from daemon: driver failed programming external connectivity on endpoint nice_ride (b57bbe2e251b70015518d00c9981e8cb8346b5c785250341a6c53e3c899875f1): Error starting userland proxy: listen tcp4 0.0.0.0:5432: bind: address already in use. +``` + +then something is already bound to port 5432. You're probably already running postgres locally. + +Once you have a postgres server running, invoke `trial` in a second terminal: + +```shell +SYNAPSE_POSTGRES=1 SYNAPSE_POSTGRES_HOST=127.0.0.1 SYNAPSE_POSTGRES_USER=postgres SYNAPSE_POSTGRES_PASSWORD=mysecretpassword poetry run trial tests +```` + +#### Using an existing Postgres installation + +If you have postgres already installed on your system, you can run `trial` with the following environment variables matching your configuration: - `SYNAPSE_POSTGRES` to anything nonempty @@ -225,8 +254,8 @@ You don't need to specify the host, user, port or password if your Postgres server is set to authenticate you over the UNIX socket (i.e. if the `psql` command works without further arguments). -Your Postgres account needs to be able to create databases. - +Your Postgres account needs to be able to create databases; see the postgres +docs for [`ALTER ROLE`](https://www.postgresql.org/docs/current/sql-alterrole.html). ## Run the integration tests ([Sytest](https://github.com/matrix-org/sytest)). @@ -266,13 +295,13 @@ COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh To run a specific test file, you can pass the test name at the end of the command. The name passed comes from the naming structure in your Complement tests. If you're unsure of the name, you can do a full run and copy it from the test output: ```sh -COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh TestBackfillingHistory +COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh -run TestImportHistoricalMessages ``` To run a specific test, you can specify the whole name structure: ```sh -COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh TestBackfillingHistory/parallel/Backfilled_historical_events_resolve_with_proper_state_in_correct_order +COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh -run TestImportHistoricalMessages/parallel/Historical_events_resolve_in_the_correct_order ``` @@ -393,8 +422,8 @@ same lightweight approach that the Linux Kernel [submitting patches process]( https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin>), [Docker](https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other -projects use: the DCO (Developer Certificate of Origin: -http://developercertificate.org/). This is a simple declaration that you wrote +projects use: the DCO ([Developer Certificate of Origin](http://developercertificate.org/)). +This is a simple declaration that you wrote the contribution or otherwise have the right to contribute it to Matrix: ``` diff --git a/docs/development/demo.md b/docs/development/demo.md index 4277252ceb60..893ed6998ebb 100644 --- a/docs/development/demo.md +++ b/docs/development/demo.md @@ -5,7 +5,7 @@ Requires you to have a [Synapse development environment setup](https://matrix-org.github.io/synapse/develop/development/contributing_guide.html#4-install-the-dependencies). The demo setup allows running three federation Synapse servers, with server -names `localhost:8080`, `localhost:8081`, and `localhost:8082`. +names `localhost:8480`, `localhost:8481`, and `localhost:8482`. You can access them via any Matrix client over HTTP at `localhost:8080`, `localhost:8081`, and `localhost:8082` or over HTTPS at `localhost:8480`, @@ -20,9 +20,10 @@ and the servers are configured in a highly insecure way, including: The servers are configured to store their data under `demo/8080`, `demo/8081`, and `demo/8082`. This includes configuration, logs, SQLite databases, and media. -Note that when joining a public room on a different HS via "#foo:bar.net", then -you are (in the current impl) joining a room with room_id "foo". This means that -it won't work if your HS already has a room with that name. +Note that when joining a public room on a different homeserver via "#foo:bar.net", +then you are (in the current implementation) joining a room with room_id "foo". +This means that it won't work if your homeserver already has a room with that +name. ## Using the demo scripts diff --git a/docs/development/dependencies.md b/docs/development/dependencies.md new file mode 100644 index 000000000000..8ef7d357d8cd --- /dev/null +++ b/docs/development/dependencies.md @@ -0,0 +1,239 @@ +# Managing dependencies with Poetry + +This is a quick cheat sheet for developers on how to use [`poetry`](https://python-poetry.org/). + +# Background + +Synapse uses a variety of third-party Python packages to function as a homeserver. +Some of these are direct dependencies, listed in `pyproject.toml` under the +`[tool.poetry.dependencies]` section. The rest are transitive dependencies (the +things that our direct dependencies themselves depend on, and so on recursively.) + +We maintain a locked list of all our dependencies (transitive included) so that +we can track exactly which version of each dependency appears in a given release. +See [here](https://github.com/matrix-org/synapse/issues/11537#issue-1074469665) +for discussion of why we wanted this for Synapse. We chose to use +[`poetry`](https://python-poetry.org/) to manage this locked list; see +[this comment](https://github.com/matrix-org/synapse/issues/11537#issuecomment-1015975819) +for the reasoning. + +The locked dependencies get included in our "self-contained" releases: namely, +our docker images and our debian packages. We also use the locked dependencies +in development and our continuous integration. + +Separately, our "broad" dependencies—the version ranges specified in +`pyproject.toml`—are included as metadata in our "sdists" and "wheels" [uploaded +to PyPI](https://pypi.org/project/matrix-synapse). Installing from PyPI or from +the Synapse source tree directly will _not_ use the locked dependencies; instead, +they'll pull in the latest version of each package available at install time. + +## Example dependency + +An example may help. We have a broad dependency on +[`phonenumbers`](https://pypi.org/project/phonenumbers/), as declared in +this snippet from pyproject.toml [as of Synapse 1.57]( +https://github.com/matrix-org/synapse/blob/release-v1.57/pyproject.toml#L133 +): + +```toml +[tool.poetry.dependencies] +# ... +phonenumbers = ">=8.2.0" +``` + +In our lockfile this is +[pinned]( https://github.com/matrix-org/synapse/blob/dfc7646504cef3e4ff396c36089e1c6f1b1634de/poetry.lock#L679-L685) +to version 8.12.44, even though +[newer versions are available](https://pypi.org/project/phonenumbers/#history). + +```toml +[[package]] +name = "phonenumbers" +version = "8.12.44" +description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." +category = "main" +optional = false +python-versions = "*" +``` + +The lockfile also includes a +[cryptographic checksum](https://github.com/matrix-org/synapse/blob/release-v1.57/poetry.lock#L2178-L2181) +of the sdists and wheels provided for this version of `phonenumbers`. + +```toml +[metadata.files] +# ... +phonenumbers = [ + {file = "phonenumbers-8.12.44-py2.py3-none-any.whl", hash = "sha256:cc1299cf37b309ecab6214297663ab86cb3d64ae37fd5b88e904fe7983a874a6"}, + {file = "phonenumbers-8.12.44.tar.gz", hash = "sha256:26cfd0257d1704fe2f88caff2caabb70d16a877b1e65b6aae51f9fbbe10aa8ce"}, +] +``` + +We can see this pinned version inside the docker image for that release: + +``` +$ docker pull matrixdotorg/synapse:v1.57.0 +... +$ docker run --entrypoint pip matrixdotorg/synapse:v1.57.0 show phonenumbers +Name: phonenumbers +Version: 8.12.44 +Summary: Python version of Google's common library for parsing, formatting, storing and validating international phone numbers. +Home-page: https://github.com/daviddrysdale/python-phonenumbers +Author: David Drysdale +Author-email: dmd@lurklurk.org +License: Apache License 2.0 +Location: /usr/local/lib/python3.9/site-packages +Requires: +Required-by: matrix-synapse +``` + +Whereas the wheel metadata just contains the broad dependencies: + +``` +$ cd /tmp +$ wget https://files.pythonhosted.org/packages/ca/5e/d722d572cc5b3092402b783d6b7185901b444427633bd8a6b00ea0dd41b7/matrix_synapse-1.57.0rc1-py3-none-any.whl +... +$ unzip -c matrix_synapse-1.57.0rc1-py3-none-any.whl matrix_synapse-1.57.0rc1.dist-info/METADATA | grep phonenumbers +Requires-Dist: phonenumbers (>=8.2.0) +``` + +# Tooling recommendation: direnv + +[`direnv`](https://direnv.net/) is a tool for activating environments in your +shell inside a given directory. Its support for poetry is unofficial (a +community wiki recipe only), but works solidly in our experience. We thoroughly +recommend it for daily use. To use it: + +1. [Install `direnv`](https://direnv.net/docs/installation.html) - it's likely + packaged for your system already. +2. Teach direnv about poetry. The [shell config here](https://github.com/direnv/direnv/wiki/Python#poetry) + needs to be added to `~/.config/direnv/direnvrc` (or more generally `$XDG_CONFIG_HOME/direnv/direnvrc`). +3. Mark the synapse checkout as a poetry project: `echo layout poetry > .envrc`. +4. Convince yourself that you trust this `.envrc` configuration and project. + Then formally confirm this to `direnv` by running `direnv allow`. + +Then whenever you navigate to the synapse checkout, you should be able to run +e.g. `mypy` instead of `poetry run mypy`; `python` instead of +`poetry run python`; and your shell commands will automatically run in the +context of poetry's venv, without having to run `poetry shell` beforehand. + + +# How do I... + +## ...reset my venv to the locked environment? + +```shell +poetry install --extras all --remove-untracked +``` + +## ...run a command in the `poetry` virtualenv? + +Use `poetry run cmd args` when you need the python virtualenv context. +To avoid typing `poetry run` all the time, you can run `poetry shell` +to start a new shell in the poetry virtualenv context. Within `poetry shell`, +`python`, `pip`, `mypy`, `trial`, etc. are all run inside the project virtualenv +and isolated from the rest o the system. + +Roughly speaking, the translation from a traditional virtualenv is: +- `env/bin/activate` -> `poetry shell`, and +- `deactivate` -> close the terminal (Ctrl-D, `exit`, etc.) + +See also the direnv recommendation above, which makes `poetry run` and +`poetry shell` unnecessary. + + +## ...inspect the `poetry` virtualenv? + +Some suggestions: + +```shell +# Current env only +poetry env info +# All envs: this allows you to have e.g. a poetry managed venv for Python 3.7, +# and another for Python 3.10. +poetry env list --full-path +poetry run pip list +``` + +Note that `poetry show` describes the abstract *lock file* rather than your +on-disk environment. With that said, `poetry show --tree` can sometimes be +useful. + + +## ...add a new dependency? + +Either: +- manually update `pyproject.toml`; then `poetry lock --no-update`; or else +- `poetry add packagename`. See `poetry add --help`; note the `--dev`, + `--extras` and `--optional` flags in particular. + - **NB**: this specifies the new package with a version given by a "caret bound". This won't get forced to its lowest version in the old deps CI job: see [this TODO](https://github.com/matrix-org/synapse/blob/4e1374373857f2f7a911a31c50476342d9070681/.ci/scripts/test_old_deps.sh#L35-L39). + +Include the updated `pyproject.toml` and `poetry.lock` files in your commit. + +## ...remove a dependency? + +This is not done often and is untested, but + +```shell +poetry remove packagename +``` + +ought to do the trick. Alternatively, manually update `pyproject.toml` and +`poetry lock --no-update`. Include the updated `pyproject.toml` and poetry.lock` +files in your commit. + +## ...update the version range for an existing dependency? + +Best done by manually editing `pyproject.toml`, then `poetry lock --no-update`. +Include the updated `pyproject.toml` and `poetry.lock` in your commit. + +## ...update a dependency in the locked environment? + +Use + +```shell +poetry update packagename +``` + +to use the latest version of `packagename` in the locked environment, without +affecting the broad dependencies listed in the wheel. + +There doesn't seem to be a way to do this whilst locking a _specific_ version of +`packagename`. We can workaround this (crudely) as follows: + +```shell +poetry add packagename==1.2.3 +# This should update pyproject.lock. + +# Now undo the changes to pyproject.toml. For example +# git restore pyproject.toml + +# Get poetry to recompute the content-hash of pyproject.toml without changing +# the locked package versions. +poetry lock --no-update +``` + +Either way, include the updated `poetry.lock` file in your commit. + +## ...export a `requirements.txt` file? + +```shell +poetry export --extras all +``` + +Be wary of bugs in `poetry export` and `pip install -r requirements.txt`. + +Note: `poetry export` will be made a plugin in Poetry 1.2. Additional config may +be required. + +## ...build a test wheel? + +I usually use + +```shell +poetry run pip install build && poetry run python -m build +``` + +because [`build`](https://github.com/pypa/build) is a standardish tool which +doesn't require poetry. (It's what we use in CI too). However, you could try +`poetry build` too. diff --git a/docs/development/synapse_architecture/cancellation.md b/docs/development/synapse_architecture/cancellation.md new file mode 100644 index 000000000000..ef9e0226353b --- /dev/null +++ b/docs/development/synapse_architecture/cancellation.md @@ -0,0 +1,392 @@ +# Cancellation +Sometimes, requests take a long time to service and clients disconnect +before Synapse produces a response. To avoid wasting resources, Synapse +can cancel request processing for select endpoints marked with the +`@cancellable` decorator. + +Synapse makes use of Twisted's `Deferred.cancel()` feature to make +cancellation work. The `@cancellable` decorator does nothing by itself +and merely acts as a flag, signalling to developers and other code alike +that a method can be cancelled. + +## Enabling cancellation for an endpoint +1. Check that the endpoint method, and any `async` functions in its call + tree handle cancellation correctly. See + [Handling cancellation correctly](#handling-cancellation-correctly) + for a list of things to look out for. +2. Add the `@cancellable` decorator to the `on_GET/POST/PUT/DELETE` + method. It's not recommended to make non-`GET` methods cancellable, + since cancellation midway through some database updates is less + likely to be handled correctly. + +## Mechanics +There are two stages to cancellation: downward propagation of a +`cancel()` call, followed by upwards propagation of a `CancelledError` +out of a blocked `await`. +Both Twisted and asyncio have a cancellation mechanism. + +| | Method | Exception | Exception inherits from | +|---------------|---------------------|-----------------------------------------|-------------------------| +| Twisted | `Deferred.cancel()` | `twisted.internet.defer.CancelledError` | `Exception` (!) | +| asyncio | `Task.cancel()` | `asyncio.CancelledError` | `BaseException` | + +### Deferred.cancel() +When Synapse starts handling a request, it runs the async method +responsible for handling it using `defer.ensureDeferred`, which returns +a `Deferred`. For example: + +```python +def do_something() -> Deferred[None]: + ... + +@cancellable +async def on_GET() -> Tuple[int, JsonDict]: + d = make_deferred_yieldable(do_something()) + await d + return 200, {} + +request = defer.ensureDeferred(on_GET()) +``` + +When a client disconnects early, Synapse checks for the presence of the +`@cancellable` decorator on `on_GET`. Since `on_GET` is cancellable, +`Deferred.cancel()` is called on the `Deferred` from +`defer.ensureDeferred`, ie. `request`. Twisted knows which `Deferred` +`request` is waiting on and passes the `cancel()` call on to `d`. + +The `Deferred` being waited on, `d`, may have its own handling for +`cancel()` and pass the call on to other `Deferred`s. + +Eventually, a `Deferred` handles the `cancel()` call by resolving itself +with a `CancelledError`. + +### CancelledError +The `CancelledError` gets raised out of the `await` and bubbles up, as +per normal Python exception handling. + +## Handling cancellation correctly +In general, when writing code that might be subject to cancellation, two +things must be considered: + * The effect of `CancelledError`s raised out of `await`s. + * The effect of `Deferred`s being `cancel()`ed. + +Examples of code that handles cancellation incorrectly include: + * `try-except` blocks which swallow `CancelledError`s. + * Code that shares the same `Deferred`, which may be cancelled, between + multiple requests. + * Code that starts some processing that's exempt from cancellation, but + uses a logging context from cancellable code. The logging context + will be finished upon cancellation, while the uncancelled processing + is still using it. + +Some common patterns are listed below in more detail. + +### `async` function calls +Most functions in Synapse are relatively straightforward from a +cancellation standpoint: they don't do anything with `Deferred`s and +purely call and `await` other `async` functions. + +An `async` function handles cancellation correctly if its own code +handles cancellation correctly and all the async function it calls +handle cancellation correctly. For example: +```python +async def do_two_things() -> None: + check_something() + await do_something() + await do_something_else() +``` +`do_two_things` handles cancellation correctly if `do_something` and +`do_something_else` handle cancellation correctly. + +That is, when checking whether a function handles cancellation +correctly, its implementation and all its `async` function calls need to +be checked, recursively. + +As `check_something` is not `async`, it does not need to be checked. + +### CancelledErrors +Because Twisted's `CancelledError`s are `Exception`s, it's easy to +accidentally catch and suppress them. Care must be taken to ensure that +`CancelledError`s are allowed to propagate upwards. + + + + + + + + + + +
+ +**Bad**: +```python +try: + await do_something() +except Exception: + # `CancelledError` gets swallowed here. + logger.info(...) +``` + + +**Good**: +```python +try: + await do_something() +except CancelledError: + raise +except Exception: + logger.info(...) +``` +
+ +**OK**: +```python +try: + check_something() + # A `CancelledError` won't ever be raised here. +except Exception: + logger.info(...) +``` + + +**Good**: +```python +try: + await do_something() +except ValueError: + logger.info(...) +``` +
+ +#### defer.gatherResults +`defer.gatherResults` produces a `Deferred` which: + * broadcasts `cancel()` calls to every `Deferred` being waited on. + * wraps the first exception it sees in a `FirstError`. + +Together, this means that `CancelledError`s will be wrapped in +a `FirstError` unless unwrapped. Such `FirstError`s are liable to be +swallowed, so they must be unwrapped. + + + + + + +
+ +**Bad**: +```python +async def do_something() -> None: + await make_deferred_yieldable( + defer.gatherResults([...], consumeErrors=True) + ) + +try: + await do_something() +except CancelledError: + raise +except Exception: + # `FirstError(CancelledError)` gets swallowed here. + logger.info(...) +``` + + + +**Good**: +```python +async def do_something() -> None: + await make_deferred_yieldable( + defer.gatherResults([...], consumeErrors=True) + ).addErrback(unwrapFirstError) + +try: + await do_something() +except CancelledError: + raise +except Exception: + logger.info(...) +``` +
+ +### Creation of `Deferred`s +If a function creates a `Deferred`, the effect of cancelling it must be considered. `Deferred`s that get shared are likely to have unintended behaviour when cancelled. + + + + + + + + + +
+ +**Bad**: +```python +cache: Dict[str, Deferred[None]] = {} + +def wait_for_room(room_id: str) -> Deferred[None]: + deferred = cache.get(room_id) + if deferred is None: + deferred = Deferred() + cache[room_id] = deferred + # `deferred` can have multiple waiters. + # All of them will observe a `CancelledError` + # if any one of them is cancelled. + return make_deferred_yieldable(deferred) + +# Request 1 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +# Request 2 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +``` + + +**Good**: +```python +cache: Dict[str, Deferred[None]] = {} + +def wait_for_room(room_id: str) -> Deferred[None]: + deferred = cache.get(room_id) + if deferred is None: + deferred = Deferred() + cache[room_id] = deferred + # `deferred` will never be cancelled now. + # A `CancelledError` will still come out of + # the `await`. + # `delay_cancellation` may also be used. + return make_deferred_yieldable(stop_cancellation(deferred)) + +# Request 1 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +# Request 2 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +``` +
+ + +**Good**: +```python +cache: Dict[str, List[Deferred[None]]] = {} + +def wait_for_room(room_id: str) -> Deferred[None]: + if room_id not in cache: + cache[room_id] = [] + # Each request gets its own `Deferred` to wait on. + deferred = Deferred() + cache[room_id]].append(deferred) + return make_deferred_yieldable(deferred) + +# Request 1 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +# Request 2 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +``` +
+ +### Uncancelled processing +Some `async` functions may kick off some `async` processing which is +intentionally protected from cancellation, by `stop_cancellation` or +other means. If the `async` processing inherits the logcontext of the +request which initiated it, care must be taken to ensure that the +logcontext is not finished before the `async` processing completes. + + + + + + + + + + +
+ +**Bad**: +```python +cache: Optional[ObservableDeferred[None]] = None + +async def do_something_else( + to_resolve: Deferred[None] +) -> None: + await ... + logger.info("done!") + to_resolve.callback(None) + +async def do_something() -> None: + if not cache: + to_resolve = Deferred() + cache = ObservableDeferred(to_resolve) + # `do_something_else` will never be cancelled and + # can outlive the `request-1` logging context. + run_in_background(do_something_else, to_resolve) + + await make_deferred_yieldable(cache.observe()) + +with LoggingContext("request-1"): + await do_something() +``` + + +**Good**: +```python +cache: Optional[ObservableDeferred[None]] = None + +async def do_something_else( + to_resolve: Deferred[None] +) -> None: + await ... + logger.info("done!") + to_resolve.callback(None) + +async def do_something() -> None: + if not cache: + to_resolve = Deferred() + cache = ObservableDeferred(to_resolve) + run_in_background(do_something_else, to_resolve) + # We'll wait until `do_something_else` is + # done before raising a `CancelledError`. + await make_deferred_yieldable( + delay_cancellation(cache.observe()) + ) + else: + await make_deferred_yieldable(cache.observe()) + +with LoggingContext("request-1"): + await do_something() +``` +
+ +**OK**: +```python +cache: Optional[ObservableDeferred[None]] = None + +async def do_something_else( + to_resolve: Deferred[None] +) -> None: + await ... + logger.info("done!") + to_resolve.callback(None) + +async def do_something() -> None: + if not cache: + to_resolve = Deferred() + cache = ObservableDeferred(to_resolve) + # `do_something_else` will get its own independent + # logging context. `request-1` will not count any + # metrics from `do_something_else`. + run_as_background_process( + "do_something_else", + do_something_else, + to_resolve, + ) + + await make_deferred_yieldable(cache.observe()) + +with LoggingContext("request-1"): + await do_something() +``` + +
diff --git a/docs/jwt.md b/docs/jwt.md index 32f58cc0cbbf..346daf78ad1e 100644 --- a/docs/jwt.md +++ b/docs/jwt.md @@ -17,9 +17,6 @@ follows: } ``` -Note that the login type of `m.login.jwt` is supported, but is deprecated. This -will be removed in a future version of Synapse. - The `token` field should include the JSON web token with the following claims: * A claim that encodes the local part of the user ID is required. By default, diff --git a/docs/modules/spam_checker_callbacks.md b/docs/modules/spam_checker_callbacks.md index 472d95718087..ad35e667ed4b 100644 --- a/docs/modules/spam_checker_callbacks.md +++ b/docs/modules/spam_checker_callbacks.md @@ -12,21 +12,27 @@ The available spam checker callbacks are: _First introduced in Synapse v1.37.0_ +_Changed in Synapse v1.60.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean or a string is now deprecated._ + ```python -async def check_event_for_spam(event: "synapse.events.EventBase") -> Union[bool, str] +async def check_event_for_spam(event: "synapse.module_api.EventBase") -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", str, bool] ``` -Called when receiving an event from a client or via federation. The callback must return -either: -- an error message string, to indicate the event must be rejected because of spam and - give a rejection reason to forward to clients; -- the boolean `True`, to indicate that the event is spammy, but not provide further details; or -- the booelan `False`, to indicate that the event is not considered spammy. +Called when receiving an event from a client or via federation. The callback must return one of: + - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still + decide to reject it. + - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case + of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. + - (deprecated) a non-`Codes` `str` to reject the operation and specify an error message. Note that clients + typically will not localize the error message to the user's preferred locale. + - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`. + - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`. If multiple modules implement this callback, they will be considered in order. If a -callback returns `False`, Synapse falls through to the next one. The value of the first -callback that does not return `False` will be used. If this happens, Synapse will not call -any of the subsequent implementations of this callback. +callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one. +The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will +be used. If this happens, Synapse will not call any of the subsequent implementations of +this callback. ### `user_may_join_room` @@ -249,6 +255,24 @@ callback returns `False`, Synapse falls through to the next one. The value of th callback that does not return `False` will be used. If this happens, Synapse will not call any of the subsequent implementations of this callback. +### `should_drop_federated_event` + +_First introduced in Synapse v1.60.0_ + +```python +async def should_drop_federated_event(event: "synapse.events.EventBase") -> bool +``` + +Called when checking whether a remote server can federate an event with us. **Returning +`True` from this function will silently drop a federated event and split-brain our view +of a room's DAG, and thus you shouldn't use this callback unless you know what you are +doing.** + +If multiple modules implement this callback, they will be considered in order. If a +callback returns `False`, Synapse falls through to the next one. The value of the first +callback that does not return `False` will be used. If this happens, Synapse will not call +any of the subsequent implementations of this callback. + ## Example The example below is a module that implements the spam checker callback diff --git a/docs/openid.md b/docs/openid.md index 19cacaafefe0..9d615a573759 100644 --- a/docs/openid.md +++ b/docs/openid.md @@ -159,7 +159,7 @@ Follow the [Getting Started Guide](https://www.keycloak.org/getting-started) to oidc_providers: - idp_id: keycloak idp_name: "My KeyCloak server" - issuer: "https://127.0.0.1:8443/auth/realms/{realm_name}" + issuer: "https://127.0.0.1:8443/realms/{realm_name}" client_id: "synapse" client_secret: "copy secret generated from above" scopes: ["openid", "profile"] @@ -293,7 +293,7 @@ can be used to retrieve information on the authenticated user. As the Synapse login mechanism needs an attribute to uniquely identify users, and that endpoint does not return a `sub` property, an alternative `subject_claim` has to be set. -1. Create a new OAuth application: https://github.com/settings/applications/new. +1. Create a new OAuth application: [https://github.com/settings/applications/new](https://github.com/settings/applications/new). 2. Set the callback URL to `[synapse public baseurl]/_synapse/client/oidc/callback`. Synapse config: @@ -322,10 +322,10 @@ oidc_providers: [Google][google-idp] is an OpenID certified authentication and authorisation provider. -1. Set up a project in the Google API Console (see - https://developers.google.com/identity/protocols/oauth2/openid-connect#appsetup). -2. Add an "OAuth Client ID" for a Web Application under "Credentials". -3. Copy the Client ID and Client Secret, and add the following to your synapse config: +1. Set up a project in the Google API Console (see + [documentation](https://developers.google.com/identity/protocols/oauth2/openid-connect#appsetup)). +3. Add an "OAuth Client ID" for a Web Application under "Credentials". +4. Copy the Client ID and Client Secret, and add the following to your synapse config: ```yaml oidc_providers: - idp_id: google @@ -501,8 +501,8 @@ As well as the private key file, you will need: * Team ID: a 10-character ID associated with your developer account. * Key ID: the 10-character identifier for the key. -https://help.apple.com/developer-account/?lang=en#/dev77c875b7e has more -documentation on setting up SiWA. +[Apple's developer documentation](https://help.apple.com/developer-account/?lang=en#/dev77c875b7e) +has more information on setting up SiWA. The synapse config will look like this: @@ -535,8 +535,8 @@ needed to add OAuth2 capabilities to your Django projects. It supports Configuration on Django's side: -1. Add an application: https://example.com/admin/oauth2_provider/application/add/ and choose parameters like this: -* `Redirect uris`: https://synapse.example.com/_synapse/client/oidc/callback +1. Add an application: `https://example.com/admin/oauth2_provider/application/add/` and choose parameters like this: +* `Redirect uris`: `https://synapse.example.com/_synapse/client/oidc/callback` * `Client type`: `Confidential` * `Authorization grant type`: `Authorization code` * `Algorithm`: `HMAC with SHA-2 256` diff --git a/docs/replication.md b/docs/replication.md index e82df0de8a30..108da9a065d8 100644 --- a/docs/replication.md +++ b/docs/replication.md @@ -35,3 +35,8 @@ See [the TCP replication documentation](tcp_replication.md). There are read-only version of the synapse storage layer in `synapse/replication/slave/storage` that use the response of the replication API to invalidate their caches. + +### The TCP Replication Module +Information about how the tcp replication module is structured, including how +the classes interact, can be found in +`synapse/replication/tcp/__init__.py` diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md index 5a0c847951a6..69caa8a73ee8 100644 --- a/docs/reverse_proxy.md +++ b/docs/reverse_proxy.md @@ -206,6 +206,28 @@ backend matrix server matrix 127.0.0.1:8008 ``` + +[Delegation](delegate.md) example: +``` +frontend https + acl matrix-well-known-client-path path /.well-known/matrix/client + acl matrix-well-known-server-path path /.well-known/matrix/server + use_backend matrix-well-known-client if matrix-well-known-client-path + use_backend matrix-well-known-server if matrix-well-known-server-path + +backend matrix-well-known-client + http-after-response set-header Access-Control-Allow-Origin "*" + http-after-response set-header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" + http-after-response set-header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization" + http-request return status 200 content-type application/json string '{"m.homeserver":{"base_url":"https://matrix.example.com"},"m.identity_server":{"base_url":"https://identity.example.com"}}' + +backend matrix-well-known-server + http-after-response set-header Access-Control-Allow-Origin "*" + http-after-response set-header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" + http-after-response set-header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization" + http-request return status 200 content-type application/json string '{"m.server":"matrix.example.com:443"}' +``` + ### Relayd ``` diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index b8d8c0dbf0a1..e0abcd3b0303 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -289,7 +289,7 @@ presence: # federation: the server-server API (/_matrix/federation). Also implies # 'media', 'keys', 'openid' # -# keys: the key discovery API (/_matrix/keys). +# keys: the key discovery API (/_matrix/key). # # media: the media API (/_matrix/media). # @@ -407,6 +407,11 @@ manhole_settings: # sign up in a short space of time never to return after their initial # session. # +# The option `mau_appservice_trial_days` is similar to `mau_trial_days`, but +# applies a different trial number if the user was registered by an appservice. +# A value of 0 means no trial days are applied. Appservices not listed in this +# dictionary use the value of `mau_trial_days` instead. +# # 'mau_limit_alerting' is a means of limiting client side alerting # should the mau limit be reached. This is useful for small instances # where the admin has 5 mau seats (say) for 5 specific people and no @@ -417,6 +422,8 @@ manhole_settings: #max_mau_value: 50 #mau_trial_days: 2 #mau_limit_alerting: false +#mau_appservice_trial_days: +# "appservice-id": 1 # If enabled, the metrics for the number of monthly active users will # be populated, however no one will be limited. If limit_usage_by_mau @@ -709,11 +716,11 @@ retention: # #allow_profile_lookup_over_federation: false -# Uncomment to disable device display name lookup over federation. By default, the -# Federation API allows other homeservers to obtain device display names of any user -# on this homeserver. Defaults to 'true'. +# Uncomment to allow device display name lookup over federation. By default, the +# Federation API prevents other homeservers from obtaining the display names of +# user devices on this homeserver. Defaults to 'false'. # -#allow_device_name_lookup_over_federation: false +#allow_device_name_lookup_over_federation: true ## Caching ## @@ -723,6 +730,12 @@ retention: # A cache 'factor' is a multiplier that can be applied to each of # Synapse's caches in order to increase or decrease the maximum # number of entries that can be stored. +# +# The configuration for cache factors (caches.global_factor and +# caches.per_cache_factors) can be reloaded while the application is running, +# by sending a SIGHUP signal to the Synapse process. Changes to other parts of +# the caching config will NOT be applied after a SIGHUP is received; a restart +# is necessary. # The number of events to cache in memory. Not affected by # caches.global_factor. @@ -771,6 +784,24 @@ caches: # #cache_entry_ttl: 30m + # This flag enables cache autotuning, and is further specified by the sub-options `max_cache_memory_usage`, + # `target_cache_memory_usage`, `min_cache_ttl`. These flags work in conjunction with each other to maintain + # a balance between cache memory usage and cache entry availability. You must be using jemalloc to utilize + # this option, and all three of the options must be specified for this feature to work. + #cache_autotuning: + # This flag sets a ceiling on much memory the cache can use before caches begin to be continuously evicted. + # They will continue to be evicted until the memory usage drops below the `target_memory_usage`, set in + # the flag below, or until the `min_cache_ttl` is hit. + #max_cache_memory_usage: 1024M + + # This flag sets a rough target for the desired memory usage of the caches. + #target_cache_memory_usage: 758M + + # 'min_cache_ttl` sets a limit under which newer cache entries are not evicted and is only applied when + # caches are actively being evicted/`max_cache_memory_usage` has been exceeded. This is to protect hot caches + # from being emptied while Synapse is evicting due to memory. + #min_cache_ttl: 5m + # Controls how long the results of a /sync request are cached for after # a successful response is returned. A higher duration can help clients with # intermittent connections, at the cost of higher memory usage. @@ -1323,6 +1354,12 @@ oembed: # #registration_requires_token: true +# Allow users to submit a token during registration to bypass any required 3pid +# steps configured in `registrations_require_3pid`. +# Defaults to false, requiring that registration tokens (if enabled) complete a 3pid flow. +# +#enable_registration_token_3pid_bypass: false + # If set, allows registration of standard or admin accounts by anyone who # has the shared secret, even if registration is otherwise disabled. # @@ -2179,7 +2216,9 @@ sso: password_config: - # Uncomment to disable password login + # Uncomment to disable password login. + # Set to `only_for_reauth` to permit reauthentication for users that + # have passwords and are already logged in. # #enabled: false @@ -2449,6 +2488,40 @@ push: # #encryption_enabled_by_default_for_room_type: invite +# Override the default power levels for rooms created on this server, per +# room creation preset. +# +# The appropriate dictionary for the room preset will be applied on top +# of the existing power levels content. +# +# Useful if you know that your users need special permissions in rooms +# that they create (e.g. to send particular types of state events without +# needing an elevated power level). This takes the same shape as the +# `power_level_content_override` parameter in the /createRoom API, but +# is applied before that parameter. +# +# Valid keys are some or all of `private_chat`, `trusted_private_chat` +# and `public_chat`. Inside each of those should be any of the +# properties allowed in `power_level_content_override` in the +# /createRoom API. If any property is missing, its default value will +# continue to be used. If any property is present, it will overwrite +# the existing default completely (so if the `events` property exists, +# the default event power levels will be ignored). +# +#default_power_level_content_override: +# private_chat: +# "events": +# "com.example.myeventtype" : 0 +# "m.room.avatar": 50 +# "m.room.canonical_alias": 50 +# "m.room.encryption": 100 +# "m.room.history_visibility": 100 +# "m.room.name": 50 +# "m.room.power_levels": 100 +# "m.room.server_acl": 100 +# "m.room.tombstone": 100 +# "events_default": 1 + # Uncomment to allow non-server-admin users to create groups on this server # diff --git a/docs/sample_log_config.yaml b/docs/sample_log_config.yaml index 2485ad25edfc..3065a0e2d986 100644 --- a/docs/sample_log_config.yaml +++ b/docs/sample_log_config.yaml @@ -62,13 +62,6 @@ loggers: # information such as access tokens. level: INFO - twisted: - # We send the twisted logging directly to the file handler, - # to work around https://github.com/matrix-org/synapse/issues/3471 - # when using "buffer" logger. Use "console" to log to stderr instead. - handlers: [file] - propagate: false - root: level: INFO diff --git a/docs/systemd-with-workers/README.md b/docs/systemd-with-workers/README.md index b160d9352840..d516501085bf 100644 --- a/docs/systemd-with-workers/README.md +++ b/docs/systemd-with-workers/README.md @@ -10,15 +10,15 @@ See the folder [system](https://github.com/matrix-org/synapse/tree/develop/docs/ for the systemd unit files. The folder [workers](https://github.com/matrix-org/synapse/tree/develop/docs/systemd-with-workers/workers/) -contains an example configuration for the `federation_reader` worker. +contains an example configuration for the `generic_worker` worker. ## Synapse configuration files See [the worker documentation](../workers.md) for information on how to set up the configuration files and reverse-proxy correctly. -Below is a sample `federation_reader` worker configuration file. +Below is a sample `generic_worker` worker configuration file. ```yaml -{{#include workers/federation_reader.yaml}} +{{#include workers/generic_worker.yaml}} ``` Systemd manages daemonization itself, so ensure that none of the configuration @@ -61,9 +61,9 @@ systemctl stop matrix-synapse.target # Restart the master alone systemctl start matrix-synapse.service -# Restart a specific worker (eg. federation_reader); the master is +# Restart a specific worker (eg. generic_worker); the master is # unaffected by this. -systemctl restart matrix-synapse-worker@federation_reader.service +systemctl restart matrix-synapse-worker@generic_worker.service # Add a new worker (assuming all configs are set up already) systemctl enable matrix-synapse-worker@federation_writer.service diff --git a/docs/systemd-with-workers/workers/background_worker.yaml b/docs/systemd-with-workers/workers/background_worker.yaml new file mode 100644 index 000000000000..9fbfbda7db94 --- /dev/null +++ b/docs/systemd-with-workers/workers/background_worker.yaml @@ -0,0 +1,8 @@ +worker_app: synapse.app.generic_worker +worker_name: background_worker + +# The replication listener on the main synapse process. +worker_replication_host: 127.0.0.1 +worker_replication_http_port: 9093 + +worker_log_config: /etc/matrix-synapse/background-worker-log.yaml diff --git a/docs/systemd-with-workers/workers/event_persister.yaml b/docs/systemd-with-workers/workers/event_persister.yaml new file mode 100644 index 000000000000..9bc6997bad99 --- /dev/null +++ b/docs/systemd-with-workers/workers/event_persister.yaml @@ -0,0 +1,23 @@ +worker_app: synapse.app.generic_worker +worker_name: event_persister1 + +# The replication listener on the main synapse process. +worker_replication_host: 127.0.0.1 +worker_replication_http_port: 9093 + +worker_listeners: + - type: http + port: 8034 + resources: + - names: [replication] + + # Enable listener if this stream writer handles endpoints for the `typing` or + # `to_device` streams. Uses a different port to the `replication` listener to + # avoid exposing the `replication` listener publicly. + # + #- type: http + # port: 8035 + # resources: + # - names: [client] + +worker_log_config: /etc/matrix-synapse/event-persister-log.yaml diff --git a/docs/systemd-with-workers/workers/federation_reader.yaml b/docs/systemd-with-workers/workers/federation_reader.yaml deleted file mode 100644 index 13e69e62c9db..000000000000 --- a/docs/systemd-with-workers/workers/federation_reader.yaml +++ /dev/null @@ -1,13 +0,0 @@ -worker_app: synapse.app.federation_reader -worker_name: federation_reader1 - -worker_replication_host: 127.0.0.1 -worker_replication_http_port: 9093 - -worker_listeners: - - type: http - port: 8011 - resources: - - names: [federation] - -worker_log_config: /etc/matrix-synapse/federation-reader-log.yaml diff --git a/docs/systemd-with-workers/workers/generic_worker.yaml b/docs/systemd-with-workers/workers/generic_worker.yaml new file mode 100644 index 000000000000..a82f9c161f81 --- /dev/null +++ b/docs/systemd-with-workers/workers/generic_worker.yaml @@ -0,0 +1,14 @@ +worker_app: synapse.app.generic_worker +worker_name: generic_worker1 + +# The replication listener on the main synapse process. +worker_replication_host: 127.0.0.1 +worker_replication_http_port: 9093 + +worker_listeners: + - type: http + port: 8083 + resources: + - names: [client, federation] + +worker_log_config: /etc/matrix-synapse/generic-worker-log.yaml diff --git a/docs/turn-howto.md b/docs/turn-howto.md index 3a2cd04e36a9..37a311ad9cc7 100644 --- a/docs/turn-howto.md +++ b/docs/turn-howto.md @@ -302,14 +302,14 @@ Here are a few things to try: (Understanding the output is beyond the scope of this document!) - * You can test your Matrix homeserver TURN setup with https://test.voip.librepush.net/. + * You can test your Matrix homeserver TURN setup with . Note that this test is not fully reliable yet, so don't be discouraged if the test fails. [Here](https://github.com/matrix-org/voip-tester) is the github repo of the source of the tester, where you can file bug reports. * There is a WebRTC test tool at - https://webrtc.github.io/samples/src/content/peerconnection/trickle-ice/. To + . To use it, you will need a username/password for your TURN server. You can either: diff --git a/docs/upgrade.md b/docs/upgrade.md index 023872490e2b..e3c64da17fd8 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -19,32 +19,36 @@ this document. packages](setup/installation.md#prebuilt-packages), you will need to follow the normal process for upgrading those packages. +- If Synapse was installed using pip then upgrade to the latest + version by running: + + ```bash + pip install --upgrade matrix-synapse + ``` + - If Synapse was installed from source, then: - 1. Activate the virtualenv before upgrading. For example, if - Synapse is installed in a virtualenv in `~/synapse/env` then + 1. Obtain the latest version of the source code. Git users can run + `git pull` to do this. + + 2. If you're running Synapse in a virtualenv, make sure to activate it before + upgrading. For example, if Synapse is installed in a virtualenv in `~/synapse/env` then run: ```bash source ~/synapse/env/bin/activate + pip install --upgrade . ``` + Include any relevant extras between square brackets, e.g. `pip install --upgrade ".[postgres,oidc]"`. - 2. If Synapse was installed using pip then upgrade to the latest - version by running: - + 3. If you're using `poetry` to manage a Synapse installation, run: ```bash - pip install --upgrade matrix-synapse + poetry install ``` + Include any relevant extras with `--extras`, e.g. `poetry install --extras postgres --extras oidc`. + It's probably easiest to run `poetry install --extras all`. - If Synapse was installed using git then upgrade to the latest - version by running: - - ```bash - git pull - pip install --upgrade . - ``` - - 3. Restart Synapse: + 4. Restart Synapse: ```bash synctl restart @@ -85,6 +89,177 @@ process, for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb ``` +# Upgrading to v1.60.0 + +## Adding a new unique index to `state_group_edges` could fail if your database is corrupted + +This release of Synapse will add a unique index to the `state_group_edges` table, in order +to prevent accidentally introducing duplicate information (for example, because a database +backup was restored multiple times). + +Duplicate rows being present in this table could cause drastic performance problems; see +[issue 11779](https://github.com/matrix-org/synapse/issues/11779) for more details. + +If your Synapse database already has had duplicate rows introduced into this table, +this could fail, with either of these errors: + + +**On Postgres:** +``` +synapse.storage.background_updates - 623 - INFO - background_updates-0 - Adding index state_group_edges_unique_idx to state_group_edges +synapse.storage.background_updates - 282 - ERROR - background_updates-0 - Error doing update +... +psycopg2.errors.UniqueViolation: could not create unique index "state_group_edges_unique_idx" +DETAIL: Key (state_group, prev_state_group)=(2, 1) is duplicated. +``` +(The numbers may be different.) + +**On SQLite:** +``` +synapse.storage.background_updates - 623 - INFO - background_updates-0 - Adding index state_group_edges_unique_idx to state_group_edges +synapse.storage.background_updates - 282 - ERROR - background_updates-0 - Error doing update +... +sqlite3.IntegrityError: UNIQUE constraint failed: state_group_edges.state_group, state_group_edges.prev_state_group +``` + + +
+Expand this section for steps to resolve this problem + +### On Postgres + +Connect to your database with `psql`. + +```sql +BEGIN; +DELETE FROM state_group_edges WHERE (ctid, state_group, prev_state_group) IN ( + SELECT row_id, state_group, prev_state_group + FROM ( + SELECT + ctid AS row_id, + MIN(ctid) OVER (PARTITION BY state_group, prev_state_group) AS min_row_id, + state_group, + prev_state_group + FROM state_group_edges + ) AS t1 + WHERE row_id <> min_row_id +); +COMMIT; +``` + + +### On SQLite + +At the command-line, use `sqlite3 path/to/your-homeserver-database.db`: + +```sql +BEGIN; +DELETE FROM state_group_edges WHERE (rowid, state_group, prev_state_group) IN ( + SELECT row_id, state_group, prev_state_group + FROM ( + SELECT + rowid AS row_id, + MIN(rowid) OVER (PARTITION BY state_group, prev_state_group) AS min_row_id, + state_group, + prev_state_group + FROM state_group_edges + ) + WHERE row_id <> min_row_id +); +COMMIT; +``` + + +### For more details + +[This comment on issue 11779](https://github.com/matrix-org/synapse/issues/11779#issuecomment-1131545970) +has queries that can be used to check a database for this problem in advance. + +
+ +## New signature for the spam checker callback `check_event_for_spam` + +The previous signature has been deprecated. + +Whereas `check_event_for_spam` callbacks used to return `Union[str, bool]`, they should now return `Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes"]`. + +This is part of an ongoing refactoring of the SpamChecker API to make it less ambiguous and more powerful. + +If your module implements `check_event_for_spam` as follows: + +```python +async def check_event_for_spam(event): + if ...: + # Event is spam + return True + # Event is not spam + return False +``` + +you should rewrite it as follows: + +```python +async def check_event_for_spam(event): + if ...: + # Event is spam, mark it as forbidden (you may use some more precise error + # code if it is useful). + return synapse.module_api.errors.Codes.FORBIDDEN + # Event is not spam, mark it as such. + return synapse.module_api.NOT_SPAM +``` + +# Upgrading to v1.59.0 + +## Device name lookup over federation has been disabled by default + +The names of user devices are no longer visible to users on other homeservers by default. +Device IDs are unaffected, as these are necessary to facilitate end-to-end encryption. + +To re-enable this functionality, set the +[`allow_device_name_lookup_over_federation`](https://matrix-org.github.io/synapse/v1.59/usage/configuration/config_documentation.html#federation) +homeserver config option to `true`. + + +## Deprecation of the `synapse.app.appservice` and `synapse.app.user_dir` worker application types + +The `synapse.app.appservice` worker application type allowed you to configure a +single worker to use to notify application services of new events, as long +as this functionality was disabled on the main process with `notify_appservices: False`. +Further, the `synapse.app.user_dir` worker application type allowed you to configure +a single worker to be responsible for updating the user directory, as long as this +was disabled on the main process with `update_user_directory: False`. + +To unify Synapse's worker types, the `synapse.app.appservice` worker application +type and the `notify_appservices` configuration option have been deprecated. +The `synapse.app.user_dir` worker application type and `update_user_directory` +configuration option have also been deprecated. + +To get the same functionality as was provided by the deprecated options, it's now recommended that the `synapse.app.generic_worker` +worker application type is used and that the `notify_appservices_from_worker` and/or +`update_user_directory_from_worker` options are set to the name of a worker. + +For the time being, the old options can be used alongside the new options to make +it easier to transition between the two configurations, however please note that: + +- the options must not contradict each other (otherwise Synapse won't start); and +- the `notify_appservices` and `update_user_directory` options will be removed in a future release of Synapse. + +Please see the [*Notifying Application Services*][v1_59_notify_ases_from] and +[*Updating the User Directory*][v1_59_update_user_dir] sections of the worker +documentation for more information. + +[v1_59_notify_ases_from]: workers.md#notifying-application-services +[v1_59_update_user_dir]: workers.md#updating-the-user-directory + + +# Upgrading to v1.58.0 + +## Groups/communities feature has been disabled by default + +The non-standard groups/communities feature in Synapse has been disabled by default +and will be removed in Synapse v1.61.0. + + # Upgrading to v1.57.0 ## Changes to database schema for application services diff --git a/docs/usage/administration/request_log.md b/docs/usage/administration/request_log.md index 316304c7348a..adb5f4f5f353 100644 --- a/docs/usage/administration/request_log.md +++ b/docs/usage/administration/request_log.md @@ -28,7 +28,7 @@ See the following for how to decode the dense data available from the default lo | NNNN | Total time waiting for response to DB queries across all parallel DB work from this request | | OOOO | Count of DB transactions performed | | PPPP | Response body size | -| QQQQ | Response status code (prefixed with ! if the socket was closed before the response was generated) | +| QQQQ | Response status code
Suffixed with `!` if the socket was closed before the response was generated.
A `499!` status code indicates that Synapse also cancelled request processing after the socket was closed.
| | RRRR | Request | | SSSS | User-agent | | TTTT | Events fetched from DB to service this request (note that this does not include events fetched from the cache) | diff --git a/docs/usage/administration/useful_sql_for_admins.md b/docs/usage/administration/useful_sql_for_admins.md index d4aada3272d0..f3b97f957677 100644 --- a/docs/usage/administration/useful_sql_for_admins.md +++ b/docs/usage/administration/useful_sql_for_admins.md @@ -1,7 +1,10 @@ ## Some useful SQL queries for Synapse Admins ## Size of full matrix db -`SELECT pg_size_pretty( pg_database_size( 'matrix' ) );` +```sql +SELECT pg_size_pretty( pg_database_size( 'matrix' ) ); +``` + ### Result example: ``` pg_size_pretty @@ -9,39 +12,19 @@ pg_size_pretty 6420 MB (1 row) ``` -## Show top 20 larger rooms by state events count -```sql -SELECT r.name, s.room_id, s.current_state_events - FROM room_stats_current s - LEFT JOIN room_stats_state r USING (room_id) - ORDER BY current_state_events DESC - LIMIT 20; -``` - -and by state_group_events count: -```sql -SELECT rss.name, s.room_id, count(s.room_id) FROM state_groups_state s -LEFT JOIN room_stats_state rss USING (room_id) -GROUP BY s.room_id, rss.name -ORDER BY count(s.room_id) DESC -LIMIT 20; -``` -plus same, but with join removed for performance reasons: -```sql -SELECT s.room_id, count(s.room_id) FROM state_groups_state s -GROUP BY s.room_id -ORDER BY count(s.room_id) DESC -LIMIT 20; -``` ## Show top 20 larger tables by row count ```sql -SELECT relname, n_live_tup as rows - FROM pg_stat_user_tables +SELECT relname, n_live_tup AS "rows" + FROM pg_stat_user_tables ORDER BY n_live_tup DESC LIMIT 20; ``` -This query is quick, but may be very approximate, for exact number of rows use `SELECT COUNT(*) FROM `. +This query is quick, but may be very approximate, for exact number of rows use: +```sql +SELECT COUNT(*) FROM ; +``` + ### Result example: ``` state_groups_state - 161687170 @@ -66,46 +49,19 @@ device_lists_stream - 326903 user_directory_search - 316433 ``` -## Show top 20 rooms by new events count in last 1 day: -```sql -SELECT e.room_id, r.name, COUNT(e.event_id) cnt FROM events e -LEFT JOIN room_stats_state r USING (room_id) -WHERE e.origin_server_ts >= DATE_PART('epoch', NOW() - INTERVAL '1 day') * 1000 GROUP BY e.room_id, r.name ORDER BY cnt DESC LIMIT 20; -``` - -## Show top 20 users on homeserver by sent events (messages) at last month: -```sql -SELECT user_id, SUM(total_events) - FROM user_stats_historical - WHERE TO_TIMESTAMP(end_ts/1000) AT TIME ZONE 'UTC' > date_trunc('day', now() - interval '1 month') - GROUP BY user_id - ORDER BY SUM(total_events) DESC - LIMIT 20; -``` - -## Show last 100 messages from needed user, with room names: -```sql -SELECT e.room_id, r.name, e.event_id, e.type, e.content, j.json FROM events e - LEFT JOIN event_json j USING (room_id) - LEFT JOIN room_stats_state r USING (room_id) - WHERE sender = '@LOGIN:example.com' - AND e.type = 'm.room.message' - ORDER BY stream_ordering DESC - LIMIT 100; -``` - ## Show top 20 larger tables by storage size ```sql SELECT nspname || '.' || relname AS "relation", - pg_size_pretty(pg_total_relation_size(C.oid)) AS "total_size" - FROM pg_class C - LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) + pg_size_pretty(pg_total_relation_size(c.oid)) AS "total_size" + FROM pg_class c + LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) WHERE nspname NOT IN ('pg_catalog', 'information_schema') - AND C.relkind <> 'i' + AND c.relkind <> 'i' AND nspname !~ '^pg_toast' - ORDER BY pg_total_relation_size(C.oid) DESC + ORDER BY pg_total_relation_size(c.oid) DESC LIMIT 20; ``` + ### Result example: ``` public.state_groups_state - 27 GB @@ -130,8 +86,93 @@ public.device_lists_remote_cache - 124 MB public.state_group_edges - 122 MB ``` +## Show top 20 larger rooms by state events count +You get the same information when you use the +[admin API](../../admin_api/rooms.md#list-room-api) +and set parameter `order_by=state_events`. + +```sql +SELECT r.name, s.room_id, s.current_state_events + FROM room_stats_current s + LEFT JOIN room_stats_state r USING (room_id) + ORDER BY current_state_events DESC + LIMIT 20; +``` + +and by state_group_events count: +```sql +SELECT rss.name, s.room_id, COUNT(s.room_id) + FROM state_groups_state s + LEFT JOIN room_stats_state rss USING (room_id) + GROUP BY s.room_id, rss.name + ORDER BY COUNT(s.room_id) DESC + LIMIT 20; +``` + +plus same, but with join removed for performance reasons: +```sql +SELECT s.room_id, COUNT(s.room_id) + FROM state_groups_state s + GROUP BY s.room_id + ORDER BY COUNT(s.room_id) DESC + LIMIT 20; +``` + +## Show top 20 rooms by new events count in last 1 day: +```sql +SELECT e.room_id, r.name, COUNT(e.event_id) cnt + FROM events e + LEFT JOIN room_stats_state r USING (room_id) + WHERE e.origin_server_ts >= DATE_PART('epoch', NOW() - INTERVAL '1 day') * 1000 + GROUP BY e.room_id, r.name + ORDER BY cnt DESC + LIMIT 20; +``` + +## Show top 20 users on homeserver by sent events (messages) at last month: +Caution. This query does not use any indexes, can be slow and create load on the database. +```sql +SELECT COUNT(*), sender + FROM events + WHERE (type = 'm.room.encrypted' OR type = 'm.room.message') + AND origin_server_ts >= DATE_PART('epoch', NOW() - INTERVAL '1 month') * 1000 + GROUP BY sender + ORDER BY COUNT(*) DESC + LIMIT 20; +``` + +## Show last 100 messages from needed user, with room names: +```sql +SELECT e.room_id, r.name, e.event_id, e.type, e.content, j.json + FROM events e + LEFT JOIN event_json j USING (room_id) + LEFT JOIN room_stats_state r USING (room_id) + WHERE sender = '@LOGIN:example.com' + AND e.type = 'm.room.message' + ORDER BY stream_ordering DESC + LIMIT 100; +``` + ## Show rooms with names, sorted by events in this rooms -`echo "select event_json.room_id,room_stats_state.name from event_json,room_stats_state where room_stats_state.room_id=event_json.room_id" | psql synapse | sort | uniq -c | sort -n` + +**Sort and order with bash** +```bash +echo "SELECT event_json.room_id, room_stats_state.name FROM event_json, room_stats_state \ +WHERE room_stats_state.room_id = event_json.room_id" | psql -d synapse -h localhost -U synapse_user -t \ +| sort | uniq -c | sort -n +``` +Documentation for `psql` command line parameters: https://www.postgresql.org/docs/current/app-psql.html + +**Sort and order with SQL** +```sql +SELECT COUNT(*), event_json.room_id, room_stats_state.name + FROM event_json, room_stats_state + WHERE room_stats_state.room_id = event_json.room_id + GROUP BY event_json.room_id, room_stats_state.name + ORDER BY COUNT(*) DESC + LIMIT 50; +``` + ### Result example: ``` 9459 !FPUfgzXYWTKgIrwKxW:matrix.org | This Week in Matrix @@ -145,12 +186,22 @@ public.state_group_edges - 122 MB ``` ## Lookup room state info by list of room_id +You get the same information when you use the +[admin API](../../admin_api/rooms.md#room-details-api). ```sql -SELECT rss.room_id, rss.name, rss.canonical_alias, rss.topic, rss.encryption, rsc.joined_members, rsc.local_users_in_room, rss.join_rules -FROM room_stats_state rss -LEFT JOIN room_stats_current rsc USING (room_id) -WHERE room_id IN (WHERE room_id IN ( - '!OGEhHVWSdvArJzumhm:matrix.org', - '!YTvKGNlinIzlkMTVRl:matrix.org' -) -``` \ No newline at end of file +SELECT rss.room_id, rss.name, rss.canonical_alias, rss.topic, rss.encryption, + rsc.joined_members, rsc.local_users_in_room, rss.join_rules + FROM room_stats_state rss + LEFT JOIN room_stats_current rsc USING (room_id) + WHERE room_id IN ( WHERE room_id IN ( + '!OGEhHVWSdvArJzumhm:matrix.org', + '!YTvKGNlinIzlkMTVRl:matrix.org' + ); +``` + +## Show users and devices that have not been online for a while +```sql +SELECT user_id, device_id, user_agent, TO_TIMESTAMP(last_seen / 1000) AS "last_seen" + FROM devices + WHERE last_seen < DATE_PART('epoch', NOW() - INTERVAL '3 month') * 1000; +``` diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md new file mode 100644 index 000000000000..295cece7e894 --- /dev/null +++ b/docs/usage/configuration/config_documentation.md @@ -0,0 +1,3554 @@ +# Configuring Synapse + +This is intended as a guide to the Synapse configuration. The behavior of a Synapse instance can be modified +through the many configuration settings documented here — each config option is explained, +including what the default is, how to change the default and what sort of behaviour the setting governs. +Also included is an example configuration for each setting. If you don't want to spend a lot of time +thinking about options, the config as generated sets sensible defaults for all values. Do note however that the +database defaults to SQLite, which is not recommended for production usage. You can read more on this subject +[here](../../setup/installation.md#using-postgresql). + +## Config Conventions + +Configuration options that take a time period can be set using a number +followed by a letter. Letters have the following meanings: + +* `s` = second +* `m` = minute +* `h` = hour +* `d` = day +* `w` = week +* `y` = year + +For example, setting `redaction_retention_period: 5m` would remove redacted +messages from the database after 5 minutes, rather than 5 months. + +In addition, configuration options referring to size use the following suffixes: + +* `M` = MiB, or 1,048,576 bytes +* `K` = KiB, or 1024 bytes + +For example, setting `max_avatar_size: 10M` means that Synapse will not accept files larger than 10,485,760 bytes +for a user avatar. + +### YAML +The configuration file is a [YAML](https://yaml.org/) file, which means that certain syntax rules +apply if you want your config file to be read properly. A few helpful things to know: +* `#` before any option in the config will comment out that setting and either a default (if available) will + be applied or Synapse will ignore the setting. Thus, in example #1 below, the setting will be read and + applied, but in example #2 the setting will not be read and a default will be applied. + + Example #1: + ```yaml + pid_file: DATADIR/homeserver.pid + ``` + Example #2: + ```yaml + #pid_file: DATADIR/homeserver.pid + ``` +* Indentation matters! The indentation before a setting + will determine whether a given setting is read as part of another + setting, or considered on its own. Thus, in example #1, the `enabled` setting + is read as a sub-option of the `presence` setting, and will be properly applied. + + However, the lack of indentation before the `enabled` setting in example #2 means + that when reading the config, Synapse will consider both `presence` and `enabled` as + different settings. In this case, `presence` has no value, and thus a default applied, and `enabled` + is an option that Synapse doesn't recognize and thus ignores. + + Example #1: + ```yaml + presence: + enabled: false + ``` + Example #2: + ```yaml + presence: + enabled: false + ``` + In this manual, all top-level settings (ones with no indentation) are identified + at the beginning of their section (i.e. "Config option: `example_setting`") and + the sub-options, if any, are identified and listed in the body of the section. + In addition, each setting has an example of its usage, with the proper indentation + shown. + +## Contents +[Modules](#modules) + +[Server](#server) + +[Homeserver Blocking](#homeserver-blocking) + +[TLS](#tls) + +[Federation](#federation) + +[Caching](#caching) + +[Database](#database) + +[Logging](#logging) + +[Ratelimiting](#ratelimiting) + +[Media Store](#media-store) + +[Captcha](#captcha) + +[TURN](#turn) + +[Registration](#registration) + +[API Configuration](#api-configuration) + +[Signing Keys](#signing-keys) + +[Single Sign On Integration](#single-sign-on-integration) + +[Push](#push) + +[Rooms](#rooms) + +[Opentracing](#opentracing) + +[Workers](#workers) + +[Background Updates](#background-updates) + +## Modules + +Server admins can expand Synapse's functionality with external modules. + +See [here](../../modules/index.md) for more +documentation on how to configure or create custom modules for Synapse. + + +--- +Config option: `modules` + +Use the `module` sub-option to add modules under this option to extend functionality. +The `module` setting then has a sub-option, `config`, which can be used to define some configuration +for the `module`. + +Defaults to none. + +Example configuration: +```yaml +modules: + - module: my_super_module.MySuperClass + config: + do_thing: true + - module: my_other_super_module.SomeClass + config: {} +``` +--- +## Server ## + +Define your homeserver name and other base options. + +--- +Config option: `server_name` + +This sets the public-facing domain of the server. + +The `server_name` name will appear at the end of usernames and room addresses +created on your server. For example if the `server_name` was example.com, +usernames on your server would be in the format `@user:example.com` + +In most cases you should avoid using a matrix specific subdomain such as +matrix.example.com or synapse.example.com as the `server_name` for the same +reasons you wouldn't use user@email.example.com as your email address. +See [here](../../delegate.md) +for information on how to host Synapse on a subdomain while preserving +a clean `server_name`. + +The `server_name` cannot be changed later so it is important to +configure this correctly before you start Synapse. It should be all +lowercase and may contain an explicit port. + +There is no default for this option. + +Example configuration #1: +```yaml +server_name: matrix.org +``` +Example configuration #2: +```yaml +server_name: localhost:8080 +``` +--- +Config option: `pid_file` + +When running Synapse as a daemon, the file to store the pid in. Defaults to none. + +Example configuration: +```yaml +pid_file: DATADIR/homeserver.pid +``` +--- +Config option: `web_client_location` + +The absolute URL to the web client which `/` will redirect to. Defaults to none. + +Example configuration: +```yaml +web_client_location: https://riot.example.com/ +``` +--- +Config option: `public_baseurl` + +The public-facing base URL that clients use to access this Homeserver (not +including _matrix/...). This is the same URL a user might enter into the +'Custom Homeserver URL' field on their client. If you use Synapse with a +reverse proxy, this should be the URL to reach Synapse via the proxy. +Otherwise, it should be the URL to reach Synapse's client HTTP listener (see +'listeners' below). + +Defaults to `https:///`. + +Example configuration: +```yaml +public_baseurl: https://example.com/ +``` +--- +Config option: `serve_server_wellknown` + +By default, other servers will try to reach our server on port 8448, which can +be inconvenient in some environments. + +Provided `https:///` on port 443 is routed to Synapse, this +option configures Synapse to serve a file at `https:///.well-known/matrix/server`. +This will tell other servers to send traffic to port 443 instead. + +This option currently defaults to false. + +See https://matrix-org.github.io/synapse/latest/delegate.html for more +information. + +Example configuration: +```yaml +serve_server_wellknown: true +``` +--- +Config option: `soft_file_limit` + +Set the soft limit on the number of file descriptors synapse can use. +Zero is used to indicate synapse should set the soft limit to the hard limit. +Defaults to 0. + +Example configuration: +```yaml +soft_file_limit: 3 +``` +--- +Config option: `presence` + +Presence tracking allows users to see the state (e.g online/offline) +of other local and remote users. Set the `enabled` sub-option to false to +disable presence tracking on this homeserver. Defaults to true. +This option replaces the previous top-level 'use_presence' option. + +Example configuration: +```yaml +presence: + enabled: false +``` +--- +Config option: `require_auth_for_profile_requests` + +Whether to require authentication to retrieve profile data (avatars, display names) of other +users through the client API. Defaults to false. Note that profile data is also available +via the federation API, unless `allow_profile_lookup_over_federation` is set to false. + +Example configuration: +```yaml +require_auth_for_profile_requests: true +``` +--- +Config option: `limit_profile_requests_to_users_who_share_rooms` + +Use this option to require a user to share a room with another user in order +to retrieve their profile information. Only checked on Client-Server +requests. Profile requests from other servers should be checked by the +requesting server. Defaults to false. + +Example configuration: +```yaml +limit_profile_requests_to_users_who_share_rooms: true +``` +--- +Config option: `include_profile_data_on_invite` + +Use this option to prevent a user's profile data from being retrieved and +displayed in a room until they have joined it. By default, a user's +profile data is included in an invite event, regardless of the values +of the above two settings, and whether or not the users share a server. +Defaults to true. + +Example configuration: +```yaml +include_profile_data_on_invite: false +``` +--- +Config option: `allow_public_rooms_without_auth` + +If set to true, removes the need for authentication to access the server's +public rooms directory through the client API, meaning that anyone can +query the room directory. Defaults to false. + +Example configuration: +```yaml +allow_public_rooms_without_auth: true +``` +--- +Config option: `allow_public_rooms_without_auth` + +If set to true, allows any other homeserver to fetch the server's public +rooms directory via federation. Defaults to false. + +Example configuration: +```yaml +allow_public_rooms_over_federation: true +``` +--- +Config option: `default_room_version` + +The default room version for newly created rooms on this server. + +Known room versions are listed [here](https://spec.matrix.org/latest/rooms/#complete-list-of-room-versions) + +For example, for room version 1, `default_room_version` should be set +to "1". + +Currently defaults to "9". + +Example configuration: +```yaml +default_room_version: "8" +``` +--- +Config option: `gc_thresholds` + +The garbage collection threshold parameters to pass to `gc.set_threshold`, if defined. +Defaults to none. + +Example configuration: +```yaml +gc_thresholds: [700, 10, 10] +``` +--- +Config option: `gc_min_interval` + +The minimum time in seconds between each GC for a generation, regardless of +the GC thresholds. This ensures that we don't do GC too frequently. A value of `[1s, 10s, 30s]` +indicates that a second must pass between consecutive generation 0 GCs, etc. + +Defaults to `[1s, 10s, 30s]`. + +Example configuration: +```yaml +gc_min_interval: [0.5s, 30s, 1m] +``` +--- +Config option: `filter_timeline_limit` + +Set the limit on the returned events in the timeline in the get +and sync operations. Defaults to 100. A value of -1 means no upper limit. + + +Example configuration: +```yaml +filter_timeline_limit: 5000 +``` +--- +Config option: `block_non_admin_invites` + +Whether room invites to users on this server should be blocked +(except those sent by local server admins). Defaults to false. + +Example configuration: +```yaml +block_non_admin_invites: true +``` +--- +Config option: `enable_search` + +If set to false, new messages will not be indexed for searching and users +will receive errors when searching for messages. Defaults to true. + +Example configuration: +```yaml +enable_search: false +``` +--- +Config option: `ip_range_blacklist` + +This option prevents outgoing requests from being sent to the specified blacklisted IP address +CIDR ranges. If this option is not specified then it defaults to private IP +address ranges (see the example below). + +The blacklist applies to the outbound requests for federation, identity servers, +push servers, and for checking key validity for third-party invite events. + +(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly +listed here, since they correspond to unroutable addresses.) + +This option replaces `federation_ip_range_blacklist` in Synapse v1.25.0. + +Note: The value is ignored when an HTTP proxy is in use. + +Example configuration: +```yaml +ip_range_blacklist: + - '127.0.0.0/8' + - '10.0.0.0/8' + - '172.16.0.0/12' + - '192.168.0.0/16' + - '100.64.0.0/10' + - '192.0.0.0/24' + - '169.254.0.0/16' + - '192.88.99.0/24' + - '198.18.0.0/15' + - '192.0.2.0/24' + - '198.51.100.0/24' + - '203.0.113.0/24' + - '224.0.0.0/4' + - '::1/128' + - 'fe80::/10' + - 'fc00::/7' + - '2001:db8::/32' + - 'ff00::/8' + - 'fec0::/10' +``` +--- +Config option: `ip_range_whitelist` + +List of IP address CIDR ranges that should be allowed for federation, +identity servers, push servers, and for checking key validity for +third-party invite events. This is useful for specifying exceptions to +wide-ranging blacklisted target IP ranges - e.g. for communication with +a push server only visible in your network. + +This whitelist overrides `ip_range_blacklist` and defaults to an empty +list. + +Example configuration: +```yaml +ip_range_whitelist: + - '192.168.1.1' +``` +--- +Config option: `listeners` + +List of ports that Synapse should listen on, their purpose and their +configuration. + +Sub-options for each listener include: + +* `port`: the TCP port to bind to. + +* `bind_addresses`: a list of local addresses to listen on. The default is + 'all local interfaces'. + +* `type`: the type of listener. Normally `http`, but other valid options are: + + * `manhole`: (see the docs [here](../../manhole.md)), + + * `metrics`: (see the docs [here](../../metrics-howto.md)), + + * `replication`: (see the docs [here](../../workers.md)). + +* `tls`: set to true to enable TLS for this listener. Will use the TLS key/cert specified in tls_private_key_path / tls_certificate_path. + +* `x_forwarded`: Only valid for an 'http' listener. Set to true to use the X-Forwarded-For header as the client IP. Useful when Synapse is + behind a reverse-proxy. + +* `resources`: Only valid for an 'http' listener. A list of resources to host + on this port. Sub-options for each resource are: + + * `names`: a list of names of HTTP resources. See below for a list of valid resource names. + + * `compress`: set to true to enable HTTP compression for this resource. + +* `additional_resources`: Only valid for an 'http' listener. A map of + additional endpoints which should be loaded via dynamic modules. + +Valid resource names are: + +* `client`: the client-server API (/_matrix/client), and the synapse admin API (/_synapse/admin). Also implies `media` and `static`. + +* `consent`: user consent forms (/_matrix/consent). See [here](../../consent_tracking.md) for more. + +* `federation`: the server-server API (/_matrix/federation). Also implies `media`, `keys`, `openid` + +* `keys`: the key discovery API (/_matrix/key). + +* `media`: the media API (/_matrix/media). + +* `metrics`: the metrics interface. See [here](../../metrics-howto.md). + +* `openid`: OpenID authentication. See [here](../../openid.md). + +* `replication`: the HTTP replication API (/_synapse/replication). See [here](../../workers.md). + +* `static`: static resources under synapse/static (/_matrix/static). (Mostly useful for 'fallback authentication'.) + +Example configuration #1: +```yaml +listeners: + # TLS-enabled listener: for when matrix traffic is sent directly to synapse. + # + # (Note that you will also need to give Synapse a TLS key and certificate: see the TLS section + # below.) + # + - port: 8448 + type: http + tls: true + resources: + - names: [client, federation] +``` +Example configuration #2: +```yaml +listeners: + # Unsecure HTTP listener: for when matrix traffic passes through a reverse proxy + # that unwraps TLS. + # + # If you plan to use a reverse proxy, please see + # https://matrix-org.github.io/synapse/latest/reverse_proxy.html. + # + - port: 8008 + tls: false + type: http + x_forwarded: true + bind_addresses: ['::1', '127.0.0.1'] + + resources: + - names: [client, federation] + compress: false + + # example additional_resources: + additional_resources: + "/_matrix/my/custom/endpoint": + module: my_module.CustomRequestHandler + config: {} + + # Turn on the twisted ssh manhole service on localhost on the given + # port. + - port: 9000 + bind_addresses: ['::1', '127.0.0.1'] + type: manhole +``` +--- +Config option: `manhole_settings` + +Connection settings for the manhole. You can find more information +on the manhole [here](../../manhole.md). Manhole sub-options include: +* `username` : the username for the manhole. This defaults to 'matrix'. +* `password`: The password for the manhole. This defaults to 'rabbithole'. +* `ssh_priv_key_path` and `ssh_pub_key_path`: The private and public SSH key pair used to encrypt the manhole traffic. + If these are left unset, then hardcoded and non-secret keys are used, + which could allow traffic to be intercepted if sent over a public network. + +Example configuration: +```yaml +manhole_settings: + username: manhole + password: mypassword + ssh_priv_key_path: CONFDIR/id_rsa + ssh_pub_key_path: CONFDIR/id_rsa.pub +``` +--- +Config option: `dummy_events_threshold` + +Forward extremities can build up in a room due to networking delays between +homeservers. Once this happens in a large room, calculation of the state of +that room can become quite expensive. To mitigate this, once the number of +forward extremities reaches a given threshold, Synapse will send an +`org.matrix.dummy_event` event, which will reduce the forward extremities +in the room. + +This setting defines the threshold (i.e. number of forward extremities in the room) at which dummy events are sent. +The default value is 10. + +Example configuration: +```yaml +dummy_events_threshold: 5 +``` +--- +## Homeserver blocking ## +Useful options for Synapse admins. + +--- + +Config option: `admin_contact` + +How to reach the server admin, used in `ResourceLimitError`. Defaults to none. + +Example configuration: +```yaml +admin_contact: 'mailto:admin@server.com' +``` +--- +Config option: `hs_disabled` and `hs_disabled_message` + +Blocks users from connecting to the homeserver and provides a human-readable reason +why the connection was blocked. Defaults to false. + +Example configuration: +```yaml +hs_disabled: true +hs_disabled_message: 'Reason for why the HS is blocked' +``` +--- +Config option: `limit_usage_by_mau` + +This option disables/enables monthly active user blocking. Used in cases where the admin or +server owner wants to limit to the number of monthly active users. When enabled and a limit is +reached the server returns a `ResourceLimitError` with error type `Codes.RESOURCE_LIMIT_EXCEEDED`. +Defaults to false. If this is enabled, a value for `max_mau_value` must also be set. + +Example configuration: +```yaml +limit_usage_by_mau: true +``` +--- +Config option: `max_mau_value` + +This option sets the hard limit of monthly active users above which the server will start +blocking user actions if `limit_usage_by_mau` is enabled. Defaults to 0. + +Example configuration: +```yaml +max_mau_value: 50 +``` +--- +Config option: `mau_trial_days` + +The option `mau_trial_days` is a means to add a grace period for active users. It +means that users must be active for the specified number of days before they +can be considered active and guards against the case where lots of users +sign up in a short space of time never to return after their initial +session. Defaults to 0. + +Example configuration: +```yaml +mau_trial_days: 5 +``` +--- +Config option: `mau_appservice_trial_days` + +The option `mau_appservice_trial_days` is similar to `mau_trial_days`, but applies a different +trial number if the user was registered by an appservice. A value +of 0 means no trial days are applied. Appservices not listed in this dictionary +use the value of `mau_trial_days` instead. + +Example configuration: +```yaml +mau_appservice_trial_days: + my_appservice_id: 3 + another_appservice_id: 6 +``` +--- +Config option: `mau_limit_alerting` + +The option `mau_limit_alerting` is a means of limiting client-side alerting +should the mau limit be reached. This is useful for small instances +where the admin has 5 mau seats (say) for 5 specific people and no +interest increasing the mau limit further. Defaults to true, which +means that alerting is enabled. + +Example configuration: +```yaml +mau_limit_alerting: false +``` +--- +Config option: `mau_stats_only` + +If enabled, the metrics for the number of monthly active users will +be populated, however no one will be limited based on these numbers. If `limit_usage_by_mau` +is true, this is implied to be true. Defaults to false. + +Example configuration: +```yaml +mau_stats_only: true +``` +--- +Config option: `mau_limit_reserved_threepids` + +Sometimes the server admin will want to ensure certain accounts are +never blocked by mau checking. These accounts are specified by this option. +Defaults to none. Add accounts by specifying the `medium` and `address` of the +reserved threepid (3rd party identifier). + +Example configuration: +```yaml +mau_limit_reserved_threepids: + - medium: 'email' + address: 'reserved_user@example.com' +``` +--- +Config option: `server_context` + +This option is used by phonehome stats to group together related servers. +Defaults to none. + +Example configuration: +```yaml +server_context: context +``` +--- +Config option: `limit_remote_rooms` + +When this option is enabled, the room "complexity" will be checked before a user +joins a new remote room. If it is above the complexity limit, the server will +disallow joining, or will instantly leave. This is useful for homeservers that are +resource-constrained. Options for this setting include: +* `enabled`: whether this check is enabled. Defaults to false. +* `complexity`: the limit above which rooms cannot be joined. The default is 1.0. +* `complexity_error`: override the error which is returned when the room is too complex with a + custom message. +* `admins_can_join`: allow server admins to join complex rooms. Default is false. + +Room complexity is an arbitrary measure based on factors such as the number of +users in the room. + +Example configuration: +```yaml +limit_remote_rooms: + enabled: true + complexity: 0.5 + complexity_error: "I can't let you do that, Dave." + admins_can_join: true +``` +--- +Config option: `require_membership_for_aliases` + +Whether to require a user to be in the room to add an alias to it. +Defaults to true. + +Example configuration: +```yaml +require_membership_for_aliases: false +``` +--- +Config option: `allow_per_room_profiles` + +Whether to allow per-room membership profiles through the sending of membership +events with profile information that differs from the target's global profile. +Defaults to true. + +Example configuration: +```yaml +allow_per_room_profiles: false +``` +--- +Config option: `max_avatar_size` + +The largest permissible file size in bytes for a user avatar. Defaults to no restriction. +Use M for MB and K for KB. + +Note that user avatar changes will not work if this is set without using Synapse's media repository. + +Example configuration: +```yaml +max_avatar_size: 10M +``` +--- +Config option: `allowed_avatar_mimetypes` + +The MIME types allowed for user avatars. Defaults to no restriction. + +Note that user avatar changes will not work if this is set without +using Synapse's media repository. + +Example configuration: +```yaml +allowed_avatar_mimetypes: ["image/png", "image/jpeg", "image/gif"] +``` +--- +Config option: `redaction_retention_period` + +How long to keep redacted events in unredacted form in the database. After +this period redacted events get replaced with their redacted form in the DB. + +Defaults to `7d`. Set to `null` to disable. + +Example configuration: +```yaml +redaction_retention_period: 28d +``` +--- +Config option: `user_ips_max_age` + +How long to track users' last seen time and IPs in the database. + +Defaults to `28d`. Set to `null` to disable clearing out of old rows. + +Example configuration: +```yaml +user_ips_max_age: 14d +``` +--- +Config option: `request_token_inhibit_3pid_errors` + +Inhibits the `/requestToken` endpoints from returning an error that might leak +information about whether an e-mail address is in use or not on this +homeserver. Defaults to false. +Note that for some endpoints the error situation is the e-mail already being +used, and for others the error is entering the e-mail being unused. +If this option is enabled, instead of returning an error, these endpoints will +act as if no error happened and return a fake session ID ('sid') to clients. + +Example configuration: +```yaml +request_token_inhibit_3pid_errors: true +``` +--- +Config option: `next_link_domain_whitelist` + +A list of domains that the domain portion of `next_link` parameters +must match. + +This parameter is optionally provided by clients while requesting +validation of an email or phone number, and maps to a link that +users will be automatically redirected to after validation +succeeds. Clients can make use this parameter to aid the validation +process. + +The whitelist is applied whether the homeserver or an identity server is handling validation. + +The default value is no whitelist functionality; all domains are +allowed. Setting this value to an empty list will instead disallow +all domains. + +Example configuration: +```yaml +next_link_domain_whitelist: ["matrix.org"] +``` +--- +Config option: `templates` and `custom_template_directory` + +These options define templates to use when generating email or HTML page contents. +The `custom_template_directory` determines which directory Synapse will try to +find template files in to use to generate email or HTML page contents. +If not set, or a file is not found within the template directory, a default +template from within the Synapse package will be used. + +See [here](../../templates.md) for more +information about using custom templates. + +Example configuration: +```yaml +templates: + custom_template_directory: /path/to/custom/templates/ +``` +--- +Config option: `retention` + +This option and the associated options determine message retention policy at the +server level. + +Room admins and mods can define a retention period for their rooms using the +`m.room.retention` state event, and server admins can cap this period by setting +the `allowed_lifetime_min` and `allowed_lifetime_max` config options. + +If this feature is enabled, Synapse will regularly look for and purge events +which are older than the room's maximum retention period. Synapse will also +filter events received over federation so that events that should have been +purged are ignored and not stored again. + +The message retention policies feature is disabled by default. + +This setting has the following sub-options: +* `default_policy`: Default retention policy. If set, Synapse will apply it to rooms that lack the + 'm.room.retention' state event. This option is further specified by the + `min_lifetime` and `max_lifetime` sub-options associated with it. Note that the + value of `min_lifetime` doesn't matter much because Synapse doesn't take it into account yet. + +* `allowed_lifetime_min` and `allowed_lifetime_max`: Retention policy limits. If + set, and the state of a room contains a `m.room.retention` event in its state + which contains a `min_lifetime` or a `max_lifetime` that's out of these bounds, + Synapse will cap the room's policy to these limits when running purge jobs. + +* `purge_jobs` and the associated `shortest_max_lifetime` and `longest_max_lifetime` sub-options: + Server admins can define the settings of the background jobs purging the + events whose lifetime has expired under the `purge_jobs` section. + + If no configuration is provided for this option, a single job will be set up to delete + expired events in every room daily. + + Each job's configuration defines which range of message lifetimes the job + takes care of. For example, if `shortest_max_lifetime` is '2d' and + `longest_max_lifetime` is '3d', the job will handle purging expired events in + rooms whose state defines a `max_lifetime` that's both higher than 2 days, and + lower than or equal to 3 days. Both the minimum and the maximum value of a + range are optional, e.g. a job with no `shortest_max_lifetime` and a + `longest_max_lifetime` of '3d' will handle every room with a retention policy + whose `max_lifetime` is lower than or equal to three days. + + The rationale for this per-job configuration is that some rooms might have a + retention policy with a low `max_lifetime`, where history needs to be purged + of outdated messages on a more frequent basis than for the rest of the rooms + (e.g. every 12h), but not want that purge to be performed by a job that's + iterating over every room it knows, which could be heavy on the server. + + If any purge job is configured, it is strongly recommended to have at least + a single job with neither `shortest_max_lifetime` nor `longest_max_lifetime` + set, or one job without `shortest_max_lifetime` and one job without + `longest_max_lifetime` set. Otherwise some rooms might be ignored, even if + `allowed_lifetime_min` and `allowed_lifetime_max` are set, because capping a + room's policy to these values is done after the policies are retrieved from + Synapse's database (which is done using the range specified in a purge job's + configuration). + +Example configuration: +```yaml +retention: + enabled: true + default_policy: + min_lifetime: 1d + max_lifetime: 1y + allowed_lifetime_min: 1d + allowed_lifetime_max: 1y + purge_jobs: + - longest_max_lifetime: 3d + interval: 12h + - shortest_max_lifetime: 3d + interval: 1d +``` +--- +## TLS ## + +Options related to TLS. + +--- +Config option: `tls_certificate_path` + +This option specifies a PEM-encoded X509 certificate for TLS. +This certificate, as of Synapse 1.0, will need to be a valid and verifiable +certificate, signed by a recognised Certificate Authority. Defaults to none. + +Be sure to use a `.pem` file that includes the full certificate chain including +any intermediate certificates (for instance, if using certbot, use +`fullchain.pem` as your certificate, not `cert.pem`). + +Example configuration: +```yaml +tls_certificate_path: "CONFDIR/SERVERNAME.tls.crt" +``` +--- +Config option: `tls_private_key_path` + +PEM-encoded private key for TLS. Defaults to none. + +Example configuration: +```yaml +tls_private_key_path: "CONFDIR/SERVERNAME.tls.key" +``` +--- +Config option: `federation_verify_certificates` +Whether to verify TLS server certificates for outbound federation requests. + +Defaults to true. To disable certificate verification, set the option to false. + +Example configuration: +```yaml +federation_verify_certificates: false +``` +--- +Config option: `federation_client_minimum_tls_version` + +The minimum TLS version that will be used for outbound federation requests. + +Defaults to `1`. Configurable to `1`, `1.1`, `1.2`, or `1.3`. Note +that setting this value higher than `1.2` will prevent federation to most +of the public Matrix network: only configure it to `1.3` if you have an +entirely private federation setup and you can ensure TLS 1.3 support. + +Example configuration: +```yaml +federation_client_minimum_tls_version: 1.2 +``` +--- +Config option: `federation_certificate_verification_whitelist` + +Skip federation certificate verification on a given whitelist +of domains. + +This setting should only be used in very specific cases, such as +federation over Tor hidden services and similar. For private networks +of homeservers, you likely want to use a private CA instead. + +Only effective if `federation_verify_certicates` is `true`. + +Example configuration: +```yaml +federation_certificate_verification_whitelist: + - lon.example.com + - "*.domain.com" + - "*.onion" +``` +--- +Config option: `federation_custom_ca_list` + +List of custom certificate authorities for federation traffic. + +This setting should only normally be used within a private network of +homeservers. + +Note that this list will replace those that are provided by your +operating environment. Certificates must be in PEM format. + +Example configuration: +```yaml +federation_custom_ca_list: + - myCA1.pem + - myCA2.pem + - myCA3.pem +``` +--- +## Federation ## + +Options related to federation. + +--- +Config option: `federation_domain_whitelist` + +Restrict federation to the given whitelist of domains. +N.B. we recommend also firewalling your federation listener to limit +inbound federation traffic as early as possible, rather than relying +purely on this application-layer restriction. If not specified, the +default is to whitelist everything. + +Example configuration: +```yaml +federation_domain_whitelist: + - lon.example.com + - nyc.example.com + - syd.example.com +``` +--- +Config option: `federation_metrics_domains` + +Report prometheus metrics on the age of PDUs being sent to and received from +the given domains. This can be used to give an idea of "delay" on inbound +and outbound federation, though be aware that any delay can be due to problems +at either end or with the intermediate network. + +By default, no domains are monitored in this way. + +Example configuration: +```yaml +federation_metrics_domains: + - matrix.org + - example.com +``` +--- +Config option: `allow_profile_lookup_over_federation` + +Set to false to disable profile lookup over federation. By default, the +Federation API allows other homeservers to obtain profile data of any user +on this homeserver. + +Example configuration: +```yaml +allow_profile_lookup_over_federation: false +``` +--- +Config option: `allow_device_name_lookup_over_federation` + +Set this option to true to allow device display name lookup over federation. By default, the +Federation API prevents other homeservers from obtaining the display names of any user devices +on this homeserver. + +Example configuration: +```yaml +allow_device_name_lookup_over_federation: true +``` +--- +## Caching ## + +Options related to caching + +--- +Config option: `event_cache_size` + +The number of events to cache in memory. Not affected by +`caches.global_factor`. Defaults to 10K. + +Example configuration: +```yaml +event_cache_size: 15K +``` +--- +Config option: `cache` and associated values + +A cache 'factor' is a multiplier that can be applied to each of +Synapse's caches in order to increase or decrease the maximum +number of entries that can be stored. + +Caching can be configured through the following sub-options: + +* `global_factor`: Controls the global cache factor, which is the default cache factor + for all caches if a specific factor for that cache is not otherwise + set. + + This can also be set by the `SYNAPSE_CACHE_FACTOR` environment + variable. Setting by environment variable takes priority over + setting through the config file. + + Defaults to 0.5, which will halve the size of all caches. + +* `per_cache_factors`: A dictionary of cache name to cache factor for that individual + cache. Overrides the global cache factor for a given cache. + + These can also be set through environment variables comprised + of `SYNAPSE_CACHE_FACTOR_` + the name of the cache in capital + letters and underscores. Setting by environment variable + takes priority over setting through the config file. + Ex. `SYNAPSE_CACHE_FACTOR_GET_USERS_WHO_SHARE_ROOM_WITH_USER=2.0` + + Some caches have '*' and other characters that are not + alphanumeric or underscores. These caches can be named with or + without the special characters stripped. For example, to specify + the cache factor for `*stateGroupCache*` via an environment + variable would be `SYNAPSE_CACHE_FACTOR_STATEGROUPCACHE=2.0`. + +* `expire_caches`: Controls whether cache entries are evicted after a specified time + period. Defaults to true. Set to false to disable this feature. Note that never expiring + caches may result in excessive memory usage. + +* `cache_entry_ttl`: If `expire_caches` is enabled, this flag controls how long an entry can + be in a cache without having been accessed before being evicted. + Defaults to 30m. + +* `sync_response_cache_duration`: Controls how long the results of a /sync request are + cached for after a successful response is returned. A higher duration can help clients + with intermittent connections, at the cost of higher memory usage. + By default, this is zero, which means that sync responses are not cached + at all. +* `cache_autotuning` and its sub-options `max_cache_memory_usage`, `target_cache_memory_usage`, and + `min_cache_ttl` work in conjunction with each other to maintain a balance between cache memory + usage and cache entry availability. You must be using [jemalloc](https://github.com/matrix-org/synapse#help-synapse-is-slow-and-eats-all-my-ramcpu) + to utilize this option, and all three of the options must be specified for this feature to work. This option + defaults to off, enable it by providing values for the sub-options listed below. Please note that the feature will not work + and may cause unstable behavior (such as excessive emptying of caches or exceptions) if all of the values are not provided. + Please see the [Config Conventions](#config-conventions) for information on how to specify memory size and cache expiry + durations. + * `max_cache_memory_usage` sets a ceiling on how much memory the cache can use before caches begin to be continuously evicted. + They will continue to be evicted until the memory usage drops below the `target_memory_usage`, set in + the setting below, or until the `min_cache_ttl` is hit. There is no default value for this option. + * `target_memory_usage` sets a rough target for the desired memory usage of the caches. There is no default value + for this option. + * `min_cache_ttl` sets a limit under which newer cache entries are not evicted and is only applied when + caches are actively being evicted/`max_cache_memory_usage` has been exceeded. This is to protect hot caches + from being emptied while Synapse is evicting due to memory. There is no default value for this option. + +Example configuration: +```yaml +caches: + global_factor: 1.0 + per_cache_factors: + get_users_who_share_room_with_user: 2.0 + sync_response_cache_duration: 2m + cache_autotuning: + max_cache_memory_usage: 1024M + target_cache_memory_usage: 758M + min_cache_ttl: 5m +``` + +### Reloading cache factors + +The cache factors (i.e. `caches.global_factor` and `caches.per_cache_factors`) may be reloaded at any time by sending a +[`SIGHUP`](https://en.wikipedia.org/wiki/SIGHUP) signal to Synapse using e.g. + +```commandline +kill -HUP [PID_OF_SYNAPSE_PROCESS] +``` + +If you are running multiple workers, you must individually update the worker +config file and send this signal to each worker process. + +If you're using the [example systemd service](https://github.com/matrix-org/synapse/blob/develop/contrib/systemd/matrix-synapse.service) +file in Synapse's `contrib` directory, you can send a `SIGHUP` signal by using +`systemctl reload matrix-synapse`. + +--- +## Database ## +Config options related to database settings. + +--- +Config option: `database` + +The `database` setting defines the database that synapse uses to store all of +its data. + +Associated sub-options: + +* `name`: this option specifies the database engine to use: either `sqlite3` (for SQLite) + or `psycopg2` (for PostgreSQL). If no name is specified Synapse will default to SQLite. + +* `txn_limit` gives the maximum number of transactions to run per connection + before reconnecting. Defaults to 0, which means no limit. + +* `allow_unsafe_locale` is an option specific to Postgres. Under the default behavior, Synapse will refuse to + start if the postgres db is set to a non-C locale. You can override this behavior (which is *not* recommended) + by setting `allow_unsafe_locale` to true. Note that doing so may corrupt your database. You can find more information + [here](../../postgres.md#fixing-incorrect-collate-or-ctype) and [here](https://wiki.postgresql.org/wiki/Locale_data_changes). + +* `args` gives options which are passed through to the database engine, + except for options starting with `cp_`, which are used to configure the Twisted + connection pool. For a reference to valid arguments, see: + * for [sqlite](https://docs.python.org/3/library/sqlite3.html#sqlite3.connect) + * for [postgres](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS) + * for [the connection pool](https://twistedmatrix.com/documents/current/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__) + +For more information on using Synapse with Postgres, +see [here](../../postgres.md). + +Example SQLite configuration: +```yaml +database: + name: sqlite3 + args: + database: /path/to/homeserver.db +``` + +Example Postgres configuration: +```yaml +database: + name: psycopg2 + txn_limit: 10000 + args: + user: synapse_user + password: secretpassword + database: synapse + host: localhost + port: 5432 + cp_min: 5 + cp_max: 10 +``` +--- +## Logging ## +Config options related to logging. + +--- +Config option: `log_config` + +This option specifies a yaml python logging config file as described [here](https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema). + +Example configuration: +```yaml +log_config: "CONFDIR/SERVERNAME.log.config" +``` +--- +## Ratelimiting ## +Options related to ratelimiting in Synapse. + +Each ratelimiting configuration is made of two parameters: + - `per_second`: number of requests a client can send per second. + - `burst_count`: number of requests a client can send before being throttled. +--- +Config option: `rc_message` + + +Ratelimiting settings for client messaging. + +This is a ratelimiting option for messages that ratelimits sending based on the account the client +is using. It defaults to: `per_second: 0.2`, `burst_count: 10`. + +Example configuration: +```yaml +rc_message: + per_second: 0.5 + burst_count: 15 +``` +--- +Config option: `rc_registration` + +This option ratelimits registration requests based on the client's IP address. +It defaults to `per_second: 0.17`, `burst_count: 3`. + +Example configuration: +```yaml +rc_registration: + per_second: 0.15 + burst_count: 2 +``` +--- +Config option: `rc_registration_token_validity` + +This option checks the validity of registration tokens that ratelimits requests based on +the client's IP address. +Defaults to `per_second: 0.1`, `burst_count: 5`. + +Example configuration: +```yaml +rc_registration_token_validity: + per_second: 0.3 + burst_count: 6 +``` +--- +Config option: `rc_login` + +This option specifies several limits for login: +* `address` ratelimits login requests based on the client's IP + address. Defaults to `per_second: 0.17`, `burst_count: 3`. + +* `account` ratelimits login requests based on the account the + client is attempting to log into. Defaults to `per_second: 0.17`, + `burst_count: 3`. + +* `failted_attempts` ratelimits login requests based on the account the + client is attempting to log into, based on the amount of failed login + attempts for this account. Defaults to `per_second: 0.17`, `burst_count: 3`. + +Example configuration: +```yaml +rc_login: + address: + per_second: 0.15 + burst_count: 5 + account: + per_second: 0.18 + burst_count: 4 + failed_attempts: + per_second: 0.19 + burst_count: 7 +``` +--- +Config option: `rc_admin_redaction` + +This option sets ratelimiting redactions by room admins. If this is not explicitly +set then it uses the same ratelimiting as per `rc_message`. This is useful +to allow room admins to deal with abuse quickly. + +Example configuration: +```yaml +rc_admin_redaction: + per_second: 1 + burst_count: 50 +``` +--- +Config option: `rc_joins` + +This option allows for ratelimiting number of rooms a user can join. This setting has the following sub-options: + +* `local`: ratelimits when users are joining rooms the server is already in. + Defaults to `per_second: 0.1`, `burst_count: 10`. + +* `remote`: ratelimits when users are trying to join rooms not on the server (which + can be more computationally expensive than restricting locally). Defaults to + `per_second: 0.01`, `burst_count: 10` + +Example configuration: +```yaml +rc_joins: + local: + per_second: 0.2 + burst_count: 15 + remote: + per_second: 0.03 + burst_count: 12 +``` +--- +Config option: `rc_3pid_validation` + +This option ratelimits how often a user or IP can attempt to validate a 3PID. +Defaults to `per_second: 0.003`, `burst_count: 5`. + +Example configuration: +```yaml +rc_3pid_validation: + per_second: 0.003 + burst_count: 5 +``` +--- +Config option: `rc_invites` + +This option sets ratelimiting how often invites can be sent in a room or to a +specific user. `per_room` defaults to `per_second: 0.3`, `burst_count: 10` and +`per_user` defaults to `per_second: 0.003`, `burst_count: 5`. + +Client requests that invite user(s) when [creating a +room](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3createroom) +will count against the `rc_invites.per_room` limit, whereas +client requests to [invite a single user to a +room](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3roomsroomidinvite) +will count against both the `rc_invites.per_user` and `rc_invites.per_room` limits. + +Federation requests to invite a user will count against the `rc_invites.per_user` +limit only, as Synapse presumes ratelimiting by room will be done by the sending server. + +The `rc_invites.per_user` limit applies to the *receiver* of the invite, rather than the +sender, meaning that a `rc_invite.per_user.burst_count` of 5 mandates that a single user +cannot *receive* more than a burst of 5 invites at a time. + +Example configuration: +```yaml +rc_invites: + per_room: + per_second: 0.5 + burst_count: 5 + per_user: + per_second: 0.004 + burst_count: 3 +``` +--- +Config option: `rc_third_party_invite` + +This option ratelimits 3PID invites (i.e. invites sent to a third-party ID +such as an email address or a phone number) based on the account that's +sending the invite. Defaults to `per_second: 0.2`, `burst_count: 10`. + +Example configuration: +```yaml +rc_third_party_invite: + per_second: 0.2 + burst_count: 10 +``` +--- +Config option: `rc_federation` + +Defines limits on federation requests. + +The `rc_federation` configuration has the following sub-options: +* `window_size`: window size in milliseconds. Defaults to 1000. +* `sleep_limit`: number of federation requests from a single server in + a window before the server will delay processing the request. Defaults to 10. +* `sleep_delay`: duration in milliseconds to delay processing events + from remote servers by if they go over the sleep limit. Defaults to 500. +* `reject_limit`: maximum number of concurrent federation requests + allowed from a single server. Defaults to 50. +* `concurrent`: number of federation requests to concurrently process + from a single server. Defaults to 3. + +Example configuration: +```yaml +rc_federation: + window_size: 750 + sleep_limit: 15 + sleep_delay: 400 + reject_limit: 40 + concurrent: 5 +``` +--- +Config option: `federation_rr_transactions_per_room_per_second` + +Sets outgoing federation transaction frequency for sending read-receipts, +per-room. + +If we end up trying to send out more read-receipts, they will get buffered up +into fewer transactions. Defaults to 50. + +Example configuration: +```yaml +federation_rr_transactions_per_room_per_second: 40 +``` +--- +## Media Store ## +Config options relating to Synapse media store. + +--- +Config option: `enable_media_repo` + +Enable the media store service in the Synapse master. Defaults to true. +Set to false if you are using a separate media store worker. + +Example configuration: +```yaml +enable_media_repo: false +``` +--- +Config option: `media_store_path` + +Directory where uploaded images and attachments are stored. + +Example configuration: +```yaml +media_store_path: "DATADIR/media_store" +``` +--- +Config option: `media_storage_providers` + +Media storage providers allow media to be stored in different +locations. Defaults to none. Associated sub-options are: +* `module`: type of resource, e.g. `file_system`. +* `store_local`: whether to store newly uploaded local files +* `store_remote`: whether to store newly downloaded local files +* `store_synchronous`: whether to wait for successful storage for local uploads +* `config`: sets a path to the resource through the `directory` option + +Example configuration: +```yaml +media_storage_providers: + - module: file_system + store_local: false + store_remote: false + store_synchronous: false + config: + directory: /mnt/some/other/directory +``` +--- +Config option: `max_upload_size` + +The largest allowed upload size in bytes. + +If you are using a reverse proxy you may also need to set this value in +your reverse proxy's config. Defaults to 50M. Notably Nginx has a small max body size by default. +See [here](../../reverse_proxy.md) for more on using a reverse proxy with Synapse. + +Example configuration: +```yaml +max_upload_size: 60M +``` +--- +Config option: `max_image_pixels` + +Maximum number of pixels that will be thumbnailed. Defaults to 32M. + +Example configuration: +```yaml +max_image_pixels: 35M +``` +--- +Config option: `dynamic_thumbnails` + +Whether to generate new thumbnails on the fly to precisely match +the resolution requested by the client. If true then whenever +a new resolution is requested by the client the server will +generate a new thumbnail. If false the server will pick a thumbnail +from a precalculated list. Defaults to false. + +Example configuration: +```yaml +dynamic_thumbnails: true +``` +--- +Config option: `thumbnail_sizes` + +List of thumbnails to precalculate when an image is uploaded. Associated sub-options are: +* `width` +* `height` +* `method`: i.e. `crop`, `scale`, etc. + +Example configuration: +```yaml +thumbnail_sizes: + - width: 32 + height: 32 + method: crop + - width: 96 + height: 96 + method: crop + - width: 320 + height: 240 + method: scale + - width: 640 + height: 480 + method: scale + - width: 800 + height: 600 + method: scale +``` +Config option: `url_preview_enabled` + +This setting determines whether the preview URL API is enabled. +It is disabled by default. Set to true to enable. If enabled you must specify a +`url_preview_ip_range_blacklist` blacklist. + +Example configuration: +```yaml +url_preview_enabled: true +``` +--- +Config option: `url_preview_ip_range_blacklist` + +List of IP address CIDR ranges that the URL preview spider is denied +from accessing. There are no defaults: you must explicitly +specify a list for URL previewing to work. You should specify any +internal services in your network that you do not want synapse to try +to connect to, otherwise anyone in any Matrix room could cause your +synapse to issue arbitrary GET requests to your internal services, +causing serious security issues. + +(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly +listed here, since they correspond to unroutable addresses.) + +This must be specified if `url_preview_enabled` is set. It is recommended that +you use the following example list as a starting point. + +Note: The value is ignored when an HTTP proxy is in use. + +Example configuration: +```yaml +url_preview_ip_range_blacklist: + - '127.0.0.0/8' + - '10.0.0.0/8' + - '172.16.0.0/12' + - '192.168.0.0/16' + - '100.64.0.0/10' + - '192.0.0.0/24' + - '169.254.0.0/16' + - '192.88.99.0/24' + - '198.18.0.0/15' + - '192.0.2.0/24' + - '198.51.100.0/24' + - '203.0.113.0/24' + - '224.0.0.0/4' + - '::1/128' + - 'fe80::/10' + - 'fc00::/7' + - '2001:db8::/32' + - 'ff00::/8' + - 'fec0::/10' +``` +---- +Config option: `url_preview_ip_range_whitelist` + +This option sets a list of IP address CIDR ranges that the URL preview spider is allowed +to access even if they are specified in `url_preview_ip_range_blacklist`. +This is useful for specifying exceptions to wide-ranging blacklisted +target IP ranges - e.g. for enabling URL previews for a specific private +website only visible in your network. Defaults to none. + +Example configuration: +```yaml +url_preview_ip_range_whitelist: + - '192.168.1.1' +``` +--- +Config option: `url_preview_url_blacklist` + +Optional list of URL matches that the URL preview spider is +denied from accessing. You should use `url_preview_ip_range_blacklist` +in preference to this, otherwise someone could define a public DNS +entry that points to a private IP address and circumvent the blacklist. +This is more useful if you know there is an entire shape of URL that +you know that will never want synapse to try to spider. + +Each list entry is a dictionary of url component attributes as returned +by urlparse.urlsplit as applied to the absolute form of the URL. See +[here](https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit) for more +information. Some examples are: + +* `username` +* `netloc` +* `scheme` +* `path` + +The values of the dictionary are treated as a filename match pattern +applied to that component of URLs, unless they start with a ^ in which +case they are treated as a regular expression match. If all the +specified component matches for a given list item succeed, the URL is +blacklisted. + +Example configuration: +```yaml +url_preview_url_blacklist: + # blacklist any URL with a username in its URI + - username: '*' + + # blacklist all *.google.com URLs + - netloc: 'google.com' + - netloc: '*.google.com' + + # blacklist all plain HTTP URLs + - scheme: 'http' + + # blacklist http(s)://www.acme.com/foo + - netloc: 'www.acme.com' + path: '/foo' + + # blacklist any URL with a literal IPv4 address + - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' +``` +--- +Config option: `max_spider_size` + +The largest allowed URL preview spidering size in bytes. Defaults to 10M. + +Example configuration: +```yaml +max_spider_size: 8M +``` +--- +Config option: `url_preview_language` + +A list of values for the Accept-Language HTTP header used when +downloading webpages during URL preview generation. This allows +Synapse to specify the preferred languages that URL previews should +be in when communicating with remote servers. + +Each value is a IETF language tag; a 2-3 letter identifier for a +language, optionally followed by subtags separated by '-', specifying +a country or region variant. + +Multiple values can be provided, and a weight can be added to each by +using quality value syntax (;q=). '*' translates to any language. + +Defaults to "en". + +Example configuration: +```yaml + url_preview_accept_language: + - 'en-UK' + - 'en-US;q=0.9' + - 'fr;q=0.8' + - '*;q=0.7' +``` +---- +Config option: `oembed` + +oEmbed allows for easier embedding content from a website. It can be +used for generating URLs previews of services which support it. A default list of oEmbed providers +is included with Synapse. Set `disable_default_providers` to true to disable using +these default oEmbed URLs. Use `additional_providers` to specify additional files with oEmbed configuration (each +should be in the form of providers.json). By default this list is empty. + +Example configuration: +```yaml +oembed: + disable_default_providers: true + additional_providers: + - oembed/my_providers.json +``` +--- +## Captcha ## + +See [here](../../CAPTCHA_SETUP.md) for full details on setting up captcha. + +--- +Config option: `recaptcha_public_key` + +This homeserver's ReCAPTCHA public key. Must be specified if `enable_registration_captcha` is +enabled. + +Example configuration: +```yaml +recaptcha_public_key: "YOUR_PUBLIC_KEY" +``` +--- +Config option: `recaptcha_private_key` + +This homeserver's ReCAPTCHA private key. Must be specified if `enable_registration_captcha` is +enabled. + +Example configuration: +```yaml +recaptcha_private_key: "YOUR_PRIVATE_KEY" +``` +--- +Config option: `enable_registration_captcha` + +Set to true to enable ReCaptcha checks when registering, preventing signup +unless a captcha is answered. Requires a valid ReCaptcha public/private key. +Defaults to false. + +Example configuration: +```yaml +enable_registration_captcha: true +``` +--- +Config option: `recaptcha_siteverify_api` + +The API endpoint to use for verifying `m.login.recaptcha` responses. +Defaults to `https://www.recaptcha.net/recaptcha/api/siteverify`. + +Example configuration: +```yaml +recaptcha_siteverify_api: "https://my.recaptcha.site" +``` +--- +## TURN ## +Options related to adding a TURN server to Synapse. + +--- +Config option: `turn_uris` + +The public URIs of the TURN server to give to clients. + +Example configuration: +```yaml +turn_uris: [turn:example.org] +``` +--- +Config option: `turn_shared_secret` + +The shared secret used to compute passwords for the TURN server. + +Example configuration: +```yaml +turn_shared_secret: "YOUR_SHARED_SECRET" +``` +---- +Config options: `turn_username` and `turn_password` + +The Username and password if the TURN server needs them and does not use a token. + +Example configuration: +```yaml +turn_username: "TURNSERVER_USERNAME" +turn_password: "TURNSERVER_PASSWORD" +``` +--- +Config option: `turn_user_lifetime` + +How long generated TURN credentials last. Defaults to 1h. + +Example configuration: +```yaml +turn_user_lifetime: 2h +``` +--- +Config option: `turn_allow_guests` + +Whether guests should be allowed to use the TURN server. This defaults to true, otherwise +VoIP will be unreliable for guests. However, it does introduce a slight security risk as +it allows users to connect to arbitrary endpoints without having first signed up for a valid account (e.g. by passing a CAPTCHA). + +Example configuration: +```yaml +turn_allow_guests: false +``` +--- +## Registration ## + +Registration can be rate-limited using the parameters in the [Ratelimiting](#ratelimiting) section of this manual. + +--- +Config option: `enable_registration` + +Enable registration for new users. Defaults to false. It is highly recommended that if you enable registration, +you use either captcha, email, or token-based verification to verify that new users are not bots. In order to enable registration +without any verification, you must also set `enable_registration_without_verification` to true. + +Example configuration: +```yaml +enable_registration: true +``` +--- +Config option: `enable_registration_without_verification` +Enable registration without email or captcha verification. Note: this option is *not* recommended, +as registration without verification is a known vector for spam and abuse. Defaults to false. Has no effect +unless `enable_registration` is also enabled. + +Example configuration: +```yaml +enable_registration_without_verification: true +``` +--- +Config option: `session_lifetime` + +Time that a user's session remains valid for, after they log in. + +Note that this is not currently compatible with guest logins. + +Note also that this is calculated at login time: changes are not applied retrospectively to users who have already +logged in. + +By default, this is infinite. + +Example configuration: +```yaml +session_lifetime: 24h +``` +---- +Config option: `refresh_access_token_lifetime` + +Time that an access token remains valid for, if the session is using refresh tokens. + +For more information about refresh tokens, please see the [manual](user_authentication/refresh_tokens.md). + +Note that this only applies to clients which advertise support for refresh tokens. + +Note also that this is calculated at login time and refresh time: changes are not applied to +existing sessions until they are refreshed. + +By default, this is 5 minutes. + +Example configuration: +```yaml +refreshable_access_token_lifetime: 10m +``` +--- +Config option: `refresh_token_lifetime: 24h` + +Time that a refresh token remains valid for (provided that it is not +exchanged for another one first). +This option can be used to automatically log-out inactive sessions. +Please see the manual for more information. + +Note also that this is calculated at login time and refresh time: +changes are not applied to existing sessions until they are refreshed. + +By default, this is infinite. + +Example configuration: +```yaml +refresh_token_lifetime: 24h +``` +--- +Config option: `nonrefreshable_access_token_lifetime` + +Time that an access token remains valid for, if the session is NOT +using refresh tokens. + +Please note that not all clients support refresh tokens, so setting +this to a short value may be inconvenient for some users who will +then be logged out frequently. + +Note also that this is calculated at login time: changes are not applied +retrospectively to existing sessions for users that have already logged in. + +By default, this is infinite. + +Example configuration: +```yaml +nonrefreshable_access_token_lifetime: 24h +``` +--- +Config option: `registrations_require_3pid` + +If this is set, the user must provide all of the specified types of 3PID when registering. + +Example configuration: +```yaml +registrations_require_3pid: + - email + - msisdn +``` +--- +Config option: `disable_msisdn_registration` + +Explicitly disable asking for MSISDNs from the registration +flow (overrides `registrations_require_3pid` if MSISDNs are set as required). + +Example configuration: +```yaml +disable_msisdn_registration: true +``` +--- +Config option: `allowed_local_3pids` + +Mandate that users are only allowed to associate certain formats of +3PIDs with accounts on this server, as specified by the `medium` and `pattern` sub-options. + +Example configuration: +```yaml +allowed_local_3pids: + - medium: email + pattern: '^[^@]+@matrix\.org$' + - medium: email + pattern: '^[^@]+@vector\.im$' + - medium: msisdn + pattern: '\+44' +``` +--- +Config option: `enable_3pid_lookup` + +Enable 3PIDs lookup requests to identity servers from this server. Defaults to true. + +Example configuration: +```yaml +enable_3pid_lookup: false +``` +--- +Config option: `registration_requires_token` + +Require users to submit a token during registration. +Tokens can be managed using the admin [API](../administration/admin_api/registration_tokens.md). +Note that `enable_registration` must be set to true. +Disabling this option will not delete any tokens previously generated. +Defaults to false. Set to true to enable. + +Example configuration: +```yaml +registration_requires_token: true +``` +--- +Config option: `registration_shared_secret` + +If set, allows registration of standard or admin accounts by anyone who +has the shared secret, even if registration is otherwise disabled. + +Example configuration: +```yaml +registration_shared_secret: +``` +--- +Config option: `bcrypt_rounds` + +Set the number of bcrypt rounds used to generate password hash. +Larger numbers increase the work factor needed to generate the hash. +The default number is 12 (which equates to 2^12 rounds). +N.B. that increasing this will exponentially increase the time required +to register or login - e.g. 24 => 2^24 rounds which will take >20 mins. +Example configuration: +```yaml +bcrypt_rounds: 14 +``` +--- +Config option: `allow_guest_access` + +Allows users to register as guests without a password/email/etc, and +participate in rooms hosted on this server which have been made +accessible to anonymous users. Defaults to false. + +Example configuration: +```yaml +allow_guest_access: true +``` +--- +Config option: `default_identity_server` + +The identity server which we suggest that clients should use when users log +in on this server. + +(By default, no suggestion is made, so it is left up to the client. +This setting is ignored unless `public_baseurl` is also explicitly set.) + +Example configuration: +```yaml +default_identity_server: https://matrix.org +``` +--- +Config option: `account_threepid_delegates` + +Handle threepid (email/phone etc) registration and password resets through a set of +*trusted* identity servers. Note that this allows the configured identity server to +reset passwords for accounts! + +Be aware that if `email` is not set, and SMTP options have not been +configured in the email config block, registration and user password resets via +email will be globally disabled. + +Additionally, if `msisdn` is not set, registration and password resets via msisdn +will be disabled regardless, and users will not be able to associate an msisdn +identifier to their account. This is due to Synapse currently not supporting +any method of sending SMS messages on its own. + +To enable using an identity server for operations regarding a particular third-party +identifier type, set the value to the URL of that identity server as shown in the +examples below. + +Servers handling the these requests must answer the `/requestToken` endpoints defined +by the Matrix Identity Service API [specification](https://matrix.org/docs/spec/identity_service/latest). + +Example configuration: +```yaml +account_threepid_delegates: + email: https://example.com # Delegate email sending to example.com + msisdn: http://localhost:8090 # Delegate SMS sending to this local process +``` +--- +Config option: `enable_set_displayname` + +Whether users are allowed to change their displayname after it has +been initially set. Useful when provisioning users based on the +contents of a third-party directory. + +Does not apply to server administrators. Defaults to true. + +Example configuration: +```yaml +enable_set_displayname: false +``` +--- +Config option: `enable_set_avatar_url` + +Whether users are allowed to change their avatar after it has been +initially set. Useful when provisioning users based on the contents +of a third-party directory. + +Does not apply to server administrators. Defaults to true. + +Example configuration: +```yaml +enable_set_avatar_url: false +``` +--- +Config option: `enable_3pid_changes` + +Whether users can change the third-party IDs associated with their accounts +(email address and msisdn). + +Defaults to true. + +Example configuration: +```yaml +enable_3pid_changes: false +``` +--- +Config option: `auto_join_rooms` + +Users who register on this homeserver will automatically be joined +to the rooms listed under this option. + +By default, any room aliases included in this list will be created +as a publicly joinable room when the first user registers for the +homeserver. If the room already exists, make certain it is a publicly joinable +room, i.e. the join rule of the room must be set to 'public'. You can find more options +relating to auto-joining rooms below. + +Example configuration: +```yaml +auto_join_rooms: + - "#exampleroom:example.com" + - "#anotherexampleroom:example.com" +``` +--- +Config option: `autocreate_auto_join_rooms` + +Where `auto_join_rooms` are specified, setting this flag ensures that +the rooms exist by creating them when the first user on the +homeserver registers. + +By default the auto-created rooms are publicly joinable from any federated +server. Use the `autocreate_auto_join_rooms_federated` and +`autocreate_auto_join_room_preset` settings to customise this behaviour. + +Setting to false means that if the rooms are not manually created, +users cannot be auto-joined since they do not exist. + +Defaults to true. + +Example configuration: +```yaml +autocreate_auto_join_rooms: false +``` +--- +Config option: `autocreate_auto_join_rooms_federated` + +Whether the rooms listen in `auto_join_rooms` that are auto-created are available +via federation. Only has an effect if `autocreate_auto_join_rooms` is true. + +Note that whether a room is federated cannot be modified after +creation. + +Defaults to true: the room will be joinable from other servers. +Set to false to prevent users from other homeservers from +joining these rooms. + +Example configuration: +```yaml +autocreate_auto_join_rooms_federated: false +``` +--- +Config option: `autocreate_auto_join_room_preset` + +The room preset to use when auto-creating one of `auto_join_rooms`. Only has an +effect if `autocreate_auto_join_rooms` is true. + +Possible values for this option are: +* "public_chat": the room is joinable by anyone, including + federated servers if `autocreate_auto_join_rooms_federated` is true (the default). +* "private_chat": an invitation is required to join these rooms. +* "trusted_private_chat": an invitation is required to join this room and the invitee is + assigned a power level of 100 upon joining the room. + +If a value of "private_chat" or "trusted_private_chat" is used then +`auto_join_mxid_localpart` must also be configured. + +Defaults to "public_chat". + +Example configuration: +```yaml +autocreate_auto_join_room_preset: private_chat +``` +--- +Config option: `auto_join_mxid_localpart` + +The local part of the user id which is used to create `auto_join_rooms` if +`autocreate_auto_join_rooms` is true. If this is not provided then the +initial user account that registers will be used to create the rooms. + +The user id is also used to invite new users to any auto-join rooms which +are set to invite-only. + +It *must* be configured if `autocreate_auto_join_room_preset` is set to +"private_chat" or "trusted_private_chat". + +Note that this must be specified in order for new users to be correctly +invited to any auto-join rooms which have been set to invite-only (either +at the time of creation or subsequently). + +Note that, if the room already exists, this user must be joined and +have the appropriate permissions to invite new members. + +Example configuration: +```yaml +auto_join_mxid_localpart: system +``` +--- +Config option: `auto_join_rooms_for_guests` + +When `auto_join_rooms` is specified, setting this flag to false prevents +guest accounts from being automatically joined to the rooms. + +Defaults to true. + +Example configuration: +```yaml +auto_join_rooms_for_guests: false +``` +--- +Config option: `inhibit_user_in_use_error` + +Whether to inhibit errors raised when registering a new account if the user ID +already exists. If turned on, requests to `/register/available` will always +show a user ID as available, and Synapse won't raise an error when starting +a registration with a user ID that already exists. However, Synapse will still +raise an error if the registration completes and the username conflicts. + +Defaults to false. + +Example configuration: +```yaml +inhibit_user_in_use_error: true +``` +--- +## Metrics ### +Config options related to metrics. + +--- +Config option: `enable_metrics` + +Set to true to enable collection and rendering of performance metrics. +Defaults to false. + +Example configuration: +```yaml +enable_metrics: true +``` +--- +Config option: `sentry` + +Use this option to enable sentry integration. Provide the DSN assigned to you by sentry +with the `dsn` setting. + +NOTE: While attempts are made to ensure that the logs don't contain +any sensitive information, this cannot be guaranteed. By enabling +this option the sentry server may therefore receive sensitive +information, and it in turn may then disseminate sensitive information +through insecure notification channels if so configured. + +Example configuration: +```yaml +sentry: + dsn: "..." +``` +--- +Config option: `metrics_flags` + +Flags to enable Prometheus metrics which are not suitable to be +enabled by default, either for performance reasons or limited use. +Currently the only option is `known_servers`, which publishes +`synapse_federation_known_servers`, a gauge of the number of +servers this homeserver knows about, including itself. May cause +performance problems on large homeservers. + +Example configuration: +```yaml +metrics_flags: + known_servers: true +``` +--- +Config option: `report_stats` + +Whether or not to report anonymized homeserver usage statistics. This is originally +set when generating the config. Set this option to true or false to change the current +behavior. + +Example configuration: +```yaml +report_stats: true +``` +--- +Config option: `report_stats_endpoint` + +The endpoint to report the anonymized homeserver usage statistics to. +Defaults to https://matrix.org/report-usage-stats/push + +Example configuration: +```yaml +report_stats_endpoint: https://example.com/report-usage-stats/push +``` +--- +## API Configuration ## +Config settings related to the client/server API + +--- +Config option: `room_prejoin_state:` + +Controls for the state that is shared with users who receive an invite +to a room. By default, the following state event types are shared with users who +receive invites to the room: +- m.room.join_rules +- m.room.canonical_alias +- m.room.avatar +- m.room.encryption +- m.room.name +- m.room.create +- m.room.topic + +To change the default behavior, use the following sub-options: +* `disable_default_event_types`: set to true to disable the above defaults. If this + is enabled, only the event types listed in `additional_event_types` are shared. + Defaults to false. +* `additional_event_types`: Additional state event types to share with users when they are invited + to a room. By default, this list is empty (so only the default event types are shared). + +Example configuration: +```yaml +room_prejoin_state: + disable_default_event_types: true + additional_event_types: + - org.example.custom.event.type + - m.room.join_rules +``` +--- +Config option: `track_puppeted_user_ips` + +We record the IP address of clients used to access the API for various +reasons, including displaying it to the user in the "Where you're signed in" +dialog. + +By default, when puppeting another user via the admin API, the client IP +address is recorded against the user who created the access token (ie, the +admin user), and *not* the puppeted user. + +Set this option to true to also record the IP address against the puppeted +user. (This also means that the puppeted user will count as an "active" user +for the purpose of monthly active user tracking - see `limit_usage_by_mau` etc +above.) + +Example configuration: +```yaml +track_puppeted_user_ips: true +``` +--- +Config option: `app_service_config_files` + +A list of application service config files to use. + +Example configuration: +```yaml +app_service_config_files: + - app_service_1.yaml + - app_service_2.yaml +``` +--- +Config option: `track_appservice_user_ips` + +Defaults to false. Set to true to enable tracking of application service IP addresses. +Implicitly enables MAU tracking for application service users. + +Example configuration: +```yaml +track_appservice_user_ips: true +``` +--- +Config option: `macaroon_secret_key` + +A secret which is used to sign access tokens. If none is specified, +the `registration_shared_secret` is used, if one is given; otherwise, +a secret key is derived from the signing key. + +Example configuration: +```yaml +macaroon_secret_key: +``` +--- +Config option: `form_secret` + +A secret which is used to calculate HMACs for form values, to stop +falsification of values. Must be specified for the User Consent +forms to work. + +Example configuration: +```yaml +form_secret: +``` +--- +## Signing Keys ## +Config options relating to signing keys + +--- +Config option: `signing_key_path` + +Path to the signing key to sign messages with. + +Example configuration: +```yaml +signing_key_path: "CONFDIR/SERVERNAME.signing.key" +``` +--- +Config option: `old_signing_keys` + +The keys that the server used to sign messages with but won't use +to sign new messages. For each key, `key` should be the base64-encoded public key, and +`expired_ts`should be the time (in milliseconds since the unix epoch) that +it was last used. + +It is possible to build an entry from an old `signing.key` file using the +`export_signing_key` script which is provided with synapse. + +Example configuration: +```yaml +old_signing_keys: + "ed25519:id": { key: "base64string", expired_ts: 123456789123 } +``` +--- +Config option: `key_refresh_interval` + +How long key response published by this server is valid for. +Used to set the `valid_until_ts` in `/key/v2` APIs. +Determines how quickly servers will query to check which keys +are still valid. Defaults to 1d. + +Example configuration: +```yaml +key_refresh_interval: 2d +``` +--- +Config option: `trusted_key_servers:` + +The trusted servers to download signing keys from. + +When we need to fetch a signing key, each server is tried in parallel. + +Normally, the connection to the key server is validated via TLS certificates. +Additional security can be provided by configuring a `verify key`, which +will make synapse check that the response is signed by that key. + +This setting supercedes an older setting named `perspectives`. The old format +is still supported for backwards-compatibility, but it is deprecated. + +`trusted_key_servers` defaults to matrix.org, but using it will generate a +warning on start-up. To suppress this warning, set +`suppress_key_server_warning` to true. + +Options for each entry in the list include: +* `server_name`: the name of the server. Required. +* `verify_keys`: an optional map from key id to base64-encoded public key. + If specified, we will check that the response is signed by at least + one of the given keys. +* `accept_keys_insecurely`: a boolean. Normally, if `verify_keys` is unset, + and `federation_verify_certificates` is not `true`, synapse will refuse + to start, because this would allow anyone who can spoof DNS responses + to masquerade as the trusted key server. If you know what you are doing + and are sure that your network environment provides a secure connection + to the key server, you can set this to `true` to override this behaviour. + +Example configuration #1: +```yaml +trusted_key_servers: + - server_name: "my_trusted_server.example.com" + verify_keys: + "ed25519:auto": "abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr" + - server_name: "my_other_trusted_server.example.com" +``` +Example configuration #2: +```yaml +trusted_key_servers: + - server_name: "matrix.org" +``` +--- +Config option: `suppress_key_server_warning` + +Set the following to true to disable the warning that is emitted when the +`trusted_key_servers` include 'matrix.org'. See above. + +Example configuration: +```yaml +suppress_key_server_warning: true +``` +--- +Config option: `key_server_signing_keys_path` + +The signing keys to use when acting as a trusted key server. If not specified +defaults to the server signing key. + +Can contain multiple keys, one per line. + +Example configuration: +```yaml +key_server_signing_keys_path: "key_server_signing_keys.key" +``` +--- +## Single sign-on integration ## + +The following settings can be used to make Synapse use a single sign-on +provider for authentication, instead of its internal password database. + +You will probably also want to set the following options to false to +disable the regular login/registration flows: + * `enable_registration` + * `password_config.enabled` + +You will also want to investigate the settings under the "sso" configuration +section below. + +--- +Config option: `saml2_config` + +Enable SAML2 for registration and login. Uses pysaml2. To learn more about pysaml and +to find a full list options for configuring pysaml, read the docs [here](https://pysaml2.readthedocs.io/en/latest/). + +At least one of `sp_config` or `config_path` must be set in this section to +enable SAML login. You can either put your entire pysaml config inline using the `sp_config` +option, or you can specify a path to a psyaml config file with the sub-option `config_path`. +This setting has the following sub-options: + +* `sp_config`: the configuration for the pysaml2 Service Provider. See pysaml2 docs for format of config. + Default values will be used for the `entityid` and `service` settings, + so it is not normally necessary to specify them unless you need to + override them. Here are a few useful sub-options for configuring pysaml: + * `metadata`: Point this to the IdP's metadata. You must provide either a local + file via the `local` attribute or (preferably) a URL via the + `remote` attribute. + * `accepted_time_diff: 3`: Allowed clock difference in seconds between the homeserver and IdP. + Defaults to 0. + * `service`: By default, the user has to go to our login page first. If you'd like + to allow IdP-initiated login, set `allow_unsolicited` to true under `sp` in the `service` + section. +* `config_path`: specify a separate pysaml2 configuration file thusly: + `config_path: "CONFDIR/sp_conf.py"` +* `saml_session_lifetime`: The lifetime of a SAML session. This defines how long a user has to + complete the authentication process, if `allow_unsolicited` is unset. The default is 15 minutes. +* `user_mapping_provider`: Using this option, an external module can be provided as a + custom solution to mapping attributes returned from a saml provider onto a matrix user. The + `user_mapping_provider` has the following attributes: + * `module`: The custom module's class. + * `config`: Custom configuration values for the module. Use the values provided in the + example if you are using the built-in user_mapping_provider, or provide your own + config values for a custom class if you are using one. This section will be passed as a Python + dictionary to the module's `parse_config` method. The built-in provider takes the following two + options: + * `mxid_source_attribute`: The SAML attribute (after mapping via the attribute maps) to use + to derive the Matrix ID from. It is 'uid' by default. Note: This used to be configured by the + `saml2_config.mxid_source_attribute option`. If that is still defined, its value will be used instead. + * `mxid_mapping`: The mapping system to use for mapping the saml attribute onto a + matrix ID. Options include: `hexencode` (which maps unpermitted characters to '=xx') + and `dotreplace` (which replaces unpermitted characters with '.'). + The default is `hexencode`. Note: This used to be configured by the + `saml2_config.mxid_mapping option`. If that is still defined, its value will be used instead. +* `grandfathered_mxid_source_attribute`: In previous versions of synapse, the mapping from SAML attribute to + MXID was always calculated dynamically rather than stored in a table. For backwards- compatibility, we will look for `user_ids` + matching such a pattern before creating a new account. This setting controls the SAML attribute which will be used for this + backwards-compatibility lookup. Typically it should be 'uid', but if the attribute maps are changed, it may be necessary to change it. + The default is 'uid'. +* `attribute_requirements`: It is possible to configure Synapse to only allow logins if SAML attributes + match particular values. The requirements can be listed under + `attribute_requirements` as shown in the example. All of the listed attributes must + match for the login to be permitted. +* `idp_entityid`: If the metadata XML contains multiple IdP entities then the `idp_entityid` + option must be set to the entity to redirect users to. + Most deployments only have a single IdP entity and so should omit this option. + + +Once SAML support is enabled, a metadata file will be exposed at +`https://:/_synapse/client/saml2/metadata.xml`, which you may be able to +use to configure your SAML IdP with. Alternatively, you can manually configure +the IdP to use an ACS location of +`https://:/_synapse/client/saml2/authn_response`. + +Example configuration: +```yaml +saml2_config: + sp_config: + metadata: + local: ["saml2/idp.xml"] + remote: + - url: https://our_idp/metadata.xml + accepted_time_diff: 3 + + service: + sp: + allow_unsolicited: true + + # The examples below are just used to generate our metadata xml, and you + # may well not need them, depending on your setup. Alternatively you + # may need a whole lot more detail - see the pysaml2 docs! + description: ["My awesome SP", "en"] + name: ["Test SP", "en"] + + ui_info: + display_name: + - lang: en + text: "Display Name is the descriptive name of your service." + description: + - lang: en + text: "Description should be a short paragraph explaining the purpose of the service." + information_url: + - lang: en + text: "https://example.com/terms-of-service" + privacy_statement_url: + - lang: en + text: "https://example.com/privacy-policy" + keywords: + - lang: en + text: ["Matrix", "Element"] + logo: + - lang: en + text: "https://example.com/logo.svg" + width: "200" + height: "80" + + organization: + name: Example com + display_name: + - ["Example co", "en"] + url: "http://example.com" + + contact_person: + - given_name: Bob + sur_name: "the Sysadmin" + email_address": ["admin@example.com"] + contact_type": technical + + saml_session_lifetime: 5m + + user_mapping_provider: + # Below options are intended for the built-in provider, they should be + # changed if using a custom module. + config: + mxid_source_attribute: displayName + mxid_mapping: dotreplace + + grandfathered_mxid_source_attribute: upn + + attribute_requirements: + - attribute: userGroup + value: "staff" + - attribute: department + value: "sales" + + idp_entityid: 'https://our_idp/entityid' +``` +--- +Config option: `oidc_providers` + +List of OpenID Connect (OIDC) / OAuth 2.0 identity providers, for registration +and login. See [here](../../openid.md) +for information on how to configure these options. + +For backwards compatibility, it is also possible to configure a single OIDC +provider via an `oidc_config` setting. This is now deprecated and admins are +advised to migrate to the `oidc_providers` format. (When doing that migration, +use `oidc` for the `idp_id` to ensure that existing users continue to be +recognised.) + +Options for each entry include: +* `idp_id`: a unique identifier for this identity provider. Used internally + by Synapse; should be a single word such as 'github'. + Note that, if this is changed, users authenticating via that provider + will no longer be recognised as the same user! + (Use "oidc" here if you are migrating from an old `oidc_config` configuration.) + +* `idp_name`: A user-facing name for this identity provider, which is used to + offer the user a choice of login mechanisms. + +* `idp_icon`: An optional icon for this identity provider, which is presented + by clients and Synapse's own IdP picker page. If given, must be an + MXC URI of the format mxc:///. (An easy way to + obtain such an MXC URI is to upload an image to an (unencrypted) room + and then copy the "url" from the source of the event.) + +* `idp_brand`: An optional brand for this identity provider, allowing clients + to style the login flow according to the identity provider in question. + See the [spec](https://spec.matrix.org/latest/) for possible options here. + +* `discover`: set to false to disable the use of the OIDC discovery mechanism + to discover endpoints. Defaults to true. + +* `issuer`: Required. The OIDC issuer. Used to validate tokens and (if discovery + is enabled) to discover the provider's endpoints. + +* `client_id`: Required. oauth2 client id to use. + +* `client_secret`: oauth2 client secret to use. May be omitted if + `client_secret_jwt_key` is given, or if `client_auth_method` is 'none'. + +* `client_secret_jwt_key`: Alternative to client_secret: details of a key used + to create a JSON Web Token to be used as an OAuth2 client secret. If + given, must be a dictionary with the following properties: + + * `key`: a pem-encoded signing key. Must be a suitable key for the + algorithm specified. Required unless `key_file` is given. + + * `key_file`: the path to file containing a pem-encoded signing key file. + Required unless `key` is given. + + * `jwt_header`: a dictionary giving properties to include in the JWT + header. Must include the key `alg`, giving the algorithm used to + sign the JWT, such as "ES256", using the JWA identifiers in + RFC7518. + + * `jwt_payload`: an optional dictionary giving properties to include in + the JWT payload. Normally this should include an `iss` key. + +* `client_auth_method`: auth method to use when exchanging the token. Valid + values are `client_secret_basic` (default), `client_secret_post` and + `none`. + +* `scopes`: list of scopes to request. This should normally include the "openid" + scope. Defaults to ["openid"]. + +* `authorization_endpoint`: the oauth2 authorization endpoint. Required if + provider discovery is disabled. + +* `token_endpoint`: the oauth2 token endpoint. Required if provider discovery is + disabled. + +* `userinfo_endpoint`: the OIDC userinfo endpoint. Required if discovery is + disabled and the 'openid' scope is not requested. + +* `jwks_uri`: URI where to fetch the JWKS. Required if discovery is disabled and + the 'openid' scope is used. + +* `skip_verification`: set to 'true' to skip metadata verification. Use this if + you are connecting to a provider that is not OpenID Connect compliant. + Defaults to false. Avoid this in production. + +* `user_profile_method`: Whether to fetch the user profile from the userinfo + endpoint, or to rely on the data returned in the id_token from the `token_endpoint`. + Valid values are: `auto` or `userinfo_endpoint`. + Defaults to `auto`, which uses the userinfo endpoint if `openid` is + not included in `scopes`. Set to `userinfo_endpoint` to always use the + userinfo endpoint. + +* `allow_existing_users`: set to true to allow a user logging in via OIDC to + match a pre-existing account instead of failing. This could be used if + switching from password logins to OIDC. Defaults to false. + +* `user_mapping_provider`: Configuration for how attributes returned from a OIDC + provider are mapped onto a matrix user. This setting has the following + sub-properties: + + * `module`: The class name of a custom mapping module. Default is + `synapse.handlers.oidc.JinjaOidcMappingProvider`. + See https://matrix-org.github.io/synapse/latest/sso_mapping_providers.html#openid-mapping-providers + for information on implementing a custom mapping provider. + + * `config`: Configuration for the mapping provider module. This section will + be passed as a Python dictionary to the user mapping provider + module's `parse_config` method. + + For the default provider, the following settings are available: + + * subject_claim: name of the claim containing a unique identifier + for the user. Defaults to 'sub', which OpenID Connect + compliant providers should provide. + + * `localpart_template`: Jinja2 template for the localpart of the MXID. + If this is not set, the user will be prompted to choose their + own username (see the documentation for the `sso_auth_account_details.html` + template). This template can use the `localpart_from_email` filter. + + * `confirm_localpart`: Whether to prompt the user to validate (or + change) the generated localpart (see the documentation for the + 'sso_auth_account_details.html' template), instead of + registering the account right away. + + * `display_name_template`: Jinja2 template for the display name to set + on first login. If unset, no displayname will be set. + + * `email_template`: Jinja2 template for the email address of the user. + If unset, no email address will be added to the account. + + * `extra_attributes`: a map of Jinja2 templates for extra attributes + to send back to the client during login. Note that these are non-standard and clients will ignore them + without modifications. + + When rendering, the Jinja2 templates are given a 'user' variable, + which is set to the claims returned by the UserInfo Endpoint and/or + in the ID Token. + + +It is possible to configure Synapse to only allow logins if certain attributes +match particular values in the OIDC userinfo. The requirements can be listed under +`attribute_requirements` as shown here: +```yaml +attribute_requirements: + - attribute: family_name + value: "Stephensson" + - attribute: groups + value: "admin" +``` +All of the listed attributes must match for the login to be permitted. Additional attributes can be added to +userinfo by expanding the `scopes` section of the OIDC config to retrieve +additional information from the OIDC provider. + +If the OIDC claim is a list, then the attribute must match any value in the list. +Otherwise, it must exactly match the value of the claim. Using the example +above, the `family_name` claim MUST be "Stephensson", but the `groups` +claim MUST contain "admin". + +Example configuration: +```yaml +oidc_providers: + # Generic example + # + - idp_id: my_idp + idp_name: "My OpenID provider" + idp_icon: "mxc://example.com/mediaid" + discover: false + issuer: "https://accounts.example.com/" + client_id: "provided-by-your-issuer" + client_secret: "provided-by-your-issuer" + client_auth_method: client_secret_post + scopes: ["openid", "profile"] + authorization_endpoint: "https://accounts.example.com/oauth2/auth" + token_endpoint: "https://accounts.example.com/oauth2/token" + userinfo_endpoint: "https://accounts.example.com/userinfo" + jwks_uri: "https://accounts.example.com/.well-known/jwks.json" + skip_verification: true + user_mapping_provider: + config: + subject_claim: "id" + localpart_template: "{{ user.login }}" + display_name_template: "{{ user.name }}" + email_template: "{{ user.email }}" + attribute_requirements: + - attribute: userGroup + value: "synapseUsers" +``` +--- +Config option: `cas_config` + +Enable Central Authentication Service (CAS) for registration and login. +Has the following sub-options: +* `enabled`: Set this to true to enable authorization against a CAS server. + Defaults to false. +* `server_url`: The URL of the CAS authorization endpoint. +* `displayname_attribute`: The attribute of the CAS response to use as the display name. + If no name is given here, no displayname will be set. +* `required_attributes`: It is possible to configure Synapse to only allow logins if CAS attributes + match particular values. All of the keys given below must exist + and the values must match the given value. Alternately if the given value + is `None` then any value is allowed (the attribute just must exist). + All of the listed attributes must match for the login to be permitted. + +Example configuration: +```yaml +cas_config: + enabled: true + server_url: "https://cas-server.com" + displayname_attribute: name + required_attributes: + userGroup: "staff" + department: None +``` +--- +Config option: `sso` + +Additional settings to use with single-sign on systems such as OpenID Connect, +SAML2 and CAS. + +Server admins can configure custom templates for pages related to SSO. See +[here](../../templates.md) for more information. + +Options include: +* `client_whitelist`: A list of client URLs which are whitelisted so that the user does not + have to confirm giving access to their account to the URL. Any client + whose URL starts with an entry in the following list will not be subject + to an additional confirmation step after the SSO login is completed. + WARNING: An entry such as "https://my.client" is insecure, because it + will also match "https://my.client.evil.site", exposing your users to + phishing attacks from evil.site. To avoid this, include a slash after the + hostname: "https://my.client/". + The login fallback page (used by clients that don't natively support the + required login flows) is whitelisted in addition to any URLs in this list. + By default, this list contains only the login fallback page. +* `update_profile_information`: Use this setting to keep a user's profile fields in sync with information from + the identity provider. Currently only syncing the displayname is supported. Fields + are checked on every SSO login, and are updated if necessary. + Note that enabling this option will override user profile information, + regardless of whether users have opted-out of syncing that + information when first signing in. Defaults to false. + + +Example configuration: +```yaml +sso: + client_whitelist: + - https://riot.im/develop + - https://my.custom.client/ + update_profile_information: true +``` +--- +Config option: `jwt_config` + +JSON web token integration. The following settings can be used to make +Synapse JSON web tokens for authentication, instead of its internal +password database. + +Each JSON Web Token needs to contain a "sub" (subject) claim, which is +used as the localpart of the mxid. + +Additionally, the expiration time ("exp"), not before time ("nbf"), +and issued at ("iat") claims are validated if present. + +Note that this is a non-standard login type and client support is +expected to be non-existent. + +See [here](../../jwt.md) for more. + +Additional sub-options for this setting include: +* `enabled`: Set to true to enable authorization using JSON web + tokens. Defaults to false. +* `secret`: This is either the private shared secret or the public key used to + decode the contents of the JSON web token. Required if `enabled` is set to true. +* `algorithm`: The algorithm used to sign the JSON web token. Supported algorithms are listed at + https://pyjwt.readthedocs.io/en/latest/algorithms.html Required if `enabled` is set to true. +* `subject_claim`: Name of the claim containing a unique identifier for the user. + Optional, defaults to `sub`. +* `issuer`: The issuer to validate the "iss" claim against. Optional. If provided the + "iss" claim will be required and validated for all JSON web tokens. +* `audiences`: A list of audiences to validate the "aud" claim against. Optional. + If provided the "aud" claim will be required and validated for all JSON web tokens. + Note that if the "aud" claim is included in a JSON web token then + validation will fail without configuring audiences. + +Example configuration: +```yaml +jwt_config: + enabled: true + secret: "provided-by-your-issuer" + algorithm: "provided-by-your-issuer" + subject_claim: "name_of_claim" + issuer: "provided-by-your-issuer" + audiences: + - "provided-by-your-issuer" +``` +--- +Config option: `password_config` + +Use this setting to enable password-based logins. + +This setting has the following sub-options: +* `enabled`: Defaults to true. + Set to false to disable password authentication. + Set to `only_for_reauth` to allow users with existing passwords to use them + to log in and reauthenticate, whilst preventing new users from setting passwords. +* `localdb_enabled`: Set to false to disable authentication against the local password + database. This is ignored if `enabled` is false, and is only useful + if you have other `password_providers`. Defaults to true. +* `pepper`: Set the value here to a secret random string for extra security. # Uncomment and change to a secret random string for extra security. + DO NOT CHANGE THIS AFTER INITIAL SETUP! +* `policy`: Define and enforce a password policy, such as minimum lengths for passwords, etc. + Each parameter is optional. This is an implementation of MSC2000. Parameters are as follows: + * `enabled`: Defaults to false. Set to true to enable. + * `minimum_length`: Minimum accepted length for a password. Defaults to 0. + * `require_digit`: Whether a password must contain at least one digit. + Defaults to false. + * `require_symbol`: Whether a password must contain at least one symbol. + A symbol is any character that's not a number or a letter. Defaults to false. + * `require_lowercase`: Whether a password must contain at least one lowercase letter. + Defaults to false. + * `require_uppercase`: Whether a password must contain at least one uppercase letter. + Defaults to false. + + +Example configuration: +```yaml +password_config: + enabled: false + localdb_enabled: false + pepper: "EVEN_MORE_SECRET" + + policy: + enabled: true + minimum_length: 15 + require_digit: true + require_symbol: true + require_lowercase: true + require_uppercase: true +``` +--- +Config option: `ui_auth` + +The amount of time to allow a user-interactive authentication session to be active. + +This defaults to 0, meaning the user is queried for their credentials +before every action, but this can be overridden to allow a single +validation to be re-used. This weakens the protections afforded by +the user-interactive authentication process, by allowing for multiple +(and potentially different) operations to use the same validation session. + +This is ignored for potentially "dangerous" operations (including +deactivating an account, modifying an account password, and +adding a 3PID). + +Use the `session_timeout` sub-option here to change the time allowed for credential validation. + +Example configuration: +```yaml +ui_auth: + session_timeout: "15s" +``` +--- +Config option: `email` + +Configuration for sending emails from Synapse. + +Server admins can configure custom templates for email content. See +[here](../../templates.md) for more information. + +This setting has the following sub-options: +* `smtp_host`: The hostname of the outgoing SMTP server to use. Defaults to 'localhost'. +* `smtp_port`: The port on the mail server for outgoing SMTP. Defaults to 25. +* `smtp_user` and `smtp_pass`: Username/password for authentication to the SMTP server. By default, no + authentication is attempted. +* `require_transport_security`: Set to true to require TLS transport security for SMTP. + By default, Synapse will connect over plain text, and will then switch to + TLS via STARTTLS *if the SMTP server supports it*. If this option is set, + Synapse will refuse to connect unless the server supports STARTTLS. +* `enable_tls`: By default, if the server supports TLS, it will be used, and the server + must present a certificate that is valid for 'smtp_host'. If this option + is set to false, TLS will not be used. +* `notif_from`: defines the "From" address to use when sending emails. + It must be set if email sending is enabled. The placeholder '%(app)s' will be replaced by the application name, + which is normally set in `app_name`, but may be overridden by the + Matrix client application. Note that the placeholder must be written '%(app)s', including the + trailing 's'. +* `app_name`: `app_name` defines the default value for '%(app)s' in `notif_from` and email + subjects. It defaults to 'Matrix'. +* `enable_notifs`: Set to true to enable sending emails for messages that the user + has missed. Disabled by default. +* `notif_for_new_users`: Set to false to disable automatic subscription to email + notifications for new users. Enabled by default. +* `client_base_url`: Custom URL for client links within the email notifications. By default + links will be based on "https://matrix.to". (This setting used to be called `riot_base_url`; + the old name is still supported for backwards-compatibility but is now deprecated.) +* `validation_token_lifetime`: Configures the time that a validation email will expire after sending. + Defaults to 1h. +* `invite_client_location`: The web client location to direct users to during an invite. This is passed + to the identity server as the `org.matrix.web_client_location` key. Defaults + to unset, giving no guidance to the identity server. +* `subjects`: Subjects to use when sending emails from Synapse. The placeholder '%(app)s' will + be replaced with the value of the `app_name` setting, or by a value dictated by the Matrix client application. + In addition, each subject can use the following placeholders: '%(person)s', which will be replaced by the displayname + of the user(s) that sent the message(s), e.g. "Alice and Bob", and '%(room)s', which will be replaced by the name of the room the + message(s) have been sent to, e.g. "My super room". In addition, emails related to account administration will + can use the '%(server_name)s' placeholder, which will be replaced by the value of the + `server_name` setting in your Synapse configuration. + + Here is a list of subjects for notification emails that can be set: + * `message_from_person_in_room`: Subject to use to notify about one message from one or more user(s) in a + room which has a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..." + * `message_from_person`: Subject to use to notify about one message from one or more user(s) in a + room which doesn't have a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s..." + * `messages_from_person`: Subject to use to notify about multiple messages from one or more users in + a room which doesn't have a name. Defaults to "[%(app)s] You have messages on %(app)s from %(person)s..." + * `messages_in_room`: Subject to use to notify about multiple messages in a room which has a + name. Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room..." + * `messages_in_room_and_others`: Subject to use to notify about multiple messages in multiple rooms. + Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room and others..." + * `messages_from_person_and_others`: Subject to use to notify about multiple messages from multiple persons in + multiple rooms. This is similar to the setting above except it's used when + the room in which the notification was triggered has no name. Defaults to + "[%(app)s] You have messages on %(app)s from %(person)s and others..." + * `invite_from_person_to_room`: Subject to use to notify about an invite to a room which has a name. + Defaults to "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..." + * `invite_from_person`: Subject to use to notify about an invite to a room which doesn't have a + name. Defaults to "[%(app)s] %(person)s has invited you to chat on %(app)s..." + * `password_reset`: Subject to use when sending a password reset email. Defaults to "[%(server_name)s] Password reset" + * `email_validation`: Subject to use when sending a verification email to assert an address's + ownership. Defaults to "[%(server_name)s] Validate your email" + +Example configuration: +```yaml +email: + smtp_host: mail.server + smtp_port: 587 + smtp_user: "exampleusername" + smtp_pass: "examplepassword" + require_transport_security: true + enable_tls: false + notif_from: "Your Friendly %(app)s homeserver " + app_name: my_branded_matrix_server + enable_notifs: true + notif_for_new_users: false + client_base_url: "http://localhost/riot" + validation_token_lifetime: 15m + invite_client_location: https://app.element.io + + subjects: + message_from_person_in_room: "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..." + message_from_person: "[%(app)s] You have a message on %(app)s from %(person)s..." + messages_from_person: "[%(app)s] You have messages on %(app)s from %(person)s..." + messages_in_room: "[%(app)s] You have messages on %(app)s in the %(room)s room..." + messages_in_room_and_others: "[%(app)s] You have messages on %(app)s in the %(room)s room and others..." + messages_from_person_and_others: "[%(app)s] You have messages on %(app)s from %(person)s and others..." + invite_from_person_to_room: "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..." + invite_from_person: "[%(app)s] %(person)s has invited you to chat on %(app)s..." + password_reset: "[%(server_name)s] Password reset" + email_validation: "[%(server_name)s] Validate your email" +``` +--- +## Push ## +Configuration settings related to push notifications + +--- +Config option: `push` + +This setting defines options for push notifications. + +This option has a number of sub-options. They are as follows: +* `include_content`: Clients requesting push notifications can either have the body of + the message sent in the notification poke along with other details + like the sender, or just the event ID and room ID (`event_id_only`). + If clients choose the to have the body sent, this option controls whether the + notification request includes the content of the event (other details + like the sender are still included). If `event_id_only` is enabled, it + has no effect. + For modern android devices the notification content will still appear + because it is loaded by the app. iPhone, however will send a + notification saying only that a message arrived and who it came from. + Defaults to true. Set to false to only include the event ID and room ID in push notification payloads. +* `group_unread_count_by_room: false`: When a push notification is received, an unread count is also sent. + This number can either be calculated as the number of unread messages for the user, or the number of *rooms* the + user has unread messages in. Defaults to true, meaning push clients will see the number of + rooms with unread messages in them. Set to false to instead send the number + of unread messages. + +Example configuration: +```yaml +push: + include_content: false + group_unread_count_by_room: false +``` +--- +## Rooms ## +Config options relating to rooms. + +--- +Config option: `encryption_enabled_by_default` + +Controls whether locally-created rooms should be end-to-end encrypted by +default. + +Possible options are "all", "invite", and "off". They are defined as: + +* "all": any locally-created room +* "invite": any room created with the `private_chat` or `trusted_private_chat` + room creation presets +* "off": this option will take no effect + +The default value is "off". + +Note that this option will only affect rooms created after it is set. It +will also not affect rooms created by other servers. + +Example configuration: +```yaml +encryption_enabled_by_default_for_room_type: invite +``` +--- +Config option: `enable_group_creation` + +Set to true to allow non-server-admin users to create groups on this server + +Example configuration: +```yaml +enable_group_creation: true +``` +--- +Config option: `group_creation_prefix` + +If enabled/present, non-server admins can only create groups with local parts +starting with this prefix. + +Example configuration: +```yaml +group_creation_prefix: "unofficial_" +``` +--- +Config option: `user_directory` + +This setting defines options related to the user directory. + +This option has the following sub-options: +* `enabled`: Defines whether users can search the user directory. If false then + empty responses are returned to all queries. Defaults to true. +* `search_all_users`: Defines whether to search all users visible to your HS when searching + the user directory. If false, search results will only contain users + visible in public rooms and users sharing a room with the requester. + Defaults to false. + NB. If you set this to true, and the last time the user_directory search + indexes were (re)built was before Synapse 1.44, you'll have to + rebuild the indexes in order to search through all known users. + These indexes are built the first time Synapse starts; admins can + manually trigger a rebuild via API following the instructions at + https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/background_updates.html#run + Set to true to return search results containing all known users, even if that + user does not share a room with the requester. +* `prefer_local_users`: Defines whether to prefer local users in search query results. + If set to true, local users are more likely to appear above remote users when searching the + user directory. Defaults to false. + +Example configuration: +```yaml +user_directory: + enabled: false + search_all_users: true + prefer_local_users: true +``` +--- +Config option: `user_consent` + +For detailed instructions on user consent configuration, see [here](../../consent_tracking.md). + +Parts of this section are required if enabling the `consent` resource under +`listeners`, in particular `template_dir` and `version`. # TODO: link `listeners` + +* `template_dir`: gives the location of the templates for the HTML forms. + This directory should contain one subdirectory per language (eg, `en`, `fr`), + and each language directory should contain the policy document (named as + .html) and a success page (success.html). + +* `version`: specifies the 'current' version of the policy document. It defines + the version to be served by the consent resource if there is no 'v' + parameter. + +* `server_notice_content`: if enabled, will send a user a "Server Notice" + asking them to consent to the privacy policy. The `server_notices` section ##TODO: link + must also be configured for this to work. Notices will *not* be sent to + guest users unless `send_server_notice_to_guests` is set to true. + +* `block_events_error`, if set, will block any attempts to send events + until the user consents to the privacy policy. The value of the setting is + used as the text of the error. + +* `require_at_registration`, if enabled, will add a step to the registration + process, similar to how captcha works. Users will be required to accept the + policy before their account is created. + +* `policy_name` is the display name of the policy users will see when registering + for an account. Has no effect unless `require_at_registration` is enabled. + Defaults to "Privacy Policy". + +Example configuration: +```yaml +user_consent: + template_dir: res/templates/privacy + version: 1.0 + server_notice_content: + msgtype: m.text + body: >- + To continue using this homeserver you must review and agree to the + terms and conditions at %(consent_uri)s + send_server_notice_to_guests: true + block_events_error: >- + To continue using this homeserver you must review and agree to the + terms and conditions at %(consent_uri)s + require_at_registration: false + policy_name: Privacy Policy +``` +--- +Config option: `stats` + +Settings for local room and user statistics collection. See [here](../../room_and_user_statistics.md) +for more. + +* `enabled`: Set to false to disable room and user statistics. Note that doing + so may cause certain features (such as the room directory) not to work + correctly. Defaults to true. + +Example configuration: +```yaml +stats: + enabled: false +``` +--- +Config option: `server_notices` + +Use this setting to enable a room which can be used to send notices +from the server to users. It is a special room which users cannot leave; notices +in the room come from a special "notices" user id. + +If you use this setting, you *must* define the `system_mxid_localpart` +sub-setting, which defines the id of the user which will be used to send the +notices. + +Sub-options for this setting include: +* `system_mxid_display_name`: set the display name of the "notices" user +* `system_mxid_avatar_url`: set the avatar for the "notices" user +* `room_name`: set the room name of the server notices room + +Example configuration: +```yaml +server_notices: + system_mxid_localpart: notices + system_mxid_display_name: "Server Notices" + system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ" + room_name: "Server Notices" +``` +--- +Config option: `enable_room_list_search` + +Set to false to disable searching the public room list. When disabled +blocks searching local and remote room lists for local and remote +users by always returning an empty list for all queries. Defaults to true. + +Example configuration: +```yaml +enable_room_list_search: false +``` +--- +Config option: `alias_creation` + +The `alias_creation` option controls who is allowed to create aliases +on this server. + +The format of this option is a list of rules that contain globs that +match against user_id, room_id and the new alias (fully qualified with +server name). The action in the first rule that matches is taken, +which can currently either be "allow" or "deny". + +Missing user_id/room_id/alias fields default to "*". + +If no rules match the request is denied. An empty list means no one +can create aliases. + +Options for the rules include: +* `user_id`: Matches against the creator of the alias. Defaults to "*". +* `alias`: Matches against the alias being created. Defaults to "*". +* `room_id`: Matches against the room ID the alias is being pointed at. Defaults to "*" +* `action`: Whether to "allow" or "deny" the request if the rule matches. Defaults to allow. + +Example configuration: +```yaml +alias_creation_rules: + - user_id: "bad_user" + alias: "spammy_alias" + room_id: "*" + action: deny +``` +--- +Config options: `room_list_publication_rules` + +The `room_list_publication_rules` option controls who can publish and +which rooms can be published in the public room list. + +The format of this option is the same as that for +`alias_creation_rules`. + +If the room has one or more aliases associated with it, only one of +the aliases needs to match the alias rule. If there are no aliases +then only rules with `alias: *` match. + +If no rules match the request is denied. An empty list means no one +can publish rooms. + +Options for the rules include: +* `user_id`: Matches against the creator of the alias. Defaults to "*". +* `alias`: Matches against any current local or canonical aliases associated with the room. Defaults to "*". +* `room_id`: Matches against the room ID being published. Defaults to "*". +* `action`: Whether to "allow" or "deny" the request if the rule matches. Defaults to allow. + +Example configuration: +```yaml +room_list_publication_rules: + - user_id: "*" + alias: "*" + room_id: "*" + action: allow +``` + +--- +Config option: `default_power_level_content_override` + +The `default_power_level_content_override` option controls the default power +levels for rooms. + +Useful if you know that your users need special permissions in rooms +that they create (e.g. to send particular types of state events without +needing an elevated power level). This takes the same shape as the +`power_level_content_override` parameter in the /createRoom API, but +is applied before that parameter. + +Note that each key provided inside a preset (for example `events` in the example +below) will overwrite all existing defaults inside that key. So in the example +below, newly-created private_chat rooms will have no rules for any event types +except `com.example.foo`. + +Example configuration: +```yaml +default_power_level_content_override: + private_chat: { "events": { "com.example.foo" : 0 } } + trusted_private_chat: null + public_chat: null +``` + +--- +## Opentracing ## +Configuration options related to Opentracing support. + +--- +Config option: `opentracing` + +These settings enable and configure opentracing, which implements distributed tracing. +This allows you to observe the causal chains of events across servers +including requests, key lookups etc., across any server running +synapse or any other services which support opentracing +(specifically those implemented with Jaeger). + +Sub-options include: +* `enabled`: whether tracing is enabled. Set to true to enable. Disabled by default. +* `homeserver_whitelist`: The list of homeservers we wish to send and receive span contexts and span baggage. + See [here](../../opentracing.md) for more. + This is a list of regexes which are matched against the `server_name` of the homeserver. + By default, it is empty, so no servers are matched. +* `force_tracing_for_users`: # A list of the matrix IDs of users whose requests will always be traced, + even if the tracing system would otherwise drop the traces due to probabilistic sampling. + By default, the list is empty. +* `jaeger_config`: Jaeger can be configured to sample traces at different rates. + All configuration options provided by Jaeger can be set here. Jaeger's configuration is + mostly related to trace sampling which is documented [here](https://www.jaegertracing.io/docs/latest/sampling/). + +Example configuration: +```yaml +opentracing: + enabled: true + homeserver_whitelist: + - ".*" + force_tracing_for_users: + - "@user1:server_name" + - "@user2:server_name" + + jaeger_config: + sampler: + type: const + param: 1 + logging: + false +``` +--- +## Workers ## +Configuration options related to workers. + +--- +Config option: `send_federation` + +Controls sending of outbound federation transactions on the main process. +Set to false if using a federation sender worker. Defaults to true. + +Example configuration: +```yaml +send_federation: false +``` +--- +Config option: `federation_sender_instances` + +It is possible to run multiple federation sender workers, in which case the +work is balanced across them. Use this setting to list the senders. + +This configuration setting must be shared between all federation sender workers, and if +changed all federation sender workers must be stopped at the same time and then +started, to ensure that all instances are running with the same config (otherwise +events may be dropped). + +Example configuration: +```yaml +federation_sender_instances: + - federation_sender1 +``` +--- +Config option: `instance_map` + +When using workers this should be a map from worker name to the +HTTP replication listener of the worker, if configured. + +Example configuration: +```yaml +instance_map: + worker1: + host: localhost + port: 8034 +``` +--- +Config option: `stream_writers` + +Experimental: When using workers you can define which workers should +handle event persistence and typing notifications. Any worker +specified here must also be in the `instance_map`. + +Example configuration: +```yaml +stream_writers: + events: worker1 + typing: worker1 +``` +--- +Config option: `run_background_tasks_on` + +The worker that is used to run background tasks (e.g. cleaning up expired +data). If not provided this defaults to the main process. + +Example configuration: +```yaml +run_background_tasks_on: worker1 +``` +--- +Config option: `worker_replication_secret` + +A shared secret used by the replication APIs to authenticate HTTP requests +from workers. + +By default this is unused and traffic is not authenticated. + +Example configuration: +```yaml +worker_replication_secret: "secret_secret" +``` +Config option: `redis` + +Configuration for Redis when using workers. This *must* be enabled when +using workers (unless using old style direct TCP configuration). +This setting has the following sub-options: +* `enabled`: whether to use Redis support. Defaults to false. +* `host` and `port`: Optional host and port to use to connect to redis. Defaults to + localhost and 6379 +* `password`: Optional password if configured on the Redis instance. + +Example configuration: +```yaml +redis: + enabled: true + host: localhost + port: 6379 + password: +``` +## Background Updates ## +Configuration settings related to background updates. + +--- +Config option: `background_updates` + +Background updates are database updates that are run in the background in batches. +The duration, minimum batch size, default batch size, whether to sleep between batches and if so, how long to +sleep can all be configured. This is helpful to speed up or slow down the updates. +This setting has the following sub-options: +* `background_update_duration_ms`: How long in milliseconds to run a batch of background updates for. Defaults to 100. + Set a different time to change the default. +* `sleep_enabled`: Whether to sleep between updates. Defaults to true. Set to false to change the default. +* `sleep_duration_ms`: If sleeping between updates, how long in milliseconds to sleep for. Defaults to 1000. + Set a duration to change the default. +* `min_batch_size`: Minimum size a batch of background updates can be. Must be greater than 0. Defaults to 1. + Set a size to change the default. +* `default_batch_size`: The batch size to use for the first iteration of a new background update. The default is 100. + Set a size to change the default. + +Example configuration: +```yaml +background_updates: + background_update_duration_ms: 500 + sleep_enabled: false + sleep_duration_ms: 300 + min_batch_size: 10 + default_batch_size: 50 +``` diff --git a/docs/website_files/table-of-contents.js b/docs/website_files/table-of-contents.js index 0de5960b22b1..772da97fb9de 100644 --- a/docs/website_files/table-of-contents.js +++ b/docs/website_files/table-of-contents.js @@ -75,6 +75,20 @@ function setTocEntry() { * Populate sidebar on load */ window.addEventListener('load', () => { + // Prevent rendering the table of contents of the "print book" page, as it + // will end up being rendered into the output (in a broken-looking way) + + // Get the name of the current page (i.e. 'print.html') + const pageNameExtension = window.location.pathname.split('/').pop(); + + // Split off the extension (as '.../print' is also a valid page name), which + // should result in 'print' + const pageName = pageNameExtension.split('.')[0]; + if (pageName === "print") { + // Don't render the table of contents on this page + return; + } + // Only create table of contents if there is more than one header on the page if (headers.length <= 1) { return; diff --git a/docs/welcome_and_overview.md b/docs/welcome_and_overview.md index aab2d6b4f0f6..451759f06ec6 100644 --- a/docs/welcome_and_overview.md +++ b/docs/welcome_and_overview.md @@ -7,10 +7,10 @@ team. ## Installing and using Synapse This documentation covers topics for **installation**, **configuration** and -**maintainence** of your Synapse process: +**maintenance** of your Synapse process: * Learn how to [install](setup/installation.md) and - [configure](usage/configuration/index.html) your own instance, perhaps with [Single + [configure](usage/configuration/config_documentation.md) your own instance, perhaps with [Single Sign-On](usage/configuration/user_authentication/index.html). * See how to [upgrade](upgrade.md) between Synapse versions. @@ -65,7 +65,7 @@ following documentation: Want to help keep Synapse going but don't know how to code? Synapse is a [Matrix.org Foundation](https://matrix.org) project. Consider becoming a -supportor on [Liberapay](https://liberapay.com/matrixdotorg), +supporter on [Liberapay](https://liberapay.com/matrixdotorg), [Patreon](https://patreon.com/matrixdotorg) or through [PayPal](https://paypal.me/matrixdotorg) via a one-time donation. diff --git a/docs/workers.md b/docs/workers.md index caef44b614de..779069b8177f 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -138,22 +138,7 @@ as the `listeners` option in the shared config. For example: ```yaml -worker_app: synapse.app.generic_worker -worker_name: worker1 - -# The replication listener on the main synapse process. -worker_replication_host: 127.0.0.1 -worker_replication_http_port: 9093 - -worker_listeners: - - type: http - port: 8083 - resources: - - names: - - client - - federation - -worker_log_config: /home/matrix/synapse/config/worker1_log_config.yaml +{{#include systemd-with-workers/workers/generic_worker.yaml}} ``` ...is a full configuration for a generic worker instance, which will expose a @@ -266,6 +251,8 @@ information. # Presence requests ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/ + # User directory search requests + ^/_matrix/client/(r0|v3|unstable)/user_directory/search$ Additionally, the following REST endpoints can be handled for GET requests: @@ -343,9 +330,9 @@ effects of bursts of events from that bridge on events sent by normal users. #### Stream writers -Additionally, there is *experimental* support for moving writing of specific -streams (such as events) off of the main process to a particular worker. (This -is only supported with Redis-based replication.) +Additionally, the writing of specific streams (such as events) can be moved off +of the main process to a particular worker. +(This is only supported with Redis-based replication.) To enable this, the worker must have a HTTP replication listener configured, have a `worker_name` and be listed in the `instance_map` config. The same worker @@ -365,6 +352,12 @@ stream_writers: events: event_persister1 ``` +An example for a stream writer instance: + +```yaml +{{#include systemd-with-workers/workers/event_persister.yaml}} +``` + Some of the streams have associated endpoints which, for maximum efficiency, should be routed to the workers handling that stream. See below for the currently supported streams and the endpoints associated with them: @@ -422,7 +415,7 @@ the stream writer for the `presence` stream: #### Background tasks -There is also *experimental* support for moving background tasks to a separate +There is also support for moving background tasks to a separate worker. Background tasks are run periodically or started via replication. Exactly which tasks are configured to run depends on your Synapse configuration (e.g. if stats is enabled). @@ -435,9 +428,57 @@ the shared configuration would include: run_background_tasks_on: background_worker ``` -You might also wish to investigate the `update_user_directory` and +You might also wish to investigate the `update_user_directory_from_worker` and `media_instance_running_background_jobs` settings. +An example for a dedicated background worker instance: + +```yaml +{{#include systemd-with-workers/workers/background_worker.yaml}} +``` + +#### Updating the User Directory + +You can designate one generic worker to update the user directory. + +Specify its name in the shared configuration as follows: + +```yaml +update_user_directory_from_worker: worker_name +``` + +This work cannot be load-balanced; please ensure the main process is restarted +after setting this option in the shared configuration! + +User directory updates allow REST endpoints matching the following regular +expressions to work: + + ^/_matrix/client/(r0|v3|unstable)/user_directory/search$ + +The above endpoints can be routed to any worker, though you may choose to route +it to the chosen user directory worker. + +This style of configuration supersedes the legacy `synapse.app.user_dir` +worker application type. + + +#### Notifying Application Services + +You can designate one generic worker to send output traffic to Application Services. + +Specify its name in the shared configuration as follows: + +```yaml +notify_appservices_from_worker: worker_name +``` + +This work cannot be load-balanced; please ensure the main process is restarted +after setting this option in the shared configuration! + +This style of configuration supersedes the legacy `synapse.app.appservice` +worker application type. + + ### `synapse.app.pusher` Handles sending push notifications to sygnal and email. Doesn't handle any @@ -456,6 +497,9 @@ pusher_instances: ### `synapse.app.appservice` +**Deprecated as of Synapse v1.59.** [Use `synapse.app.generic_worker` with the +`notify_appservices_from_worker` option instead.](#notifying-application-services) + Handles sending output traffic to Application Services. Doesn't handle any REST endpoints itself, but you should set `notify_appservices: False` in the shared configuration file to stop the main synapse sending appservice notifications. @@ -523,6 +567,9 @@ Note that if a reverse proxy is used , then `/_matrix/media/` must be routed for ### `synapse.app.user_dir` +**Deprecated as of Synapse v1.59.** [Use `synapse.app.generic_worker` with the +`update_user_directory_from_worker` option instead.](#updating-the-user-directory) + Handles searches in the user directory. It can handle REST endpoints matching the following regular expressions: @@ -617,14 +664,14 @@ The following shows an example setup using Redis and a reverse proxy: | Main | | Generic | | Generic | | Event | | Process | | Worker 1 | | Worker 2 | | Persister | +--------------+ +--------------+ +--------------+ +--------------+ - ^ ^ | ^ | | ^ | ^ ^ - | | | | | | | | | | - | | | | | HTTP | | | | | - | +----------+<--|---|---------+ | | | | - | | +-------------|-->+----------+ | - | | | | - | | | | - v v v v -==================================================================== + ^ ^ | ^ | | ^ | | ^ ^ + | | | | | | | | | | | + | | | | | HTTP | | | | | | + | +----------+<--|---|---------+<--|---|---------+ | | + | | +-------------|-->+-------------+ | + | | | | + | | | | + v v v v +====================================================================== Redis pub/sub channel ``` diff --git a/mypy.ini b/mypy.ini index 5246f987c009..fe3e3f9b8efd 100644 --- a/mypy.ini +++ b/mypy.ini @@ -7,13 +7,14 @@ show_error_codes = True show_traceback = True mypy_path = stubs warn_unreachable = True +warn_unused_ignores = True local_partial_types = True no_implicit_optional = True +disallow_untyped_defs = True files = docker/, scripts-dev/, - setup.py, synapse/, tests/ @@ -24,16 +25,9 @@ files = # https://docs.python.org/3/library/re.html#re.X exclude = (?x) ^( - |scripts-dev/build_debian_packages.py - |scripts-dev/federation_client.py - |scripts-dev/release.py - |synapse/storage/databases/__init__.py |synapse/storage/databases/main/cache.py |synapse/storage/databases/main/devices.py - |synapse/storage/databases/main/event_federation.py - |synapse/storage/databases/main/push_rule.py - |synapse/storage/databases/main/roommember.py |synapse/storage/schema/ |tests/api/test_auth.py @@ -47,16 +41,11 @@ exclude = (?x) |tests/events/test_utils.py |tests/federation/test_federation_catch_up.py |tests/federation/test_federation_sender.py - |tests/federation/test_federation_server.py |tests/federation/transport/test_knocking.py - |tests/federation/transport/test_server.py |tests/handlers/test_typing.py |tests/http/federation/test_matrix_federation_agent.py |tests/http/federation/test_srv_resolver.py - |tests/http/test_fedclient.py |tests/http/test_proxyagent.py - |tests/http/test_servlet.py - |tests/http/test_site.py |tests/logging/__init__.py |tests/logging/test_terse_json.py |tests/module_api/test_api.py @@ -65,12 +54,9 @@ exclude = (?x) |tests/push/test_push_rule_evaluator.py |tests/rest/client/test_transactions.py |tests/rest/media/v1/test_media_storage.py - |tests/scripts/test_new_matrix_user.py |tests/server.py |tests/server_notices/test_resource_limits_server_notices.py |tests/state/test_v2.py - |tests/storage/test_base.py - |tests/storage/test_roommember.py |tests/test_metrics.py |tests/test_server.py |tests/test_state.py @@ -93,124 +79,37 @@ exclude = (?x) |tests/utils.py )$ -[mypy-synapse._scripts.*] -disallow_untyped_defs = True - -[mypy-synapse.api.*] -disallow_untyped_defs = True - -[mypy-synapse.app.*] -disallow_untyped_defs = True - -[mypy-synapse.appservice.*] -disallow_untyped_defs = True - -[mypy-synapse.config.*] -disallow_untyped_defs = True - -[mypy-synapse.crypto.*] -disallow_untyped_defs = True - -[mypy-synapse.event_auth] -disallow_untyped_defs = True - -[mypy-synapse.events.*] -disallow_untyped_defs = True - -[mypy-synapse.federation.*] -disallow_untyped_defs = True - [mypy-synapse.federation.transport.client] disallow_untyped_defs = False -[mypy-synapse.handlers.*] -disallow_untyped_defs = True - -[mypy-synapse.http.server] -disallow_untyped_defs = True - -[mypy-synapse.logging.context] -disallow_untyped_defs = True - -[mypy-synapse.metrics.*] -disallow_untyped_defs = True - -[mypy-synapse.module_api.*] -disallow_untyped_defs = True - -[mypy-synapse.notifier] -disallow_untyped_defs = True - -[mypy-synapse.push.*] -disallow_untyped_defs = True - -[mypy-synapse.replication.*] -disallow_untyped_defs = True - -[mypy-synapse.rest.*] -disallow_untyped_defs = True - -[mypy-synapse.server_notices.*] -disallow_untyped_defs = True - -[mypy-synapse.state.*] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.account_data] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.client_ips] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.directory] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.e2e_room_keys] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.end_to_end_keys] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.event_push_actions] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.events_bg_updates] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.events_worker] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.room] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.room_batch] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.profile] -disallow_untyped_defs = True +[mypy-synapse.http.client] +disallow_untyped_defs = False -[mypy-synapse.storage.databases.main.stats] -disallow_untyped_defs = True +[mypy-synapse.http.matrixfederationclient] +disallow_untyped_defs = False -[mypy-synapse.storage.databases.main.state_deltas] -disallow_untyped_defs = True +[mypy-synapse.logging.opentracing] +disallow_untyped_defs = False -[mypy-synapse.storage.databases.main.transactions] -disallow_untyped_defs = True +[mypy-synapse.logging.scopecontextmanager] +disallow_untyped_defs = False -[mypy-synapse.storage.databases.main.user_erasure_store] -disallow_untyped_defs = True +[mypy-synapse.metrics._reactor_metrics] +disallow_untyped_defs = False +# This module imports select.epoll. That exists on Linux, but doesn't on macOS. +# See https://github.com/matrix-org/synapse/pull/11771. +warn_unused_ignores = False -[mypy-synapse.storage.util.*] -disallow_untyped_defs = True +[mypy-synapse.util.caches.treecache] +disallow_untyped_defs = False -[mypy-synapse.streams.*] -disallow_untyped_defs = True +[mypy-synapse.server] +disallow_untyped_defs = False -[mypy-synapse.util.*] -disallow_untyped_defs = True +[mypy-synapse.storage.database] +disallow_untyped_defs = False -[mypy-synapse.util.caches.treecache] +[mypy-tests.*] disallow_untyped_defs = False [mypy-tests.handlers.test_user_directory] @@ -234,69 +133,32 @@ disallow_untyped_defs = True ;; The `typeshed` project maintains stubs here: ;; https://github.com/python/typeshed/tree/master/stubs ;; and for each package `foo` there's a corresponding `types-foo` package on PyPI, -;; which we can pull in as a dev dependency by adding to `setup.py`'s -;; `CONDITIONAL_REQUIREMENTS["mypy"]` list. +;; which we can pull in as a dev dependency by adding to `pyproject.toml`'s +;; `[tool.poetry.dev-dependencies]` list. [mypy-authlib.*] ignore_missing_imports = True -[mypy-bcrypt] -ignore_missing_imports = True - [mypy-canonicaljson] ignore_missing_imports = True -[mypy-constantly] -ignore_missing_imports = True - -[mypy-daemonize] -ignore_missing_imports = True - -[mypy-h11] -ignore_missing_imports = True - -[mypy-hiredis] -ignore_missing_imports = True - -[mypy-hyperlink] -ignore_missing_imports = True - [mypy-ijson.*] ignore_missing_imports = True -[mypy-importlib_metadata.*] -ignore_missing_imports = True - -[mypy-jaeger_client.*] -ignore_missing_imports = True - -[mypy-josepy.*] -ignore_missing_imports = True - -[mypy-jwt.*] -ignore_missing_imports = True - [mypy-lxml] ignore_missing_imports = True [mypy-msgpack] ignore_missing_imports = True -[mypy-nacl.*] -ignore_missing_imports = True - +# Note: WIP stubs available at +# https://github.com/microsoft/python-type-stubs/tree/64934207f523ad6b611e6cfe039d85d7175d7d0d/netaddr [mypy-netaddr] ignore_missing_imports = True [mypy-parameterized.*] ignore_missing_imports = True -[mypy-phonenumbers.*] -ignore_missing_imports = True - -[mypy-prometheus_client.*] -ignore_missing_imports = True - [mypy-pymacaroons.*] ignore_missing_imports = True @@ -309,23 +171,14 @@ ignore_missing_imports = True [mypy-saml2.*] ignore_missing_imports = True -[mypy-sentry_sdk] -ignore_missing_imports = True - [mypy-service_identity.*] ignore_missing_imports = True -[mypy-signedjson.*] +[mypy-srvlookup.*] ignore_missing_imports = True [mypy-treq.*] ignore_missing_imports = True -[mypy-twisted.*] -ignore_missing_imports = True - -[mypy-zope] -ignore_missing_imports = True - [mypy-incremental.*] ignore_missing_imports = True diff --git a/poetry.lock b/poetry.lock index bbe8eba96d40..49a912a58962 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,11 +1,3 @@ -[[package]] -name = "appdirs" -version = "1.4.4" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" -optional = false -python-versions = "*" - [[package]] name = "attrs" version = "21.4.0" @@ -49,17 +41,6 @@ six = "*" [package.extras] visualize = ["graphviz (>0.5.1)", "Twisted (>=16.1.1)"] -[[package]] -name = "baron" -version = "0.10.1" -description = "Full Syntax Tree for python to make writing refactoring code a realist task" -category = "dev" -optional = false -python-versions = "*" - -[package.dependencies] -rply = "*" - [[package]] name = "bcrypt" version = "3.2.0" @@ -328,14 +309,15 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.14" -description = "Python Git Library" +version = "3.1.27" +description = "GitPython is a python library used to interact with Git repositories" category = "dev" optional = false -python-versions = ">=3.4" +python-versions = ">=3.7" [package.dependencies] gitdb = ">=4.0.1,<5" +typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.8\""} [[package]] name = "hiredis" @@ -558,17 +540,20 @@ test = ["tox", "twisted", "aiounittest"] [[package]] name = "matrix-synapse-ldap3" -version = "0.1.5" +version = "0.2.0" description = "An LDAP3 auth provider for Synapse" category = "main" optional = true -python-versions = "*" +python-versions = ">=3.7" [package.dependencies] ldap3 = ">=2.8" -service_identity = "*" +service-identity = "*" Twisted = ">=15.1.0" +[package.extras] +dev = ["matrix-synapse", "tox", "ldaptor", "mypy (==0.910)", "types-setuptools", "black (==21.9b0)", "flake8 (==4.0.1)", "isort (==5.9.3)"] + [[package]] name = "mccabe" version = "0.6.1" @@ -587,7 +572,7 @@ python-versions = "*" [[package]] name = "mypy" -version = "0.931" +version = "0.950" description = "Optional static typing for Python" category = "dev" optional = false @@ -595,13 +580,14 @@ python-versions = ">=3.6" [package.dependencies] mypy-extensions = ">=0.4.3" -tomli = ">=1.1.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} typed-ast = {version = ">=1.4.0,<2", markers = "python_version < \"3.8\""} typing-extensions = ">=3.10" [package.extras] dmypy = ["psutil (>=4.0)"] python2 = ["typed-ast (>=1.4.0,<2)"] +reports = ["lxml"] [[package]] name = "mypy-extensions" @@ -613,14 +599,14 @@ python-versions = "*" [[package]] name = "mypy-zope" -version = "0.3.5" +version = "0.3.7" description = "Plugin for mypy to support zope interfaces" category = "dev" optional = false python-versions = "*" [package.dependencies] -mypy = "0.931" +mypy = "0.950" "zope.interface" = "*" "zope.schema" = "*" @@ -717,7 +703,7 @@ test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock [[package]] name = "prometheus-client" -version = "0.13.1" +version = "0.14.0" description = "Python client for the Prometheus monitoring system." category = "main" optional = false @@ -981,20 +967,6 @@ Pygments = ">=2.5.1" [package.extras] md = ["cmarkgfm (>=0.8.0)"] -[[package]] -name = "redbaron" -version = "0.9.2" -description = "Abstraction on top of baron, a FST for python to make writing refactoring code a realistic task" -category = "dev" -optional = false -python-versions = "*" - -[package.dependencies] -baron = ">=0.7" - -[package.extras] -notebook = ["pygments"] - [[package]] name = "requests" version = "2.27.1" @@ -1035,17 +1007,6 @@ python-versions = ">=3.7" [package.extras] idna2008 = ["idna"] -[[package]] -name = "rply" -version = "0.7.8" -description = "A pure Python Lex/Yacc that works with RPython" -category = "dev" -optional = false -python-versions = "*" - -[package.dependencies] -appdirs = "*" - [[package]] name = "secretstorage" version = "3.3.1" @@ -1060,7 +1021,7 @@ jeepney = ">=0.6" [[package]] name = "sentry-sdk" -version = "1.5.7" +version = "1.5.11" description = "Python client for Sentry (https://sentry.io)" category = "main" optional = true @@ -1285,7 +1246,7 @@ urllib3 = ">=1.26.0" [[package]] name = "twisted" -version = "22.2.0" +version = "22.4.0" description = "An asynchronous networking framework written in Python" category = "main" optional = false @@ -1305,19 +1266,20 @@ typing-extensions = ">=3.6.5" "zope.interface" = ">=4.4.2" [package.extras] -all_non_platform = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +all_non_platform = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] conch = ["pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)"] +conch_nacl = ["pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pynacl"] contextvars = ["contextvars (>=2.4,<3)"] dev = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "python-subunit (>=1.4,<2.0)", "pydoctor (>=21.9.0,<21.10.0)"] dev_release = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pydoctor (>=21.9.0,<21.10.0)"] -http2 = ["h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)"] -macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] -mypy = ["mypy (==0.930)", "mypy-zope (==0.3.4)", "types-setuptools", "types-pyopenssl", "towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "python-subunit (>=1.4,<2.0)", "contextvars (>=2.4,<3)", "pydoctor (>=21.9.0,<21.10.0)"] -osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +http2 = ["h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)"] +macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +mypy = ["mypy (==0.930)", "mypy-zope (==0.3.4)", "types-setuptools", "types-pyopenssl", "towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pynacl", "pywin32 (!=226)", "python-subunit (>=1.4,<2.0)", "contextvars (>=2.4,<3)", "pydoctor (>=21.9.0,<21.10.0)"] +osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] serial = ["pyserial (>=3.0)", "pywin32 (!=226)"] test = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)"] tls = ["pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)"] -windows_platform = ["pywin32 (!=226)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +windows_platform = ["pywin32 (!=226)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] [[package]] name = "twisted-iocpsupport" @@ -1355,6 +1317,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "types-commonmark" +version = "0.9.2" +description = "Typing stubs for commonmark" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "types-cryptography" version = "3.3.15" @@ -1401,7 +1371,7 @@ python-versions = "*" [[package]] name = "types-pillow" -version = "9.0.6" +version = "9.0.15" description = "Typing stubs for Pillow" category = "dev" optional = false @@ -1576,7 +1546,7 @@ docs = ["sphinx", "repoze.sphinx.autointerface"] test = ["zope.i18nmessageid", "zope.testing", "zope.testrunner"] [extras] -all = ["matrix-synapse-ldap3", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pysaml2", "authlib", "lxml", "sentry-sdk", "jaeger-client", "opentracing", "pyjwt", "txredisapi", "hiredis"] +all = ["matrix-synapse-ldap3", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pysaml2", "authlib", "lxml", "sentry-sdk", "jaeger-client", "opentracing", "pyjwt", "txredisapi", "hiredis", "Pympler"] cache_memory = ["Pympler"] jwt = ["pyjwt"] matrix-synapse-ldap3 = ["matrix-synapse-ldap3"] @@ -1592,14 +1562,10 @@ url_preview = ["lxml"] [metadata] lock-version = "1.1" -python-versions = "^3.7" -content-hash = "964ad29eaf7fd02749a4e735818f3bc0ba729c2f4b9e3213f0daa02643508b16" +python-versions = "^3.7.1" +content-hash = "d39d5ac5d51c014581186b7691999b861058b569084c525523baf70b77f292b1" [metadata.files] -appdirs = [ - {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, - {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, -] attrs = [ {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, @@ -1612,10 +1578,6 @@ automat = [ {file = "Automat-20.2.0-py2.py3-none-any.whl", hash = "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111"}, {file = "Automat-20.2.0.tar.gz", hash = "sha256:7979803c74610e11ef0c0d68a2942b152df52da55336e0c9d58daf1831cbdf33"}, ] -baron = [ - {file = "baron-0.10.1-py2.py3-none-any.whl", hash = "sha256:befb33f4b9e832c7cd1e3cf0eafa6dd3cb6ed4cb2544245147c019936f4e0a8a"}, - {file = "baron-0.10.1.tar.gz", hash = "sha256:af822ad44d4eb425c8516df4239ac4fdba9fdb398ef77e4924cd7c9b4045bc2f"}, -] bcrypt = [ {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:b589229207630484aefe5899122fb938a5b017b0f4349f769b8c13e78d99a8fd"}, {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6"}, @@ -1814,8 +1776,8 @@ gitdb = [ {file = "gitdb-4.0.9.tar.gz", hash = "sha256:bac2fd45c0a1c9cf619e63a90d62bdc63892ef92387424b855792a6cabe789aa"}, ] gitpython = [ - {file = "GitPython-3.1.14-py3-none-any.whl", hash = "sha256:3283ae2fba31c913d857e12e5ba5f9a7772bbc064ae2bb09efafa71b0dd4939b"}, - {file = "GitPython-3.1.14.tar.gz", hash = "sha256:be27633e7509e58391f10207cd32b2a6cf5b908f92d9cd30da2e514e1137af61"}, + {file = "GitPython-3.1.27-py3-none-any.whl", hash = "sha256:5b68b000463593e05ff2b261acff0ff0972df8ab1b70d3cdbd41b546c8b8fc3d"}, + {file = "GitPython-3.1.27.tar.gz", hash = "sha256:1c885ce809e8ba2d88a29befeb385fcea06338d3640712b59ca623c220bb5704"}, ] hiredis = [ {file = "hiredis-2.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b4c8b0bc5841e578d5fb32a16e0c305359b987b850a06964bd5a62739d688048"}, @@ -2084,7 +2046,8 @@ matrix-common = [ {file = "matrix_common-1.1.0.tar.gz", hash = "sha256:a8238748afc2b37079818367fed5156f355771b07c8ff0a175934f47e0ff3276"}, ] matrix-synapse-ldap3 = [ - {file = "matrix-synapse-ldap3-0.1.5.tar.gz", hash = "sha256:9fdf8df7c8ec756642aa0fea53b31c0b2f1924f70d7f049a2090b523125456fe"}, + {file = "matrix-synapse-ldap3-0.2.0.tar.gz", hash = "sha256:91a0715b43a41ec3033244174fca20846836da98fda711fb01687f7199eecd2e"}, + {file = "matrix_synapse_ldap3-0.2.0-py3-none-any.whl", hash = "sha256:0128ca7c3058987adc2e8a88463bb46879915bfd3d373309632813b353e30f9f"}, ] mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, @@ -2127,34 +2090,37 @@ msgpack = [ {file = "msgpack-1.0.3.tar.gz", hash = "sha256:51fdc7fb93615286428ee7758cecc2f374d5ff363bdd884c7ea622a7a327a81e"}, ] mypy = [ - {file = "mypy-0.931-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c5b42d0815e15518b1f0990cff7a705805961613e701db60387e6fb663fe78a"}, - {file = "mypy-0.931-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c89702cac5b302f0c5d33b172d2b55b5df2bede3344a2fbed99ff96bddb2cf00"}, - {file = "mypy-0.931-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:300717a07ad09525401a508ef5d105e6b56646f7942eb92715a1c8d610149714"}, - {file = "mypy-0.931-cp310-cp310-win_amd64.whl", hash = "sha256:7b3f6f557ba4afc7f2ce6d3215d5db279bcf120b3cfd0add20a5d4f4abdae5bc"}, - {file = "mypy-0.931-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:1bf752559797c897cdd2c65f7b60c2b6969ffe458417b8d947b8340cc9cec08d"}, - {file = "mypy-0.931-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4365c60266b95a3f216a3047f1d8e3f895da6c7402e9e1ddfab96393122cc58d"}, - {file = "mypy-0.931-cp36-cp36m-win_amd64.whl", hash = "sha256:1b65714dc296a7991000b6ee59a35b3f550e0073411ac9d3202f6516621ba66c"}, - {file = "mypy-0.931-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e839191b8da5b4e5d805f940537efcaa13ea5dd98418f06dc585d2891d228cf0"}, - {file = "mypy-0.931-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:50c7346a46dc76a4ed88f3277d4959de8a2bd0a0fa47fa87a4cde36fe247ac05"}, - {file = "mypy-0.931-cp37-cp37m-win_amd64.whl", hash = "sha256:d8f1ff62f7a879c9fe5917b3f9eb93a79b78aad47b533911b853a757223f72e7"}, - {file = "mypy-0.931-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f9fe20d0872b26c4bba1c1be02c5340de1019530302cf2dcc85c7f9fc3252ae0"}, - {file = "mypy-0.931-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1b06268df7eb53a8feea99cbfff77a6e2b205e70bf31743e786678ef87ee8069"}, - {file = "mypy-0.931-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8c11003aaeaf7cc2d0f1bc101c1cc9454ec4cc9cb825aef3cafff8a5fdf4c799"}, - {file = "mypy-0.931-cp38-cp38-win_amd64.whl", hash = "sha256:d9d2b84b2007cea426e327d2483238f040c49405a6bf4074f605f0156c91a47a"}, - {file = "mypy-0.931-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ff3bf387c14c805ab1388185dd22d6b210824e164d4bb324b195ff34e322d166"}, - {file = "mypy-0.931-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5b56154f8c09427bae082b32275a21f500b24d93c88d69a5e82f3978018a0266"}, - {file = "mypy-0.931-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8ca7f8c4b1584d63c9a0f827c37ba7a47226c19a23a753d52e5b5eddb201afcd"}, - {file = "mypy-0.931-cp39-cp39-win_amd64.whl", hash = "sha256:74f7eccbfd436abe9c352ad9fb65872cc0f1f0a868e9d9c44db0893440f0c697"}, - {file = "mypy-0.931-py3-none-any.whl", hash = "sha256:1171f2e0859cfff2d366da2c7092b06130f232c636a3f7301e3feb8b41f6377d"}, - {file = "mypy-0.931.tar.gz", hash = "sha256:0038b21890867793581e4cb0d810829f5fd4441aa75796b53033af3aa30430ce"}, + {file = "mypy-0.950-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cf9c261958a769a3bd38c3e133801ebcd284ffb734ea12d01457cb09eacf7d7b"}, + {file = "mypy-0.950-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5b5bd0ffb11b4aba2bb6d31b8643902c48f990cc92fda4e21afac658044f0c0"}, + {file = "mypy-0.950-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5e7647df0f8fc947388e6251d728189cfadb3b1e558407f93254e35abc026e22"}, + {file = "mypy-0.950-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:eaff8156016487c1af5ffa5304c3e3fd183edcb412f3e9c72db349faf3f6e0eb"}, + {file = "mypy-0.950-cp310-cp310-win_amd64.whl", hash = "sha256:563514c7dc504698fb66bb1cf897657a173a496406f1866afae73ab5b3cdb334"}, + {file = "mypy-0.950-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:dd4d670eee9610bf61c25c940e9ade2d0ed05eb44227275cce88701fee014b1f"}, + {file = "mypy-0.950-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ca75ecf2783395ca3016a5e455cb322ba26b6d33b4b413fcdedfc632e67941dc"}, + {file = "mypy-0.950-cp36-cp36m-win_amd64.whl", hash = "sha256:6003de687c13196e8a1243a5e4bcce617d79b88f83ee6625437e335d89dfebe2"}, + {file = "mypy-0.950-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4c653e4846f287051599ed8f4b3c044b80e540e88feec76b11044ddc5612ffed"}, + {file = "mypy-0.950-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e19736af56947addedce4674c0971e5dceef1b5ec7d667fe86bcd2b07f8f9075"}, + {file = "mypy-0.950-cp37-cp37m-win_amd64.whl", hash = "sha256:ef7beb2a3582eb7a9f37beaf38a28acfd801988cde688760aea9e6cc4832b10b"}, + {file = "mypy-0.950-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0112752a6ff07230f9ec2f71b0d3d4e088a910fdce454fdb6553e83ed0eced7d"}, + {file = "mypy-0.950-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ee0a36edd332ed2c5208565ae6e3a7afc0eabb53f5327e281f2ef03a6bc7687a"}, + {file = "mypy-0.950-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:77423570c04aca807508a492037abbd72b12a1fb25a385847d191cd50b2c9605"}, + {file = "mypy-0.950-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5ce6a09042b6da16d773d2110e44f169683d8cc8687e79ec6d1181a72cb028d2"}, + {file = "mypy-0.950-cp38-cp38-win_amd64.whl", hash = "sha256:5b231afd6a6e951381b9ef09a1223b1feabe13625388db48a8690f8daa9b71ff"}, + {file = "mypy-0.950-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0384d9f3af49837baa92f559d3fa673e6d2652a16550a9ee07fc08c736f5e6f8"}, + {file = "mypy-0.950-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1fdeb0a0f64f2a874a4c1f5271f06e40e1e9779bf55f9567f149466fc7a55038"}, + {file = "mypy-0.950-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:61504b9a5ae166ba5ecfed9e93357fd51aa693d3d434b582a925338a2ff57fd2"}, + {file = "mypy-0.950-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a952b8bc0ae278fc6316e6384f67bb9a396eb30aced6ad034d3a76120ebcc519"}, + {file = "mypy-0.950-cp39-cp39-win_amd64.whl", hash = "sha256:eaea21d150fb26d7b4856766e7addcf929119dd19fc832b22e71d942835201ef"}, + {file = "mypy-0.950-py3-none-any.whl", hash = "sha256:a4d9898f46446bfb6405383b57b96737dcfd0a7f25b748e78ef3e8c576bba3cb"}, + {file = "mypy-0.950.tar.gz", hash = "sha256:1b333cfbca1762ff15808a0ef4f71b5d3eed8528b23ea1c3fb50543c867d68de"}, ] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] mypy-zope = [ - {file = "mypy-zope-0.3.5.tar.gz", hash = "sha256:489e7da1c2af887f2cfe3496995fc247f296512b495b57817edddda9d22308f3"}, - {file = "mypy_zope-0.3.5-py3-none-any.whl", hash = "sha256:3bd0cc9a3e5933b02931af4b214ba32a4f4ff98adb30c979ce733857db91a18b"}, + {file = "mypy-zope-0.3.7.tar.gz", hash = "sha256:9da171e78e8ef7ac8922c86af1a62f1b7f3244f121020bd94a2246bc3f33c605"}, + {file = "mypy_zope-0.3.7-py3-none-any.whl", hash = "sha256:9c7637d066e4d1bafa0651abc091c752009769098043b236446e6725be2bc9c2"}, ] netaddr = [ {file = "netaddr-0.8.0-py2.py3-none-any.whl", hash = "sha256:9666d0232c32d2656e5e5f8d735f58fd6c7457ce52fc21c98d45f2af78f990ac"}, @@ -2225,8 +2191,8 @@ platformdirs = [ {file = "platformdirs-2.5.1.tar.gz", hash = "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d"}, ] prometheus-client = [ - {file = "prometheus_client-0.13.1-py3-none-any.whl", hash = "sha256:357a447fd2359b0a1d2e9b311a0c5778c330cfbe186d880ad5a6b39884652316"}, - {file = "prometheus_client-0.13.1.tar.gz", hash = "sha256:ada41b891b79fca5638bd5cfe149efa86512eaa55987893becd2c6d8d0a5dfc5"}, + {file = "prometheus_client-0.14.0-py3-none-any.whl", hash = "sha256:f4aba3fdd1735852049f537c1f0ab177159b7ab76f271ecc4d2f45aa2a1d01f2"}, + {file = "prometheus_client-0.14.0.tar.gz", hash = "sha256:8f7a922dd5455ad524b6ba212ce8eb2b4b05e073f4ec7218287f88b1cac34750"}, ] psycopg2 = [ {file = "psycopg2-2.9.3-cp310-cp310-win32.whl", hash = "sha256:083707a696e5e1c330af2508d8fab36f9700b26621ccbcb538abe22e15485362"}, @@ -2407,10 +2373,6 @@ readme-renderer = [ {file = "readme_renderer-33.0-py3-none-any.whl", hash = "sha256:f02cee0c4de9636b5a62b6be50c9742427ba1b956aad1d938bfb087d0d72ccdf"}, {file = "readme_renderer-33.0.tar.gz", hash = "sha256:e3b53bc84bd6af054e4cc1fe3567dc1ae19f554134221043a3f8c674e22209db"}, ] -redbaron = [ - {file = "redbaron-0.9.2-py2.py3-none-any.whl", hash = "sha256:d01032b6a848b5521a8d6ef72486315c2880f420956870cdd742e2b5a09b9bab"}, - {file = "redbaron-0.9.2.tar.gz", hash = "sha256:472d0739ca6b2240bb2278ae428604a75472c9c12e86c6321e8c016139c0132f"}, -] requests = [ {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, @@ -2423,17 +2385,13 @@ rfc3986 = [ {file = "rfc3986-2.0.0-py2.py3-none-any.whl", hash = "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd"}, {file = "rfc3986-2.0.0.tar.gz", hash = "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c"}, ] -rply = [ - {file = "rply-0.7.8-py2.py3-none-any.whl", hash = "sha256:28ffd11d656c48aeb8c508eb382acd6a0bd906662624b34388751732a27807e7"}, - {file = "rply-0.7.8.tar.gz", hash = "sha256:2a808ac25a4580a9991fc304d64434e299a8fc75760574492f242cbb5bb301c9"}, -] secretstorage = [ {file = "SecretStorage-3.3.1-py3-none-any.whl", hash = "sha256:422d82c36172d88d6a0ed5afdec956514b189ddbfb72fefab0c8a1cee4eaf71f"}, {file = "SecretStorage-3.3.1.tar.gz", hash = "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195"}, ] sentry-sdk = [ - {file = "sentry-sdk-1.5.7.tar.gz", hash = "sha256:aa52da941c56b5a76fd838f8e9e92a850bf893a9eb1e33ffce6c21431d07ee30"}, - {file = "sentry_sdk-1.5.7-py2.py3-none-any.whl", hash = "sha256:411a8495bd18cf13038e5749e4710beb4efa53da6351f67b4c2f307c2d9b6d49"}, + {file = "sentry-sdk-1.5.11.tar.gz", hash = "sha256:6c01d9d0b65935fd275adc120194737d1df317dce811e642cbf0394d0d37a007"}, + {file = "sentry_sdk-1.5.11-py2.py3-none-any.whl", hash = "sha256:c17179183cac614e900cbd048dab03f49a48e2820182ec686c25e7ce46f8548f"}, ] service-identity = [ {file = "service-identity-21.1.0.tar.gz", hash = "sha256:6e6c6086ca271dc11b033d17c3a8bea9f24ebff920c587da090afc9519419d34"}, @@ -2592,8 +2550,8 @@ twine = [ {file = "twine-3.8.0.tar.gz", hash = "sha256:8efa52658e0ae770686a13b675569328f1fba9837e5de1867bfe5f46a9aefe19"}, ] twisted = [ - {file = "Twisted-22.2.0-py3-none-any.whl", hash = "sha256:5c63c149eb6b8fe1e32a0215b1cef96fabdba04f705d8efb9174b1ccf5b49d49"}, - {file = "Twisted-22.2.0.tar.gz", hash = "sha256:57f32b1f6838facb8c004c89467840367ad38e9e535f8252091345dba500b4f2"}, + {file = "Twisted-22.4.0-py3-none-any.whl", hash = "sha256:f9f7a91f94932477a9fc3b169d57f54f96c6e74a23d78d9ce54039a7f48928a2"}, + {file = "Twisted-22.4.0.tar.gz", hash = "sha256:a047990f57dfae1e0bd2b7df2526d4f16dcdc843774dc108b78c52f2a5f13680"}, ] twisted-iocpsupport = [ {file = "twisted-iocpsupport-1.0.2.tar.gz", hash = "sha256:72068b206ee809c9c596b57b5287259ea41ddb4774d86725b19f35bf56aa32a9"}, @@ -2643,6 +2601,10 @@ types-bleach = [ {file = "types-bleach-4.1.4.tar.gz", hash = "sha256:2d30c2c4fb6854088ac636471352c9a51bf6c089289800d2a8060820a01cd43a"}, {file = "types_bleach-4.1.4-py3-none-any.whl", hash = "sha256:edffe173ed6d7b6f3543036a96204a9319c3bf6c3645917b14274e43f000cc9b"}, ] +types-commonmark = [ + {file = "types-commonmark-0.9.2.tar.gz", hash = "sha256:b894b67750c52fd5abc9a40a9ceb9da4652a391d75c1b480bba9cef90f19fc86"}, + {file = "types_commonmark-0.9.2-py3-none-any.whl", hash = "sha256:56f20199a1f9a2924443211a0ef97f8b15a8a956a7f4e9186be6950bf38d6d02"}, +] types-cryptography = [ {file = "types-cryptography-3.3.15.tar.gz", hash = "sha256:a7983a75a7b88a18f88832008f0ef140b8d1097888ec1a0824ec8fb7e105273b"}, {file = "types_cryptography-3.3.15-py3-none-any.whl", hash = "sha256:d9b0dd5465d7898d400850e7f35e5518aa93a7e23d3e11757cd81b4777089046"}, @@ -2664,8 +2626,8 @@ types-opentracing = [ {file = "types_opentracing-2.4.7-py3-none-any.whl", hash = "sha256:861fb8103b07cf717f501dd400cb274ca9992552314d4d6c7a824b11a215e512"}, ] types-pillow = [ - {file = "types-Pillow-9.0.6.tar.gz", hash = "sha256:79b350b1188c080c27558429f1e119e69c9f020b877a82df761d9283070e0185"}, - {file = "types_Pillow-9.0.6-py3-none-any.whl", hash = "sha256:bd1e0a844fc718398aa265bf50fcad550fc520cc54f80e5ffeb7b3226b3cc507"}, + {file = "types-Pillow-9.0.15.tar.gz", hash = "sha256:d2e385fe5c192e75970f18accce69f5c2a9f186f3feb578a9b91cd6fdf64211d"}, + {file = "types_Pillow-9.0.15-py3-none-any.whl", hash = "sha256:c9646595dfafdf8b63d4b1443292ead17ee0fc7b18a143e497b68e0ea2dc1eb6"}, ] types-psycopg2 = [ {file = "types-psycopg2-2.9.9.tar.gz", hash = "sha256:4f9d4d52eeb343dc00fd5ed4f1513a8a5c18efba0a072eb82706d15cf4f20a2e"}, diff --git a/pyproject.toml b/pyproject.toml index 7f58c37e3fb7..75251c863d14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.57.0" +version = "1.60.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" @@ -100,7 +100,7 @@ synapse_review_recent_signups = "synapse._scripts.review_recent_signups:main" update_synapse_database = "synapse._scripts.update_synapse_database:main" [tool.poetry.dependencies] -python = "^3.7" +python = "^3.7.1" # Mandatory Dependencies # ---------------------- @@ -142,8 +142,10 @@ netaddr = ">=0.7.18" # add a lower bound to the Jinja2 dependency. Jinja2 = ">=3.0" bleach = ">=1.4.3" -# We use `ParamSpec`, which was added in `typing-extensions` 3.10.0.0. -typing-extensions = ">=3.10.0" +# We use `ParamSpec` and `Concatenate`, which were added in `typing-extensions` 3.10.0.0. +# Additionally we need https://github.com/python/typing/pull/817 to allow types to be +# generic over ParamSpecs. +typing-extensions = ">=3.10.0.1" # We enforce that we have a `cryptography` version that bundles an `openssl` # with the latest security patches. cryptography = ">=3.4.7" @@ -231,10 +233,11 @@ all = [ "jaeger-client", "opentracing", # jwt "pyjwt", - #redis - "txredisapi", "hiredis" + # redis + "txredisapi", "hiredis", + # cache_memory + "pympler", # omitted: - # - cache_memory: this is an experimental option # - test: it's useful to have this separate from dev deps in the olddeps job # - systemd: this is a system-based requirement ] @@ -248,9 +251,10 @@ flake8-bugbear = "==21.3.2" flake8 = "*" # Typechecking -mypy = "==0.931" -mypy-zope = "==0.3.5" +mypy = "*" +mypy-zope = "*" types-bleach = ">=4.1.0" +types-commonmark = ">=0.9.2" types-jsonschema = ">=3.2.0" types-opentracing = ">=2.4.2" types-Pillow = ">=8.3.4" @@ -270,8 +274,8 @@ idna = ">=2.5" # The following are used by the release script click = "==8.1.0" -redbaron = "==0.9.2" -GitPython = "==3.1.14" +# GitPython was == 3.1.14; bumped to 3.1.20, the first release with type hints. +GitPython = ">=3.1.20" commonmark = "==0.9.1" pygithub = "==1.55" # The following are executed as commands by the release script. @@ -280,5 +284,5 @@ twine = "*" towncrier = ">=18.6.0rc1" [build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/scripts-dev/build_debian_packages.py b/scripts-dev/build_debian_packages.py index 7ff96a1ee6fe..38564893e95b 100755 --- a/scripts-dev/build_debian_packages.py +++ b/scripts-dev/build_debian_packages.py @@ -17,7 +17,8 @@ import sys import threading from concurrent.futures import ThreadPoolExecutor -from typing import Optional, Sequence +from types import FrameType +from typing import Collection, Optional, Sequence, Set DISTS = ( "debian:buster", # oldstable: EOL 2022-08 @@ -26,6 +27,7 @@ "debian:sid", "ubuntu:focal", # 20.04 LTS (our EOL forced by Py38 on 2024-10-14) "ubuntu:impish", # 21.10 (EOL 2022-07) + "ubuntu:jammy", # 22.04 LTS (EOL 2027-04) ) DESC = """\ @@ -40,15 +42,17 @@ class Builder(object): def __init__( - self, redirect_stdout=False, docker_build_args: Optional[Sequence[str]] = None + self, + redirect_stdout: bool = False, + docker_build_args: Optional[Sequence[str]] = None, ): self.redirect_stdout = redirect_stdout self._docker_build_args = tuple(docker_build_args or ()) - self.active_containers = set() + self.active_containers: Set[str] = set() self._lock = threading.Lock() self._failed = False - def run_build(self, dist, skip_tests=False): + def run_build(self, dist: str, skip_tests: bool = False) -> None: """Build deb for a single distribution""" if self._failed: @@ -62,7 +66,7 @@ def run_build(self, dist, skip_tests=False): self._failed = True raise - def _inner_build(self, dist, skip_tests=False): + def _inner_build(self, dist: str, skip_tests: bool = False) -> None: tag = dist.split(":", 1)[1] # Make the dir where the debs will live. @@ -137,7 +141,7 @@ def _inner_build(self, dist, skip_tests=False): stdout.close() print("Completed build of %s" % (dist,)) - def kill_containers(self): + def kill_containers(self) -> None: with self._lock: active = list(self.active_containers) @@ -155,8 +159,10 @@ def kill_containers(self): self.active_containers.remove(c) -def run_builds(builder, dists, jobs=1, skip_tests=False): - def sig(signum, _frame): +def run_builds( + builder: Builder, dists: Collection[str], jobs: int = 1, skip_tests: bool = False +) -> None: + def sig(signum: int, _frame: Optional[FrameType]) -> None: print("Caught SIGINT") builder.kill_containers() diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index d34e9f355483..ca476d9a5e61 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -43,6 +43,10 @@ fi # Build the base Synapse image from the local checkout docker build -t matrixdotorg/synapse -f "docker/Dockerfile" . +extra_test_args=() + +test_tags="synapse_blacklist,msc2716,msc3030" + # If we're using workers, modify the docker files slightly. if [[ -n "$WORKERS" ]]; then # Build the workers docker image (from the base Synapse image). @@ -52,10 +56,21 @@ if [[ -n "$WORKERS" ]]; then COMPLEMENT_DOCKERFILE=SynapseWorkers.Dockerfile # And provide some more configuration to complement. - export COMPLEMENT_SPAWN_HS_TIMEOUT_SECS=60 + + # It can take quite a while to spin up a worker-mode Synapse for the first + # time (the main problem is that we start 14 python processes for each test, + # and complement likes to do two of them in parallel). + export COMPLEMENT_SPAWN_HS_TIMEOUT_SECS=120 + + # ... and it takes longer than 10m to run the whole suite. + extra_test_args+=("-timeout=60m") else export COMPLEMENT_BASE_IMAGE=complement-synapse COMPLEMENT_DOCKERFILE=Dockerfile + + # We only test faster room joins on monoliths, because they are purposefully + # being developed without worker support to start with. + test_tags="$test_tags,faster_joins" fi # Build the Complement image from the Synapse image we just built. @@ -64,4 +79,5 @@ docker build -t $COMPLEMENT_BASE_IMAGE -f "docker/complement/$COMPLEMENT_DOCKERF # Run the tests! echo "Images built; running complement" cd "$COMPLEMENT_DIR" -go test -v -tags synapse_blacklist,msc2716,msc3030 -count=1 "$@" ./tests/... + +go test -v -tags $test_tags -count=1 "${extra_test_args[@]}" "$@" ./tests/... diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py index c72e19f61d62..763dd02c477e 100755 --- a/scripts-dev/federation_client.py +++ b/scripts-dev/federation_client.py @@ -38,7 +38,7 @@ import base64 import json import sys -from typing import Any, Optional +from typing import Any, Dict, Optional, Tuple from urllib import parse as urlparse import requests @@ -47,13 +47,14 @@ import srvlookup import yaml from requests.adapters import HTTPAdapter +from urllib3 import HTTPConnectionPool # uncomment the following to enable debug logging of http requests # from httplib import HTTPConnection # HTTPConnection.debuglevel = 1 -def encode_base64(input_bytes): +def encode_base64(input_bytes: bytes) -> str: """Encode bytes as a base64 string without any padding.""" input_len = len(input_bytes) @@ -63,7 +64,7 @@ def encode_base64(input_bytes): return output_string -def encode_canonical_json(value): +def encode_canonical_json(value: object) -> bytes: return json.dumps( value, # Encode code-points outside of ASCII as UTF-8 rather than \u escapes @@ -124,8 +125,13 @@ def request( authorization_headers = [] for key, sig in signed_json["signatures"][origin_name].items(): - header = 'X-Matrix origin=%s,key="%s",sig="%s"' % (origin_name, key, sig) - authorization_headers.append(header.encode("ascii")) + header = 'X-Matrix origin=%s,key="%s",sig="%s",destination="%s"' % ( + origin_name, + key, + sig, + destination, + ) + authorization_headers.append(header) print("Authorization: %s" % header, file=sys.stderr) dest = "matrix://%s%s" % (destination, path) @@ -134,7 +140,10 @@ def request( s = requests.Session() s.mount("matrix://", MatrixConnectionAdapter()) - headers = {"Host": destination, "Authorization": authorization_headers[0]} + headers: Dict[str, str] = { + "Host": destination, + "Authorization": authorization_headers[0], + } if method == "POST": headers["Content-Type"] = "application/json" @@ -149,7 +158,7 @@ def request( ) -def main(): +def main() -> None: parser = argparse.ArgumentParser( description="Signs and sends a federation request to a matrix homeserver" ) @@ -207,6 +216,7 @@ def main(): if not args.server_name or not args.signing_key: read_args_from_config(args) + assert isinstance(args.signing_key, str) algorithm, version, key_base64 = args.signing_key.split() key = signedjson.key.decode_signing_key_base64(algorithm, version, key_base64) @@ -228,7 +238,7 @@ def main(): print("") -def read_args_from_config(args): +def read_args_from_config(args: argparse.Namespace) -> None: with open(args.config, "r") as fh: config = yaml.safe_load(fh) @@ -245,7 +255,7 @@ def read_args_from_config(args): class MatrixConnectionAdapter(HTTPAdapter): @staticmethod - def lookup(s, skip_well_known=False): + def lookup(s: str, skip_well_known: bool = False) -> Tuple[str, int]: if s[-1] == "]": # ipv6 literal (with no port) return s, 8448 @@ -271,7 +281,7 @@ def lookup(s, skip_well_known=False): return s, 8448 @staticmethod - def get_well_known(server_name): + def get_well_known(server_name: str) -> Optional[str]: uri = "https://%s/.well-known/matrix/server" % (server_name,) print("fetching %s" % (uri,), file=sys.stderr) @@ -294,7 +304,9 @@ def get_well_known(server_name): print("Invalid response from %s: %s" % (uri, e), file=sys.stderr) return None - def get_connection(self, url, proxies=None): + def get_connection( + self, url: str, proxies: Optional[Dict[str, str]] = None + ) -> HTTPConnectionPool: parsed = urlparse.urlparse(url) (host, port) = self.lookup(parsed.netloc) diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh index 4698d2d5be32..377348b107ea 100755 --- a/scripts-dev/lint.sh +++ b/scripts-dev/lint.sh @@ -79,8 +79,20 @@ else # If we were not asked to lint changed files, and no paths were found as a result, # then lint everything! if [[ -z ${files+x} ]]; then - # Lint all source code files and directories - files=( "." ) + # CI runs each linter on the entire checkout, e.g. `black .`. So don't + # rely on this list to *find* lint targets if that misses a file; instead; + # use it to exclude files from linters when this can't be done by config. + # + # To check which files the linters examine, use: + # black --verbose . 2>&1 | \grep -v ignored + # isort --show-files . + # flake8 --verbose . # This isn't a great option + # mypy has explicit config in mypy.ini; there is also mypy --verbose + files=( + "synapse" "docker" "tests" + "scripts-dev" + "contrib" "synmark" "stubs" ".ci" + ) fi fi diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index 1217e148747d..d08517a95382 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -16,12 +16,12 @@ can crop up, e.g the cache descriptors. """ -from typing import Callable, Optional +from typing import Callable, Optional, Type from mypy.nodes import ARG_NAMED_OPT from mypy.plugin import MethodSigContext, Plugin from mypy.typeops import bind_self -from mypy.types import CallableType, NoneType +from mypy.types import CallableType, NoneType, UnionType class SynapsePlugin(Plugin): @@ -72,13 +72,20 @@ def cached_function_method_signature(ctx: MethodSigContext) -> CallableType: # Third, we add an optional "on_invalidate" argument. # - # This is a callable which accepts no input and returns nothing. - calltyp = CallableType( - arg_types=[], - arg_kinds=[], - arg_names=[], - ret_type=NoneType(), - fallback=ctx.api.named_generic_type("builtins.function", []), + # This is a either + # - a callable which accepts no input and returns nothing, or + # - None. + calltyp = UnionType( + [ + NoneType(), + CallableType( + arg_types=[], + arg_kinds=[], + arg_names=[], + ret_type=NoneType(), + fallback=ctx.api.named_generic_type("builtins.function", []), + ), + ] ) arg_types.append(calltyp) @@ -94,8 +101,8 @@ def cached_function_method_signature(ctx: MethodSigContext) -> CallableType: return signature -def plugin(version: str): - # This is the entry point of the plugin, and let's us deal with the fact +def plugin(version: str) -> Type[SynapsePlugin]: + # This is the entry point of the plugin, and lets us deal with the fact # that the mypy plugin interface is *not* stable by looking at the version # string. # diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 685fa32b03f4..0031ba3e4b2f 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -25,19 +25,20 @@ import urllib.request from os import path from tempfile import TemporaryDirectory -from typing import List, Optional, Tuple +from typing import Any, List, Optional, cast import attr import click import commonmark import git -import redbaron from click.exceptions import ClickException from github import Github from packaging import version -def run_until_successful(command, *args, **kwargs): +def run_until_successful( + command: str, *args: Any, **kwargs: Any +) -> subprocess.CompletedProcess: while True: completed_process = subprocess.run(command, *args, **kwargs) exit_code = completed_process.returncode @@ -51,7 +52,7 @@ def run_until_successful(command, *args, **kwargs): @click.group() -def cli(): +def cli() -> None: """An interactive script to walk through the parts of creating a release. Requires the dev dependencies be installed, which can be done via: @@ -69,11 +70,12 @@ def cli(): # ... wait for assets to build ... ./scripts-dev/release.py publish + ./scripts-dev/release.py upload # Optional: generate some nice links for the announcement - ./scripts-dev/release.py upload + ./scripts-dev/release.py announce If the env var GH_TOKEN (or GITHUB_TOKEN) is set, or passed into the `tag`/`publish` command, then a new draft release will be created/published. @@ -81,25 +83,19 @@ def cli(): @cli.command() -def prepare(): +def prepare() -> None: """Do the initial stages of creating a release, including creating release branch, updating changelog and pushing to GitHub. """ # Make sure we're in a git repo. - try: - repo = git.Repo() - except git.InvalidGitRepositoryError: - raise click.ClickException("Not in Synapse repo.") - - if repo.is_dirty(): - raise click.ClickException("Uncommitted changes exist.") + repo = get_repo_and_check_clean_checkout() click.secho("Updating git repo...") repo.remote().fetch() # Get the current version and AST from root Synapse module. - current_version, parsed_synapse_ast, version_node = parse_version_from_module() + current_version = get_package_version() # Figure out what sort of release we're doing and calcuate the new version. rc = click.confirm("RC", default=True) @@ -161,22 +157,21 @@ def prepare(): click.get_current_context().abort() # Switch to the release branch. - parsed_new_version = version.parse(new_version) + # Cast safety: parse() won't return a version.LegacyVersion from our + # version string format. + parsed_new_version = cast(version.Version, version.parse(new_version)) # We assume for debian changelogs that we only do RCs or full releases. assert not parsed_new_version.is_devrelease assert not parsed_new_version.is_postrelease - release_branch_name = ( - f"release-v{parsed_new_version.major}.{parsed_new_version.minor}" - ) + release_branch_name = get_release_branch_name(parsed_new_version) release_branch = find_ref(repo, release_branch_name) if release_branch: if release_branch.is_remote(): # If the release branch only exists on the remote we check it out # locally. repo.git.checkout(release_branch_name) - release_branch = repo.active_branch else: # If a branch doesn't exist we create one. We ask which one branch it # should be based off, defaulting to sensible values depending on the @@ -198,25 +193,25 @@ def prepare(): click.get_current_context().abort() # Check out the base branch and ensure it's up to date - repo.head.reference = base_branch + repo.head.set_reference(base_branch, "check out the base branch") repo.head.reset(index=True, working_tree=True) if not base_branch.is_remote(): update_branch(repo) # Create the new release branch - release_branch = repo.create_head(release_branch_name, commit=base_branch) + # Type ignore will no longer be needed after GitPython 3.1.28. + # See https://github.com/gitpython-developers/GitPython/pull/1419 + repo.create_head(release_branch_name, commit=base_branch) # type: ignore[arg-type] - # Switch to the release branch and ensure its up to date. + # Switch to the release branch and ensure it's up to date. repo.git.checkout(release_branch_name) update_branch(repo) - # Update the `__version__` variable and write it back to the file. - version_node.value = '"' + new_version + '"' - with open("synapse/__init__.py", "w") as f: - f.write(parsed_synapse_ast.dumps()) + # Update the version specified in pyproject.toml. + subprocess.check_output(["poetry", "version", new_version]) # Generate changelogs. - generate_and_write_changelog(current_version) + generate_and_write_changelog(current_version, new_version) # Generate debian changelogs if parsed_new_version.pre is not None: @@ -229,7 +224,7 @@ def prepare(): debian_version = new_version run_until_successful( - f'dch -M -v {debian_version} "New synapse release {debian_version}."', + f'dch -M -v {debian_version} "New Synapse release {new_version}."', shell=True, ) run_until_successful('dch -M -r -D stable ""', shell=True) @@ -267,35 +262,43 @@ def prepare(): @cli.command() @click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"]) -def tag(gh_token: Optional[str]): +def tag(gh_token: Optional[str]) -> None: """Tags the release and generates a draft GitHub release""" # Make sure we're in a git repo. - try: - repo = git.Repo() - except git.InvalidGitRepositoryError: - raise click.ClickException("Not in Synapse repo.") - - if repo.is_dirty(): - raise click.ClickException("Uncommitted changes exist.") + repo = get_repo_and_check_clean_checkout() click.secho("Updating git repo...") repo.remote().fetch() # Find out the version and tag name. - current_version, _, _ = parse_version_from_module() + current_version = get_package_version() tag_name = f"v{current_version}" # Check we haven't released this version. if tag_name in repo.tags: raise click.ClickException(f"Tag {tag_name} already exists!\n") + # Check we're on the right release branch + release_branch = get_release_branch_name(current_version) + if repo.active_branch.name != release_branch: + click.echo( + f"Need to be on the release branch ({release_branch}) before tagging. " + f"Currently on ({repo.active_branch.name})." + ) + click.get_current_context().abort() + # Get the appropriate changelogs and tag. changes = get_changes_for_version(current_version) click.echo_via_pager(changes) if click.confirm("Edit text?", default=False): - changes = click.edit(changes, require_save=False) + edited_changes = click.edit(changes, require_save=False) + # This assert is for mypy's benefit. click's docs are a little unclear, but + # when `require_save=False`, not saving the temp file in the editor returns + # the original string. + assert edited_changes is not None + changes = edited_changes repo.create_tag(tag_name, message=changes, sign=True) @@ -349,22 +352,16 @@ def tag(gh_token: Optional[str]): @cli.command() @click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=True) -def publish(gh_token: str): - """Publish release.""" +def publish(gh_token: str) -> None: + """Publish release on GitHub.""" # Make sure we're in a git repo. - try: - repo = git.Repo() - except git.InvalidGitRepositoryError: - raise click.ClickException("Not in Synapse repo.") - - if repo.is_dirty(): - raise click.ClickException("Uncommitted changes exist.") + get_repo_and_check_clean_checkout() - current_version, _, _ = parse_version_from_module() + current_version = get_package_version() tag_name = f"v{current_version}" - if not click.confirm(f"Publish {tag_name}?", default=True): + if not click.confirm(f"Publish release {tag_name} on GitHub?", default=True): return # Publish the draft release @@ -392,12 +389,19 @@ def publish(gh_token: str): @cli.command() -def upload(): +def upload() -> None: """Upload release to pypi.""" - current_version, _, _ = parse_version_from_module() + current_version = get_package_version() tag_name = f"v{current_version}" + # Check we have the right tag checked out. + repo = get_repo_and_check_clean_checkout() + tag = repo.tag(f"refs/tags/{tag_name}") + if repo.head.commit != tag.commit: + click.echo("Tag {tag_name} (tag.commit) is not currently checked out!") + click.get_current_context().abort() + pypi_asset_names = [ f"matrix_synapse-{current_version}-py3-none-any.whl", f"matrix-synapse-{current_version}.tar.gz", @@ -420,17 +424,17 @@ def upload(): @cli.command() -def announce(): +def announce() -> None: """Generate markdown to announce the release.""" - current_version, _, _ = parse_version_from_module() + current_version = get_package_version() tag_name = f"v{current_version}" click.echo( f""" Hi everyone. Synapse {current_version} has just been released. -[notes](https://github.com/matrix-org/synapse/releases/tag/{tag_name}) |\ +[notes](https://github.com/matrix-org/synapse/releases/tag/{tag_name}) | \ [docker](https://hub.docker.com/r/matrixdotorg/synapse/tags?name={tag_name}) | \ [debs](https://packages.matrix.org/debian/) | \ [pypi](https://pypi.org/project/matrix-synapse/{current_version}/)""" @@ -454,53 +458,43 @@ def announce(): ) -def parse_version_from_module() -> Tuple[ - version.Version, redbaron.RedBaron, redbaron.Node -]: - # Parse the AST and load the `__version__` node so that we can edit it - # later. - with open("synapse/__init__.py") as f: - red = redbaron.RedBaron(f.read()) - - version_node = None - for node in red: - if node.type != "assignment": - continue - - if node.target.type != "name": - continue - - if node.target.value != "__version__": - continue +def get_package_version() -> version.Version: + version_string = subprocess.check_output(["poetry", "version", "--short"]).decode( + "utf-8" + ) + return version.Version(version_string) - version_node = node - break - if not version_node: - print("Failed to find '__version__' definition in synapse/__init__.py") - sys.exit(1) +def get_release_branch_name(version_number: version.Version) -> str: + return f"release-v{version_number.major}.{version_number.minor}" - # Parse the current version. - current_version = version.parse(version_node.value.value.strip('"')) - assert isinstance(current_version, version.Version) - return current_version, red, version_node +def get_repo_and_check_clean_checkout() -> git.Repo: + """Get the project repo and check it's not got any uncommitted changes.""" + try: + repo = git.Repo() + except git.InvalidGitRepositoryError: + raise click.ClickException("Not in Synapse repo.") + if repo.is_dirty(): + raise click.ClickException("Uncommitted changes exist.") + return repo def find_ref(repo: git.Repo, ref_name: str) -> Optional[git.HEAD]: """Find the branch/ref, looking first locally then in the remote.""" - if ref_name in repo.refs: - return repo.refs[ref_name] + if ref_name in repo.references: + return repo.references[ref_name] elif ref_name in repo.remote().refs: return repo.remote().refs[ref_name] else: return None -def update_branch(repo: git.Repo): +def update_branch(repo: git.Repo) -> None: """Ensure branch is up to date if it has a remote""" - if repo.active_branch.tracking_branch(): - repo.git.merge(repo.active_branch.tracking_branch().name) + tracking_branch = repo.active_branch.tracking_branch() + if tracking_branch: + repo.git.merge(tracking_branch.name) def get_changes_for_version(wanted_version: version.Version) -> str: @@ -564,11 +558,15 @@ class VersionSection: return "\n".join(version_changelog) -def generate_and_write_changelog(current_version: version.Version): +def generate_and_write_changelog( + current_version: version.Version, new_version: str +) -> None: # We do this by getting a draft so that we can edit it before writing to the # changelog. result = run_until_successful( - "python3 -m towncrier --draft", shell=True, capture_output=True + f"python3 -m towncrier build --draft --version {new_version}", + shell=True, + capture_output=True, ) new_changes = result.stdout.decode("utf-8") new_changes = new_changes.replace( @@ -584,8 +582,8 @@ def generate_and_write_changelog(current_version: version.Version): f.write(existing_content) # Remove all the news fragments - for f in glob.iglob("changelog.d/*.*"): - os.remove(f) + for filename in glob.iglob("changelog.d/*.*"): + os.remove(filename) if __name__ == "__main__": diff --git a/scripts-dev/sign_json.py b/scripts-dev/sign_json.py index 945954310610..bb217799fbcc 100755 --- a/scripts-dev/sign_json.py +++ b/scripts-dev/sign_json.py @@ -27,7 +27,7 @@ from synapse.util import json_encoder -def main(): +def main() -> None: parser = argparse.ArgumentParser( description="""Adds a signature to a JSON object. diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 6213f3265b59..000000000000 --- a/setup.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[check-manifest] -ignore = - .git-blame-ignore-revs - contrib - contrib/* - docs/* - pylint.cfg - tox.ini - diff --git a/setup.py b/setup.py deleted file mode 100755 index ecd30247ed6b..000000000000 --- a/setup.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2014-2017 OpenMarket Ltd -# Copyright 2017 Vector Creations Ltd -# Copyright 2017-2018 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -from typing import Any, Dict - -from setuptools import Command, find_packages, setup - -here = os.path.abspath(os.path.dirname(__file__)) - - -# Some notes on `setup.py test`: -# -# Once upon a time we used to try to make `setup.py test` run `tox` to run the -# tests. That's a bad idea for three reasons: -# -# 1: `setup.py test` is supposed to find out whether the tests work in the -# *current* environmentt, not whatever tox sets up. -# 2: Empirically, trying to install tox during the test run wasn't working ("No -# module named virtualenv"). -# 3: The tox documentation advises against it[1]. -# -# Even further back in time, we used to use setuptools_trial [2]. That has its -# own set of issues: for instance, it requires installation of Twisted to build -# an sdist (because the recommended mode of usage is to add it to -# `setup_requires`). That in turn means that in order to successfully run tox -# you have to have the python header files installed for whichever version of -# python tox uses (which is python3 on recent ubuntus, for example). -# -# So, for now at least, we stick with what appears to be the convention among -# Twisted projects, and don't attempt to do anything when someone runs -# `setup.py test`; instead we direct people to run `trial` directly if they -# care. -# -# [1]: http://tox.readthedocs.io/en/2.5.0/example/basic.html#integration-with-setup-py-test-command -# [2]: https://pypi.python.org/pypi/setuptools_trial -class TestCommand(Command): - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - print( - """Synapse's tests cannot be run via setup.py. To run them, try: - PYTHONPATH="." trial tests -""" - ) - - -def read_file(path_segments): - """Read a file from the package. Takes a list of strings to join to - make the path""" - file_path = os.path.join(here, *path_segments) - with open(file_path) as f: - return f.read() - - -def exec_file(path_segments): - """Execute a single python file to get the variables defined in it""" - result: Dict[str, Any] = {} - code = read_file(path_segments) - exec(code, result) - return result - - -version = exec_file(("synapse", "__init__.py"))["__version__"] -dependencies = exec_file(("synapse", "python_dependencies.py")) -long_description = read_file(("README.rst",)) - -REQUIREMENTS = dependencies["REQUIREMENTS"] -CONDITIONAL_REQUIREMENTS = dependencies["CONDITIONAL_REQUIREMENTS"] -ALL_OPTIONAL_REQUIREMENTS = dependencies["ALL_OPTIONAL_REQUIREMENTS"] - -# Make `pip install matrix-synapse[all]` install all the optional dependencies. -CONDITIONAL_REQUIREMENTS["all"] = list(ALL_OPTIONAL_REQUIREMENTS) - -# Developer dependencies should not get included in "all". -# -# We pin black so that our tests don't start failing on new releases. -CONDITIONAL_REQUIREMENTS["lint"] = [ - "isort==5.7.0", - "black==22.3.0", - "flake8-comprehensions", - "flake8-bugbear==21.3.2", - "flake8", -] - -CONDITIONAL_REQUIREMENTS["mypy"] = [ - "mypy==0.931", - "mypy-zope==0.3.5", - "types-bleach>=4.1.0", - "types-jsonschema>=3.2.0", - "types-opentracing>=2.4.2", - "types-Pillow>=8.3.4", - "types-psycopg2>=2.9.9", - "types-pyOpenSSL>=20.0.7", - "types-PyYAML>=5.4.10", - "types-requests>=2.26.0", - "types-setuptools>=57.4.0", -] - -# Dependencies which are exclusively required by unit test code. This is -# NOT a list of all modules that are necessary to run the unit tests. -# Tests assume that all optional dependencies are installed. -# -# parameterized_class decorator was introduced in parameterized 0.7.0 -CONDITIONAL_REQUIREMENTS["test"] = ["parameterized>=0.7.0", "idna>=2.5"] - -CONDITIONAL_REQUIREMENTS["dev"] = ( - CONDITIONAL_REQUIREMENTS["lint"] - + CONDITIONAL_REQUIREMENTS["mypy"] - + CONDITIONAL_REQUIREMENTS["test"] - + [ - # The following are used by the release script - "click==8.1.0", - "redbaron==0.9.2", - "GitPython==3.1.14", - "commonmark==0.9.1", - "pygithub==1.55", - # The following are executed as commands by the release script. - "twine", - "towncrier", - ] -) - -setup( - name="matrix-synapse", - version=version, - packages=find_packages(exclude=["tests", "tests.*"]), - description="Reference homeserver for the Matrix decentralised comms protocol", - install_requires=REQUIREMENTS, - extras_require=CONDITIONAL_REQUIREMENTS, - include_package_data=True, - zip_safe=False, - long_description=long_description, - long_description_content_type="text/x-rst", - python_requires="~=3.7", - entry_points={ - "console_scripts": [ - # Application - "synapse_homeserver = synapse.app.homeserver:main", - "synapse_worker = synapse.app.generic_worker:main", - "synctl = synapse._scripts.synctl:main", - # Scripts - "export_signing_key = synapse._scripts.export_signing_key:main", - "generate_config = synapse._scripts.generate_config:main", - "generate_log_config = synapse._scripts.generate_log_config:main", - "generate_signing_key = synapse._scripts.generate_signing_key:main", - "hash_password = synapse._scripts.hash_password:main", - "register_new_matrix_user = synapse._scripts.register_new_matrix_user:main", - "synapse_port_db = synapse._scripts.synapse_port_db:main", - "synapse_review_recent_signups = synapse._scripts.review_recent_signups:main", - "update_synapse_database = synapse._scripts.update_synapse_database:main", - ] - }, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Topic :: Communications :: Chat", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], - cmdclass={"test": TestCommand}, -) diff --git a/stubs/sortedcontainers/sorteddict.pyi b/stubs/sortedcontainers/sorteddict.pyi index 344d55cce118..7c399ab38d5e 100644 --- a/stubs/sortedcontainers/sorteddict.pyi +++ b/stubs/sortedcontainers/sorteddict.pyi @@ -85,12 +85,19 @@ class SortedDict(Dict[_KT, _VT]): def popitem(self, index: int = ...) -> Tuple[_KT, _VT]: ... def peekitem(self, index: int = ...) -> Tuple[_KT, _VT]: ... def setdefault(self, key: _KT, default: Optional[_VT] = ...) -> _VT: ... - @overload - def update(self, __map: Mapping[_KT, _VT], **kwargs: _VT) -> None: ... - @overload - def update(self, __iterable: Iterable[Tuple[_KT, _VT]], **kwargs: _VT) -> None: ... - @overload - def update(self, **kwargs: _VT) -> None: ... + # Mypy now reports the first overload as an error, because typeshed widened the type + # of `__map` to its internal `_typeshed.SupportsKeysAndGetItem` type in + # https://github.com/python/typeshed/pull/6653 + # Since sorteddicts don't change the signature of `update` from that of `dict`, we + # let the stubs for `update` inherit from the stubs for `dict`. (I suspect we could + # do the same for many othe methods.) We leave the stubs commented to better track + # how this file has evolved from the original stubs. + # @overload + # def update(self, __map: Mapping[_KT, _VT], **kwargs: _VT) -> None: ... + # @overload + # def update(self, __iterable: Iterable[Tuple[_KT, _VT]], **kwargs: _VT) -> None: ... + # @overload + # def update(self, **kwargs: _VT) -> None: ... def __reduce__( self, ) -> Tuple[ @@ -103,7 +110,7 @@ class SortedDict(Dict[_KT, _VT]): self, start: Optional[int] = ..., stop: Optional[int] = ..., - reverse=bool, + reverse: bool = ..., ) -> Iterator[_KT]: ... def bisect_left(self, value: _KT) -> int: ... def bisect_right(self, value: _KT) -> int: ... @@ -115,9 +122,7 @@ class SortedKeysView(KeysView[_KT_co], Sequence[_KT_co]): def __getitem__(self, index: slice) -> List[_KT_co]: ... def __delitem__(self, index: Union[int, slice]) -> None: ... -class SortedItemsView( # type: ignore - ItemsView[_KT_co, _VT_co], Sequence[Tuple[_KT_co, _VT_co]] -): +class SortedItemsView(ItemsView[_KT_co, _VT_co], Sequence[Tuple[_KT_co, _VT_co]]): def __iter__(self) -> Iterator[Tuple[_KT_co, _VT_co]]: ... @overload def __getitem__(self, index: int) -> Tuple[_KT_co, _VT_co]: ... diff --git a/stubs/sortedcontainers/sortedlist.pyi b/stubs/sortedcontainers/sortedlist.pyi index f80a3a72ce04..403897e3919e 100644 --- a/stubs/sortedcontainers/sortedlist.pyi +++ b/stubs/sortedcontainers/sortedlist.pyi @@ -81,7 +81,7 @@ class SortedList(MutableSequence[_T]): self, start: Optional[int] = ..., stop: Optional[int] = ..., - reverse=bool, + reverse: bool = ..., ) -> Iterator[_T]: ... def _islice( self, @@ -153,14 +153,14 @@ class SortedKeyList(SortedList[_T]): maximum: Optional[int] = ..., inclusive: Tuple[bool, bool] = ..., reverse: bool = ..., - ): ... + ) -> Iterator[_T]: ... def irange_key( self, min_key: Optional[Any] = ..., max_key: Optional[Any] = ..., inclusive: Tuple[bool, bool] = ..., reserve: bool = ..., - ): ... + ) -> Iterator[_T]: ... def bisect_left(self, value: _T) -> int: ... def bisect_right(self, value: _T) -> int: ... def bisect(self, value: _T) -> int: ... diff --git a/stubs/sortedcontainers/sortedset.pyi b/stubs/sortedcontainers/sortedset.pyi index f9c290838678..43c860f4221e 100644 --- a/stubs/sortedcontainers/sortedset.pyi +++ b/stubs/sortedcontainers/sortedset.pyi @@ -103,7 +103,7 @@ class SortedSet(MutableSet[_T], Sequence[_T]): self, start: Optional[int] = ..., stop: Optional[int] = ..., - reverse=bool, + reverse: bool = ..., ) -> Iterator[_T]: ... def irange( self, diff --git a/stubs/txredisapi.pyi b/stubs/txredisapi.pyi index 2d8ca018fbfc..695a2307c2c5 100644 --- a/stubs/txredisapi.pyi +++ b/stubs/txredisapi.pyi @@ -18,6 +18,8 @@ from typing import Any, List, Optional, Type, Union from twisted.internet import protocol from twisted.internet.defer import Deferred +from twisted.internet.interfaces import IAddress +from twisted.python.failure import Failure class RedisProtocol(protocol.Protocol): def publish(self, channel: str, message: bytes) -> "Deferred[None]": ... @@ -34,11 +36,14 @@ class RedisProtocol(protocol.Protocol): def get(self, key: str) -> "Deferred[Any]": ... class SubscriberProtocol(RedisProtocol): - def __init__(self, *args, **kwargs): ... + def __init__(self, *args: object, **kwargs: object): ... password: Optional[str] - def subscribe(self, channels: Union[str, List[str]]): ... - def connectionMade(self): ... - def connectionLost(self, reason): ... + def subscribe(self, channels: Union[str, List[str]]) -> "Deferred[None]": ... + def connectionMade(self) -> None: ... + # type-ignore: twisted.internet.protocol.Protocol provides a default argument for + # `reason`. txredisapi's LineReceiver Protocol doesn't. But that's fine: it's what's + # actually specified in twisted.internet.interfaces.IProtocol. + def connectionLost(self, reason: Failure) -> None: ... # type: ignore[override] def lazyConnection( host: str = ..., @@ -74,7 +79,7 @@ class RedisFactory(protocol.ReconnectingClientFactory): replyTimeout: Optional[int] = None, convertNumbers: Optional[int] = True, ): ... - def buildProtocol(self, addr) -> RedisProtocol: ... + def buildProtocol(self, addr: IAddress) -> RedisProtocol: ... class SubscriberFactory(RedisFactory): def __init__(self) -> None: ... diff --git a/synapse/__init__.py b/synapse/__init__.py index bf88f1d93d3c..161394175939 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -20,6 +20,8 @@ import os import sys +from matrix_common.versionstring import get_distribution_version_string + # Check that we're not running on an unsupported Python version. if sys.version_info < (3, 7): print("Synapse requires Python 3.7 or above.") @@ -68,7 +70,7 @@ except ImportError: pass -__version__ = "1.57.0" +__version__ = get_distribution_version_string("matrix-synapse") if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when diff --git a/synapse/_scripts/hash_password.py b/synapse/_scripts/hash_password.py index 3aa29de5bd8a..3bed367be29d 100755 --- a/synapse/_scripts/hash_password.py +++ b/synapse/_scripts/hash_password.py @@ -46,14 +46,14 @@ def main() -> None: "Path to server config file. " "Used to read in bcrypt_rounds and password_pepper." ), + required=True, ) args = parser.parse_args() - if "config" in args and args.config: - config = yaml.safe_load(args.config) - bcrypt_rounds = config.get("bcrypt_rounds", bcrypt_rounds) - password_config = config.get("password_config", None) or {} - password_pepper = password_config.get("pepper", password_pepper) + config = yaml.safe_load(args.config) + bcrypt_rounds = config.get("bcrypt_rounds", bcrypt_rounds) + password_config = config.get("password_config", None) or {} + password_pepper = password_config.get("pepper", password_pepper) password = args.password if not password: diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 01c32417d862..931750668ea2 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -187,7 +187,7 @@ async def _wrapped_get_user_by_req( Once get_user_by_req has set up the opentracing span, this does the actual work. """ try: - ip_addr = request.getClientIP() + ip_addr = request.getClientAddress().host user_agent = get_request_user_agent(request) access_token = self.get_access_token_from_request(request) @@ -356,7 +356,7 @@ async def _get_appservice_user_id_and_device_id( return None, None, None if app_service.ip_range_whitelist: - ip_address = IPAddress(request.getClientIP()) + ip_address = IPAddress(request.getClientAddress().host) if ip_address not in app_service.ip_range_whitelist: return None, None, None @@ -417,7 +417,8 @@ async def get_user_by_access_token( """ if rights == "access": - # first look in the database + # First look in the database to see if the access token is present + # as an opaque token. r = await self.store.get_user_by_access_token(token) if r: valid_until_ms = r.valid_until_ms @@ -434,7 +435,8 @@ async def get_user_by_access_token( return r - # otherwise it needs to be a valid macaroon + # If the token isn't found in the database, then it could still be a + # macaroon, so we check that here. try: user_id, guest = self._parse_and_validate_macaroon(token, rights) @@ -482,8 +484,12 @@ async def get_user_by_access_token( TypeError, ValueError, ) as e: - logger.warning("Invalid macaroon in auth: %s %s", type(e), e) - raise InvalidClientTokenError("Invalid macaroon passed.") + logger.warning( + "Invalid access token in auth: %s %s.", + type(e), + e, + ) + raise InvalidClientTokenError("Invalid access token passed.") def _parse_and_validate_macaroon( self, token: str, rights: str = "access" @@ -504,10 +510,7 @@ def _parse_and_validate_macaroon( try: macaroon = pymacaroons.Macaroon.deserialize(token) except Exception: # deserialize can throw more-or-less anything - # doesn't look like a macaroon: treat it as an opaque token which - # must be in the database. - # TODO: it would be nice to get rid of this, but apparently some - # people use access tokens which aren't macaroons + # The access token doesn't look like a macaroon. raise _InvalidMacaroonException() try: diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 92907415e651..330de21f6b80 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -65,6 +65,8 @@ class JoinRules: PRIVATE: Final = "private" # As defined for MSC3083. RESTRICTED: Final = "restricted" + # As defined for MSC3787. + KNOCK_RESTRICTED: Final = "knock_restricted" class RestrictedJoinRuleTypes: @@ -179,8 +181,6 @@ class RelationTypes: REPLACE: Final = "m.replace" REFERENCE: Final = "m.reference" THREAD: Final = "m.thread" - # TODO Remove this in Synapse >= v1.57.0. - UNSTABLE_THREAD: Final = "io.element.thread" class LimitBlockingTypes: @@ -257,7 +257,5 @@ class GuestAccess: class ReceiptTypes: READ: Final = "m.read" - - -class ReadReceiptEventFields: - MSC2285_HIDDEN: Final = "org.matrix.msc2285.hidden" + READ_PRIVATE: Final = "org.matrix.msc2285.read.private" + FULLY_READ: Final = "m.fully_read" diff --git a/synapse/api/errors.py b/synapse/api/errors.py index e92db29f6dc6..6650e826d5af 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -17,6 +17,7 @@ import logging import typing +from enum import Enum from http import HTTPStatus from typing import Any, Dict, List, Optional, Union @@ -30,7 +31,11 @@ logger = logging.getLogger(__name__) -class Codes: +class Codes(str, Enum): + """ + All known error codes, as an enum of strings. + """ + UNRECOGNIZED = "M_UNRECOGNIZED" UNAUTHORIZED = "M_UNAUTHORIZED" FORBIDDEN = "M_FORBIDDEN" @@ -79,6 +84,8 @@ class Codes: UNABLE_AUTHORISE_JOIN = "M_UNABLE_TO_AUTHORISE_JOIN" UNABLE_TO_GRANT_JOIN = "M_UNABLE_TO_GRANT_JOIN" + UNREDACTED_CONTENT_DELETED = "FI.MAU.MSC2815_UNREDACTED_CONTENT_DELETED" + class CodeMessageException(RuntimeError): """An exception with integer code and message string attributes. @@ -483,6 +490,22 @@ def __init__(self, inner_exception: BaseException, can_retry: bool): self.can_retry = can_retry +class UnredactedContentDeletedError(SynapseError): + def __init__(self, content_keep_ms: Optional[int] = None): + super().__init__( + 404, + "The content for that event has already been erased from the database", + errcode=Codes.UNREDACTED_CONTENT_DELETED, + ) + self.content_keep_ms = content_keep_ms + + def error_dict(self) -> "JsonDict": + extra = {} + if self.content_keep_ms is not None: + extra = {"fi.mau.msc2815.content_keep_ms": self.content_keep_ms} + return cs_error(self.msg, self.errcode, **extra) + + def cs_error(msg: str, code: str = Codes.UNKNOWN, **kwargs: Any) -> "JsonDict": """Utility method for constructing an error response for client-server interactions. diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 27e97d6f372d..b91ce06de7c3 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -19,6 +19,7 @@ TYPE_CHECKING, Awaitable, Callable, + Collection, Dict, Iterable, List, @@ -89,9 +90,7 @@ "org.matrix.not_labels": {"type": "array", "items": {"type": "string"}}, # MSC3440, filtering by event relations. "related_by_senders": {"type": "array", "items": {"type": "string"}}, - "io.element.relation_senders": {"type": "array", "items": {"type": "string"}}, "related_by_rel_types": {"type": "array", "items": {"type": "string"}}, - "io.element.relation_types": {"type": "array", "items": {"type": "string"}}, }, } @@ -323,16 +322,6 @@ def __init__(self, hs: "HomeServer", filter_json: JsonDict): self.related_by_senders = self.filter_json.get("related_by_senders", None) self.related_by_rel_types = self.filter_json.get("related_by_rel_types", None) - # Fallback to the unstable prefix if the stable version is not given. - if hs.config.experimental.msc3440_enabled: - self.related_by_senders = self.related_by_senders or self.filter_json.get( - "io.element.relation_senders", None - ) - self.related_by_rel_types = ( - self.related_by_rel_types - or self.filter_json.get("io.element.relation_types", None) - ) - def filters_all_types(self) -> bool: return "*" in self.not_types @@ -456,9 +445,9 @@ def filter_rooms(self, room_ids: Iterable[str]) -> Set[str]: return room_ids async def _check_event_relations( - self, events: Iterable[FilterEvent] + self, events: Collection[FilterEvent] ) -> List[FilterEvent]: - # The event IDs to check, mypy doesn't understand the ifinstance check. + # The event IDs to check, mypy doesn't understand the isinstance check. event_ids = [event.event_id for event in events if isinstance(event, EventBase)] # type: ignore[attr-defined] event_ids_to_keep = set( await self._store.events_have_relations( diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py index a747a4081497..3f85d61b4633 100644 --- a/synapse/api/room_versions.py +++ b/synapse/api/room_versions.py @@ -81,6 +81,9 @@ class RoomVersion: msc2716_historical: bool # MSC2716: Adds support for redacting "insertion", "chunk", and "marker" events msc2716_redactions: bool + # MSC3787: Adds support for a `knock_restricted` join rule, mixing concepts of + # knocks and restricted join rules into the same join condition. + msc3787_knock_restricted_join_rule: bool class RoomVersions: @@ -99,6 +102,7 @@ class RoomVersions: msc2403_knocking=False, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) V2 = RoomVersion( "2", @@ -115,6 +119,7 @@ class RoomVersions: msc2403_knocking=False, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) V3 = RoomVersion( "3", @@ -131,6 +136,7 @@ class RoomVersions: msc2403_knocking=False, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) V4 = RoomVersion( "4", @@ -147,6 +153,7 @@ class RoomVersions: msc2403_knocking=False, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) V5 = RoomVersion( "5", @@ -163,6 +170,7 @@ class RoomVersions: msc2403_knocking=False, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) V6 = RoomVersion( "6", @@ -179,6 +187,7 @@ class RoomVersions: msc2403_knocking=False, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) MSC2176 = RoomVersion( "org.matrix.msc2176", @@ -195,6 +204,7 @@ class RoomVersions: msc2403_knocking=False, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) V7 = RoomVersion( "7", @@ -211,6 +221,7 @@ class RoomVersions: msc2403_knocking=True, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) V8 = RoomVersion( "8", @@ -227,6 +238,7 @@ class RoomVersions: msc2403_knocking=True, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) V9 = RoomVersion( "9", @@ -243,6 +255,7 @@ class RoomVersions: msc2403_knocking=True, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) MSC2716v3 = RoomVersion( "org.matrix.msc2716v3", @@ -259,6 +272,24 @@ class RoomVersions: msc2403_knocking=True, msc2716_historical=True, msc2716_redactions=True, + msc3787_knock_restricted_join_rule=False, + ) + MSC3787 = RoomVersion( + "org.matrix.msc3787", + RoomDisposition.UNSTABLE, + EventFormatVersions.V3, + StateResolutionVersions.V2, + enforce_key_validity=True, + special_case_aliases_auth=False, + strict_canonicaljson=True, + limit_notifications_power_levels=True, + msc2176_redaction_rules=False, + msc3083_join_rules=True, + msc3375_redaction_rules=True, + msc2403_knocking=True, + msc2716_historical=False, + msc2716_redactions=False, + msc3787_knock_restricted_join_rule=True, ) @@ -276,6 +307,7 @@ class RoomVersions: RoomVersions.V8, RoomVersions.V9, RoomVersions.MSC2716v3, + RoomVersions.MSC3787, ) } diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 37321f913399..a3446ac6e874 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -38,6 +38,7 @@ from cryptography.utils import CryptographyDeprecationWarning from matrix_common.versionstring import get_distribution_version_string +from typing_extensions import ParamSpec import twisted from twisted.internet import defer, error, reactor as _reactor @@ -48,10 +49,12 @@ from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.python.threadpool import ThreadPool -import synapse +import synapse.util.caches from synapse.api.constants import MAX_PDU_SIZE from synapse.app import check_bind_error from synapse.app.phone_stats_home import start_phone_stats_home +from synapse.config import ConfigError +from synapse.config._base import format_config_error from synapse.config.homeserver import HomeServerConfig from synapse.config.server import ManholeConfig from synapse.crypto import context_factory @@ -60,6 +63,7 @@ from synapse.events.third_party_rules import load_legacy_third_party_event_rules from synapse.handlers.auth import load_legacy_password_auth_providers from synapse.logging.context import PreserveLoggingContext +from synapse.logging.opentracing import init_tracer from synapse.metrics import install_gc_manager, register_threadpool from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.metrics.jemalloc import setup_jemalloc_stats @@ -81,11 +85,12 @@ # list of tuples of function, args list, kwargs dict _sighup_callbacks: List[ - Tuple[Callable[..., None], Tuple[Any, ...], Dict[str, Any]] + Tuple[Callable[..., None], Tuple[object, ...], Dict[str, object]] ] = [] +P = ParamSpec("P") -def register_sighup(func: Callable[..., None], *args: Any, **kwargs: Any) -> None: +def register_sighup(func: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: """ Register a function to be called when a SIGHUP occurs. @@ -93,7 +98,9 @@ def register_sighup(func: Callable[..., None], *args: Any, **kwargs: Any) -> Non func: Function to be called when sent a SIGHUP signal. *args, **kwargs: args and kwargs to be passed to the target function. """ - _sighup_callbacks.append((func, args, kwargs)) + # This type-ignore should be redundant once we use a mypy release with + # https://github.com/python/mypy/pull/12668. + _sighup_callbacks.append((func, args, kwargs)) # type: ignore[arg-type] def start_worker_reactor( @@ -214,7 +221,9 @@ def redirect_stdio_to_logs() -> None: print("Redirected stdout/stderr to logs") -def register_start(cb: Callable[..., Awaitable], *args: Any, **kwargs: Any) -> None: +def register_start( + cb: Callable[P, Awaitable], *args: P.args, **kwargs: P.kwargs +) -> None: """Register a callback with the reactor, to be called once it is running This can be used to initialise parts of the system which require an asynchronous @@ -426,12 +435,16 @@ def run_sighup(*args: Any, **kwargs: Any) -> None: signal.signal(signal.SIGHUP, run_sighup) register_sighup(refresh_certificate, hs) + register_sighup(reload_cache_config, hs.config) + + # Apply the cache config. + hs.config.caches.resize_all_caches() # Load the certificate from disk. refresh_certificate(hs) # Start the tracer - synapse.logging.opentracing.init_tracer(hs) # type: ignore[attr-defined] # noqa + init_tracer(hs) # noqa # Instantiate the modules so they can register their web resources to the module API # before we start the listeners. @@ -480,6 +493,43 @@ def run_sighup(*args: Any, **kwargs: Any) -> None: atexit.register(gc.freeze) +def reload_cache_config(config: HomeServerConfig) -> None: + """Reload cache config from disk and immediately apply it.resize caches accordingly. + + If the config is invalid, a `ConfigError` is logged and no changes are made. + + Otherwise, this: + - replaces the `caches` section on the given `config` object, + - resizes all caches according to the new cache factors, and + + Note that the following cache config keys are read, but not applied: + - event_cache_size: used to set a max_size and _original_max_size on + EventsWorkerStore._get_event_cache when it is created. We'd have to update + the _original_max_size (and maybe + - sync_response_cache_duration: would have to update the timeout_sec attribute on + HomeServer -> SyncHandler -> ResponseCache. + - track_memory_usage. This affects synapse.util.caches.TRACK_MEMORY_USAGE which + influences Synapse's self-reported metrics. + + Also, the HTTPConnectionPool in SimpleHTTPClient sets its maxPersistentPerHost + parameter based on the global_factor. This won't be applied on a config reload. + """ + try: + previous_cache_config = config.reload_config_section("caches") + except ConfigError as e: + logger.warning("Failed to reload cache config") + for f in format_config_error(e): + logger.warning(f) + else: + logger.debug( + "New cache config. Was:\n %s\nNow:\n", + previous_cache_config.__dict__, + config.caches.__dict__, + ) + synapse.util.caches.TRACK_MEMORY_USAGE = config.caches.track_memory_usage + config.caches.resize_all_caches() + + def setup_sentry(hs: "HomeServer") -> None: """Enable sentry integration, if enabled in configuration""" diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index 2b0d92cbaedc..2a4c2e59cda3 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -210,7 +210,7 @@ def start(config_options: List[str]) -> None: config.logging.no_redirect_stdio = True # Explicitly disable background processes - config.server.update_user_directory = False + config.worker.should_update_user_directory = False config.worker.run_background_tasks = False config.worker.start_pushers = False config.worker.pusher_shard_config.instances = [] diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 1865c671f41c..2a9480a5c161 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -441,38 +441,6 @@ def start(config_options: List[str]) -> None: "synapse.app.user_dir", ) - if config.worker.worker_app == "synapse.app.appservice": - if config.appservice.notify_appservices: - sys.stderr.write( - "\nThe appservices must be disabled in the main synapse process" - "\nbefore they can be run in a separate worker." - "\nPlease add ``notify_appservices: false`` to the main config" - "\n" - ) - sys.exit(1) - - # Force the appservice to start since they will be disabled in the main config - config.appservice.notify_appservices = True - else: - # For other worker types we force this to off. - config.appservice.notify_appservices = False - - if config.worker.worker_app == "synapse.app.user_dir": - if config.server.update_user_directory: - sys.stderr.write( - "\nThe update_user_directory must be disabled in the main synapse process" - "\nbefore they can be run in a separate worker." - "\nPlease add ``update_user_directory: false`` to the main config" - "\n" - ) - sys.exit(1) - - # Force the pushers to start since they will be disabled in the main config - config.server.update_user_directory = True - else: - # For other worker types we force this to off. - config.server.update_user_directory = False - synapse.events.USE_FROZEN_DICTS = config.server.use_frozen_dicts synapse.util.caches.TRACK_MEMORY_USAGE = config.caches.track_memory_usage diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 0f75e7b9d491..4c6c0658ab14 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -16,7 +16,7 @@ import logging import os import sys -from typing import Dict, Iterable, Iterator, List +from typing import Dict, Iterable, List from matrix_common.versionstring import get_distribution_version_string @@ -45,7 +45,7 @@ redirect_stdio_to_logs, register_start, ) -from synapse.config._base import ConfigError +from synapse.config._base import ConfigError, format_config_error from synapse.config.emailconfig import ThreepidBehaviour from synapse.config.homeserver import HomeServerConfig from synapse.config.server import ListenerConfig @@ -399,38 +399,6 @@ async def start() -> None: return hs -def format_config_error(e: ConfigError) -> Iterator[str]: - """ - Formats a config error neatly - - The idea is to format the immediate error, plus the "causes" of those errors, - hopefully in a way that makes sense to the user. For example: - - Error in configuration at 'oidc_config.user_mapping_provider.config.display_name_template': - Failed to parse config for module 'JinjaOidcMappingProvider': - invalid jinja template: - unexpected end of template, expected 'end of print statement'. - - Args: - e: the error to be formatted - - Returns: An iterator which yields string fragments to be formatted - """ - yield "Error in configuration" - - if e.path: - yield " at '%s'" % (".".join(e.path),) - - yield ":\n %s" % (e.msg,) - - parent_e = e.__cause__ - indent = 1 - while parent_e: - indent += 1 - yield ":\n%s%s" % (" " * indent, str(parent_e)) - parent_e = parent_e.__cause__ - - def run(hs: HomeServer) -> None: _base.start_reactor( "synapse-homeserver", diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index d23d9221bc5b..a610fb785d38 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -42,7 +42,7 @@ # user ID -> {device ID -> {algorithm -> count}} TransactionOneTimeKeyCounts = Dict[str, Dict[str, Dict[str, int]]] -# Type for the `device_unused_fallback_keys` field in an appservice transaction +# Type for the `device_unused_fallback_key_types` field in an appservice transaction # user ID -> {device ID -> [algorithm]} TransactionUnusedFallbackKeys = Dict[str, Dict[str, List[str]]] diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 0cdbb04bfbe3..d19f8dd996b2 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple from prometheus_client import Counter +from typing_extensions import TypeGuard from synapse.api.constants import EventTypes, Membership, ThirdPartyEntityKind from synapse.api.errors import CodeMessageException @@ -66,7 +67,7 @@ def _is_valid_3pe_metadata(info: JsonDict) -> bool: return True -def _is_valid_3pe_result(r: JsonDict, field: str) -> bool: +def _is_valid_3pe_result(r: object, field: str) -> TypeGuard[JsonDict]: if not isinstance(r, dict): return False @@ -278,7 +279,7 @@ async def push_bulk( ] = one_time_key_counts if unused_fallback_keys: body[ - "org.matrix.msc3202.device_unused_fallback_keys" + "org.matrix.msc3202.device_unused_fallback_key_types" ] = unused_fallback_keys if device_list_summary: body["org.matrix.msc3202.device_lists"] = { diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 179aa7ff887e..42364fc133f1 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -16,14 +16,18 @@ import argparse import errno +import logging import os from collections import OrderedDict from hashlib import sha256 from textwrap import dedent from typing import ( Any, + ClassVar, + Collection, Dict, Iterable, + Iterator, List, MutableMapping, Optional, @@ -40,6 +44,8 @@ from synapse.util.templates import _create_mxc_to_http_filter, _format_ts_filter +logger = logging.getLogger(__name__) + class ConfigError(Exception): """Represents a problem parsing the configuration @@ -55,6 +61,38 @@ def __init__(self, msg: str, path: Optional[Iterable[str]] = None): self.path = path +def format_config_error(e: ConfigError) -> Iterator[str]: + """ + Formats a config error neatly + + The idea is to format the immediate error, plus the "causes" of those errors, + hopefully in a way that makes sense to the user. For example: + + Error in configuration at 'oidc_config.user_mapping_provider.config.display_name_template': + Failed to parse config for module 'JinjaOidcMappingProvider': + invalid jinja template: + unexpected end of template, expected 'end of print statement'. + + Args: + e: the error to be formatted + + Returns: An iterator which yields string fragments to be formatted + """ + yield "Error in configuration" + + if e.path: + yield " at '%s'" % (".".join(e.path),) + + yield ":\n %s" % (e.msg,) + + parent_e = e.__cause__ + indent = 1 + while parent_e: + indent += 1 + yield ":\n%s%s" % (" " * indent, str(parent_e)) + parent_e = parent_e.__cause__ + + # We split these messages out to allow packages to override with package # specific instructions. MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS = """\ @@ -119,7 +157,7 @@ class Config: defined in subclasses. """ - section: str + section: ClassVar[str] def __init__(self, root_config: "RootConfig" = None): self.root = root_config @@ -309,9 +347,12 @@ class RootConfig: class, lower-cased and with "Config" removed. """ - config_classes = [] + config_classes: List[Type[Config]] = [] + + def __init__(self, config_files: Collection[str] = ()): + # Capture absolute paths here, so we can reload config after we daemonize. + self.config_files = [os.path.abspath(path) for path in config_files] - def __init__(self): for config_class in self.config_classes: if config_class.section is None: raise ValueError("%r requires a section name" % (config_class,)) @@ -512,12 +553,10 @@ def load_config_with_parser( object from parser.parse_args(..)` """ - obj = cls() - config_args = parser.parse_args(argv) config_files = find_config_files(search_paths=config_args.config_path) - + obj = cls(config_files) if not config_files: parser.error("Must supply a config file.") @@ -627,7 +666,7 @@ def load_or_generate_config( generate_missing_configs = config_args.generate_missing_configs - obj = cls() + obj = cls(config_files) if config_args.generate_config: if config_args.report_stats is None: @@ -727,6 +766,34 @@ def generate_missing_files( ) -> None: self.invoke_all("generate_files", config_dict, config_dir_path) + def reload_config_section(self, section_name: str) -> Config: + """Reconstruct the given config section, leaving all others unchanged. + + This works in three steps: + + 1. Create a new instance of the relevant `Config` subclass. + 2. Call `read_config` on that instance to parse the new config. + 3. Replace the existing config instance with the new one. + + :raises ValueError: if the given `section` does not exist. + :raises ConfigError: for any other problems reloading config. + + :returns: the previous config object, which no longer has a reference to this + RootConfig. + """ + existing_config: Optional[Config] = getattr(self, section_name, None) + if existing_config is None: + raise ValueError(f"Unknown config section '{section_name}'") + logger.info("Reloading config section '%s'", section_name) + + new_config_data = read_config_files(self.config_files) + new_config = type(existing_config)(self) + new_config.read_config(new_config_data) + setattr(self, section_name, new_config) + + existing_config.root = None + return existing_config + def read_config_files(config_files: Iterable[str]) -> Dict[str, Any]: """Read the config files into a dict diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index bd092f956dde..71d6655fda4e 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -1,15 +1,19 @@ import argparse from typing import ( Any, + Collection, Dict, Iterable, + Iterator, List, + Literal, MutableMapping, Optional, Tuple, Type, TypeVar, Union, + overload, ) import jinja2 @@ -64,6 +68,8 @@ class ConfigError(Exception): self.msg = msg self.path = path +def format_config_error(e: ConfigError) -> Iterator[str]: ... + MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS: str MISSING_REPORT_STATS_SPIEL: str MISSING_SERVER_NAME: str @@ -117,7 +123,8 @@ class RootConfig: background_updates: background_updates.BackgroundUpdateConfig config_classes: List[Type["Config"]] = ... - def __init__(self) -> None: ... + config_files: List[str] + def __init__(self, config_files: Collection[str] = ...) -> None: ... def invoke_all( self, func_name: str, *args: Any, **kwargs: Any ) -> MutableMapping[str, Any]: ... @@ -157,6 +164,12 @@ class RootConfig: def generate_missing_files( self, config_dict: dict, config_dir_path: str ) -> None: ... + @overload + def reload_config_section( + self, section_name: Literal["caches"] + ) -> cache.CacheConfig: ... + @overload + def reload_config_section(self, section_name: str) -> Config: ... class Config: root: RootConfig diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index 720b90a28386..24498e794433 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -33,7 +33,6 @@ class AppServiceConfig(Config): def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.app_service_config_files = config.get("app_service_config_files", []) - self.notify_appservices = config.get("notify_appservices", True) self.track_appservice_user_ips = config.get("track_appservice_user_ips", False) def generate_config_section(cls, **kwargs: Any) -> str: @@ -56,7 +55,8 @@ def load_appservices( ) -> List[ApplicationService]: """Returns a list of Application Services from the config files.""" if not isinstance(config_files, list): - logger.warning("Expected %s to be a list of AS config files.", config_files) + # type-ignore: this function gets arbitrary json value; we do use this path. + logger.warning("Expected %s to be a list of AS config files.", config_files) # type: ignore[unreachable] return [] # Dicts of value -> filename diff --git a/synapse/config/auth.py b/synapse/config/auth.py index bb417a235946..265a554a5da6 100644 --- a/synapse/config/auth.py +++ b/synapse/config/auth.py @@ -29,7 +29,18 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: if password_config is None: password_config = {} - self.password_enabled = password_config.get("enabled", True) + passwords_enabled = password_config.get("enabled", True) + # 'only_for_reauth' allows users who have previously set a password to use it, + # even though passwords would otherwise be disabled. + passwords_for_reauth_only = passwords_enabled == "only_for_reauth" + + self.password_enabled_for_login = ( + passwords_enabled and not passwords_for_reauth_only + ) + self.password_enabled_for_reauth = ( + passwords_for_reauth_only or passwords_enabled + ) + self.password_localdb_enabled = password_config.get("localdb_enabled", True) self.password_pepper = password_config.get("pepper", "") @@ -46,7 +57,9 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: def generate_config_section(self, **kwargs: Any) -> str: return """\ password_config: - # Uncomment to disable password login + # Uncomment to disable password login. + # Set to `only_for_reauth` to permit reauthentication for users that + # have passwords and are already logged in. # #enabled: false diff --git a/synapse/config/cache.py b/synapse/config/cache.py index 94d852f413d9..d2f55534d7d1 100644 --- a/synapse/config/cache.py +++ b/synapse/config/cache.py @@ -69,11 +69,11 @@ def _canonicalise_cache_name(cache_name: str) -> str: def add_resizable_cache( cache_name: str, cache_resize_callback: Callable[[float], None] ) -> None: - """Register a cache that's size can dynamically change + """Register a cache whose size can dynamically change Args: cache_name: A reference to the cache - cache_resize_callback: A callback function that will be ran whenever + cache_resize_callback: A callback function that will run whenever the cache needs to be resized """ # Some caches have '*' in them which we strip out. @@ -96,6 +96,13 @@ class CacheConfig(Config): section = "caches" _environ = os.environ + event_cache_size: int + cache_factors: Dict[str, float] + global_factor: float + track_memory_usage: bool + expiry_time_msec: Optional[int] + sync_response_cache_duration: int + @staticmethod def reset() -> None: """Resets the caches to their defaults. Used for tests.""" @@ -115,6 +122,12 @@ def generate_config_section(self, **kwargs: Any) -> str: # A cache 'factor' is a multiplier that can be applied to each of # Synapse's caches in order to increase or decrease the maximum # number of entries that can be stored. + # + # The configuration for cache factors (caches.global_factor and + # caches.per_cache_factors) can be reloaded while the application is running, + # by sending a SIGHUP signal to the Synapse process. Changes to other parts of + # the caching config will NOT be applied after a SIGHUP is received; a restart + # is necessary. # The number of events to cache in memory. Not affected by # caches.global_factor. @@ -163,6 +176,24 @@ def generate_config_section(self, **kwargs: Any) -> str: # #cache_entry_ttl: 30m + # This flag enables cache autotuning, and is further specified by the sub-options `max_cache_memory_usage`, + # `target_cache_memory_usage`, `min_cache_ttl`. These flags work in conjunction with each other to maintain + # a balance between cache memory usage and cache entry availability. You must be using jemalloc to utilize + # this option, and all three of the options must be specified for this feature to work. + #cache_autotuning: + # This flag sets a ceiling on much memory the cache can use before caches begin to be continuously evicted. + # They will continue to be evicted until the memory usage drops below the `target_memory_usage`, set in + # the flag below, or until the `min_cache_ttl` is hit. + #max_cache_memory_usage: 1024M + + # This flag sets a rough target for the desired memory usage of the caches. + #target_cache_memory_usage: 758M + + # 'min_cache_ttl` sets a limit under which newer cache entries are not evicted and is only applied when + # caches are actively being evicted/`max_cache_memory_usage` has been exceeded. This is to protect hot caches + # from being emptied while Synapse is evicting due to memory. + #min_cache_ttl: 5m + # Controls how long the results of a /sync request are cached for after # a successful response is returned. A higher duration can help clients with # intermittent connections, at the cost of higher memory usage. @@ -174,21 +205,21 @@ def generate_config_section(self, **kwargs: Any) -> str: """ def read_config(self, config: JsonDict, **kwargs: Any) -> None: + """Populate this config object with values from `config`. + + This method does NOT resize existing or future caches: use `resize_all_caches`. + We use two separate methods so that we can reject bad config before applying it. + """ self.event_cache_size = self.parse_size( config.get("event_cache_size", _DEFAULT_EVENT_CACHE_SIZE) ) - self.cache_factors: Dict[str, float] = {} + self.cache_factors = {} cache_config = config.get("caches") or {} - self.global_factor = cache_config.get( - "global_factor", properties.default_factor_size - ) + self.global_factor = cache_config.get("global_factor", _DEFAULT_FACTOR_SIZE) if not isinstance(self.global_factor, (int, float)): raise ConfigError("caches.global_factor must be a number.") - # Set the global one so that it's reflected in new caches - properties.default_factor_size = self.global_factor - # Load cache factors from the config individual_factors = cache_config.get("per_cache_factors") or {} if not isinstance(individual_factors, dict): @@ -230,7 +261,7 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: cache_entry_ttl = cache_config.get("cache_entry_ttl", "30m") if expire_caches: - self.expiry_time_msec: Optional[int] = self.parse_duration(cache_entry_ttl) + self.expiry_time_msec = self.parse_duration(cache_entry_ttl) else: self.expiry_time_msec = None @@ -250,23 +281,38 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: ) self.expiry_time_msec = self.parse_duration(expiry_time) + self.cache_autotuning = cache_config.get("cache_autotuning") + if self.cache_autotuning: + max_memory_usage = self.cache_autotuning.get("max_cache_memory_usage") + self.cache_autotuning["max_cache_memory_usage"] = self.parse_size( + max_memory_usage + ) + + target_mem_size = self.cache_autotuning.get("target_cache_memory_usage") + self.cache_autotuning["target_cache_memory_usage"] = self.parse_size( + target_mem_size + ) + + min_cache_ttl = self.cache_autotuning.get("min_cache_ttl") + self.cache_autotuning["min_cache_ttl"] = self.parse_duration(min_cache_ttl) + self.sync_response_cache_duration = self.parse_duration( cache_config.get("sync_response_cache_duration", 0) ) - # Resize all caches (if necessary) with the new factors we've loaded - self.resize_all_caches() - - # Store this function so that it can be called from other classes without - # needing an instance of Config - properties.resize_all_caches_func = self.resize_all_caches - def resize_all_caches(self) -> None: - """Ensure all cache sizes are up to date + """Ensure all cache sizes are up-to-date. For each cache, run the mapped callback function with either a specific cache factor or the default, global one. """ + # Set the global factor size, so that new caches are appropriately sized. + properties.default_factor_size = self.global_factor + + # Store this function so that it can be called from other classes without + # needing an instance of CacheConfig + properties.resize_all_caches_func = self.resize_all_caches + # block other threads from modifying _CACHES while we iterate it. with _CACHES_LOCK: for cache_name, callback in _CACHES.items(): diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 447476fbfac5..b20d949689ec 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -26,16 +26,13 @@ class ExperimentalConfig(Config): def read_config(self, config: JsonDict, **kwargs: Any) -> None: experimental = config.get("experimental_features") or {} - # MSC3440 (thread relation) - self.msc3440_enabled: bool = experimental.get("msc3440_enabled", False) - # MSC3026 (busy presence state) self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False) # MSC2716 (importing historical messages) self.msc2716_enabled: bool = experimental.get("msc2716_enabled", False) - # MSC2285 (hidden read receipts) + # MSC2285 (private read receipts) self.msc2285_enabled: bool = experimental.get("msc2285_enabled", False) # MSC3244 (room version capabilities) @@ -77,7 +74,13 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.msc3720_enabled: bool = experimental.get("msc3720_enabled", False) # The deprecated groups feature. - self.groups_enabled: bool = experimental.get("groups_enabled", True) + self.groups_enabled: bool = experimental.get("groups_enabled", False) # MSC2654: Unread counts self.msc2654_enabled: bool = experimental.get("msc2654_enabled", False) + + # MSC2815 (allow room moderators to view redacted event content) + self.msc2815_enabled: bool = experimental.get("msc2815_enabled", False) + + # MSC3786 (Add a default push rule to ignore m.room.server_acl events) + self.msc3786_enabled: bool = experimental.get("msc3786_enabled", False) diff --git a/synapse/config/federation.py b/synapse/config/federation.py index 0e74f7078455..f83f93c0ef11 100644 --- a/synapse/config/federation.py +++ b/synapse/config/federation.py @@ -46,7 +46,7 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: ) self.allow_device_name_lookup_over_federation = config.get( - "allow_device_name_lookup_over_federation", True + "allow_device_name_lookup_over_federation", False ) def generate_config_section(self, **kwargs: Any) -> str: @@ -81,11 +81,11 @@ def generate_config_section(self, **kwargs: Any) -> str: # #allow_profile_lookup_over_federation: false - # Uncomment to disable device display name lookup over federation. By default, the - # Federation API allows other homeservers to obtain device display names of any user - # on this homeserver. Defaults to 'true'. + # Uncomment to allow device display name lookup over federation. By default, the + # Federation API prevents other homeservers from obtaining the display names of + # user devices on this homeserver. Defaults to 'false'. # - #allow_device_name_lookup_over_federation: false + #allow_device_name_lookup_over_federation: true """ diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 99db9e1e3910..470b8b44929c 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -110,13 +110,6 @@ # information such as access tokens. level: INFO - twisted: - # We send the twisted logging directly to the file handler, - # to work around https://github.com/matrix-org/synapse/issues/3471 - # when using "buffer" logger. Use "console" to log to stderr instead. - handlers: [file] - propagate: false - root: level: INFO diff --git a/synapse/config/oembed.py b/synapse/config/oembed.py index 690ffb52963e..e9edea073123 100644 --- a/synapse/config/oembed.py +++ b/synapse/config/oembed.py @@ -57,9 +57,9 @@ def _parse_and_validate_providers( """ # Whether to use the packaged providers.json file. if not oembed_config.get("disable_default_providers") or False: - providers = json.load( - pkg_resources.resource_stream("synapse", "res/providers.json") - ) + with pkg_resources.resource_stream("synapse", "res/providers.json") as s: + providers = json.load(s) + yield from self._parse_and_validate_provider( providers, config_path=("oembed",) ) diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 39e9acb62a90..d2d0425e62cb 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -43,6 +43,9 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.registration_requires_token = config.get( "registration_requires_token", False ) + self.enable_registration_token_3pid_bypass = config.get( + "enable_registration_token_3pid_bypass", False + ) self.registration_shared_secret = config.get("registration_shared_secret") self.bcrypt_rounds = config.get("bcrypt_rounds", 12) @@ -309,6 +312,12 @@ def generate_config_section( # #registration_requires_token: true + # Allow users to submit a token during registration to bypass any required 3pid + # steps configured in `registrations_require_3pid`. + # Defaults to false, requiring that registration tokens (if enabled) complete a 3pid flow. + # + #enable_registration_token_3pid_bypass: false + # If set, allows registration of standard or admin accounts by anyone who # has the shared secret, even if registration is otherwise disabled. # diff --git a/synapse/config/room.py b/synapse/config/room.py index e18a87ea37f6..462d85ac1d1e 100644 --- a/synapse/config/room.py +++ b/synapse/config/room.py @@ -63,6 +63,19 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: "Invalid value for encryption_enabled_by_default_for_room_type" ) + self.default_power_level_content_override = config.get( + "default_power_level_content_override", + None, + ) + if self.default_power_level_content_override is not None: + for preset in self.default_power_level_content_override: + if preset not in vars(RoomCreationPreset).values(): + raise ConfigError( + "Unrecognised room preset %s in default_power_level_content_override" + % preset + ) + # We validate the actual overrides when we try to apply them. + def generate_config_section(self, **kwargs: Any) -> str: return """\ ## Rooms ## @@ -83,4 +96,38 @@ def generate_config_section(self, **kwargs: Any) -> str: # will also not affect rooms created by other servers. # #encryption_enabled_by_default_for_room_type: invite + + # Override the default power levels for rooms created on this server, per + # room creation preset. + # + # The appropriate dictionary for the room preset will be applied on top + # of the existing power levels content. + # + # Useful if you know that your users need special permissions in rooms + # that they create (e.g. to send particular types of state events without + # needing an elevated power level). This takes the same shape as the + # `power_level_content_override` parameter in the /createRoom API, but + # is applied before that parameter. + # + # Valid keys are some or all of `private_chat`, `trusted_private_chat` + # and `public_chat`. Inside each of those should be any of the + # properties allowed in `power_level_content_override` in the + # /createRoom API. If any property is missing, its default value will + # continue to be used. If any property is present, it will overwrite + # the existing default completely (so if the `events` property exists, + # the default event power levels will be ignored). + # + #default_power_level_content_override: + # private_chat: + # "events": + # "com.example.myeventtype" : 0 + # "m.room.avatar": 50 + # "m.room.canonical_alias": 50 + # "m.room.encryption": 100 + # "m.room.history_visibility": 100 + # "m.room.name": 50 + # "m.room.power_levels": 100 + # "m.room.server_acl": 100 + # "m.room.tombstone": 100 + # "events_default": 1 """ diff --git a/synapse/config/server.py b/synapse/config/server.py index 415279d269d6..f73d5e1f6666 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -186,7 +186,7 @@ def generate_ip_set( class HttpResourceConfig: names: List[str] = attr.ib( factory=list, - validator=attr.validators.deep_iterable(attr.validators.in_(KNOWN_RESOURCES)), # type: ignore + validator=attr.validators.deep_iterable(attr.validators.in_(KNOWN_RESOURCES)), ) compress: bool = attr.ib( default=False, @@ -231,9 +231,7 @@ class ManholeConfig: class LimitRemoteRoomsConfig: enabled: bool = attr.ib(validator=attr.validators.instance_of(bool), default=False) complexity: Union[float, int] = attr.ib( - validator=attr.validators.instance_of( - (float, int) # type: ignore[arg-type] # noqa - ), + validator=attr.validators.instance_of((float, int)), # noqa default=1.0, ) complexity_error: str = attr.ib( @@ -321,10 +319,6 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.presence_router_config, ) = load_module(presence_router_config, ("presence", "presence_router")) - # Whether to update the user directory or not. This should be set to - # false only if we are updating the user directory in a worker - self.update_user_directory = config.get("update_user_directory", True) - # whether to enable the media repository endpoints. This should be set # to false if the media repository is running as a separate endpoint; # doing so ensures that we will not run cache cleanup jobs on the @@ -415,6 +409,7 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: ) self.mau_trial_days = config.get("mau_trial_days", 0) + self.mau_appservice_trial_days = config.get("mau_appservice_trial_days", {}) self.mau_limit_alerting = config.get("mau_limit_alerting", True) # How long to keep redacted events in the database in unredacted form @@ -680,14 +675,6 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: config.get("use_account_validity_in_account_status") or False ) - # This is a temporary option that enables fully using the new - # `device_lists_changes_in_room` without the backwards compat code. This - # is primarily for testing. If enabled the server should *not* be - # downgraded, as it may lead to missing device list updates. - self.use_new_device_lists_changes_in_room = ( - config.get("use_new_device_lists_changes_in_room") or False - ) - self.rooms_to_exclude_from_sync: List[str] = ( config.get("exclude_rooms_from_sync") or [] ) @@ -1009,7 +996,7 @@ def generate_config_section( # federation: the server-server API (/_matrix/federation). Also implies # 'media', 'keys', 'openid' # - # keys: the key discovery API (/_matrix/keys). + # keys: the key discovery API (/_matrix/key). # # media: the media API (/_matrix/media). # @@ -1115,6 +1102,11 @@ def generate_config_section( # sign up in a short space of time never to return after their initial # session. # + # The option `mau_appservice_trial_days` is similar to `mau_trial_days`, but + # applies a different trial number if the user was registered by an appservice. + # A value of 0 means no trial days are applied. Appservices not listed in this + # dictionary use the value of `mau_trial_days` instead. + # # 'mau_limit_alerting' is a means of limiting client side alerting # should the mau limit be reached. This is useful for small instances # where the admin has 5 mau seats (say) for 5 specific people and no @@ -1125,6 +1117,8 @@ def generate_config_section( #max_mau_value: 50 #mau_trial_days: 2 #mau_limit_alerting: false + #mau_appservice_trial_days: + # "appservice-id": 1 # If enabled, the metrics for the number of monthly active users will # be populated, however no one will be limited. If limit_usage_by_mau diff --git a/synapse/config/workers.py b/synapse/config/workers.py index a5479dfca98b..e1569b3c14f9 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -14,7 +14,8 @@ # limitations under the License. import argparse -from typing import Any, List, Union +import logging +from typing import Any, Dict, List, Union import attr @@ -42,6 +43,13 @@ Please add ``start_pushers: false`` to the main config """ +_DEPRECATED_WORKER_DUTY_OPTION_USED = """ +The '%s' configuration option is deprecated and will be removed in a future +Synapse version. Please use ``%s: name_of_worker`` instead. +""" + +logger = logging.getLogger(__name__) + def _instance_to_list_converter(obj: Union[str, List[str]]) -> List[str]: """Helper for allowing parsing a string or list of strings to a config @@ -296,6 +304,112 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.worker_name is None and background_tasks_instance == "master" ) or self.worker_name == background_tasks_instance + self.should_notify_appservices = self._should_this_worker_perform_duty( + config, + legacy_master_option_name="notify_appservices", + legacy_worker_app_name="synapse.app.appservice", + new_option_name="notify_appservices_from_worker", + ) + + self.should_update_user_directory = self._should_this_worker_perform_duty( + config, + legacy_master_option_name="update_user_directory", + legacy_worker_app_name="synapse.app.user_dir", + new_option_name="update_user_directory_from_worker", + ) + + def _should_this_worker_perform_duty( + self, + config: Dict[str, Any], + legacy_master_option_name: str, + legacy_worker_app_name: str, + new_option_name: str, + ) -> bool: + """ + Figures out whether this worker should perform a certain duty. + + This function is temporary and is only to deal with the complexity + of allowing old, transitional and new configurations all at once. + + Contradictions between the legacy and new part of a transitional configuration + will lead to a ConfigError. + + Parameters: + config: The config dictionary + legacy_master_option_name: The name of a legacy option, whose value is boolean, + specifying whether it's the master that should handle a certain duty. + e.g. "notify_appservices" + legacy_worker_app_name: The name of a legacy Synapse worker application + that would traditionally perform this duty. + e.g. "synapse.app.appservice" + new_option_name: The name of the new option, whose value is the name of a + designated worker to perform the duty. + e.g. "notify_appservices_from_worker" + """ + + # None means 'unspecified'; True means 'run here' and False means + # 'don't run here'. + new_option_should_run_here = None + if new_option_name in config: + designated_worker = config[new_option_name] or "master" + new_option_should_run_here = ( + designated_worker == "master" and self.worker_name is None + ) or designated_worker == self.worker_name + + legacy_option_should_run_here = None + if legacy_master_option_name in config: + run_on_master = bool(config[legacy_master_option_name]) + + legacy_option_should_run_here = ( + self.worker_name is None and run_on_master + ) or (self.worker_app == legacy_worker_app_name and not run_on_master) + + # Suggest using the new option instead. + logger.warning( + _DEPRECATED_WORKER_DUTY_OPTION_USED, + legacy_master_option_name, + new_option_name, + ) + + if self.worker_app == legacy_worker_app_name and config.get( + legacy_master_option_name, True + ): + # As an extra bit of complication, we need to check that the + # specialised worker is only used if the legacy config says the + # master isn't performing the duties. + raise ConfigError( + f"Cannot use deprecated worker app type '{legacy_worker_app_name}' whilst deprecated option '{legacy_master_option_name}' is not set to false.\n" + f"Consider setting `worker_app: synapse.app.generic_worker` and using the '{new_option_name}' option instead.\n" + f"The '{new_option_name}' option replaces '{legacy_master_option_name}'." + ) + + if new_option_should_run_here is None and legacy_option_should_run_here is None: + # Neither option specified; the fallback behaviour is to run on the main process + return self.worker_name is None + + if ( + new_option_should_run_here is not None + and legacy_option_should_run_here is not None + ): + # Both options specified; ensure they match! + if new_option_should_run_here != legacy_option_should_run_here: + update_worker_type = ( + " and set worker_app: synapse.app.generic_worker" + if self.worker_app == legacy_worker_app_name + else "" + ) + # If the values conflict, we suggest the admin removes the legacy option + # for simplicity. + raise ConfigError( + f"Conflicting configuration options: {legacy_master_option_name} (legacy), {new_option_name} (new).\n" + f"Suggestion: remove {legacy_master_option_name}{update_worker_type}.\n" + ) + + # We've already validated that these aren't conflicting; now just see if + # either is True. + # (By this point, these are either the same value or only one is not None.) + return bool(new_option_should_run_here or legacy_option_should_run_here) + def generate_config_section(self, **kwargs: Any) -> str: return """\ ## Workers ## diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 621a3efcccec..4c0b587a7643 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -414,7 +414,12 @@ def _is_membership_change_allowed( raise AuthError(403, "You are banned from this room") elif join_rule == JoinRules.PUBLIC: pass - elif room_version.msc3083_join_rules and join_rule == JoinRules.RESTRICTED: + elif ( + room_version.msc3083_join_rules and join_rule == JoinRules.RESTRICTED + ) or ( + room_version.msc3787_knock_restricted_join_rule + and join_rule == JoinRules.KNOCK_RESTRICTED + ): # This is the same as public, but the event must contain a reference # to the server who authorised the join. If the event does not contain # the proper content it is rejected. @@ -440,8 +445,13 @@ def _is_membership_change_allowed( if authorising_user_level < invite_level: raise AuthError(403, "Join event authorised by invalid server.") - elif join_rule == JoinRules.INVITE or ( - room_version.msc2403_knocking and join_rule == JoinRules.KNOCK + elif ( + join_rule == JoinRules.INVITE + or (room_version.msc2403_knocking and join_rule == JoinRules.KNOCK) + or ( + room_version.msc3787_knock_restricted_join_rule + and join_rule == JoinRules.KNOCK_RESTRICTED + ) ): if not caller_in_room and not caller_invited: raise AuthError(403, "You are not invited to this room.") @@ -462,7 +472,10 @@ def _is_membership_change_allowed( if user_level < ban_level or user_level <= target_level: raise AuthError(403, "You don't have permission to ban") elif room_version.msc2403_knocking and Membership.KNOCK == membership: - if join_rule != JoinRules.KNOCK: + if join_rule != JoinRules.KNOCK and ( + not room_version.msc3787_knock_restricted_join_rule + or join_rule != JoinRules.KNOCK_RESTRICTED + ): raise AuthError(403, "You don't have permission to knock") elif target_user_id != event.user_id: raise AuthError(403, "You cannot knock for other users") diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 9acb3c0cc454..39ad2793d98d 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -15,6 +15,7 @@ # limitations under the License. import abc +import collections.abc import os from typing import ( TYPE_CHECKING, @@ -32,9 +33,11 @@ overload, ) +import attr from typing_extensions import Literal from unpaddedbase64 import encode_base64 +from synapse.api.constants import RelationTypes from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions from synapse.types import JsonDict, RoomStreamToken from synapse.util.caches import intern_dict @@ -213,10 +216,17 @@ def is_outlier(self) -> bool: return self.outlier def is_out_of_band_membership(self) -> bool: - """Whether this is an out of band membership, like an invite or an invite - rejection. This is needed as those events are marked as outliers, but - they still need to be processed as if they're new events (e.g. updating - invite state in the database, relaying to clients, etc). + """Whether this event is an out-of-band membership. + + OOB memberships are a special case of outlier events: they are membership events + for federated rooms that we aren't full members of. Examples include invites + received over federation, and rejections for such invites. + + The concept of an OOB membership is needed because these events need to be + processed as if they're new regular events (e.g. updating membership state in + the database, relaying to clients via /sync, etc) despite being outliers. + + See also https://matrix-org.github.io/synapse/develop/development/room-dag-concepts.html#out-of-band-membership-events. (Added in synapse 0.99.0, so may be unreliable for events received before that) """ @@ -608,3 +618,45 @@ def make_event_from_dict( return event_type( event_dict, room_version, internal_metadata_dict or {}, rejected_reason ) + + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class _EventRelation: + # The target event of the relation. + parent_id: str + # The relation type. + rel_type: str + # The aggregation key. Will be None if the rel_type is not m.annotation or is + # not a string. + aggregation_key: Optional[str] + + +def relation_from_event(event: EventBase) -> Optional[_EventRelation]: + """ + Attempt to parse relation information an event. + + Returns: + The event relation information, if it is valid. None, otherwise. + """ + relation = event.content.get("m.relates_to") + if not relation or not isinstance(relation, collections.abc.Mapping): + # No relation information. + return None + + # Relations must have a type and parent event ID. + rel_type = relation.get("rel_type") + if not isinstance(rel_type, str): + return None + + parent_id = relation.get("event_id") + if not isinstance(parent_id, str): + return None + + # Annotations have a key field. + aggregation_key = None + if rel_type == RelationTypes.ANNOTATION: + aggregation_key = relation.get("key") + if not isinstance(aggregation_key, str): + aggregation_key = None + + return _EventRelation(parent_id, rel_type, aggregation_key) diff --git a/synapse/events/presence_router.py b/synapse/events/presence_router.py index a58f313e8b1c..bb4a6bd9574a 100644 --- a/synapse/events/presence_router.py +++ b/synapse/events/presence_router.py @@ -22,11 +22,16 @@ List, Optional, Set, + TypeVar, Union, ) +from typing_extensions import ParamSpec + +from twisted.internet.defer import CancelledError + from synapse.api.presence import UserPresenceState -from synapse.util.async_helpers import maybe_awaitable +from synapse.util.async_helpers import delay_cancellation, maybe_awaitable if TYPE_CHECKING: from synapse.server import HomeServer @@ -40,6 +45,10 @@ logger = logging.getLogger(__name__) +P = ParamSpec("P") +R = TypeVar("R") + + def load_legacy_presence_router(hs: "HomeServer") -> None: """Wrapper that loads a presence router module configured using the old configuration, and registers the hooks they implement. @@ -63,13 +72,15 @@ def load_legacy_presence_router(hs: "HomeServer") -> None: # All methods that the module provides should be async, but this wasn't enforced # in the old module system, so we wrap them if needed - def async_wrapper(f: Optional[Callable]) -> Optional[Callable[..., Awaitable]]: + def async_wrapper( + f: Optional[Callable[P, R]] + ) -> Optional[Callable[P, Awaitable[R]]]: # f might be None if the callback isn't implemented by the module. In this # case we don't want to register a callback at all so we return None. if f is None: return None - def run(*args: Any, **kwargs: Any) -> Awaitable: + def run(*args: P.args, **kwargs: P.kwargs) -> Awaitable[R]: # Assertion required because mypy can't prove we won't change `f` # back to `None`. See # https://mypy.readthedocs.io/en/latest/common_issues.html#narrowing-and-inner-functions @@ -80,7 +91,7 @@ def run(*args: Any, **kwargs: Any) -> Awaitable: return run # Register the hooks through the module API. - hooks = { + hooks: Dict[str, Optional[Callable[..., Any]]] = { hook: async_wrapper(getattr(presence_router, hook, None)) for hook in presence_router_methods } @@ -147,7 +158,11 @@ async def get_users_for_states( # run all the callbacks for get_users_for_states and combine the results for callback in self._get_users_for_states_callbacks: try: - result = await callback(state_updates) + # Note: result is an object here, because we don't trust modules to + # return the types they're supposed to. + result: object = await delay_cancellation(callback(state_updates)) + except CancelledError: + raise except Exception as e: logger.warning("Failed to run module API callback %s: %s", callback, e) continue @@ -199,7 +214,9 @@ async def get_interested_users(self, user_id: str) -> Union[Set[str], str]: # run all the callbacks for get_interested_users and combine the results for callback in self._get_interested_users_callbacks: try: - result = await callback(user_id) + result = await delay_cancellation(callback(user_id)) + except CancelledError: + raise except Exception as e: logger.warning("Failed to run module API callback %s: %s", callback, e) continue diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index 46042b2bf7af..7a91544119f7 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -15,17 +15,16 @@ import attr from frozendict import frozendict - -from twisted.internet.defer import Deferred +from typing_extensions import Literal from synapse.appservice import ApplicationService from synapse.events import EventBase -from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.types import JsonDict, StateMap if TYPE_CHECKING: from synapse.storage import Storage from synapse.storage.databases.main import DataStore + from synapse.storage.state import StateFilter @attr.s(slots=True, auto_attribs=True) @@ -60,6 +59,9 @@ class EventContext: If ``state_group`` is None (ie, the event is an outlier), ``state_group_before_event`` will always also be ``None``. + state_delta_due_to_event: If `state_group` and `state_group_before_event` are not None + then this is the delta of the state between the two groups. + prev_group: If it is known, ``state_group``'s prev_group. Note that this being None does not necessarily mean that ``state_group`` does not have a prev_group! @@ -78,73 +80,47 @@ class EventContext: app_service: If this event is being sent by a (local) application service, that app service. - _current_state_ids: The room state map, including this event - ie, the state - in ``state_group``. - - (type, state_key) -> event_id - - For an outlier, this is {} - - Note that this is a private attribute: it should be accessed via - ``get_current_state_ids``. _AsyncEventContext impl calculates this - on-demand: it will be None until that happens. - - _prev_state_ids: The room state map, excluding this event - ie, the state - in ``state_group_before_event``. For a non-state - event, this will be the same as _current_state_events. - - Note that it is a completely different thing to prev_group! - - (type, state_key) -> event_id - - For an outlier, this is {} - - As with _current_state_ids, this is a private attribute. It should be - accessed via get_prev_state_ids. - partial_state: if True, we may be storing this event with a temporary, incomplete state. """ - rejected: Union[bool, str] = False + _storage: "Storage" + rejected: Union[Literal[False], str] = False _state_group: Optional[int] = None state_group_before_event: Optional[int] = None + _state_delta_due_to_event: Optional[StateMap[str]] = None prev_group: Optional[int] = None delta_ids: Optional[StateMap[str]] = None app_service: Optional[ApplicationService] = None - _current_state_ids: Optional[StateMap[str]] = None - _prev_state_ids: Optional[StateMap[str]] = None - partial_state: bool = False @staticmethod def with_state( + storage: "Storage", state_group: Optional[int], state_group_before_event: Optional[int], - current_state_ids: Optional[StateMap[str]], - prev_state_ids: Optional[StateMap[str]], + state_delta_due_to_event: Optional[StateMap[str]], partial_state: bool, prev_group: Optional[int] = None, delta_ids: Optional[StateMap[str]] = None, ) -> "EventContext": return EventContext( - current_state_ids=current_state_ids, - prev_state_ids=prev_state_ids, + storage=storage, state_group=state_group, state_group_before_event=state_group_before_event, + state_delta_due_to_event=state_delta_due_to_event, prev_group=prev_group, delta_ids=delta_ids, partial_state=partial_state, ) @staticmethod - def for_outlier() -> "EventContext": + def for_outlier( + storage: "Storage", + ) -> "EventContext": """Return an EventContext instance suitable for persisting an outlier event""" - return EventContext( - current_state_ids={}, - prev_state_ids={}, - ) + return EventContext(storage=storage) async def serialize(self, event: EventBase, store: "DataStore") -> JsonDict: """Converts self to a type that can be serialized as JSON, and then @@ -157,24 +133,14 @@ async def serialize(self, event: EventBase, store: "DataStore") -> JsonDict: The serialized event. """ - # We don't serialize the full state dicts, instead they get pulled out - # of the DB on the other side. However, the other side can't figure out - # the prev_state_ids, so if we're a state event we include the event - # id that we replaced in the state. - if event.is_state(): - prev_state_ids = await self.get_prev_state_ids() - prev_state_id = prev_state_ids.get((event.type, event.state_key)) - else: - prev_state_id = None - return { - "prev_state_id": prev_state_id, - "event_type": event.type, - "event_state_key": event.get_state_key(), "state_group": self._state_group, "state_group_before_event": self.state_group_before_event, "rejected": self.rejected, "prev_group": self.prev_group, + "state_delta_due_to_event": _encode_state_dict( + self._state_delta_due_to_event + ), "delta_ids": _encode_state_dict(self.delta_ids), "app_service_id": self.app_service.id if self.app_service else None, "partial_state": self.partial_state, @@ -192,16 +158,16 @@ def deserialize(storage: "Storage", input: JsonDict) -> "EventContext": Returns: The event context. """ - context = _AsyncEventContextImpl( + context = EventContext( # We use the state_group and prev_state_id stuff to pull the # current_state_ids out of the DB and construct prev_state_ids. storage=storage, - prev_state_id=input["prev_state_id"], - event_type=input["event_type"], - event_state_key=input["event_state_key"], state_group=input["state_group"], state_group_before_event=input["state_group_before_event"], prev_group=input["prev_group"], + state_delta_due_to_event=_decode_state_dict( + input["state_delta_due_to_event"] + ), delta_ids=_decode_state_dict(input["delta_ids"]), rejected=input["rejected"], partial_state=input.get("partial_state", False), @@ -231,7 +197,9 @@ def state_group(self) -> Optional[int]: return self._state_group - async def get_current_state_ids(self) -> Optional[StateMap[str]]: + async def get_current_state_ids( + self, state_filter: Optional["StateFilter"] = None + ) -> Optional[StateMap[str]]: """ Gets the room state map, including this event - ie, the state in ``state_group`` @@ -239,6 +207,9 @@ async def get_current_state_ids(self) -> Optional[StateMap[str]]: not make it into the room state. This method will raise an exception if ``rejected`` is set. + Arg: + state_filter: specifies the type of state event to fetch from DB, example: EventTypes.JoinRules + Returns: Returns None if state_group is None, which happens when the associated event is an outlier. @@ -249,15 +220,27 @@ async def get_current_state_ids(self) -> Optional[StateMap[str]]: if self.rejected: raise RuntimeError("Attempt to access state_ids of rejected event") - await self._ensure_fetched() - return self._current_state_ids + assert self._state_delta_due_to_event is not None + + prev_state_ids = await self.get_prev_state_ids(state_filter) + + if self._state_delta_due_to_event: + prev_state_ids = dict(prev_state_ids) + prev_state_ids.update(self._state_delta_due_to_event) - async def get_prev_state_ids(self) -> StateMap[str]: + return prev_state_ids + + async def get_prev_state_ids( + self, state_filter: Optional["StateFilter"] = None + ) -> StateMap[str]: """ Gets the room state map, excluding this event. For a non-state event, this will be the same as get_current_state_ids(). + Args: + state_filter: specifies the type of state event to fetch from DB, example: EventTypes.JoinRules + Returns: Returns {} if state_group is None, which happens when the associated event is an outlier. @@ -265,94 +248,10 @@ async def get_prev_state_ids(self) -> StateMap[str]: Maps a (type, state_key) to the event ID of the state event matching this tuple. """ - await self._ensure_fetched() - # There *should* be previous state IDs now. - assert self._prev_state_ids is not None - return self._prev_state_ids - - def get_cached_current_state_ids(self) -> Optional[StateMap[str]]: - """Gets the current state IDs if we have them already cached. - - It is an error to access this for a rejected event, since rejected state should - not make it into the room state. This method will raise an exception if - ``rejected`` is set. - - Returns: - Returns None if we haven't cached the state or if state_group is None - (which happens when the associated event is an outlier). - - Otherwise, returns the the current state IDs. - """ - if self.rejected: - raise RuntimeError("Attempt to access state_ids of rejected event") - - return self._current_state_ids - - async def _ensure_fetched(self) -> None: - return None - - -@attr.s(slots=True) -class _AsyncEventContextImpl(EventContext): - """ - An implementation of EventContext which fetches _current_state_ids and - _prev_state_ids from the database on demand. - - Attributes: - - _storage - - _fetching_state_deferred: Resolves when *_state_ids have been calculated. - None if we haven't started calculating yet - - _event_type: The type of the event the context is associated with. - - _event_state_key: The state_key of the event the context is associated with. - - _prev_state_id: If the event associated with the context is a state event, - then `_prev_state_id` is the event_id of the state that was replaced. - """ - - # This needs to have a default as we're inheriting - _storage: "Storage" = attr.ib(default=None) - _prev_state_id: Optional[str] = attr.ib(default=None) - _event_type: str = attr.ib(default=None) - _event_state_key: Optional[str] = attr.ib(default=None) - _fetching_state_deferred: Optional["Deferred[None]"] = attr.ib(default=None) - - async def _ensure_fetched(self) -> None: - if not self._fetching_state_deferred: - self._fetching_state_deferred = run_in_background(self._fill_out_state) - - await make_deferred_yieldable(self._fetching_state_deferred) - - async def _fill_out_state(self) -> None: - """Called to populate the _current_state_ids and _prev_state_ids - attributes by loading from the database. - """ - if self.state_group is None: - # No state group means the event is an outlier. Usually the state_ids dicts are also - # pre-set to empty dicts, but they get reset when the context is serialized, so set - # them to empty dicts again here. - self._current_state_ids = {} - self._prev_state_ids = {} - return - - current_state_ids = await self._storage.state.get_state_ids_for_group( - self.state_group + assert self.state_group_before_event is not None + return await self._storage.state.get_state_ids_for_group( + self.state_group_before_event, state_filter ) - # Set this separately so mypy knows current_state_ids is not None. - self._current_state_ids = current_state_ids - if self._event_state_key is not None: - self._prev_state_ids = dict(current_state_ids) - - key = (self._event_type, self._event_state_key) - if self._prev_state_id: - self._prev_state_ids[key] = self._prev_state_id - else: - self._prev_state_ids.pop(key, None) - else: - self._prev_state_ids = current_state_ids def _encode_state_dict( diff --git a/synapse/events/spamcheck.py b/synapse/events/spamcheck.py index cd80fcf9d13a..1048b4c825f6 100644 --- a/synapse/events/spamcheck.py +++ b/synapse/events/spamcheck.py @@ -27,11 +27,13 @@ Union, ) +from synapse.api.errors import Codes from synapse.rest.media.v1._base import FileInfo from synapse.rest.media.v1.media_storage import ReadableFileWrapper from synapse.spam_checker_api import RegistrationBehaviour from synapse.types import RoomAlias, UserProfile -from synapse.util.async_helpers import maybe_awaitable +from synapse.util.async_helpers import delay_cancellation, maybe_awaitable +from synapse.util.metrics import Measure if TYPE_CHECKING: import synapse.events @@ -39,7 +41,18 @@ logger = logging.getLogger(__name__) + CHECK_EVENT_FOR_SPAM_CALLBACK = Callable[ + ["synapse.events.EventBase"], + Awaitable[ + Union[ + str, + # Deprecated + bool, + ] + ], +] +SHOULD_DROP_FEDERATED_EVENT_CALLBACK = Callable[ ["synapse.events.EventBase"], Awaitable[Union[bool, str]], ] @@ -162,8 +175,16 @@ def run(*args: Any, **kwargs: Any) -> Awaitable: class SpamChecker: - def __init__(self) -> None: + NOT_SPAM = "NOT_SPAM" + + def __init__(self, hs: "synapse.server.HomeServer") -> None: + self.hs = hs + self.clock = hs.get_clock() + self._check_event_for_spam_callbacks: List[CHECK_EVENT_FOR_SPAM_CALLBACK] = [] + self._should_drop_federated_event_callbacks: List[ + SHOULD_DROP_FEDERATED_EVENT_CALLBACK + ] = [] self._user_may_join_room_callbacks: List[USER_MAY_JOIN_ROOM_CALLBACK] = [] self._user_may_invite_callbacks: List[USER_MAY_INVITE_CALLBACK] = [] self._user_may_send_3pid_invite_callbacks: List[ @@ -187,6 +208,9 @@ def __init__(self) -> None: def register_callbacks( self, check_event_for_spam: Optional[CHECK_EVENT_FOR_SPAM_CALLBACK] = None, + should_drop_federated_event: Optional[ + SHOULD_DROP_FEDERATED_EVENT_CALLBACK + ] = None, user_may_join_room: Optional[USER_MAY_JOIN_ROOM_CALLBACK] = None, user_may_invite: Optional[USER_MAY_INVITE_CALLBACK] = None, user_may_send_3pid_invite: Optional[USER_MAY_SEND_3PID_INVITE_CALLBACK] = None, @@ -205,6 +229,11 @@ def register_callbacks( if check_event_for_spam is not None: self._check_event_for_spam_callbacks.append(check_event_for_spam) + if should_drop_federated_event is not None: + self._should_drop_federated_event_callbacks.append( + should_drop_federated_event + ) + if user_may_join_room is not None: self._user_may_join_room_callbacks.append(user_may_join_room) @@ -238,9 +267,7 @@ def register_callbacks( if check_media_file_for_spam is not None: self._check_media_file_for_spam_callbacks.append(check_media_file_for_spam) - async def check_event_for_spam( - self, event: "synapse.events.EventBase" - ) -> Union[bool, str]: + async def check_event_for_spam(self, event: "synapse.events.EventBase") -> str: """Checks if a given event is considered "spammy" by this server. If the server considers an event spammy, then it will be rejected if @@ -251,11 +278,65 @@ async def check_event_for_spam( event: the event to be checked Returns: - True or a string if the event is spammy. If a string is returned it - will be used as the error message returned to the user. + - `NOT_SPAM` if the event is considered good (non-spammy) and should be let + through. Other spamcheck filters may still reject it. + - A `Code` if the event is considered spammy and is rejected with a specific + error message/code. + - A string that isn't `NOT_SPAM` if the event is considered spammy and the + string should be used as the client-facing error message. This usage is + generally discouraged as it doesn't support internationalization. """ for callback in self._check_event_for_spam_callbacks: - res: Union[bool, str] = await callback(event) + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + res = await delay_cancellation(callback(event)) + if res is False or res == self.NOT_SPAM: + # This spam-checker accepts the event. + # Other spam-checkers may reject it, though. + continue + elif res is True: + # This spam-checker rejects the event with deprecated + # return value `True` + return Codes.FORBIDDEN + elif not isinstance(res, str): + # mypy complains that we can't reach this code because of the + # return type in CHECK_EVENT_FOR_SPAM_CALLBACK, but we don't know + # for sure that the module actually returns it. + logger.warning( # type: ignore[unreachable] + "Module returned invalid value, rejecting message as spam" + ) + res = "This message has been rejected as probable spam" + else: + # The module rejected the event either with a `Codes` + # or some other `str`. In either case, we stop here. + pass + + return res + + # No spam-checker has rejected the event, let it pass. + return self.NOT_SPAM + + async def should_drop_federated_event( + self, event: "synapse.events.EventBase" + ) -> Union[bool, str]: + """Checks if a given federated event is considered "spammy" by this + server. + + If the server considers an event spammy, it will be silently dropped, + and in doing so will split-brain our view of the room's DAG. + + Args: + event: the event to be checked + + Returns: + True if the event should be silently dropped + """ + for callback in self._should_drop_federated_event_callbacks: + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + res: Union[bool, str] = await delay_cancellation(callback(event)) if res: return res @@ -276,7 +357,13 @@ async def user_may_join_room( Whether the user may join the room """ for callback in self._user_may_join_room_callbacks: - if await callback(user_id, room_id, is_invited) is False: + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + may_join_room = await delay_cancellation( + callback(user_id, room_id, is_invited) + ) + if may_join_room is False: return False return True @@ -297,7 +384,13 @@ async def user_may_invite( True if the user may send an invite, otherwise False """ for callback in self._user_may_invite_callbacks: - if await callback(inviter_userid, invitee_userid, room_id) is False: + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + may_invite = await delay_cancellation( + callback(inviter_userid, invitee_userid, room_id) + ) + if may_invite is False: return False return True @@ -322,7 +415,13 @@ async def user_may_send_3pid_invite( True if the user may send the invite, otherwise False """ for callback in self._user_may_send_3pid_invite_callbacks: - if await callback(inviter_userid, medium, address, room_id) is False: + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + may_send_3pid_invite = await delay_cancellation( + callback(inviter_userid, medium, address, room_id) + ) + if may_send_3pid_invite is False: return False return True @@ -339,7 +438,11 @@ async def user_may_create_room(self, userid: str) -> bool: True if the user may create a room, otherwise False """ for callback in self._user_may_create_room_callbacks: - if await callback(userid) is False: + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + may_create_room = await delay_cancellation(callback(userid)) + if may_create_room is False: return False return True @@ -359,7 +462,13 @@ async def user_may_create_room_alias( True if the user may create a room alias, otherwise False """ for callback in self._user_may_create_room_alias_callbacks: - if await callback(userid, room_alias) is False: + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + may_create_room_alias = await delay_cancellation( + callback(userid, room_alias) + ) + if may_create_room_alias is False: return False return True @@ -377,7 +486,11 @@ async def user_may_publish_room(self, userid: str, room_id: str) -> bool: True if the user may publish the room, otherwise False """ for callback in self._user_may_publish_room_callbacks: - if await callback(userid, room_id) is False: + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + may_publish_room = await delay_cancellation(callback(userid, room_id)) + if may_publish_room is False: return False return True @@ -398,9 +511,13 @@ async def check_username_for_spam(self, user_profile: UserProfile) -> bool: True if the user is spammy. """ for callback in self._check_username_for_spam_callbacks: - # Make a copy of the user profile object to ensure the spam checker cannot - # modify it. - if await callback(user_profile.copy()): + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + # Make a copy of the user profile object to ensure the spam checker cannot + # modify it. + res = await delay_cancellation(callback(user_profile.copy())) + if res: return True return False @@ -428,9 +545,12 @@ async def check_registration_for_spam( """ for callback in self._check_registration_for_spam_callbacks: - behaviour = await ( - callback(email_threepid, username, request_info, auth_provider_id) - ) + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + behaviour = await delay_cancellation( + callback(email_threepid, username, request_info, auth_provider_id) + ) assert isinstance(behaviour, RegistrationBehaviour) if behaviour != RegistrationBehaviour.ALLOW: return behaviour @@ -472,7 +592,10 @@ async def check_media_file_for_spam( """ for callback in self._check_media_file_for_spam_callbacks: - spam = await callback(file_wrapper, file_info) + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + spam = await delay_cancellation(callback(file_wrapper, file_info)) if spam: return True diff --git a/synapse/events/third_party_rules.py b/synapse/events/third_party_rules.py index ef68e2028220..9f4ff9799c00 100644 --- a/synapse/events/third_party_rules.py +++ b/synapse/events/third_party_rules.py @@ -14,12 +14,14 @@ import logging from typing import TYPE_CHECKING, Any, Awaitable, Callable, List, Optional, Tuple +from twisted.internet.defer import CancelledError + from synapse.api.errors import ModuleFailedException, SynapseError from synapse.events import EventBase from synapse.events.snapshot import EventContext from synapse.storage.roommember import ProfileInfo from synapse.types import Requester, StateMap -from synapse.util.async_helpers import maybe_awaitable +from synapse.util.async_helpers import delay_cancellation, maybe_awaitable if TYPE_CHECKING: from synapse.server import HomeServer @@ -263,7 +265,11 @@ async def check_event_allowed( for callback in self._check_event_allowed_callbacks: try: - res, replacement_data = await callback(event, state_events) + res, replacement_data = await delay_cancellation( + callback(event, state_events) + ) + except CancelledError: + raise except SynapseError as e: # FIXME: Being able to throw SynapseErrors is relied upon by # some modules. PR #10386 accidentally broke this ability. @@ -333,8 +339,13 @@ async def check_threepid_can_be_invited( for callback in self._check_threepid_can_be_invited_callbacks: try: - if await callback(medium, address, state_events) is False: + threepid_can_be_invited = await delay_cancellation( + callback(medium, address, state_events) + ) + if threepid_can_be_invited is False: return False + except CancelledError: + raise except Exception as e: logger.warning("Failed to run module API callback %s: %s", callback, e) @@ -361,8 +372,13 @@ async def check_visibility_can_be_modified( for callback in self._check_visibility_can_be_modified_callbacks: try: - if await callback(room_id, state_events, new_visibility) is False: + visibility_can_be_modified = await delay_cancellation( + callback(room_id, state_events, new_visibility) + ) + if visibility_can_be_modified is False: return False + except CancelledError: + raise except Exception as e: logger.warning("Failed to run module API callback %s: %s", callback, e) @@ -400,8 +416,11 @@ async def check_can_shutdown_room(self, user_id: str, room_id: str) -> bool: """ for callback in self._check_can_shutdown_room_callbacks: try: - if await callback(user_id, room_id) is False: + can_shutdown_room = await delay_cancellation(callback(user_id, room_id)) + if can_shutdown_room is False: return False + except CancelledError: + raise except Exception as e: logger.exception( "Failed to run module API callback %s: %s", callback, e @@ -422,8 +441,13 @@ async def check_can_deactivate_user( """ for callback in self._check_can_deactivate_user_callbacks: try: - if await callback(user_id, by_admin) is False: + can_deactivate_user = await delay_cancellation( + callback(user_id, by_admin) + ) + if can_deactivate_user is False: return False + except CancelledError: + raise except Exception as e: logger.exception( "Failed to run module API callback %s: %s", callback, e diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 918e87ed9cf1..ac91c5eb57d0 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -22,12 +22,12 @@ Iterable, List, Mapping, + MutableMapping, Optional, Union, ) import attr -from frozendict import frozendict from synapse.api.constants import EventContentFields, EventTypes, RelationTypes from synapse.api.errors import Codes, SynapseError @@ -39,7 +39,6 @@ if TYPE_CHECKING: from synapse.handlers.relations import BundledAggregations - from synapse.server import HomeServer # Split strings on "." but not "\." This uses a negative lookbehind assertion for '\' @@ -205,7 +204,9 @@ def _copy_field(src: JsonDict, dst: JsonDict, field: List[str]) -> None: key_to_move = field.pop(-1) sub_dict = src for sub_field in field: # e.g. sub_field => "content" - if sub_field in sub_dict and type(sub_dict[sub_field]) in [dict, frozendict]: + if sub_field in sub_dict and isinstance( + sub_dict[sub_field], collections.abc.Mapping + ): sub_dict = sub_dict[sub_field] else: return @@ -396,9 +397,6 @@ class EventClientSerializer: clients. """ - def __init__(self, hs: "HomeServer"): - self._msc3440_enabled = hs.config.experimental.msc3440_enabled - def serialize_event( self, event: Union[JsonDict, EventBase], @@ -406,6 +404,7 @@ def serialize_event( *, config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, bundle_aggregations: Optional[Dict[str, "BundledAggregations"]] = None, + apply_edits: bool = True, ) -> JsonDict: """Serializes a single event. @@ -413,10 +412,10 @@ def serialize_event( event: The event being serialized. time_now: The current time in milliseconds config: Event serialization config - bundle_aggregations: Whether to include the bundled aggregations for this - event. Only applies to non-state events. (State events never include - bundled aggregations.) - + bundle_aggregations: A map from event_id to the aggregations to be bundled + into the event. + apply_edits: Whether the content of the event should be modified to reflect + any replacement in `bundle_aggregations[].replace`. Returns: The serialized event """ @@ -428,14 +427,14 @@ def serialize_event( # Check if there are any bundled aggregations to include with the event. if bundle_aggregations: - event_aggregations = bundle_aggregations.get(event.event_id) - if event_aggregations: + if event.event_id in bundle_aggregations: self._inject_bundled_aggregations( event, time_now, config, - bundle_aggregations[event.event_id], + bundle_aggregations, serialized_event, + apply_edits=apply_edits, ) return serialized_event @@ -472,31 +471,49 @@ def _inject_bundled_aggregations( event: EventBase, time_now: int, config: SerializeEventConfig, - aggregations: "BundledAggregations", + bundled_aggregations: Dict[str, "BundledAggregations"], serialized_event: JsonDict, + apply_edits: bool, ) -> None: """Potentially injects bundled aggregations into the unsigned portion of the serialized event. Args: event: The event being serialized. time_now: The current time in milliseconds - aggregations: The bundled aggregation to serialize. - serialized_event: The serialized event which may be modified. config: Event serialization config + bundled_aggregations: Bundled aggregations to be injected. + A map from event_id to aggregation data. Must contain at least an + entry for `event`. + While serializing the bundled aggregations this map may be searched + again for additional events in a recursive manner. + serialized_event: The serialized event which may be modified. + apply_edits: Whether the content of the event should be modified to reflect + any replacement in `aggregations.replace`. """ + + # We have already checked that aggregations exist for this event. + event_aggregations = bundled_aggregations[event.event_id] + + # The JSON dictionary to be added under the unsigned property of the event + # being serialized. serialized_aggregations = {} - if aggregations.annotations: - serialized_aggregations[RelationTypes.ANNOTATION] = aggregations.annotations + if event_aggregations.annotations: + serialized_aggregations[ + RelationTypes.ANNOTATION + ] = event_aggregations.annotations - if aggregations.references: - serialized_aggregations[RelationTypes.REFERENCE] = aggregations.references + if event_aggregations.references: + serialized_aggregations[ + RelationTypes.REFERENCE + ] = event_aggregations.references - if aggregations.replace: - # If there is an edit, apply it to the event. - edit = aggregations.replace - self._apply_edit(event, serialized_event, edit) + if event_aggregations.replace: + # If there is an edit, optionally apply it to the event. + edit = event_aggregations.replace + if apply_edits: + self._apply_edit(event, serialized_event, edit) # Include information about it in the relations dict. serialized_aggregations[RelationTypes.REPLACE] = { @@ -505,19 +522,16 @@ def _inject_bundled_aggregations( "sender": edit.sender, } - # If this event is the start of a thread, include a summary of the replies. - if aggregations.thread: - thread = aggregations.thread + # Include any threaded replies to this event. + if event_aggregations.thread: + thread = event_aggregations.thread - # Don't bundle aggregations as this could recurse forever. - serialized_latest_event = serialize_event( - thread.latest_event, time_now, config=config + serialized_latest_event = self.serialize_event( + thread.latest_event, + time_now, + config=config, + bundle_aggregations=bundled_aggregations, ) - # Manually apply an edit, if one exists. - if thread.latest_edit: - self._apply_edit( - thread.latest_event, serialized_latest_event, thread.latest_edit - ) thread_summary = { "latest_event": serialized_latest_event, @@ -525,8 +539,6 @@ def _inject_bundled_aggregations( "current_user_participated": thread.current_user_participated, } serialized_aggregations[RelationTypes.THREAD] = thread_summary - if self._msc3440_enabled: - serialized_aggregations[RelationTypes.UNSTABLE_THREAD] = thread_summary # Include the bundled aggregations in the event. if serialized_aggregations: @@ -569,10 +581,20 @@ def serialize_events( ] -def copy_power_levels_contents( - old_power_levels: Mapping[str, Union[int, Mapping[str, int]]] +_PowerLevel = Union[str, int] + + +def copy_and_fixup_power_levels_contents( + old_power_levels: Mapping[str, Union[_PowerLevel, Mapping[str, _PowerLevel]]] ) -> Dict[str, Union[int, Dict[str, int]]]: - """Copy the content of a power_levels event, unfreezing frozendicts along the way + """Copy the content of a power_levels event, unfreezing frozendicts along the way. + + We accept as input power level values which are strings, provided they represent an + integer, e.g. `"`100"` instead of 100. Such strings are converted to integers + in the returned dictionary (hence "fixup" in the function name). + + Note that future room versions will outlaw such stringy power levels (see + https://github.com/matrix-org/matrix-spec/issues/853). Raises: TypeError if the input does not look like a valid power levels event content @@ -581,29 +603,47 @@ def copy_power_levels_contents( raise TypeError("Not a valid power-levels content: %r" % (old_power_levels,)) power_levels: Dict[str, Union[int, Dict[str, int]]] = {} - for k, v in old_power_levels.items(): - - if isinstance(v, int): - power_levels[k] = v - continue + for k, v in old_power_levels.items(): if isinstance(v, collections.abc.Mapping): h: Dict[str, int] = {} power_levels[k] = h for k1, v1 in v.items(): - # we should only have one level of nesting - if not isinstance(v1, int): - raise TypeError( - "Invalid power_levels value for %s.%s: %r" % (k, k1, v1) - ) - h[k1] = v1 - continue + _copy_power_level_value_as_integer(v1, h, k1) - raise TypeError("Invalid power_levels value for %s: %r" % (k, v)) + else: + _copy_power_level_value_as_integer(v, power_levels, k) return power_levels +def _copy_power_level_value_as_integer( + old_value: object, + power_levels: MutableMapping[str, Any], + key: str, +) -> None: + """Set `power_levels[key]` to the integer represented by `old_value`. + + :raises TypeError: if `old_value` is not an integer, nor a base-10 string + representation of an integer. + """ + if isinstance(old_value, int): + power_levels[key] = old_value + return + + if isinstance(old_value, str): + try: + parsed_value = int(old_value, base=10) + except ValueError: + # Fall through to the final TypeError. + pass + else: + power_levels[key] = parsed_value + return + + raise TypeError(f"Invalid power_levels value for {key}: {old_value}") + + def validate_canonicaljson(value: Any) -> None: """ Ensure that the JSON object is valid according to the rules of canonical JSON. @@ -623,7 +663,7 @@ def validate_canonicaljson(value: Any) -> None: # Note that Infinity, -Infinity, and NaN are also considered floats. raise SynapseError(400, "Bad JSON value: float", Codes.BAD_JSON) - elif isinstance(value, (dict, frozendict)): + elif isinstance(value, collections.abc.Mapping): for v in value.values(): validate_canonicaljson(v) diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 41ac49fdc8bf..7bc54b99881e 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -98,9 +98,9 @@ async def _check_sigs_and_hash( ) return redacted_event - result = await self.spam_checker.check_event_for_spam(pdu) + spam_check = await self.spam_checker.check_event_for_spam(pdu) - if result: + if spam_check != self.spam_checker.NOT_SPAM: logger.warning("Event contains spam, soft-failing %s", pdu.event_id) # we redact (to save disk space) as well as soft-failing (to stop # using the event in prev_events). diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 6a59cb4b713e..17eff60909a2 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -618,7 +618,7 @@ def _is_unknown_endpoint( # # Dendrite returns a 404 (with a body of "404 page not found"); # Conduit returns a 404 (with no body); and Synapse returns a 400 - # with M_UNRECOGNISED. + # with M_UNRECOGNIZED. # # This needs to be rather specific as some endpoints truly do return 404 # errors. @@ -1426,6 +1426,8 @@ async def send_request( room = res.get("room") if not isinstance(room, dict): raise InvalidResponseError("'room' must be a dict") + if room.get("room_id") != room_id: + raise InvalidResponseError("wrong room returned in hierarchy response") # Validate children_state of the room. children_state = room.pop("children_state", []) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 69d833585f2a..b8232e5257d2 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -110,6 +110,7 @@ def __init__(self, hs: "HomeServer"): self.handler = hs.get_federation_handler() self.storage = hs.get_storage() + self._spam_checker = hs.get_spam_checker() self._federation_event_handler = hs.get_federation_event_handler() self.state = hs.get_state_handler() self._event_auth_handler = hs.get_event_auth_handler() @@ -268,8 +269,8 @@ async def on_incoming_transaction( transaction_id=transaction_id, destination=destination, origin=origin, - origin_server_ts=transaction_data.get("origin_server_ts"), # type: ignore - pdus=transaction_data.get("pdus"), # type: ignore + origin_server_ts=transaction_data.get("origin_server_ts"), # type: ignore[arg-type] + pdus=transaction_data.get("pdus"), edus=transaction_data.get("edus"), ) @@ -515,7 +516,7 @@ async def _process_edu(edu_dict: JsonDict) -> None: ) async def on_room_state_request( - self, origin: str, room_id: str, event_id: Optional[str] + self, origin: str, room_id: str, event_id: str ) -> Tuple[int, JsonDict]: origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, room_id) @@ -530,18 +531,13 @@ async def on_room_state_request( # - but that's non-trivial to get right, and anyway somewhat defeats # the point of the linearizer. async with self._server_linearizer.queue((origin, room_id)): - resp: JsonDict = dict( - await self._state_resp_cache.wrap( - (room_id, event_id), - self._on_context_state_request_compute, - room_id, - event_id, - ) + resp = await self._state_resp_cache.wrap( + (room_id, event_id), + self._on_context_state_request_compute, + room_id, + event_id, ) - room_version = await self.store.get_room_version_id(room_id) - resp["room_version"] = room_version - return 200, resp async def on_state_ids_request( @@ -574,14 +570,11 @@ async def _on_state_ids_request_compute( return {"pdu_ids": state_ids, "auth_chain_ids": list(auth_chain_ids)} async def _on_context_state_request_compute( - self, room_id: str, event_id: Optional[str] + self, room_id: str, event_id: str ) -> Dict[str, list]: pdus: Collection[EventBase] - if event_id: - event_ids = await self.handler.get_state_ids_for_pdu(room_id, event_id) - pdus = await self.store.get_events_as_list(event_ids) - else: - pdus = (await self.state.get_current_state(room_id)).values() + event_ids = await self.handler.get_state_ids_for_pdu(room_id, event_id) + pdus = await self.store.get_events_as_list(event_ids) auth_chain = await self.store.get_auth_chain( room_id, [pdu.event_id for pdu in pdus] @@ -687,8 +680,6 @@ async def on_send_join_request( time_now = self._clock.time_msec() event_json = event.get_pdu_json(time_now) resp = { - # TODO Remove the unstable prefix when servers have updated. - "org.matrix.msc3083.v2.event": event_json, "event": event_json, "state": [p.get_pdu_json(time_now) for p in state_events], "auth_chain": [p.get_pdu_json(time_now) for p in auth_chain_events], @@ -1029,6 +1020,12 @@ async def _handle_received_pdu(self, origin: str, pdu: EventBase) -> None: except SynapseError as e: raise FederationError("ERROR", e.code, e.msg, affected=pdu.event_id) + if await self._spam_checker.should_drop_federated_event(pdu): + logger.warning( + "Unstaged federated event contains spam, dropping %s", pdu.event_id + ) + return + # Add the event to our staging area await self.store.insert_received_event_to_staging(origin, pdu) @@ -1042,6 +1039,41 @@ async def _handle_received_pdu(self, origin: str, pdu: EventBase) -> None: pdu.room_id, room_version, lock, origin, pdu ) + async def _get_next_nonspam_staged_event_for_room( + self, room_id: str, room_version: RoomVersion + ) -> Optional[Tuple[str, EventBase]]: + """Fetch the first non-spam event from staging queue. + + Args: + room_id: the room to fetch the first non-spam event in. + room_version: the version of the room. + + Returns: + The first non-spam event in that room. + """ + + while True: + # We need to do this check outside the lock to avoid a race between + # a new event being inserted by another instance and it attempting + # to acquire the lock. + next = await self.store.get_next_staged_event_for_room( + room_id, room_version + ) + + if next is None: + return None + + origin, event = next + + if await self._spam_checker.should_drop_federated_event(event): + logger.warning( + "Staged federated event contains spam, dropping %s", + event.event_id, + ) + continue + + return next + @wrap_as_background_process("_process_incoming_pdus_in_room_inner") async def _process_incoming_pdus_in_room_inner( self, @@ -1119,12 +1151,10 @@ async def _process_incoming_pdus_in_room_inner( (self._clock.time_msec() - received_ts) / 1000 ) - # We need to do this check outside the lock to avoid a race between - # a new event being inserted by another instance and it attempting - # to acquire the lock. - next = await self.store.get_next_staged_event_for_room( + next = await self._get_next_nonspam_staged_event_for_room( room_id, room_version ) + if not next: break diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 30e2421efc6d..dbe303ed9be8 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -15,7 +15,17 @@ import abc import logging from collections import OrderedDict -from typing import TYPE_CHECKING, Dict, Hashable, Iterable, List, Optional, Set, Tuple +from typing import ( + TYPE_CHECKING, + Collection, + Dict, + Hashable, + Iterable, + List, + Optional, + Set, + Tuple, +) import attr from prometheus_client import Counter @@ -343,9 +353,16 @@ async def _process_event_queue_loop(self) -> None: last_token, self._last_poked_id, limit=100 ) - logger.debug("Handling %s -> %s", last_token, next_token) + logger.debug( + "Handling %i -> %i: %i events to send (current id %i)", + last_token, + next_token, + len(events), + self._last_poked_id, + ) if not events and next_token >= self._last_poked_id: + logger.debug("All events processed") break async def handle_event(event: EventBase) -> None: @@ -353,12 +370,56 @@ async def handle_event(event: EventBase) -> None: send_on_behalf_of = event.internal_metadata.get_send_on_behalf_of() is_mine = self.is_mine_id(event.sender) if not is_mine and send_on_behalf_of is None: + logger.debug("Not sending remote-origin event %s", event) + return + + # We also want to not send out-of-band membership events. + # + # OOB memberships are used in three (and a half) situations: + # + # (1) invite events which we have received over federation. Those + # will have a `sender` on a different server, so will be + # skipped by the "is_mine" test above anyway. + # + # (2) rejections of invites to federated rooms - either remotely + # or locally generated. (Such rejections are normally + # created via federation, in which case the remote server is + # responsible for sending out the rejection. If that fails, + # we'll create a leave event locally, but that's only really + # for the benefit of the invited user - we don't have enough + # information to send it out over federation). + # + # (2a) rescinded knocks. These are identical to rejected invites. + # + # (3) knock events which we have sent over federation. As with + # invite rejections, the remote server should send them out to + # the federation. + # + # So, in all the above cases, we want to ignore such events. + # + # OOB memberships are always(?) outliers anyway, so if we *don't* + # ignore them, we'll get an exception further down when we try to + # fetch the membership list for the room. + # + # Arguably, we could equivalently ignore all outliers here, since + # in theory the only way for an outlier with a local `sender` to + # exist is by being an OOB membership (via one of (2), (2a) or (3) + # above). + # + if event.internal_metadata.is_out_of_band_membership(): + logger.debug("Not sending OOB membership event %s", event) return + # Finally, there are some other events that we should not send out + # until someone asks for them. They are explicitly flagged as such + # with `proactively_send: False`. if not event.internal_metadata.should_proactively_send(): + logger.debug( + "Not sending event with proactively_send=false: %s", event + ) return - destinations: Optional[Set[str]] = None + destinations: Optional[Collection[str]] = None if not event.prev_event_ids(): # If there are no prev event IDs then the state is empty # and so no remote servers in the room @@ -393,7 +454,7 @@ async def handle_event(event: EventBase) -> None: ) return - destinations = { + sharded_destinations = { d for d in destinations if self._federation_shard_config.should_handle( @@ -405,12 +466,12 @@ async def handle_event(event: EventBase) -> None: # If we are sending the event on behalf of another server # then it already has the event and there is no reason to # send the event to it. - destinations.discard(send_on_behalf_of) + sharded_destinations.discard(send_on_behalf_of) - logger.debug("Sending %s to %r", event, destinations) + logger.debug("Sending %s to %r", event, sharded_destinations) - if destinations: - await self._send_pdu(event, destinations) + if sharded_destinations: + await self._send_pdu(event, sharded_destinations) now = self.clock.time_msec() ts = await self.store.get_received_ts(event.event_id) @@ -419,7 +480,10 @@ async def handle_event(event: EventBase) -> None: "federation_sender" ).observe((now - ts) / 1000) - async def handle_room_events(events: Iterable[EventBase]) -> None: + async def handle_room_events(events: List[EventBase]) -> None: + logger.debug( + "Handling %i events in room %s", len(events), events[0].room_id + ) with Measure(self.clock, "handle_room_events"): for event in events: await handle_event(event) @@ -438,6 +502,7 @@ async def handle_room_events(events: Iterable[EventBase]) -> None: ) ) + logger.debug("Successfully handled up to %i", next_token) await self.store.update_federation_out_pos("events", next_token) if events: diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 01dc5ca94f99..2686ee2e511d 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -229,21 +229,21 @@ async def send_transaction( """ logger.debug( "send_data dest=%s, txid=%s", - transaction.destination, # type: ignore - transaction.transaction_id, # type: ignore + transaction.destination, + transaction.transaction_id, ) - if transaction.destination == self.server_name: # type: ignore + if transaction.destination == self.server_name: raise RuntimeError("Transport layer cannot send to itself!") # FIXME: This is only used by the tests. The actual json sent is # generated by the json_data_callback. json_data = transaction.get_dict() - path = _create_v1_path("/send/%s", transaction.transaction_id) # type: ignore + path = _create_v1_path("/send/%s", transaction.transaction_id) return await self.client.put_json( - transaction.destination, # type: ignore + transaction.destination, path=path, data=json_data, json_data_callback=json_data_callback, @@ -1363,7 +1363,7 @@ class SendJoinParser(ByteParser[SendJoinResponse]): def __init__(self, room_version: RoomVersion, v1_api: bool): self._response = SendJoinResponse([], [], event_dict={}) self._room_version = room_version - self._coros = [] + self._coros: List[Generator[None, bytes, None]] = [] # The V1 API has the shape of `[200, {...}]`, which we handle by # prefixing with `item.*`. @@ -1380,16 +1380,6 @@ def __init__(self, room_version: RoomVersion, v1_api: bool): prefix + "auth_chain.item", use_float=True, ), - # TODO Remove the unstable prefix when servers have updated. - # - # By re-using the same event dictionary this will cause the parsing of - # org.matrix.msc3083.v2.event and event to stomp over each other. - # Generally this should be fine. - ijson.kvitems_coro( - _event_parser(self._response.event_dict), - prefix + "org.matrix.msc3083.v2.event", - use_float=True, - ), ijson.kvitems_coro( _event_parser(self._response.event_dict), prefix + "event", @@ -1421,6 +1411,9 @@ def write(self, data: bytes) -> int: return len(data) def finish(self) -> SendJoinResponse: + for c in self._coros: + c.close() + if self._response.event_dict: self._response.event = make_event_from_dict( self._response.event_dict, self._room_version @@ -1440,7 +1433,7 @@ class _StateParser(ByteParser[StateRequestResponse]): def __init__(self, room_version: RoomVersion): self._response = StateRequestResponse([], []) self._room_version = room_version - self._coros = [ + self._coros: List[Generator[None, bytes, None]] = [ ijson.items_coro( _event_list_parser(room_version, self._response.state), "pdus.item", @@ -1459,4 +1452,6 @@ def write(self, data: bytes) -> int: return len(data) def finish(self) -> StateRequestResponse: + for c in self._coros: + c.close() return self._response diff --git a/synapse/federation/transport/server/_base.py b/synapse/federation/transport/server/_base.py index 2529dee613aa..84100a5a5257 100644 --- a/synapse/federation/transport/server/_base.py +++ b/synapse/federation/transport/server/_base.py @@ -16,11 +16,12 @@ import logging import re import time -from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional, Tuple, cast +from http import HTTPStatus +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Optional, Tuple, cast from synapse.api.errors import Codes, FederationDeniedError, SynapseError from synapse.api.urls import FEDERATION_V1_PREFIX -from synapse.http.server import HttpServer, ServletCallback +from synapse.http.server import HttpServer, ServletCallback, is_method_cancellable from synapse.http.servlet import parse_json_object_from_request from synapse.http.site import SynapseRequest from synapse.logging.context import run_in_background @@ -86,15 +87,24 @@ async def authenticate_request( if not auth_headers: raise NoAuthenticationError( - 401, "Missing Authorization headers", Codes.UNAUTHORIZED + HTTPStatus.UNAUTHORIZED, + "Missing Authorization headers", + Codes.UNAUTHORIZED, ) for auth in auth_headers: if auth.startswith(b"X-Matrix"): - (origin, key, sig) = _parse_auth_header(auth) + (origin, key, sig, destination) = _parse_auth_header(auth) json_request["origin"] = origin json_request["signatures"].setdefault(origin, {})[key] = sig + # if the origin_server sent a destination along it needs to match our own server_name + if destination is not None and destination != self.server_name: + raise AuthenticationError( + HTTPStatus.UNAUTHORIZED, + "Destination mismatch in auth header", + Codes.UNAUTHORIZED, + ) if ( self.federation_domain_whitelist is not None and origin not in self.federation_domain_whitelist @@ -103,7 +113,9 @@ async def authenticate_request( if origin is None or not json_request["signatures"]: raise NoAuthenticationError( - 401, "Missing Authorization headers", Codes.UNAUTHORIZED + HTTPStatus.UNAUTHORIZED, + "Missing Authorization headers", + Codes.UNAUTHORIZED, ) await self.keyring.verify_json_for_server( @@ -142,13 +154,14 @@ async def reset_retry_timings(self, origin: str) -> None: logger.exception("Error resetting retry timings on %s", origin) -def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]: +def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str, Optional[str]]: """Parse an X-Matrix auth header Args: header_bytes: header value Returns: + origin, key id, signature, destination. origin, key id, signature. Raises: @@ -156,12 +169,16 @@ def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]: """ try: header_str = header_bytes.decode("utf-8") - params = header_str.split(" ")[1].split(",") - param_dict = {k: v for k, v in (kv.split("=", maxsplit=1) for kv in params)} + params = re.split(" +", header_str)[1].split(",") + param_dict: Dict[str, str] = { + k.lower(): v for k, v in [param.split("=", maxsplit=1) for param in params] + } def strip_quotes(value: str) -> str: if value.startswith('"'): - return value[1:-1] + return re.sub( + "\\\\(.)", lambda matchobj: matchobj.group(1), value[1:-1] + ) else: return value @@ -172,7 +189,15 @@ def strip_quotes(value: str) -> str: key = strip_quotes(param_dict["key"]) sig = strip_quotes(param_dict["sig"]) - return origin, key, sig + + # get the destination server_name from the auth header if it exists + destination = param_dict.get("destination") + if destination is not None: + destination = strip_quotes(destination) + else: + destination = None + + return origin, key, sig, destination except Exception as e: logger.warning( "Error parsing auth header '%s': %s", @@ -180,7 +205,7 @@ def strip_quotes(value: str) -> str: e, ) raise AuthenticationError( - 400, "Malformed Authorization header", Codes.UNAUTHORIZED + HTTPStatus.BAD_REQUEST, "Malformed Authorization header", Codes.UNAUTHORIZED ) @@ -350,6 +375,17 @@ def register(self, server: HttpServer) -> None: if code is None: continue + if is_method_cancellable(code): + # The wrapper added by `self._wrap` will inherit the cancellable flag, + # but the wrapper itself does not support cancellation yet. + # Once resolved, the cancellation tests in + # `tests/federation/transport/server/test__base.py` can be re-enabled. + raise Exception( + f"{self.__class__.__name__}.on_{method} has been marked as " + "cancellable, but federation servlets do not support cancellation " + "yet." + ) + server.register_paths( method, (pattern,), diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py index aed3d5069ca1..6fbc7b5f15a7 100644 --- a/synapse/federation/transport/server/federation.py +++ b/synapse/federation/transport/server/federation.py @@ -160,7 +160,7 @@ async def on_GET( return await self.handler.on_room_state_request( origin, room_id, - parse_string_from_args(query, "event_id", None, required=False), + parse_string_from_args(query, "event_id", None, required=True), ) diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py index 4c3a5a6e24d1..dfd24af695ab 100644 --- a/synapse/groups/groups_server.py +++ b/synapse/groups/groups_server.py @@ -934,7 +934,7 @@ async def delete_group(self, group_id: str, requester_user_id: str) -> None: # Before deleting the group lets kick everyone out of it users = await self.store.get_users_in_group(group_id, include_private=True) - async def _kick_user_from_group(user_id): + async def _kick_user_from_group(user_id: str) -> None: if self.hs.is_mine_id(user_id): groups_local = self.hs.get_groups_local_handler() assert isinstance( diff --git a/synapse/handlers/account_data.py b/synapse/handlers/account_data.py index 4af9fbc5d10a..0478448b47ea 100644 --- a/synapse/handlers/account_data.py +++ b/synapse/handlers/account_data.py @@ -23,7 +23,7 @@ ReplicationUserAccountDataRestServlet, ) from synapse.streams import EventSource -from synapse.types import JsonDict, UserID +from synapse.types import JsonDict, StreamKeyType, UserID if TYPE_CHECKING: from synapse.server import HomeServer @@ -105,7 +105,7 @@ async def add_account_data_to_room( ) self._notifier.on_new_event( - "account_data_key", max_stream_id, users=[user_id] + StreamKeyType.ACCOUNT_DATA, max_stream_id, users=[user_id] ) await self._notify_modules(user_id, room_id, account_data_type, content) @@ -141,7 +141,7 @@ async def add_account_data_for_user( ) self._notifier.on_new_event( - "account_data_key", max_stream_id, users=[user_id] + StreamKeyType.ACCOUNT_DATA, max_stream_id, users=[user_id] ) await self._notify_modules(user_id, None, account_data_type, content) @@ -176,7 +176,7 @@ async def add_tag_to_room( ) self._notifier.on_new_event( - "account_data_key", max_stream_id, users=[user_id] + StreamKeyType.ACCOUNT_DATA, max_stream_id, users=[user_id] ) return max_stream_id else: @@ -201,7 +201,7 @@ async def remove_tag_from_room(self, user_id: str, room_id: str, tag: str) -> in ) self._notifier.on_new_event( - "account_data_key", max_stream_id, users=[user_id] + StreamKeyType.ACCOUNT_DATA, max_stream_id, users=[user_id] ) return max_stream_id else: diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py index 05a138410e25..33e45e3a1136 100644 --- a/synapse/handlers/account_validity.py +++ b/synapse/handlers/account_validity.py @@ -23,6 +23,7 @@ from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.types import UserID from synapse.util import stringutils +from synapse.util.async_helpers import delay_cancellation if TYPE_CHECKING: from synapse.server import HomeServer @@ -150,7 +151,7 @@ async def is_user_expired(self, user_id: str) -> bool: Whether the user has expired. """ for callback in self._is_user_expired_callbacks: - expired = await callback(user_id) + expired = await delay_cancellation(callback(user_id)) if expired is not None: return expired diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 1b5784050621..1da7bcc85b5c 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -38,6 +38,7 @@ JsonDict, RoomAlias, RoomStreamToken, + StreamKeyType, UserID, ) from synapse.util.async_helpers import Linearizer @@ -59,7 +60,7 @@ def __init__(self, hs: "HomeServer"): self.scheduler = hs.get_application_service_scheduler() self.started_scheduler = False self.clock = hs.get_clock() - self.notify_appservices = hs.config.appservice.notify_appservices + self.notify_appservices = hs.config.worker.should_notify_appservices self.event_sources = hs.get_event_sources() self._msc2409_to_device_messages_enabled = ( hs.config.experimental.msc2409_to_device_messages_enabled @@ -213,8 +214,8 @@ def notify_interested_services_ephemeral( Args: stream_key: The stream the event came from. - `stream_key` can be "typing_key", "receipt_key", "presence_key", - "to_device_key" or "device_list_key". Any other value for `stream_key` + `stream_key` can be StreamKeyType.TYPING, StreamKeyType.RECEIPT, StreamKeyType.PRESENCE, + StreamKeyType.TO_DEVICE or StreamKeyType.DEVICE_LIST. Any other value for `stream_key` will cause this function to return early. Ephemeral events will only be pushed to appservices that have opted into @@ -235,11 +236,11 @@ def notify_interested_services_ephemeral( # Only the following streams are currently supported. # FIXME: We should use constants for these values. if stream_key not in ( - "typing_key", - "receipt_key", - "presence_key", - "to_device_key", - "device_list_key", + StreamKeyType.TYPING, + StreamKeyType.RECEIPT, + StreamKeyType.PRESENCE, + StreamKeyType.TO_DEVICE, + StreamKeyType.DEVICE_LIST, ): return @@ -258,14 +259,14 @@ def notify_interested_services_ephemeral( # Ignore to-device messages if the feature flag is not enabled if ( - stream_key == "to_device_key" + stream_key == StreamKeyType.TO_DEVICE and not self._msc2409_to_device_messages_enabled ): return # Ignore device lists if the feature flag is not enabled if ( - stream_key == "device_list_key" + stream_key == StreamKeyType.DEVICE_LIST and not self._msc3202_transaction_extensions_enabled ): return @@ -283,15 +284,15 @@ def notify_interested_services_ephemeral( if ( stream_key in ( - "typing_key", - "receipt_key", - "presence_key", - "to_device_key", + StreamKeyType.TYPING, + StreamKeyType.RECEIPT, + StreamKeyType.PRESENCE, + StreamKeyType.TO_DEVICE, ) and service.supports_ephemeral ) or ( - stream_key == "device_list_key" + stream_key == StreamKeyType.DEVICE_LIST and service.msc3202_transaction_extensions ) ] @@ -317,7 +318,7 @@ async def _notify_interested_services_ephemeral( logger.debug("Checking interested services for %s", stream_key) with Measure(self.clock, "notify_interested_services_ephemeral"): for service in services: - if stream_key == "typing_key": + if stream_key == StreamKeyType.TYPING: # Note that we don't persist the token (via set_appservice_stream_type_pos) # for typing_key due to performance reasons and due to their highly # ephemeral nature. @@ -333,7 +334,7 @@ async def _notify_interested_services_ephemeral( async with self._ephemeral_events_linearizer.queue( (service.id, stream_key) ): - if stream_key == "receipt_key": + if stream_key == StreamKeyType.RECEIPT: events = await self._handle_receipts(service, new_token) self.scheduler.enqueue_for_appservice(service, ephemeral=events) @@ -342,7 +343,7 @@ async def _notify_interested_services_ephemeral( service, "read_receipt", new_token ) - elif stream_key == "presence_key": + elif stream_key == StreamKeyType.PRESENCE: events = await self._handle_presence(service, users, new_token) self.scheduler.enqueue_for_appservice(service, ephemeral=events) @@ -351,7 +352,7 @@ async def _notify_interested_services_ephemeral( service, "presence", new_token ) - elif stream_key == "to_device_key": + elif stream_key == StreamKeyType.TO_DEVICE: # Retrieve a list of to-device message events, as well as the # maximum stream token of the messages we were able to retrieve. to_device_messages = await self._get_to_device_messages( @@ -366,7 +367,7 @@ async def _notify_interested_services_ephemeral( service, "to_device", new_token ) - elif stream_key == "device_list_key": + elif stream_key == StreamKeyType.DEVICE_LIST: device_list_summary = await self._get_device_list_summary( service, new_token ) @@ -416,7 +417,7 @@ async def _handle_typing( return typing async def _handle_receipts( - self, service: ApplicationService, new_token: Optional[int] + self, service: ApplicationService, new_token: int ) -> List[JsonDict]: """ Return the latest read receipts that the given application service should receive. @@ -447,7 +448,7 @@ async def _handle_receipts( receipts_source = self.event_sources.sources.receipt receipts, _ = await receipts_source.get_new_events_as( - service=service, from_key=from_key + service=service, from_key=from_key, to_key=new_token ) return receipts diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 86991d26ce79..fbafbbee6b0b 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -41,6 +41,7 @@ import unpaddedbase64 from pymacaroons.exceptions import MacaroonVerificationFailedException +from twisted.internet.defer import CancelledError from twisted.web.server import Request from synapse.api.constants import LoginType @@ -67,7 +68,7 @@ from synapse.storage.roommember import ProfileInfo from synapse.types import JsonDict, Requester, UserID from synapse.util import stringutils as stringutils -from synapse.util.async_helpers import maybe_awaitable +from synapse.util.async_helpers import delay_cancellation, maybe_awaitable from synapse.util.macaroons import get_value_from_macaroon, satisfy_expiry from synapse.util.msisdn import phone_number_to_msisdn from synapse.util.stringutils import base62_encode @@ -209,7 +210,8 @@ def __init__(self, hs: "HomeServer"): self.hs = hs # FIXME better possibility to access registrationHandler later? self.macaroon_gen = hs.get_macaroon_generator() - self._password_enabled = hs.config.auth.password_enabled + self._password_enabled_for_login = hs.config.auth.password_enabled_for_login + self._password_enabled_for_reauth = hs.config.auth.password_enabled_for_reauth self._password_localdb_enabled = hs.config.auth.password_localdb_enabled self._third_party_rules = hs.get_third_party_event_rules() @@ -386,13 +388,13 @@ def get_new_session_data() -> JsonDict: return params, session_id async def _get_available_ui_auth_types(self, user: UserID) -> Iterable[str]: - """Get a list of the authentication types this user can use""" + """Get a list of the user-interactive authentication types this user can use.""" ui_auth_types = set() # if the HS supports password auth, and the user has a non-null password, we # support password auth - if self._password_localdb_enabled and self._password_enabled: + if self._password_localdb_enabled and self._password_enabled_for_reauth: lookupres = await self._find_user_id_and_pwd_hash(user.to_string()) if lookupres: _, password_hash = lookupres @@ -401,7 +403,7 @@ async def _get_available_ui_auth_types(self, user: UserID) -> Iterable[str]: # also allow auth from password providers for t in self.password_auth_provider.get_supported_login_types().keys(): - if t == LoginType.PASSWORD and not self._password_enabled: + if t == LoginType.PASSWORD and not self._password_enabled_for_reauth: continue ui_auth_types.add(t) @@ -481,7 +483,7 @@ async def check_ui_auth( sid = authdict["session"] # Convert the URI and method to strings. - uri = request.uri.decode("utf-8") # type: ignore + uri = request.uri.decode("utf-8") method = request.method.decode("utf-8") # If there's no session ID, create a new session. @@ -551,7 +553,7 @@ async def check_ui_auth( await self.store.set_ui_auth_clientdict(sid, clientdict) user_agent = get_request_user_agent(request) - clientip = request.getClientIP() + clientip = request.getClientAddress().host await self.store.add_user_agent_ip_to_ui_auth_session( session.session_id, user_agent, clientip @@ -709,7 +711,7 @@ async def _check_auth_dict( return res # fall back to the v1 login flow - canonical_id, _ = await self.validate_login(authdict) + canonical_id, _ = await self.validate_login(authdict, is_reauth=True) return canonical_id def _get_params_recaptcha(self) -> dict: @@ -1063,7 +1065,7 @@ def can_change_password(self) -> bool: Returns: Whether users on this server are allowed to change or set a password """ - return self._password_enabled and self._password_localdb_enabled + return self._password_enabled_for_login and self._password_localdb_enabled def get_supported_login_types(self) -> Iterable[str]: """Get a the login types supported for the /login API @@ -1088,9 +1090,9 @@ def get_supported_login_types(self) -> Iterable[str]: # that comes first, where it's present. if LoginType.PASSWORD in types: types.remove(LoginType.PASSWORD) - if self._password_enabled: + if self._password_enabled_for_login: types.insert(0, LoginType.PASSWORD) - elif self._password_localdb_enabled and self._password_enabled: + elif self._password_localdb_enabled and self._password_enabled_for_login: types.insert(0, LoginType.PASSWORD) return types @@ -1099,6 +1101,7 @@ async def validate_login( self, login_submission: Dict[str, Any], ratelimit: bool = False, + is_reauth: bool = False, ) -> Tuple[str, Optional[Callable[["LoginResponse"], Awaitable[None]]]]: """Authenticates the user for the /login API @@ -1109,6 +1112,9 @@ async def validate_login( login_submission: the whole of the login submission (including 'type' and other relevant fields) ratelimit: whether to apply the failed_login_attempt ratelimiter + is_reauth: whether this is part of a User-Interactive Authorisation + flow to reauthenticate for a privileged action (rather than a + new login) Returns: A tuple of the canonical user id, and optional callback to be called once the access token and device id are issued @@ -1131,8 +1137,14 @@ async def validate_login( # special case to check for "password" for the check_password interface # for the auth providers password = login_submission.get("password") + if login_type == LoginType.PASSWORD: - if not self._password_enabled: + if is_reauth: + passwords_allowed_here = self._password_enabled_for_reauth + else: + passwords_allowed_here = self._password_enabled_for_login + + if not passwords_allowed_here: raise SynapseError(400, "Password login has been disabled.") if not isinstance(password, str): raise SynapseError(400, "Bad parameter: password", Codes.INVALID_PARAM) @@ -2202,7 +2214,11 @@ async def check_auth( # other than None (i.e. until a callback returns a success) for callback in self.auth_checker_callbacks[login_type]: try: - result = await callback(username, login_type, login_dict) + result = await delay_cancellation( + callback(username, login_type, login_dict) + ) + except CancelledError: + raise except Exception as e: logger.warning("Failed to run module API callback %s: %s", callback, e) continue @@ -2263,7 +2279,9 @@ async def check_3pid_auth( for callback in self.check_3pid_auth_callbacks: try: - result = await callback(medium, address, password) + result = await delay_cancellation(callback(medium, address, password)) + except CancelledError: + raise except Exception as e: logger.warning("Failed to run module API callback %s: %s", callback, e) continue @@ -2345,7 +2363,7 @@ async def get_username_for_registration( """ for callback in self.get_username_for_registration_callbacks: try: - res = await callback(uia_results, params) + res = await delay_cancellation(callback(uia_results, params)) if isinstance(res, str): return res @@ -2359,6 +2377,8 @@ async def get_username_for_registration( callback, res, ) + except CancelledError: + raise except Exception as e: logger.error( "Module raised an exception in get_username_for_registration: %s", @@ -2388,7 +2408,7 @@ async def get_displayname_for_registration( """ for callback in self.get_displayname_for_registration_callbacks: try: - res = await callback(uia_results, params) + res = await delay_cancellation(callback(uia_results, params)) if isinstance(res, str): return res @@ -2402,6 +2422,8 @@ async def get_displayname_for_registration( callback, res, ) + except CancelledError: + raise except Exception as e: logger.error( "Module raised an exception in get_displayname_for_registration: %s", @@ -2429,7 +2451,7 @@ async def is_3pid_allowed( """ for callback in self.is_3pid_allowed_callbacks: try: - res = await callback(medium, address, registration) + res = await delay_cancellation(callback(medium, address, registration)) if res is False: return res @@ -2443,6 +2465,8 @@ async def is_3pid_allowed( callback, res, ) + except CancelledError: + raise except Exception as e: logger.error("Module raised an exception in is_3pid_allowed: %s", e) raise SynapseError(code=500, msg="Internal Server Error") diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index ffa28b2a3077..1d6d1f8a9248 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -43,6 +43,7 @@ ) from synapse.types import ( JsonDict, + StreamKeyType, StreamToken, UserID, get_domain_from_id, @@ -291,12 +292,6 @@ def __init__(self, hs: "HomeServer"): # On start up check if there are any updates pending. hs.get_reactor().callWhenRunning(self._handle_new_device_update_async) - # Used to decide if we calculate outbound pokes up front or not. By - # default we do to allow safely downgrading Synapse. - self.use_new_device_lists_changes_in_room = ( - hs.config.server.use_new_device_lists_changes_in_room - ) - def _check_device_name_length(self, name: Optional[str]) -> None: """ Checks whether a device name is longer than the maximum allowed length. @@ -490,23 +485,9 @@ async def notify_device_update( room_ids = await self.store.get_rooms_for_user(user_id) - hosts: Optional[Set[str]] = None - if not self.use_new_device_lists_changes_in_room: - hosts = set() - - if self.hs.is_mine_id(user_id): - for room_id in room_ids: - joined_users = await self.store.get_users_in_room(room_id) - hosts.update(get_domain_from_id(u) for u in joined_users) - - set_tag("target_hosts", hosts) - - hosts.discard(self.server_name) - position = await self.store.add_device_change_to_streams( user_id, device_ids, - hosts=hosts, room_ids=room_ids, ) @@ -522,19 +503,12 @@ async def notify_device_update( # specify the user ID too since the user should always get their own device list # updates, even if they aren't in any rooms. self.notifier.on_new_event( - "device_list_key", position, users={user_id}, rooms=room_ids + StreamKeyType.DEVICE_LIST, position, users={user_id}, rooms=room_ids ) - # We may need to do some processing asynchronously. - self._handle_new_device_update_async() - - if hosts: - logger.info( - "Sending device list update notif for %r to: %r", user_id, hosts - ) - for host in hosts: - self.federation_sender.send_device_messages(host, immediate=False) - log_kv({"message": "sent device update to host", "host": host}) + # We may need to do some processing asynchronously for local user IDs. + if self.hs.is_mine_id(user_id): + self._handle_new_device_update_async() async def notify_user_signature_update( self, from_user_id: str, user_ids: List[str] @@ -550,7 +524,9 @@ async def notify_user_signature_update( from_user_id, user_ids ) - self.notifier.on_new_event("device_list_key", position, users=[from_user_id]) + self.notifier.on_new_event( + StreamKeyType.DEVICE_LIST, position, users=[from_user_id] + ) async def user_left_room(self, user: UserID, room_id: str) -> None: user_id = user.to_string() @@ -677,9 +653,13 @@ async def _handle_new_device_update_async(self) -> None: return for user_id, device_id, room_id, stream_id, opentracing_context in rows: - joined_user_ids = await self.store.get_users_in_room(room_id) - hosts = {get_domain_from_id(u) for u in joined_user_ids} - hosts.discard(self.server_name) + hosts = set() + + # Ignore any users that aren't ours + if self.hs.is_mine_id(user_id): + joined_user_ids = await self.store.get_users_in_room(room_id) + hosts = {get_domain_from_id(u) for u in joined_user_ids} + hosts.discard(self.server_name) # Check if we've already sent this update to some hosts if current_stream_id == stream_id: @@ -707,9 +687,12 @@ async def _handle_new_device_update_async(self) -> None: self.federation_sender.send_device_messages( host, immediate=False ) - log_kv( - {"message": "sent device update to host", "host": host} - ) + # TODO: when called, this isn't in a logging context. + # This leads to log spam, sentry event spam, and massive + # memory usage. See #12552. + # log_kv( + # {"message": "sent device update to host", "host": host} + # ) if current_stream_id != stream_id: # Clear the set of hosts we've already sent to as we're diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index 4cb725d027c7..53668cce3bb4 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -26,7 +26,7 @@ set_tag, ) from synapse.replication.http.devices import ReplicationUserDevicesResyncRestServlet -from synapse.types import JsonDict, Requester, UserID, get_domain_from_id +from synapse.types import JsonDict, Requester, StreamKeyType, UserID, get_domain_from_id from synapse.util import json_encoder from synapse.util.stringutils import random_string @@ -151,7 +151,7 @@ async def on_direct_to_device_edu(self, origin: str, content: JsonDict) -> None: # Notify listeners that there are new to-device messages to process, # handing them the latest stream id. self.notifier.on_new_event( - "to_device_key", last_stream_id, users=local_messages.keys() + StreamKeyType.TO_DEVICE, last_stream_id, users=local_messages.keys() ) async def _check_for_unknown_devices( @@ -285,7 +285,7 @@ async def send_device_message( # Notify listeners that there are new to-device messages to process, # handing them the latest stream id. self.notifier.on_new_event( - "to_device_key", last_stream_id, users=local_messages.keys() + StreamKeyType.TO_DEVICE, last_stream_id, users=local_messages.keys() ) if self.federation_sender: diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 33d827a45b33..4aa33df884ac 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -71,6 +71,9 @@ async def _create_association( if wchar in room_alias.localpart: raise SynapseError(400, "Invalid characters in room alias") + if ":" in room_alias.localpart: + raise SynapseError(400, "Invalid character in room alias localpart: ':'.") + if not self.hs.is_mine(room_alias): raise SynapseError(400, "Room alias must be local") # TODO(erikj): Change this. diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index d6714228ef41..e6c2cfb8c8e7 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -15,7 +15,7 @@ # limitations under the License. import logging -from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple import attr from canonicaljson import encode_canonical_json @@ -1105,22 +1105,19 @@ async def _get_e2e_cross_signing_verify_key( # can request over federation raise NotFoundError("No %s key found for %s" % (key_type, user_id)) - ( - key, - key_id, - verify_key, - ) = await self._retrieve_cross_signing_keys_for_remote_user(user, key_type) - - if key is None: + cross_signing_keys = await self._retrieve_cross_signing_keys_for_remote_user( + user, key_type + ) + if cross_signing_keys is None: raise NotFoundError("No %s key found for %s" % (key_type, user_id)) - return key, key_id, verify_key + return cross_signing_keys async def _retrieve_cross_signing_keys_for_remote_user( self, user: UserID, desired_key_type: str, - ) -> Tuple[Optional[dict], Optional[str], Optional[VerifyKey]]: + ) -> Optional[Tuple[Dict[str, Any], str, VerifyKey]]: """Queries cross-signing keys for a remote user and saves them to the database Only the key specified by `key_type` will be returned, while all retrieved keys @@ -1146,12 +1143,10 @@ async def _retrieve_cross_signing_keys_for_remote_user( type(e), e, ) - return None, None, None + return None # Process each of the retrieved cross-signing keys - desired_key = None - desired_key_id = None - desired_verify_key = None + desired_key_data = None retrieved_device_ids = [] for key_type in ["master", "self_signing"]: key_content = remote_result.get(key_type + "_key") @@ -1196,9 +1191,7 @@ async def _retrieve_cross_signing_keys_for_remote_user( # If this is the desired key type, save it and its ID/VerifyKey if key_type == desired_key_type: - desired_key = key_content - desired_verify_key = verify_key - desired_key_id = key_id + desired_key_data = key_content, key_id, verify_key # At the same time, store this key in the db for subsequent queries await self.store.set_e2e_cross_signing_key( @@ -1212,7 +1205,7 @@ async def _retrieve_cross_signing_keys_for_remote_user( user.to_string(), retrieved_device_ids ) - return desired_key, desired_key_id, desired_verify_key + return desired_key_data def _check_cross_signing_key( diff --git a/synapse/handlers/event_auth.py b/synapse/handlers/event_auth.py index d441ebb0ab3d..6bed46435135 100644 --- a/synapse/handlers/event_auth.py +++ b/synapse/handlers/event_auth.py @@ -241,7 +241,15 @@ async def has_restricted_join_rules( # If the join rule is not restricted, this doesn't apply. join_rules_event = await self._store.get_event(join_rules_event_id) - return join_rules_event.content.get("join_rule") == JoinRules.RESTRICTED + content_join_rule = join_rules_event.content.get("join_rule") + if content_join_rule == JoinRules.RESTRICTED: + return True + + # also check for MSC3787 behaviour + if room_version.msc3787_knock_restricted_join_rule: + return content_join_rule == JoinRules.KNOCK_RESTRICTED + + return False async def get_rooms_that_allow_join( self, state_ids: StateMap[str] diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index d2ccb5c5d311..82a5aac3dda6 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -16,11 +16,12 @@ import random from typing import TYPE_CHECKING, Iterable, List, Optional -from synapse.api.constants import EduTypes, EventTypes, Membership +from synapse.api.constants import EduTypes, EventTypes, Membership, PresenceState from synapse.api.errors import AuthError, SynapseError from synapse.events import EventBase from synapse.events.utils import SerializeEventConfig from synapse.handlers.presence import format_user_presence_state +from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.streams.config import PaginationConfig from synapse.types import JsonDict, UserID from synapse.visibility import filter_events_for_client @@ -67,7 +68,9 @@ async def get_stream( presence_handler = self.hs.get_presence_handler() context = await presence_handler.user_syncing( - auth_user_id, affect_presence=affect_presence + auth_user_id, + affect_presence=affect_presence, + presence_state=PresenceState.ONLINE, ) with context: if timeout: @@ -139,7 +142,11 @@ def __init__(self, hs: "HomeServer"): self.storage = hs.get_storage() async def get_event( - self, user: UserID, room_id: Optional[str], event_id: str + self, + user: UserID, + room_id: Optional[str], + event_id: str, + show_redacted: bool = False, ) -> Optional[EventBase]: """Retrieve a single specified event. @@ -148,6 +155,7 @@ async def get_event( room_id: The expected room id. We'll return None if the event's room does not match. event_id: The event ID to obtain. + show_redacted: Should the full content of redacted events be returned? Returns: An event, or None if there is no event matching this ID. Raises: @@ -155,7 +163,12 @@ async def get_event( AuthError if the user does not have the rights to inspect this event. """ - event = await self.store.get_event(event_id, check_room_id=room_id) + redact_behaviour = ( + EventRedactBehaviour.as_is if show_redacted else EventRedactBehaviour.redact + ) + event = await self.store.get_event( + event_id, check_room_id=room_id, redact_behaviour=redact_behaviour + ) if not event: return None diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 78d149905f52..0386d0a07bba 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1,4 +1,4 @@ -# Copyright 2014-2021 The Matrix.org Foundation C.I.C. +# Copyright 2014-2022 The Matrix.org Foundation C.I.C. # Copyright 2020 Sorunome # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,10 +15,14 @@ """Contains handlers for federation events.""" +import enum +import itertools import logging +from enum import Enum from http import HTTPStatus from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, Union +import attr from signedjson.key import decode_verify_key_bytes from signedjson.sign import verify_signed_json from unpaddedbase64 import decode_base64 @@ -50,6 +54,7 @@ ReplicationStoreRoomOnOutlierMembershipRestServlet, ) from synapse.storage.databases.main.events_worker import EventRedactBehaviour +from synapse.storage.state import StateFilter from synapse.types import JsonDict, StateMap, get_domain_from_id from synapse.util.async_helpers import Linearizer from synapse.util.retryutils import NotRetryingDestination @@ -92,6 +97,24 @@ def get_domains_from_state(state: StateMap[EventBase]) -> List[Tuple[str, int]]: return sorted(joined_domains.items(), key=lambda d: d[1]) +class _BackfillPointType(Enum): + # a regular backwards extremity (ie, an event which we don't yet have, but which + # is referred to by other events in the DAG) + BACKWARDS_EXTREMITY = enum.auto() + + # an MSC2716 "insertion event" + INSERTION_PONT = enum.auto() + + +@attr.s(slots=True, auto_attribs=True, frozen=True) +class _BackfillPoint: + """A potential point we might backfill from""" + + event_id: str + depth: int + type: _BackfillPointType + + class FederationHandler: """Handles general incoming federation requests @@ -157,89 +180,51 @@ async def maybe_backfill( async def _maybe_backfill_inner( self, room_id: str, current_depth: int, limit: int ) -> bool: - oldest_events_with_depth = ( - await self.store.get_oldest_event_ids_with_depth_in_room(room_id) - ) + backwards_extremities = [ + _BackfillPoint(event_id, depth, _BackfillPointType.BACKWARDS_EXTREMITY) + for event_id, depth in await self.store.get_oldest_event_ids_with_depth_in_room( + room_id + ) + ] - insertion_events_to_be_backfilled: Dict[str, int] = {} + insertion_events_to_be_backfilled: List[_BackfillPoint] = [] if self.hs.config.experimental.msc2716_enabled: - insertion_events_to_be_backfilled = ( - await self.store.get_insertion_event_backward_extremities_in_room( + insertion_events_to_be_backfilled = [ + _BackfillPoint(event_id, depth, _BackfillPointType.INSERTION_PONT) + for event_id, depth in await self.store.get_insertion_event_backward_extremities_in_room( room_id ) - ) + ] logger.debug( - "_maybe_backfill_inner: extremities oldest_events_with_depth=%s insertion_events_to_be_backfilled=%s", - oldest_events_with_depth, + "_maybe_backfill_inner: backwards_extremities=%s insertion_events_to_be_backfilled=%s", + backwards_extremities, insertion_events_to_be_backfilled, ) - if not oldest_events_with_depth and not insertion_events_to_be_backfilled: + if not backwards_extremities and not insertion_events_to_be_backfilled: logger.debug("Not backfilling as no extremeties found.") return False - # We only want to paginate if we can actually see the events we'll get, - # as otherwise we'll just spend a lot of resources to get redacted - # events. - # - # We do this by filtering all the backwards extremities and seeing if - # any remain. Given we don't have the extremity events themselves, we - # need to actually check the events that reference them. - # - # *Note*: the spec wants us to keep backfilling until we reach the start - # of the room in case we are allowed to see some of the history. However - # in practice that causes more issues than its worth, as a) its - # relatively rare for there to be any visible history and b) even when - # there is its often sufficiently long ago that clients would stop - # attempting to paginate before backfill reached the visible history. - # - # TODO: If we do do a backfill then we should filter the backwards - # extremities to only include those that point to visible portions of - # history. - # - # TODO: Correctly handle the case where we are allowed to see the - # forward event but not the backward extremity, e.g. in the case of - # initial join of the server where we are allowed to see the join - # event but not anything before it. This would require looking at the - # state *before* the event, ignoring the special casing certain event - # types have. - - forward_event_ids = await self.store.get_successor_events( - list(oldest_events_with_depth) + # we now have a list of potential places to backpaginate from. We prefer to + # start with the most recent (ie, max depth), so let's sort the list. + sorted_backfill_points: List[_BackfillPoint] = sorted( + itertools.chain( + backwards_extremities, + insertion_events_to_be_backfilled, + ), + key=lambda e: -int(e.depth), ) - extremities_events = await self.store.get_events( - forward_event_ids, - redact_behaviour=EventRedactBehaviour.AS_IS, - get_prev_content=False, - ) - - # We set `check_history_visibility_only` as we might otherwise get false - # positives from users having been erased. - filtered_extremities = await filter_events_for_server( - self.storage, - self.server_name, - list(extremities_events.values()), - redact=False, - check_history_visibility_only=True, - ) logger.debug( - "_maybe_backfill_inner: filtered_extremities %s", filtered_extremities + "_maybe_backfill_inner: room_id: %s: current_depth: %s, limit: %s, " + "backfill points (%d): %s", + room_id, + current_depth, + limit, + len(sorted_backfill_points), + sorted_backfill_points, ) - if not filtered_extremities and not insertion_events_to_be_backfilled: - return False - - extremities = { - **oldest_events_with_depth, - # TODO: insertion_events_to_be_backfilled is currently skipping the filtered_extremities checks - **insertion_events_to_be_backfilled, - } - - # Check if we reached a point where we should start backfilling. - sorted_extremeties_tuple = sorted(extremities.items(), key=lambda e: -int(e[1])) - max_depth = sorted_extremeties_tuple[0][1] - # If we're approaching an extremity we trigger a backfill, otherwise we # no-op. # @@ -249,6 +234,11 @@ async def _maybe_backfill_inner( # chose more than one times the limit in case of failure, but choosing a # much larger factor will result in triggering a backfill request much # earlier than necessary. + # + # XXX: shouldn't we do this *after* the filter by depth below? Again, we don't + # care about events that have happened after our current position. + # + max_depth = sorted_backfill_points[0].depth if current_depth - 2 * limit > max_depth: logger.debug( "Not backfilling as we don't need to. %d < %d - 2 * %d", @@ -265,31 +255,98 @@ async def _maybe_backfill_inner( # 2. we have likely previously tried and failed to backfill from that # extremity, so to avoid getting "stuck" requesting the same # backfill repeatedly we drop those extremities. - filtered_sorted_extremeties_tuple = [ - t for t in sorted_extremeties_tuple if int(t[1]) <= current_depth - ] - - logger.debug( - "room_id: %s, backfill: current_depth: %s, limit: %s, max_depth: %s, extrems (%d): %s filtered_sorted_extremeties_tuple: %s", - room_id, - current_depth, - limit, - max_depth, - len(sorted_extremeties_tuple), - sorted_extremeties_tuple, - filtered_sorted_extremeties_tuple, - ) - + # # However, we need to check that the filtered extremities are non-empty. # If they are empty then either we can a) bail or b) still attempt to # backfill. We opt to try backfilling anyway just in case we do get # relevant events. - if filtered_sorted_extremeties_tuple: - sorted_extremeties_tuple = filtered_sorted_extremeties_tuple + # + filtered_sorted_backfill_points = [ + t for t in sorted_backfill_points if t.depth <= current_depth + ] + if filtered_sorted_backfill_points: + logger.debug( + "_maybe_backfill_inner: backfill points before current depth: %s", + filtered_sorted_backfill_points, + ) + sorted_backfill_points = filtered_sorted_backfill_points + else: + logger.debug( + "_maybe_backfill_inner: all backfill points are *after* current depth. Backfilling anyway." + ) + + # For performance's sake, we only want to paginate from a particular extremity + # if we can actually see the events we'll get. Otherwise, we'd just spend a lot + # of resources to get redacted events. We check each extremity in turn and + # ignore those which users on our server wouldn't be able to see. + # + # Additionally, we limit ourselves to backfilling from at most 5 extremities, + # for two reasons: + # + # - The check which determines if we can see an extremity's events can be + # expensive (we load the full state for the room at each of the backfill + # points, or (worse) their successors) + # - We want to avoid the server-server API request URI becoming too long. + # + # *Note*: the spec wants us to keep backfilling until we reach the start + # of the room in case we are allowed to see some of the history. However, + # in practice that causes more issues than its worth, as (a) it's + # relatively rare for there to be any visible history and (b) even when + # there is it's often sufficiently long ago that clients would stop + # attempting to paginate before backfill reached the visible history. + + extremities_to_request: List[str] = [] + for bp in sorted_backfill_points: + if len(extremities_to_request) >= 5: + break + + # For regular backwards extremities, we don't have the extremity events + # themselves, so we need to actually check the events that reference them - + # their "successor" events. + # + # TODO: Correctly handle the case where we are allowed to see the + # successor event but not the backward extremity, e.g. in the case of + # initial join of the server where we are allowed to see the join + # event but not anything before it. This would require looking at the + # state *before* the event, ignoring the special casing certain event + # types have. + if bp.type == _BackfillPointType.INSERTION_PONT: + event_ids_to_check = [bp.event_id] + else: + event_ids_to_check = await self.store.get_successor_events(bp.event_id) + + events_to_check = await self.store.get_events_as_list( + event_ids_to_check, + redact_behaviour=EventRedactBehaviour.as_is, + get_prev_content=False, + ) + + # We set `check_history_visibility_only` as we might otherwise get false + # positives from users having been erased. + filtered_extremities = await filter_events_for_server( + self.storage, + self.server_name, + events_to_check, + redact=False, + check_history_visibility_only=True, + ) + if filtered_extremities: + extremities_to_request.append(bp.event_id) + else: + logger.debug( + "_maybe_backfill_inner: skipping extremity %s as it would not be visible", + bp, + ) + + if not extremities_to_request: + logger.debug( + "_maybe_backfill_inner: found no extremities which would be visible" + ) + return False - # We don't want to specify too many extremities as it causes the backfill - # request URI to be too long. - extremities = dict(sorted_extremeties_tuple[:5]) + logger.debug( + "_maybe_backfill_inner: extremities_to_request %s", extremities_to_request + ) # Now we need to decide which hosts to hit first. @@ -309,7 +366,7 @@ async def try_backfill(domains: List[str]) -> bool: for dom in domains: try: await self._federation_event_handler.backfill( - dom, room_id, limit=100, extremities=extremities + dom, room_id, limit=100, extremities=extremities_to_request ) # If this succeeded then we probably already have the # appropriate stuff. @@ -466,6 +523,8 @@ async def do_invite_join( ) if ret.partial_state: + # TODO(faster_joins): roll this back if we don't manage to start the + # background resync (eg process_remote_join fails) await self.store.store_partial_state_room(room_id, ret.servers_in_room) max_stream_id = await self._federation_event_handler.process_remote_join( @@ -478,6 +537,18 @@ async def do_invite_join( partial_state=ret.partial_state, ) + if ret.partial_state: + # Kick off the process of asynchronously fetching the state for this + # room. + # + # TODO(faster_joins): pick this up again on restart + run_as_background_process( + desc="sync_partial_state_room", + func=self._sync_partial_state_room, + destination=origin, + room_id=room_id, + ) + # We wait here until this instance has seen the events come down # replication (if we're using replication) as the below uses caches. await self._replication.wait_for_stream_position( @@ -589,7 +660,7 @@ async def do_knock( # in the invitee's sync stream. It is stripped out for all other local users. event.unsigned["knock_room_state"] = stripped_room_state["knock_state_events"] - context = EventContext.for_outlier() + context = EventContext.for_outlier(self.storage) stream_id = await self._federation_event_handler.persist_events_and_notify( event.room_id, [(event, context)] ) @@ -778,7 +849,7 @@ async def on_invite_request( ) ) - context = EventContext.for_outlier() + context = EventContext.for_outlier(self.storage) await self._federation_event_handler.persist_events_and_notify( event.room_id, [(event, context)] ) @@ -807,7 +878,7 @@ async def do_remotely_reject_invite( await self.federation_client.send_leave(host_list, event) - context = EventContext.for_outlier() + context = EventContext.for_outlier(self.storage) stream_id = await self._federation_event_handler.persist_events_and_notify( event.room_id, [(event, context)] ) @@ -1189,7 +1260,9 @@ async def add_display_name_to_third_party_invite( event.content["third_party_invite"]["signed"]["token"], ) original_invite = None - prev_state_ids = await context.get_prev_state_ids() + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types([(EventTypes.ThirdPartyInvite, None)]) + ) original_invite_id = prev_state_ids.get(key) if original_invite_id: original_invite = await self.store.get_event( @@ -1238,7 +1311,9 @@ async def _check_signature(self, event: EventBase, context: EventContext) -> Non signed = event.content["third_party_invite"]["signed"] token = signed["token"] - prev_state_ids = await context.get_prev_state_ids() + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types([(EventTypes.ThirdPartyInvite, None)]) + ) invite_event_id = prev_state_ids.get((EventTypes.ThirdPartyInvite, token)) invite_event = None @@ -1370,3 +1445,64 @@ async def get_room_complexity( # We fell off the bottom, couldn't get the complexity from anyone. Oh # well. return None + + async def _sync_partial_state_room( + self, + destination: str, + room_id: str, + ) -> None: + """Background process to resync the state of a partial-state room + + Args: + destination: homeserver to pull the state from + room_id: room to be resynced + """ + + # TODO(faster_joins): do we need to lock to avoid races? What happens if other + # worker processes kick off a resync in parallel? Perhaps we should just elect + # a single worker to do the resync. + # + # TODO(faster_joins): what happens if we leave the room during a resync? if we + # really leave, that might mean we have difficulty getting the room state over + # federation. + # + # TODO(faster_joins): try other destinations if the one we have fails + + logger.info("Syncing state for room %s via %s", room_id, destination) + + # we work through the queue in order of increasing stream ordering. + while True: + batch = await self.store.get_partial_state_events_batch(room_id) + if not batch: + # all the events are updated, so we can update current state and + # clear the lazy-loading flag. + logger.info("Updating current state for %s", room_id) + assert ( + self.storage.persistence is not None + ), "TODO(faster_joins): support for workers" + await self.storage.persistence.update_current_state(room_id) + + logger.info("Clearing partial-state flag for %s", room_id) + success = await self.store.clear_partial_state_room(room_id) + if success: + logger.info("State resync complete for %s", room_id) + + # TODO(faster_joins) update room stats and user directory? + return + + # we raced against more events arriving with partial state. Go round + # the loop again. We've already logged a warning, so no need for more. + # TODO(faster_joins): there is still a race here, whereby incoming events which raced + # with us will fail to be persisted after the call to `clear_partial_state_room` due to + # having partial state. + continue + + events = await self.store.get_events_as_list( + batch, + redact_behaviour=EventRedactBehaviour.as_is, + allow_rejected=True, + ) + for event in events: + await self._federation_event_handler.update_state_for_partial_state_event( + destination, event + ) diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 03c1197c997f..ca82df8a6d9e 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -30,6 +30,7 @@ from prometheus_client import Counter +from synapse import event_auth from synapse.api.constants import ( EventContentFields, EventTypes, @@ -63,6 +64,7 @@ ) from synapse.state import StateResolutionStore from synapse.storage.databases.main.events_worker import EventRedactBehaviour +from synapse.storage.state import StateFilter from synapse.types import ( PersistedEventPosition, RoomStreamToken, @@ -103,7 +105,7 @@ def __init__(self, hs: "HomeServer"): self._event_creation_handler = hs.get_event_creation_handler() self._event_auth_handler = hs.get_event_auth_handler() self._message_handler = hs.get_message_handler() - self._action_generator = hs.get_action_generator() + self._bulk_push_rule_evaluator = hs.get_bulk_push_rule_evaluator() self._state_resolution_handler = hs.get_state_resolution_handler() # avoid a circular dependency by deferring execution here self._get_room_member_handler = hs.get_room_member_handler @@ -475,7 +477,63 @@ async def process_remote_join( # and discover that we do not have it. event.internal_metadata.proactively_send = False - return await self.persist_events_and_notify(room_id, [(event, context)]) + stream_id_after_persist = await self.persist_events_and_notify( + room_id, [(event, context)] + ) + + # If we're joining the room again, check if there is new marker + # state indicating that there is new history imported somewhere in + # the DAG. Multiple markers can exist in the current state with + # unique state_keys. + # + # Do this after the state from the remote join was persisted (via + # `persist_events_and_notify`). Otherwise we can run into a + # situation where the create event doesn't exist yet in the + # `current_state_events` + for e in state: + await self._handle_marker_event(origin, e) + + return stream_id_after_persist + + async def update_state_for_partial_state_event( + self, destination: str, event: EventBase + ) -> None: + """Recalculate the state at an event as part of a de-partial-stating process + + Args: + destination: server to request full state from + event: partial-state event to be de-partial-stated + """ + logger.info("Updating state for %s", event.event_id) + with nested_logging_context(suffix=event.event_id): + # if we have all the event's prev_events, then we can work out the + # state based on their states. Otherwise, we request it from the destination + # server. + # + # This is the same operation as we do when we receive a regular event + # over federation. + state = await self._resolve_state_at_missing_prevs(destination, event) + + # build a new state group for it if need be + context = await self._state_handler.compute_event_context( + event, + old_state=state, + ) + if context.partial_state: + # this can happen if some or all of the event's prev_events still have + # partial state - ie, an event has an earlier stream_ordering than one + # or more of its prev_events, so we de-partial-state it before its + # prev_events. + # + # TODO(faster_joins): we probably need to be more intelligent, and + # exclude partial-state prev_events from consideration + logger.warning( + "%s still has partial state: can't de-partial-state it yet", + event.event_id, + ) + return + await self._store.update_state_for_partial_state_event(event, context) + self._state_store.notify_event_un_partial_stated(event.event_id) async def backfill( self, dest: str, room_id: str, limit: int, extremities: Collection[str] @@ -820,7 +878,7 @@ async def _resolve_state_at_missing_prevs( evs = await self._store.get_events( list(state_map.values()), get_prev_content=False, - redact_behaviour=EventRedactBehaviour.AS_IS, + redact_behaviour=EventRedactBehaviour.as_is, ) event_map.update(evs) @@ -1188,6 +1246,14 @@ async def _handle_marker_event(self, origin: str, marker_event: EventBase) -> No # Nothing to retrieve then (invalid marker) return + already_seen_insertion_event = await self._store.have_seen_event( + marker_event.room_id, insertion_event_id + ) + if already_seen_insertion_event: + # No need to process a marker again if we have already seen the + # insertion event that it was pointing to + return + logger.debug( "_handle_marker_event: backfilling insertion event %s", insertion_event_id ) @@ -1383,7 +1449,7 @@ def prep(event: EventBase) -> Optional[Tuple[EventBase, EventContext]]: # we're not bothering about room state, so flag the event as an outlier. event.internal_metadata.outlier = True - context = EventContext.for_outlier() + context = EventContext.for_outlier(self._storage) try: validate_event_for_room_version(room_version_obj, event) check_auth_rules_for_event(room_version_obj, event, auth) @@ -1460,7 +1526,11 @@ async def _check_event_auth( return context # now check auth against what we think the auth events *should* be. - prev_state_ids = await context.get_prev_state_ids() + event_types = event_auth.auth_types_for_event(event.room_version, event) + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types(event_types) + ) + auth_events_ids = self._event_auth_handler.compute_auth_events( event, prev_state_ids, for_verification=True ) @@ -1834,10 +1904,10 @@ async def _update_context_for_auth_events( ) return EventContext.with_state( + storage=self._storage, state_group=state_group, state_group_before_event=context.state_group_before_event, - current_state_ids=current_state_ids, - prev_state_ids=prev_state_ids, + state_delta_due_to_event=state_updates, prev_group=prev_group, delta_ids=state_updates, partial_state=context.partial_state, @@ -1873,7 +1943,7 @@ async def _run_push_actions_and_persist_event( min_depth, ) else: - await self._action_generator.handle_push_actions_for_event( + await self._bulk_push_rule_evaluator.action_for_event_by_user( event, context ) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index c183e9c46523..9bca2bc4b24e 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -92,7 +92,7 @@ async def ratelimit_request_token_requests( """ await self._3pid_validation_ratelimiter_ip.ratelimit( - None, (medium, request.getClientIP()) + None, (medium, request.getClientAddress().host) ) await self._3pid_validation_ratelimiter_address.ratelimit( None, (medium, address) diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index a7db8feb57eb..d79248ad905b 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -30,6 +30,7 @@ Requester, RoomStreamToken, StateMap, + StreamKeyType, StreamToken, UserID, ) @@ -143,7 +144,7 @@ async def _snapshot_all_rooms( to_key=int(now_token.receipt_key), ) if self.hs.config.experimental.msc2285_enabled: - receipt = ReceiptEventSource.filter_out_hidden(receipt, user_id) + receipt = ReceiptEventSource.filter_out_private_receipts(receipt, user_id) tags_by_room = await self.store.get_tags_for_user(user_id) @@ -220,8 +221,10 @@ async def handle_room(event: RoomsForUser) -> None: self.storage, user_id, messages ) - start_token = now_token.copy_and_replace("room_key", token) - end_token = now_token.copy_and_replace("room_key", room_end_token) + start_token = now_token.copy_and_replace(StreamKeyType.ROOM, token) + end_token = now_token.copy_and_replace( + StreamKeyType.ROOM, room_end_token + ) time_now = self.clock.time_msec() d["messages"] = { @@ -369,8 +372,8 @@ async def _room_initial_sync_parted( self.storage, user_id, messages, is_peeking=is_peeking ) - start_token = StreamToken.START.copy_and_replace("room_key", token) - end_token = StreamToken.START.copy_and_replace("room_key", stream_token) + start_token = StreamToken.START.copy_and_replace(StreamKeyType.ROOM, token) + end_token = StreamToken.START.copy_and_replace(StreamKeyType.ROOM, stream_token) time_now = self.clock.time_msec() @@ -449,7 +452,9 @@ async def get_receipts() -> List[JsonDict]: if not receipts: return [] if self.hs.config.experimental.msc2285_enabled: - receipts = ReceiptEventSource.filter_out_hidden(receipts, user_id) + receipts = ReceiptEventSource.filter_out_private_receipts( + receipts, user_id + ) return receipts presence, receipts, (messages, token) = await make_deferred_yieldable( @@ -472,7 +477,7 @@ async def get_receipts() -> List[JsonDict]: self.storage, user_id, messages, is_peeking=is_peeking ) - start_token = now_token.copy_and_replace("room_key", token) + start_token = now_token.copy_and_replace(StreamKeyType.ROOM, token) end_token = now_token time_now = self.clock.time_msec() diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7db6905c6165..22cdad3f3353 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -44,7 +44,7 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.api.urls import ConsentURIBuilder from synapse.event_auth import validate_event_for_room_version -from synapse.events import EventBase +from synapse.events import EventBase, relation_from_event from synapse.events.builder import EventBuilder from synapse.events.snapshot import EventContext from synapse.events.validator import EventValidator @@ -175,17 +175,13 @@ async def get_state_events( state_filter = state_filter or StateFilter.all() if at_token: - # FIXME this claims to get the state at a stream position, but - # get_recent_events_for_room operates by topo ordering. This therefore - # does not reliably give you the state at the given stream position. - # (https://github.com/matrix-org/synapse/issues/3305) - last_events, _ = await self.store.get_recent_events_for_room( - room_id, end_token=at_token.room_key, limit=1 + last_event = await self.store.get_last_event_in_room_before_stream_ordering( + room_id, + end_token=at_token.room_key, ) - if not last_events: + if not last_event: raise NotFoundError("Can't find event for token %s" % (at_token,)) - last_event = last_events[0] # check whether the user is in the room at that time to determine # whether they should be treated as peeking. @@ -204,7 +200,7 @@ async def get_state_events( visible_events = await filter_events_for_client( self.storage, user_id, - last_events, + [last_event], filter_send_to_client=False, is_peeking=is_peeking, ) @@ -430,7 +426,7 @@ def __init__(self, hs: "HomeServer"): # This is to stop us from diverging history *too* much. self.limiter = Linearizer(max_count=5, name="room_event_creation_limit") - self.action_generator = hs.get_action_generator() + self._bulk_push_rule_evaluator = hs.get_bulk_push_rule_evaluator() self.spam_checker = hs.get_spam_checker() self.third_party_event_rules: "ThirdPartyEventRules" = ( @@ -638,7 +634,9 @@ async def create_event( # federation as well as those created locally. As of room v3, aliases events # can be created by users that are not in the room, therefore we have to # tolerate them in event_auth.check(). - prev_state_ids = await context.get_prev_state_ids() + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types([(EventTypes.Member, None)]) + ) prev_event_id = prev_state_ids.get((EventTypes.Member, event.sender)) prev_event = ( await self.store.get_event(prev_event_id, allow_none=True) @@ -761,7 +759,13 @@ async def deduplicate_state_event( The previous version of the event is returned, if it is found in the event context. Otherwise, None is returned. """ - prev_state_ids = await context.get_prev_state_ids() + if event.internal_metadata.is_outlier(): + # This can happen due to out of band memberships + return None + + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types([(event.type, None)]) + ) prev_event_id = prev_state_ids.get((event.type, event.state_key)) if not prev_event_id: return None @@ -881,11 +885,23 @@ async def create_and_send_nonmember_event( event.sender, ) - spam_error = await self.spam_checker.check_event_for_spam(event) - if spam_error: - if not isinstance(spam_error, str): - spam_error = "Spam is not permitted here" - raise SynapseError(403, spam_error, Codes.FORBIDDEN) + spam_check_result = await self.spam_checker.check_event_for_spam(event) + if spam_check_result != self.spam_checker.NOT_SPAM: + if isinstance(spam_check_result, Codes): + raise SynapseError( + 403, + "This message has been rejected as probable spam", + spam_check_result, + ) + + # Backwards compatibility: if the return value is not an error code, it + # means the module returned an error message to be included in the + # SynapseError (which is now deprecated). + raise SynapseError( + 403, + spam_check_result, + Codes.FORBIDDEN, + ) ev = await self.handle_new_client_event( requester=requester, @@ -1005,7 +1021,7 @@ async def create_new_client_event( # after it is created if builder.internal_metadata.outlier: event.internal_metadata.outlier = True - context = EventContext.for_outlier() + context = EventContext.for_outlier(self.storage) elif ( event.type == EventTypes.MSC2716_INSERTION and state_event_ids @@ -1060,20 +1076,11 @@ async def _validate_event_relation(self, event: EventBase) -> None: SynapseError if the event is invalid. """ - relation = event.content.get("m.relates_to") + relation = relation_from_event(event) if not relation: return - relation_type = relation.get("rel_type") - if not relation_type: - return - - # Ensure the parent is real. - relates_to = relation.get("event_id") - if not relates_to: - return - - parent_event = await self.store.get_event(relates_to, allow_none=True) + parent_event = await self.store.get_event(relation.parent_id, allow_none=True) if parent_event: # And in the same room. if parent_event.room_id != event.room_id: @@ -1082,31 +1089,31 @@ async def _validate_event_relation(self, event: EventBase) -> None: else: # There must be some reason that the client knows the event exists, # see if there are existing relations. If so, assume everything is fine. - if not await self.store.event_is_target_of_relation(relates_to): + if not await self.store.event_is_target_of_relation(relation.parent_id): # Otherwise, the client can't know about the parent event! raise SynapseError(400, "Can't send relation to unknown event") # If this event is an annotation then we check that that the sender # can't annotate the same way twice (e.g. stops users from liking an # event multiple times). - if relation_type == RelationTypes.ANNOTATION: - aggregation_key = relation["key"] + if relation.rel_type == RelationTypes.ANNOTATION: + aggregation_key = relation.aggregation_key + + if aggregation_key is None: + raise SynapseError(400, "Missing aggregation key") if len(aggregation_key) > 500: raise SynapseError(400, "Aggregation key is too long") already_exists = await self.store.has_user_annotated_event( - relates_to, event.type, aggregation_key, event.sender + relation.parent_id, event.type, aggregation_key, event.sender ) if already_exists: raise SynapseError(400, "Can't send same reaction twice") # Don't attempt to start a thread if the parent event is a relation. - elif ( - relation_type == RelationTypes.THREAD - or relation_type == RelationTypes.UNSTABLE_THREAD - ): - if await self.store.event_includes_relation(relates_to): + elif relation.rel_type == RelationTypes.THREAD: + if await self.store.event_includes_relation(relation.parent_id): raise SynapseError( 400, "Cannot start threads from an event with a relation" ) @@ -1252,7 +1259,9 @@ async def _persist_event( # and `state_groups` because they have `prev_events` that aren't persisted yet # (historical messages persisted in reverse-chronological order). if not event.internal_metadata.is_historical(): - await self.action_generator.handle_push_actions_for_event(event, context) + await self._bulk_push_rule_evaluator.action_for_event_by_user( + event, context + ) try: # If we're a worker we need to hit out to the master. @@ -1414,7 +1423,7 @@ async def persist_and_notify_client_event( original_event = await self.store.get_event( event.redacts, - redact_behaviour=EventRedactBehaviour.AS_IS, + redact_behaviour=EventRedactBehaviour.as_is, get_prev_content=False, allow_rejected=False, allow_none=True, @@ -1434,7 +1443,7 @@ async def persist_and_notify_client_event( # Validate a newly added alias or newly added alt_aliases. original_alias = None - original_alt_aliases: List[str] = [] + original_alt_aliases: object = [] original_event_id = event.unsigned.get("replaces_state") if original_event_id: @@ -1462,6 +1471,7 @@ async def persist_and_notify_client_event( # If the old version of alt_aliases is of an unknown form, # completely replace it. if not isinstance(original_alt_aliases, (list, tuple)): + # TODO: check that the original_alt_aliases' entries are all strings original_alt_aliases = [] # Check that each alias is currently valid. @@ -1511,7 +1521,7 @@ async def persist_and_notify_client_event( original_event = await self.store.get_event( event.redacts, - redact_behaviour=EventRedactBehaviour.AS_IS, + redact_behaviour=EventRedactBehaviour.as_is, get_prev_content=False, allow_rejected=False, allow_none=True, @@ -1553,7 +1563,11 @@ async def persist_and_notify_client_event( "Redacting MSC2716 events is not supported in this room version", ) - prev_state_ids = await context.get_prev_state_ids() + event_types = event_auth.auth_types_for_event(event.room_version, event) + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types(event_types) + ) + auth_events_ids = self._event_auth_handler.compute_auth_events( event, prev_state_ids, for_verification=True ) diff --git a/synapse/handlers/oidc.py b/synapse/handlers/oidc.py index 724b9cfcb4bb..9de61d554f41 100644 --- a/synapse/handlers/oidc.py +++ b/synapse/handlers/oidc.py @@ -224,7 +224,7 @@ async def handle_oidc_callback(self, request: SynapseRequest) -> None: self._sso_handler.render_error(request, "invalid_session", str(e)) return except MacaroonInvalidSignatureException as e: - logger.exception("Could not verify session for OIDC callback") + logger.warning("Could not verify session for OIDC callback: %s", e) self._sso_handler.render_error(request, "mismatching_session", str(e)) return @@ -827,7 +827,7 @@ async def handle_oidc_callback( logger.debug("Exchanging OAuth2 code for a token") token = await self._exchange_code(code) except OidcError as e: - logger.exception("Could not exchange OAuth2 code") + logger.warning("Could not exchange OAuth2 code: %s", e) self._sso_handler.render_error(request, e.error, e.error_description) return @@ -966,7 +966,7 @@ async def oidc_response_to_user_attributes(failures: int) -> UserAttributes: "Mapping provider does not support de-duplicating Matrix IDs" ) - attributes = await self._user_mapping_provider.map_user_attributes( # type: ignore + attributes = await self._user_mapping_provider.map_user_attributes( userinfo, token ) diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 7ee334037376..19a440705027 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -27,7 +27,7 @@ from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.state import StateFilter from synapse.streams.config import PaginationConfig -from synapse.types import JsonDict, Requester +from synapse.types import JsonDict, Requester, StreamKeyType from synapse.util.async_helpers import ReadWriteLock from synapse.util.stringutils import random_string from synapse.visibility import filter_events_for_client @@ -239,7 +239,7 @@ async def purge_history_for_rooms_in_range( # defined in the server's configuration, we can safely assume that's the # case and use it for this room. max_lifetime = ( - retention_policy["max_lifetime"] or self._retention_default_max_lifetime + retention_policy.max_lifetime or self._retention_default_max_lifetime ) # Cap the effective max_lifetime to be within the range allowed in the @@ -448,7 +448,7 @@ async def get_messages( ) # We expect `/messages` to use historic pagination tokens by default but # `/messages` should still works with live tokens when manually provided. - assert from_token.room_key.topological + assert from_token.room_key.topological is not None if pagin_config.limit is None: # This shouldn't happen as we've set a default limit before this @@ -491,7 +491,7 @@ async def get_messages( if leave_token.topological < curr_topo: from_token = from_token.copy_and_replace( - "room_key", leave_token + StreamKeyType.ROOM, leave_token ) await self.hs.get_federation_handler().maybe_backfill( @@ -513,7 +513,7 @@ async def get_messages( event_filter=event_filter, ) - next_token = from_token.copy_and_replace("room_key", next_key) + next_token = from_token.copy_and_replace(StreamKeyType.ROOM, next_key) if events: if event_filter: diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 215ca0585052..2f9c9e109023 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -66,7 +66,7 @@ from synapse.replication.tcp.streams import PresenceFederationStream, PresenceStream from synapse.storage.databases.main import DataStore from synapse.streams import EventSource -from synapse.types import JsonDict, UserID, get_domain_from_id +from synapse.types import JsonDict, StreamKeyType, UserID, get_domain_from_id from synapse.util.async_helpers import Linearizer from synapse.util.caches.descriptors import _CacheContext, cached from synapse.util.metrics import Measure @@ -151,7 +151,7 @@ def __init__(self, hs: "HomeServer"): @abc.abstractmethod async def user_syncing( - self, user_id: str, affect_presence: bool + self, user_id: str, affect_presence: bool, presence_state: str ) -> ContextManager[None]: """Returns a context manager that should surround any stream requests from the user. @@ -165,6 +165,7 @@ async def user_syncing( affect_presence: If false this function will be a no-op. Useful for streams that are not associated with an actual client that is being used by a user. + presence_state: The presence state indicated in the sync request """ @abc.abstractmethod @@ -228,6 +229,11 @@ async def current_state_for_users( return states + async def current_state_for_user(self, user_id: str) -> UserPresenceState: + """Get the current presence state for a user.""" + res = await self.current_state_for_users([user_id]) + return res[user_id] + @abc.abstractmethod async def set_state( self, @@ -461,7 +467,7 @@ def send_stop_syncing(self) -> None: self.send_user_sync(user_id, False, last_sync_ms) async def user_syncing( - self, user_id: str, affect_presence: bool + self, user_id: str, affect_presence: bool, presence_state: str ) -> ContextManager[None]: """Record that a user is syncing. @@ -471,6 +477,17 @@ async def user_syncing( if not affect_presence or not self._presence_enabled: return _NullContextManager() + prev_state = await self.current_state_for_user(user_id) + if prev_state != PresenceState.BUSY: + # We set state here but pass ignore_status_msg = True as we don't want to + # cause the status message to be cleared. + # Note that this causes last_active_ts to be incremented which is not + # what the spec wants: see comment in the BasePresenceHandler version + # of this function. + await self.set_state( + UserID.from_string(user_id), {"presence": presence_state}, True + ) + curr_sync = self._user_to_num_current_syncs.get(user_id, 0) self._user_to_num_current_syncs[user_id] = curr_sync + 1 @@ -505,7 +522,7 @@ async def notify_from_replication( room_ids_to_states, users_to_states = parties self.notifier.on_new_event( - "presence_key", + StreamKeyType.PRESENCE, stream_id, rooms=room_ids_to_states.keys(), users=users_to_states.keys(), @@ -642,27 +659,28 @@ def __init__(self, hs: "HomeServer"): ) now = self.clock.time_msec() - for state in self.user_to_current_state.values(): - self.wheel_timer.insert( - now=now, obj=state.user_id, then=state.last_active_ts + IDLE_TIMER - ) - self.wheel_timer.insert( - now=now, - obj=state.user_id, - then=state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, - ) - if self.is_mine_id(state.user_id): + if self._presence_enabled: + for state in self.user_to_current_state.values(): self.wheel_timer.insert( - now=now, - obj=state.user_id, - then=state.last_federation_update_ts + FEDERATION_PING_INTERVAL, + now=now, obj=state.user_id, then=state.last_active_ts + IDLE_TIMER ) - else: self.wheel_timer.insert( now=now, obj=state.user_id, - then=state.last_federation_update_ts + FEDERATION_TIMEOUT, + then=state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, ) + if self.is_mine_id(state.user_id): + self.wheel_timer.insert( + now=now, + obj=state.user_id, + then=state.last_federation_update_ts + FEDERATION_PING_INTERVAL, + ) + else: + self.wheel_timer.insert( + now=now, + obj=state.user_id, + then=state.last_federation_update_ts + FEDERATION_TIMEOUT, + ) # Set of users who have presence in the `user_to_current_state` that # have not yet been persisted @@ -787,6 +805,13 @@ async def _update_states( This is currently used to bump the max presence stream ID without changing any user's presence (see PresenceHandler.add_users_to_send_full_presence_to). """ + if not self._presence_enabled: + # We shouldn't get here if presence is disabled, but we check anyway + # to ensure that we don't a) send out presence federation and b) + # don't add things to the wheel timer that will never be handled. + logger.warning("Tried to update presence states when presence is disabled") + return + now = self.clock.time_msec() with Measure(self.clock, "presence_update_states"): @@ -942,7 +967,10 @@ async def bump_presence_active_time(self, user: UserID) -> None: await self._update_states([prev_state.copy_and_replace(**new_fields)]) async def user_syncing( - self, user_id: str, affect_presence: bool = True + self, + user_id: str, + affect_presence: bool = True, + presence_state: str = PresenceState.ONLINE, ) -> ContextManager[None]: """Returns a context manager that should surround any stream requests from the user. @@ -956,6 +984,7 @@ async def user_syncing( affect_presence: If false this function will be a no-op. Useful for streams that are not associated with an actual client that is being used by a user. + presence_state: The presence state indicated in the sync request """ # Override if it should affect the user's presence, if presence is # disabled. @@ -967,9 +996,25 @@ async def user_syncing( self.user_to_num_current_syncs[user_id] = curr_sync + 1 prev_state = await self.current_state_for_user(user_id) + + # If they're busy then they don't stop being busy just by syncing, + # so just update the last sync time. + if prev_state.state != PresenceState.BUSY: + # XXX: We set_state separately here and just update the last_active_ts above + # This keeps the logic as similar as possible between the worker and single + # process modes. Using set_state will actually cause last_active_ts to be + # updated always, which is not what the spec calls for, but synapse has done + # this for... forever, I think. + await self.set_state( + UserID.from_string(user_id), {"presence": presence_state}, True + ) + # Retrieve the new state for the logic below. This should come from the + # in-memory cache. + prev_state = await self.current_state_for_user(user_id) + + # To keep the single process behaviour consistent with worker mode, run the + # same logic as `update_external_syncs_row`, even though it looks weird. if prev_state.state == PresenceState.OFFLINE: - # If they're currently offline then bring them online, otherwise - # just update the last sync times. await self._update_states( [ prev_state.copy_and_replace( @@ -979,6 +1024,10 @@ async def user_syncing( ) ] ) + # otherwise, set the new presence state & update the last sync time, + # but don't update last_active_ts as this isn't an indication that + # they've been active (even though it's probably been updated by + # set_state above) else: await self._update_states( [ @@ -1086,11 +1135,6 @@ async def update_external_syncs_clear(self, process_id: str) -> None: ) self.external_process_last_updated_ms.pop(process_id, None) - async def current_state_for_user(self, user_id: str) -> UserPresenceState: - """Get the current presence state for a user.""" - res = await self.current_state_for_users([user_id]) - return res[user_id] - async def _persist_and_notify(self, states: List[UserPresenceState]) -> None: """Persist states in the database, poke the notifier and send to interested remote servers @@ -1101,7 +1145,7 @@ async def _persist_and_notify(self, states: List[UserPresenceState]) -> None: room_ids_to_states, users_to_states = parties self.notifier.on_new_event( - "presence_key", + StreamKeyType.PRESENCE, stream_id, rooms=room_ids_to_states.keys(), users=[UserID.from_string(u) for u in users_to_states], @@ -1193,6 +1237,10 @@ async def set_state( ): raise SynapseError(400, "Invalid presence state") + # If presence is disabled, no-op + if not self.hs.config.server.use_presence: + return + user_id = target_user.to_string() prev_state = await self.current_state_for_user(user_id) diff --git a/synapse/handlers/push_rules.py b/synapse/handlers/push_rules.py new file mode 100644 index 000000000000..2599160bcc00 --- /dev/null +++ b/synapse/handlers/push_rules.py @@ -0,0 +1,138 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING, List, Optional, Union + +import attr + +from synapse.api.errors import SynapseError, UnrecognizedRequestError +from synapse.push.baserules import BASE_RULE_IDS +from synapse.storage.push_rule import RuleNotFoundException +from synapse.types import JsonDict + +if TYPE_CHECKING: + from synapse.server import HomeServer + + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class RuleSpec: + scope: str + template: str + rule_id: str + attr: Optional[str] + + +class PushRulesHandler: + """A class to handle changes in push rules for users.""" + + def __init__(self, hs: "HomeServer"): + self._notifier = hs.get_notifier() + self._main_store = hs.get_datastores().main + + async def set_rule_attr( + self, user_id: str, spec: RuleSpec, val: Union[bool, JsonDict] + ) -> None: + """Set an attribute (enabled or actions) on an existing push rule. + + Notifies listeners (e.g. sync handler) of the change. + + Args: + user_id: the user for which to modify the push rule. + spec: the spec of the push rule to modify. + val: the value to change the attribute to. + + Raises: + RuleNotFoundException if the rule being modified doesn't exist. + SynapseError(400) if the value is malformed. + UnrecognizedRequestError if the attribute to change is unknown. + InvalidRuleException if we're trying to change the actions on a rule but + the provided actions aren't compliant with the spec. + """ + if spec.attr not in ("enabled", "actions"): + # for the sake of potential future expansion, shouldn't report + # 404 in the case of an unknown request so check it corresponds to + # a known attribute first. + raise UnrecognizedRequestError() + + namespaced_rule_id = f"global/{spec.template}/{spec.rule_id}" + rule_id = spec.rule_id + is_default_rule = rule_id.startswith(".") + if is_default_rule: + if namespaced_rule_id not in BASE_RULE_IDS: + raise RuleNotFoundException("Unknown rule %r" % (namespaced_rule_id,)) + if spec.attr == "enabled": + if isinstance(val, dict) and "enabled" in val: + val = val["enabled"] + if not isinstance(val, bool): + # Legacy fallback + # This should *actually* take a dict, but many clients pass + # bools directly, so let's not break them. + raise SynapseError(400, "Value for 'enabled' must be boolean") + await self._main_store.set_push_rule_enabled( + user_id, namespaced_rule_id, val, is_default_rule + ) + elif spec.attr == "actions": + if not isinstance(val, dict): + raise SynapseError(400, "Value must be a dict") + actions = val.get("actions") + if not isinstance(actions, list): + raise SynapseError(400, "Value for 'actions' must be dict") + check_actions(actions) + rule_id = spec.rule_id + is_default_rule = rule_id.startswith(".") + if is_default_rule: + if namespaced_rule_id not in BASE_RULE_IDS: + raise RuleNotFoundException( + "Unknown rule %r" % (namespaced_rule_id,) + ) + await self._main_store.set_push_rule_actions( + user_id, namespaced_rule_id, actions, is_default_rule + ) + else: + raise UnrecognizedRequestError() + + self.notify_user(user_id) + + def notify_user(self, user_id: str) -> None: + """Notify listeners about a push rule change. + + Args: + user_id: the user ID the change is for. + """ + stream_id = self._main_store.get_max_push_rules_stream_id() + self._notifier.on_new_event("push_rules_key", stream_id, users=[user_id]) + + +def check_actions(actions: List[Union[str, JsonDict]]) -> None: + """Check if the given actions are spec compliant. + + Args: + actions: the actions to check. + + Raises: + InvalidRuleException if the rules aren't compliant with the spec. + """ + if not isinstance(actions, list): + raise InvalidRuleException("No actions found") + + for a in actions: + if a in ["notify", "dont_notify", "coalesce"]: + pass + elif isinstance(a, dict) and "set_tweak" in a: + pass + else: + raise InvalidRuleException("Unrecognised action %s" % a) + + +class InvalidRuleException(Exception): + pass diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 6250bb3bdf2b..e6a35f1d093c 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -14,10 +14,16 @@ import logging from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple -from synapse.api.constants import ReadReceiptEventFields, ReceiptTypes +from synapse.api.constants import ReceiptTypes from synapse.appservice import ApplicationService from synapse.streams import EventSource -from synapse.types import JsonDict, ReadReceipt, UserID, get_domain_from_id +from synapse.types import ( + JsonDict, + ReadReceipt, + StreamKeyType, + UserID, + get_domain_from_id, +) if TYPE_CHECKING: from synapse.server import HomeServer @@ -112,7 +118,7 @@ async def _handle_new_receipts(self, receipts: List[ReadReceipt]) -> bool: ) if not res: - # res will be None if this read receipt is 'old' + # res will be None if this receipt is 'old' continue stream_id, max_persisted_id = res @@ -129,7 +135,9 @@ async def _handle_new_receipts(self, receipts: List[ReadReceipt]) -> bool: affected_room_ids = list({r.room_id for r in receipts}) - self.notifier.on_new_event("receipt_key", max_batch_id, rooms=affected_room_ids) + self.notifier.on_new_event( + StreamKeyType.RECEIPT, max_batch_id, rooms=affected_room_ids + ) # Note that the min here shouldn't be relied upon to be accurate. await self.hs.get_pusherpool().on_new_receipts( min_batch_id, max_batch_id, affected_room_ids @@ -138,7 +146,7 @@ async def _handle_new_receipts(self, receipts: List[ReadReceipt]) -> bool: return True async def received_client_receipt( - self, room_id: str, receipt_type: str, user_id: str, event_id: str, hidden: bool + self, room_id: str, receipt_type: str, user_id: str, event_id: str ) -> None: """Called when a client tells us a local user has read up to the given event_id in the room. @@ -148,16 +156,14 @@ async def received_client_receipt( receipt_type=receipt_type, user_id=user_id, event_ids=[event_id], - data={"ts": int(self.clock.time_msec()), "hidden": hidden}, + data={"ts": int(self.clock.time_msec())}, ) is_new = await self._handle_new_receipts([receipt]) if not is_new: return - if self.federation_sender and not ( - self.hs.config.experimental.msc2285_enabled and hidden - ): + if self.federation_sender and receipt_type != ReceiptTypes.READ_PRIVATE: await self.federation_sender.send_read_receipt(receipt) @@ -167,52 +173,69 @@ def __init__(self, hs: "HomeServer"): self.config = hs.config @staticmethod - def filter_out_hidden(events: List[JsonDict], user_id: str) -> List[JsonDict]: - visible_events = [] - - # filter out hidden receipts the user shouldn't see - for event in events: - content = event.get("content", {}) - new_event = event.copy() - new_event["content"] = {} - - for event_id in content.keys(): - event_content = content.get(event_id, {}) - m_read = event_content.get(ReceiptTypes.READ, {}) - - # If m_read is missing copy over the original event_content as there is nothing to process here - if not m_read: - new_event["content"][event_id] = event_content.copy() - continue - - new_users = {} - for rr_user_id, user_rr in m_read.items(): - try: - hidden = user_rr.get("hidden") - except AttributeError: - # Due to https://github.com/matrix-org/synapse/issues/10376 - # there are cases where user_rr is a string, in those cases - # we just ignore the read receipt - continue + def filter_out_private_receipts( + rooms: List[JsonDict], user_id: str + ) -> List[JsonDict]: + """ + Filters a list of serialized receipts (as returned by /sync and /initialSync) + and removes private read receipts of other users. - if hidden is not True or rr_user_id == user_id: - new_users[rr_user_id] = user_rr.copy() - # If hidden has a value replace hidden with the correct prefixed key - if hidden is not None: - new_users[rr_user_id].pop("hidden") - new_users[rr_user_id][ - ReadReceiptEventFields.MSC2285_HIDDEN - ] = hidden + This operates on the return value of get_linearized_receipts_for_rooms(), + which is wrapped in a cache. Care must be taken to ensure that the input + values are not modified. - # Set new users unless empty - if len(new_users.keys()) > 0: - new_event["content"][event_id] = {ReceiptTypes.READ: new_users} + Args: + rooms: A list of mappings, each mapping has a `content` field, which + is a map of event ID -> receipt type -> user ID -> receipt information. - # Append new_event to visible_events unless empty - if len(new_event["content"].keys()) > 0: - visible_events.append(new_event) + Returns: + The same as rooms, but filtered. + """ - return visible_events + result = [] + + # Iterate through each room's receipt content. + for room in rooms: + # The receipt content with other user's private read receipts removed. + content = {} + + # Iterate over each event ID / receipts for that event. + for event_id, orig_event_content in room.get("content", {}).items(): + event_content = orig_event_content + # If there are private read receipts, additional logic is necessary. + if ReceiptTypes.READ_PRIVATE in event_content: + # Make a copy without private read receipts to avoid leaking + # other user's private read receipts.. + event_content = { + receipt_type: receipt_value + for receipt_type, receipt_value in event_content.items() + if receipt_type != ReceiptTypes.READ_PRIVATE + } + + # Copy the current user's private read receipt from the + # original content, if it exists. + user_private_read_receipt = orig_event_content[ + ReceiptTypes.READ_PRIVATE + ].get(user_id, None) + if user_private_read_receipt: + event_content[ReceiptTypes.READ_PRIVATE] = { + user_id: user_private_read_receipt + } + + # Include the event if there is at least one non-private read + # receipt or the current user has a private read receipt. + if event_content: + content[event_id] = event_content + + # Include the event if there is at least one non-private read receipt + # or the current user has a private read receipt. + if content: + # Build a new event to avoid mutating the cache. + new_room = {k: v for k, v in room.items() if k != "content"} + new_room["content"] = content + result.append(new_room) + + return result async def get_new_events( self, @@ -234,18 +257,21 @@ async def get_new_events( ) if self.config.experimental.msc2285_enabled: - events = ReceiptEventSource.filter_out_hidden(events, user.to_string()) + events = ReceiptEventSource.filter_out_private_receipts( + events, user.to_string() + ) return events, to_key async def get_new_events_as( - self, from_key: int, service: ApplicationService + self, from_key: int, to_key: int, service: ApplicationService ) -> Tuple[List[JsonDict], int]: """Returns a set of new read receipt events that an appservice may be interested in. Args: from_key: the stream position at which events should be fetched from + to_key: the stream position up to which events should be fetched to service: The appservice which may be interested Returns: @@ -255,7 +281,6 @@ async def get_new_events_as( * The current read receipt stream token. """ from_key = int(from_key) - to_key = self.get_current_key() if from_key == to_key: return [], to_key diff --git a/synapse/handlers/relations.py b/synapse/handlers/relations.py index 0be231957750..ab7e54857d56 100644 --- a/synapse/handlers/relations.py +++ b/synapse/handlers/relations.py @@ -24,11 +24,10 @@ ) import attr -from frozendict import frozendict from synapse.api.constants import RelationTypes from synapse.api.errors import SynapseError -from synapse.events import EventBase +from synapse.events import EventBase, relation_from_event from synapse.storage.databases.main.relations import _RelatedEvent from synapse.types import JsonDict, Requester, StreamToken, UserID from synapse.visibility import filter_events_for_client @@ -44,8 +43,6 @@ class _ThreadAggregation: # The latest event in the thread. latest_event: EventBase - # The latest edit to the latest event in the thread. - latest_edit: Optional[EventBase] # The total number of events in the thread. count: int # True if the current user has sent an event to the thread. @@ -256,64 +253,6 @@ async def get_annotations_for_event( return filtered_results - async def _get_bundled_aggregation_for_event( - self, event: EventBase, ignored_users: FrozenSet[str] - ) -> Optional[BundledAggregations]: - """Generate bundled aggregations for an event. - - Note that this does not use a cache, but depends on cached methods. - - Args: - event: The event to calculate bundled aggregations for. - ignored_users: The users ignored by the requesting user. - - Returns: - The bundled aggregations for an event, if bundled aggregations are - enabled and the event can have bundled aggregations. - """ - - # Do not bundle aggregations for an event which represents an edit or an - # annotation. It does not make sense for them to have related events. - relates_to = event.content.get("m.relates_to") - if isinstance(relates_to, (dict, frozendict)): - relation_type = relates_to.get("rel_type") - if relation_type in (RelationTypes.ANNOTATION, RelationTypes.REPLACE): - return None - - event_id = event.event_id - room_id = event.room_id - - # The bundled aggregations to include, a mapping of relation type to a - # type-specific value. Some types include the direct return type here - # while others need more processing during serialization. - aggregations = BundledAggregations() - - annotations = await self.get_annotations_for_event( - event_id, room_id, ignored_users=ignored_users - ) - if annotations: - aggregations.annotations = {"chunk": annotations} - - references, next_token = await self.get_relations_for_event( - event_id, - event, - room_id, - RelationTypes.REFERENCE, - ignored_users=ignored_users, - ) - if references: - aggregations.references = { - "chunk": [{"event_id": event.event_id} for event in references] - } - - if next_token: - aggregations.references["next_batch"] = await next_token.to_string( - self._main_store - ) - - # Store the bundled aggregations in the event metadata for later use. - return aggregations - async def get_threads_for_events( self, event_ids: Collection[str], user_id: str, ignored_users: FrozenSet[str] ) -> Dict[str, _ThreadAggregation]: @@ -353,7 +292,7 @@ async def get_threads_for_events( for event_id, summary in summaries.items(): if summary: - thread_count, latest_thread_event, edit = summary + thread_count, latest_thread_event = summary # Subtract off the count of any ignored users. for ignored_user in ignored_users: @@ -398,7 +337,6 @@ async def get_threads_for_events( results[event_id] = _ThreadAggregation( latest_event=latest_thread_event, - latest_edit=edit, count=thread_count, # If there's a thread summary it must also exist in the # participated dictionary. @@ -417,15 +355,38 @@ async def get_bundled_aggregations( user_id: The user requesting the bundled aggregations. Returns: - A map of event ID to the bundled aggregation for the event. Not all - events may have bundled aggregations in the results. + A map of event ID to the bundled aggregations for the event. + + Not all requested events may exist in the results (if they don't have + bundled aggregations). + + The results may include additional events which are related to the + requested events. """ - # De-duplicate events by ID to handle the same event requested multiple times. - # - # State events do not get bundled aggregations. - events_by_id = { - event.event_id: event for event in events if not event.is_state() - } + # De-duplicated events by ID to handle the same event requested multiple times. + events_by_id = {} + # A map of event ID to the relation in that event, if there is one. + relations_by_id: Dict[str, str] = {} + for event in events: + # State events do not get bundled aggregations. + if event.is_state(): + continue + + relates_to = relation_from_event(event) + if relates_to: + # An event which is a replacement (ie edit) or annotation (ie, + # reaction) may not have any other event related to it. + if relates_to.rel_type in ( + RelationTypes.ANNOTATION, + RelationTypes.REPLACE, + ): + continue + + # Track the event's relation information for later. + relations_by_id[event.event_id] = relates_to.rel_type + + # The event should get bundled aggregations. + events_by_id[event.event_id] = event # event ID -> bundled aggregation in non-serialized form. results: Dict[str, BundledAggregations] = {} @@ -433,13 +394,60 @@ async def get_bundled_aggregations( # Fetch any ignored users of the requesting user. ignored_users = await self._main_store.ignored_users(user_id) + # Threads are special as the latest event of a thread might cause additional + # events to be fetched. Thus, we check those first! + + # Fetch thread summaries (but only for the directly requested events). + threads = await self.get_threads_for_events( + # It is not valid to start a thread on an event which itself relates to another event. + [eid for eid in events_by_id.keys() if eid not in relations_by_id], + user_id, + ignored_users, + ) + for event_id, thread in threads.items(): + results.setdefault(event_id, BundledAggregations()).thread = thread + + # If the latest event in a thread is not already being fetched, + # add it. This ensures that the bundled aggregations for the + # latest thread event is correct. + latest_thread_event = thread.latest_event + if latest_thread_event and latest_thread_event.event_id not in events_by_id: + events_by_id[latest_thread_event.event_id] = latest_thread_event + # Keep relations_by_id in sync with events_by_id: + # + # We know that the latest event in a thread has a thread relation + # (as that is what makes it part of the thread). + relations_by_id[latest_thread_event.event_id] = RelationTypes.THREAD + # Fetch other relations per event. for event in events_by_id.values(): - event_result = await self._get_bundled_aggregation_for_event( - event, ignored_users + # Fetch any annotations (ie, reactions) to bundle with this event. + annotations = await self.get_annotations_for_event( + event.event_id, event.room_id, ignored_users=ignored_users + ) + if annotations: + results.setdefault( + event.event_id, BundledAggregations() + ).annotations = {"chunk": annotations} + + # Fetch any references to bundle with this event. + references, next_token = await self.get_relations_for_event( + event.event_id, + event, + event.room_id, + RelationTypes.REFERENCE, + ignored_users=ignored_users, ) - if event_result: - results[event.event_id] = event_result + if references: + aggregations = results.setdefault(event.event_id, BundledAggregations()) + aggregations.references = { + "chunk": [{"event_id": ev.event_id} for ev in references] + } + + if next_token: + aggregations.references["next_batch"] = await next_token.to_string( + self._main_store + ) # Fetch any edits (but not for redacted events). # @@ -455,10 +463,4 @@ async def get_bundled_aggregations( for event_id, edit in edits.items(): results.setdefault(event_id, BundledAggregations()).replace = edit - threads = await self.get_threads_for_events( - events_by_id.keys(), user_id, ignored_users - ) - for event_id, thread in threads.items(): - results.setdefault(event_id, BundledAggregations()).thread = thread - return results diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index b31f00b517a9..92e1de050071 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -33,6 +33,7 @@ import attr from typing_extensions import TypedDict +import synapse.events.snapshot from synapse.api.constants import ( EventContentFields, EventTypes, @@ -57,7 +58,7 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.event_auth import validate_event_for_room_version from synapse.events import EventBase -from synapse.events.utils import copy_power_levels_contents +from synapse.events.utils import copy_and_fixup_power_levels_contents from synapse.federation.federation_client import InvalidResponseError from synapse.handlers.federation import get_domains_from_state from synapse.handlers.relations import BundledAggregations @@ -72,12 +73,12 @@ RoomID, RoomStreamToken, StateMap, + StreamKeyType, StreamToken, UserID, create_requester, ) from synapse.util import stringutils -from synapse.util.async_helpers import Linearizer from synapse.util.caches.response_cache import ResponseCache from synapse.util.stringutils import parse_and_validate_server_name from synapse.visibility import filter_events_for_client @@ -149,10 +150,11 @@ def __init__(self, hs: "HomeServer"): ) preset_config["encrypted"] = encrypted - self._replication = hs.get_replication_data_handler() + self._default_power_level_content_override = ( + self.config.room.default_power_level_content_override + ) - # linearizer to stop two upgrades happening at once - self._upgrade_linearizer = Linearizer("room_upgrade_linearizer") + self._replication = hs.get_replication_data_handler() # If a user tries to update the same room multiple times in quick # succession, only process the first attempt and return its result to @@ -196,6 +198,39 @@ async def upgrade_room( 400, "An upgrade for this room is currently in progress" ) + # Check whether the room exists and 404 if it doesn't. + # We could go straight for the auth check, but that will raise a 403 instead. + old_room = await self.store.get_room(old_room_id) + if old_room is None: + raise NotFoundError("Unknown room id %s" % (old_room_id,)) + + new_room_id = self._generate_room_id() + + # Check whether the user has the power level to carry out the upgrade. + # `check_auth_rules_from_context` will check that they are in the room and have + # the required power level to send the tombstone event. + ( + tombstone_event, + tombstone_context, + ) = await self.event_creation_handler.create_event( + requester, + { + "type": EventTypes.Tombstone, + "state_key": "", + "room_id": old_room_id, + "sender": user_id, + "content": { + "body": "This room has been replaced", + "replacement_room": new_room_id, + }, + }, + ) + old_room_version = await self.store.get_room_version(old_room_id) + validate_event_for_room_version(old_room_version, tombstone_event) + await self._event_auth_handler.check_auth_rules_from_context( + old_room_version, tombstone_event, tombstone_context + ) + # Upgrade the room # # If this user has sent multiple upgrade requests for the same room @@ -206,19 +241,35 @@ async def upgrade_room( self._upgrade_room, requester, old_room_id, - new_version, # args for _upgrade_room + old_room, # args for _upgrade_room + new_room_id, + new_version, + tombstone_event, + tombstone_context, ) return ret async def _upgrade_room( - self, requester: Requester, old_room_id: str, new_version: RoomVersion + self, + requester: Requester, + old_room_id: str, + old_room: Dict[str, Any], + new_room_id: str, + new_version: RoomVersion, + tombstone_event: EventBase, + tombstone_context: synapse.events.snapshot.EventContext, ) -> str: """ Args: requester: the user requesting the upgrade old_room_id: the id of the room to be replaced - new_versions: the version to upgrade the room to + old_room: a dict containing room information for the room to be replaced, + as returned by `RoomWorkerStore.get_room`. + new_room_id: the id of the replacement room + new_version: the version to upgrade the room to + tombstone_event: the tombstone event to send to the old room + tombstone_context: the context for the tombstone event Raises: ShadowBanError if the requester is shadow-banned. @@ -226,40 +277,15 @@ async def _upgrade_room( user_id = requester.user.to_string() assert self.hs.is_mine_id(user_id), "User must be our own: %s" % (user_id,) - # start by allocating a new room id - r = await self.store.get_room(old_room_id) - if r is None: - raise NotFoundError("Unknown room id %s" % (old_room_id,)) - new_room_id = await self._generate_room_id( - creator_id=user_id, - is_public=r["is_public"], - room_version=new_version, - ) - logger.info("Creating new room %s to replace %s", new_room_id, old_room_id) - # we create and auth the tombstone event before properly creating the new - # room, to check our user has perms in the old room. - ( - tombstone_event, - tombstone_context, - ) = await self.event_creation_handler.create_event( - requester, - { - "type": EventTypes.Tombstone, - "state_key": "", - "room_id": old_room_id, - "sender": user_id, - "content": { - "body": "This room has been replaced", - "replacement_room": new_room_id, - }, - }, - ) - old_room_version = await self.store.get_room_version(old_room_id) - validate_event_for_room_version(old_room_version, tombstone_event) - await self._event_auth_handler.check_auth_rules_from_context( - old_room_version, tombstone_event, tombstone_context + # create the new room. may raise a `StoreError` in the exceedingly unlikely + # event of a room ID collision. + await self.store.store_room( + room_id=new_room_id, + room_creator_user_id=user_id, + is_public=old_room["is_public"], + room_version=new_version, ) await self.clone_existing_room( @@ -277,7 +303,10 @@ async def _upgrade_room( context=tombstone_context, ) - old_room_state = await tombstone_context.get_current_state_ids() + state_filter = StateFilter.from_types( + [(EventTypes.CanonicalAlias, ""), (EventTypes.PowerLevels, "")] + ) + old_room_state = await tombstone_context.get_current_state_ids(state_filter) # We know the tombstone event isn't an outlier so it has current state. assert old_room_state is not None @@ -337,13 +366,13 @@ async def _update_upgraded_room_pls( # 50, but if the default PL in a room is 50 or more, then we set the # required PL above that. - pl_content = dict(old_room_pl_state.content) - users_default = int(pl_content.get("users_default", 0)) + pl_content = copy_and_fixup_power_levels_contents(old_room_pl_state.content) + users_default: int = pl_content.get("users_default", 0) # type: ignore[assignment] restricted_level = max(users_default + 1, 50) updated = False for v in ("invite", "events_default"): - current = int(pl_content.get(v, 0)) + current: int = pl_content.get(v, 0) # type: ignore[assignment] if current < restricted_level: logger.debug( "Setting level for %s in %s to %i (was %i)", @@ -380,7 +409,9 @@ async def _update_upgraded_room_pls( "state_key": "", "room_id": new_room_id, "sender": requester.user.to_string(), - "content": old_room_pl_state.content, + "content": copy_and_fixup_power_levels_contents( + old_room_pl_state.content + ), }, ratelimit=False, ) @@ -399,7 +430,7 @@ async def clone_existing_room( requester: the user requesting the upgrade old_room_id : the id of the room to be replaced new_room_id: the id to give the new room (should already have been - created with _gemerate_room_id()) + created with _generate_room_id()) new_room_version: the new room version to use tombstone_event_id: the ID of the tombstone event in the old room. """ @@ -441,14 +472,14 @@ async def clone_existing_room( (EventTypes.PowerLevels, ""), ] - # If the old room was a space, copy over the room type and the rooms in - # the space. - if ( - old_room_create_event.content.get(EventContentFields.ROOM_TYPE) - == RoomTypes.SPACE - ): - creation_content[EventContentFields.ROOM_TYPE] = RoomTypes.SPACE - types_to_copy.append((EventTypes.SpaceChild, None)) + # Copy the room type as per MSC3818. + room_type = old_room_create_event.content.get(EventContentFields.ROOM_TYPE) + if room_type is not None: + creation_content[EventContentFields.ROOM_TYPE] = room_type + + # If the old room was a space, copy over the rooms in the space. + if room_type == RoomTypes.SPACE: + types_to_copy.append((EventTypes.SpaceChild, None)) old_room_state_ids = await self.store.get_filtered_current_state_ids( old_room_id, StateFilter.from_types(types_to_copy) @@ -471,7 +502,7 @@ async def clone_existing_room( # dict so we can't just copy.deepcopy it. initial_state[ (EventTypes.PowerLevels, "") - ] = power_levels = copy_power_levels_contents( + ] = power_levels = copy_and_fixup_power_levels_contents( initial_state[(EventTypes.PowerLevels, "")] ) @@ -723,6 +754,21 @@ async def create_room( if wchar in config["room_alias_name"]: raise SynapseError(400, "Invalid characters in room alias") + if ":" in config["room_alias_name"]: + # Prevent someone from trying to pass in a full alias here. + # Note that it's permissible for a room alias to have multiple + # hash symbols at the start (notably bridged over from IRC, too), + # but the first colon in the alias is defined to separate the local + # part from the server name. + # (remember server names can contain port numbers, also separated + # by a colon. But under no circumstances should the local part be + # allowed to contain a colon!) + raise SynapseError( + 400, + "':' is not permitted in the room alias name. " + "Please note this expects a local part — 'wombat', not '#wombat:example.com'.", + ) + room_alias = RoomAlias(config["room_alias_name"], self.hs.hostname) mapping = await self.store.get_association_from_room_alias(room_alias) @@ -776,7 +822,7 @@ async def create_room( visibility = config.get("visibility", "private") is_public = visibility == "public" - room_id = await self._generate_room_id( + room_id = await self._generate_and_create_room_id( creator_id=user_id, is_public=is_public, room_version=room_version, @@ -1040,9 +1086,19 @@ async def send(etype: str, content: JsonDict, **kwargs: Any) -> int: for invitee in invite_list: power_level_content["users"][invitee] = 100 - # Power levels overrides are defined per chat preset + # If the user supplied a preset name e.g. "private_chat", + # we apply that preset power_level_content.update(config["power_level_content_override"]) + # If the server config contains default_power_level_content_override, + # and that contains information for this room preset, apply it. + if self._default_power_level_content_override: + override = self._default_power_level_content_override.get(preset_config) + if override is not None: + power_level_content.update(override) + + # Finally, if the user supplied specific permissions for this room, + # apply those. if power_level_content_override: power_level_content.update(power_level_content_override) @@ -1088,7 +1144,26 @@ async def send(etype: str, content: JsonDict, **kwargs: Any) -> int: return last_sent_stream_id - async def _generate_room_id( + def _generate_room_id(self) -> str: + """Generates a random room ID. + + Room IDs look like "!opaque_id:domain" and are case-sensitive as per the spec + at https://spec.matrix.org/v1.2/appendices/#room-ids-and-event-ids. + + Does not check for collisions with existing rooms or prevent future calls from + returning the same room ID. To ensure the uniqueness of a new room ID, use + `_generate_and_create_room_id` instead. + + Synapse's room IDs are 18 [a-zA-Z] characters long, which comes out to around + 102 bits. + + Returns: + A random room ID of the form "!opaque_id:domain". + """ + random_string = stringutils.random_string(18) + return RoomID(random_string, self.hs.hostname).to_string() + + async def _generate_and_create_room_id( self, creator_id: str, is_public: bool, @@ -1099,8 +1174,7 @@ async def _generate_room_id( attempts = 0 while attempts < 5: try: - random_string = stringutils.random_string(18) - gen_room_id = RoomID(random_string, self.hs.hostname).to_string() + gen_room_id = self._generate_room_id() await self.store.store_room( room_id=gen_room_id, room_creator_user_id=creator_id, @@ -1237,10 +1311,10 @@ async def filter_evts(events: List[EventBase]) -> List[EventBase]: events_after=events_after, state=await filter_evts(state_events), aggregations=aggregations, - start=await token.copy_and_replace("room_key", results.start).to_string( - self.store - ), - end=await token.copy_and_replace("room_key", results.end).to_string( + start=await token.copy_and_replace( + StreamKeyType.ROOM, results.start + ).to_string(self.store), + end=await token.copy_and_replace(StreamKeyType.ROOM, results.end).to_string( self.store ), ) diff --git a/synapse/handlers/room_batch.py b/synapse/handlers/room_batch.py index 78e299d3a5c5..fbfd7484065c 100644 --- a/synapse/handlers/room_batch.py +++ b/synapse/handlers/room_batch.py @@ -53,8 +53,9 @@ async def inherit_depth_from_prev_ids(self, prev_event_ids: List[str]) -> int: # We want to use the successor event depth so they appear after `prev_event` because # it has a larger `depth` but before the successor event because the `stream_ordering` # is negative before the successor event. + assert most_recent_prev_event_id is not None successor_event_ids = await self.store.get_successor_events( - [most_recent_prev_event_id] + most_recent_prev_event_id ) # If we can't find any successor events, then it's a forward extremity of @@ -139,6 +140,7 @@ async def get_most_recent_full_state_ids_from_event_id_list( _, ) = await self.store.get_max_depth_of(event_ids) # mapping from (type, state_key) -> state_event_id + assert most_recent_event_id is not None prev_state_map = await self.state_store.get_state_ids_for_event( most_recent_event_id ) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 802e57c4d0cc..ea876c168de7 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -38,6 +38,7 @@ from synapse.events import EventBase from synapse.events.snapshot import EventContext from synapse.handlers.profile import MAX_AVATAR_URL_LEN, MAX_DISPLAYNAME_LEN +from synapse.storage.state import StateFilter from synapse.types import ( JsonDict, Requester, @@ -362,7 +363,9 @@ async def _local_membership_update( historical=historical, ) - prev_state_ids = await context.get_prev_state_ids() + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types([(EventTypes.Member, None)]) + ) prev_member_event_id = prev_state_ids.get((EventTypes.Member, user_id), None) @@ -1160,7 +1163,9 @@ async def send_membership_event( else: requester = types.create_requester(target_user) - prev_state_ids = await context.get_prev_state_ids() + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types([(EventTypes.GuestAccess, None)]) + ) if event.membership == Membership.JOIN: if requester.is_guest: guest_can_join = await self._can_guest_join(prev_state_ids) diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index 486145f48aca..af83de319348 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -105,6 +105,7 @@ def __init__(self, hs: "HomeServer"): hs.get_clock(), "get_room_hierarchy", ) + self._msc3266_enabled = hs.config.experimental.msc3266_enabled async def get_room_hierarchy( self, @@ -561,8 +562,13 @@ async def _is_local_room_accessible( if join_rules_event_id: join_rules_event = await self._store.get_event(join_rules_event_id) join_rule = join_rules_event.content.get("join_rule") - if join_rule == JoinRules.PUBLIC or ( - room_version.msc2403_knocking and join_rule == JoinRules.KNOCK + if ( + join_rule == JoinRules.PUBLIC + or (room_version.msc2403_knocking and join_rule == JoinRules.KNOCK) + or ( + room_version.msc3787_knock_restricted_join_rule + and join_rule == JoinRules.KNOCK_RESTRICTED + ) ): return True @@ -630,7 +636,7 @@ async def _is_local_room_accessible( return False async def _is_remote_room_accessible( - self, requester: str, room_id: str, room: JsonDict + self, requester: Optional[str], room_id: str, room: JsonDict ) -> bool: """ Calculate whether the room received over federation should be shown to the requester. @@ -645,7 +651,8 @@ async def _is_remote_room_accessible( due to an invite, etc. Args: - requester: The user requesting the summary. + requester: The user requesting the summary. If not passed only world + readability is checked. room_id: The room ID returned over federation. room: The summary of the room returned over federation. @@ -659,6 +666,8 @@ async def _is_remote_room_accessible( or room.get("world_readable") is True ): return True + elif not requester: + return False # Check if the user is a member of any of the allowed rooms from the response. allowed_rooms = room.get("allowed_room_ids") @@ -715,6 +724,10 @@ async def _build_room_entry(self, room_id: str, for_federation: bool) -> JsonDic "room_type": create_event.content.get(EventContentFields.ROOM_TYPE), } + if self._msc3266_enabled: + entry["im.nheko.summary.version"] = stats["version"] + entry["im.nheko.summary.encryption"] = stats["encryption"] + # Federation requests need to provide additional information so the # requested server is able to filter the response appropriately. if for_federation: @@ -812,9 +825,45 @@ async def get_room_summary( room_summary["membership"] = membership or "leave" else: - # TODO federation API, descoped from initial unstable implementation - # as MSC needs more maturing on that side. - raise SynapseError(400, "Federation is not currently supported.") + # Reuse the hierarchy query over federation + if remote_room_hosts is None: + raise SynapseError(400, "Missing via to query remote room") + + ( + room_entry, + children_room_entries, + inaccessible_children, + ) = await self._summarize_remote_room_hierarchy( + _RoomQueueEntry(room_id, remote_room_hosts), + suggested_only=True, + ) + + # The results over federation might include rooms that we, as the + # requesting server, are allowed to see, but the requesting user is + # not permitted to see. + # + # Filter the returned results to only what is accessible to the user. + if not room_entry or not await self._is_remote_room_accessible( + requester, room_entry.room_id, room_entry.room + ): + raise NotFoundError("Room not found or is not accessible") + + room = dict(room_entry.room) + room.pop("allowed_room_ids", None) + + # If there was a requester, add their membership. + # We keep the membership in the local membership table unless the + # room is purged even for remote rooms. + if requester: + ( + membership, + _, + ) = await self._store.get_local_current_membership_for_user_in_room( + requester, room_id + ) + room["membership"] = membership or "leave" + + return room return room_summary diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 102dd4b57dea..cd1c47dae8b1 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -24,7 +24,7 @@ from synapse.api.filtering import Filter from synapse.events import EventBase from synapse.storage.state import StateFilter -from synapse.types import JsonDict, UserID +from synapse.types import JsonDict, StreamKeyType, UserID from synapse.visibility import filter_events_for_client if TYPE_CHECKING: @@ -357,7 +357,7 @@ async def _search( itertools.chain( # The events_before and events_after for each context. itertools.chain.from_iterable( - itertools.chain(context["events_before"], context["events_after"]) # type: ignore[arg-type] + itertools.chain(context["events_before"], context["events_after"]) for context in contexts.values() ), # The returned events. @@ -373,10 +373,10 @@ async def _search( for context in contexts.values(): context["events_before"] = self._event_serializer.serialize_events( - context["events_before"], time_now, bundle_aggregations=aggregations # type: ignore[arg-type] + context["events_before"], time_now, bundle_aggregations=aggregations ) context["events_after"] = self._event_serializer.serialize_events( - context["events_after"], time_now, bundle_aggregations=aggregations # type: ignore[arg-type] + context["events_after"], time_now, bundle_aggregations=aggregations ) results = [ @@ -655,11 +655,11 @@ async def _calculate_event_contexts( "events_before": events_before, "events_after": events_after, "start": await now_token.copy_and_replace( - "room_key", res.start + StreamKeyType.ROOM, res.start + ).to_string(self.store), + "end": await now_token.copy_and_replace( + StreamKeyType.ROOM, res.end ).to_string(self.store), - "end": await now_token.copy_and_replace("room_key", res.end).to_string( - self.store - ), } if include_profile: diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index e4fe94e557ad..1e171f3f7115 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -468,7 +468,7 @@ async def complete_sso_login_request( auth_provider_id, remote_user_id, get_request_user_agent(request), - request.getClientIP(), + request.getClientAddress().host, ) new_user = True elif self._sso_update_profile_information: @@ -928,7 +928,7 @@ async def register_sso_user(self, request: Request, session_id: str) -> None: session.auth_provider_id, session.remote_user_id, get_request_user_agent(request), - request.getClientIP(), + request.getClientAddress().host, ) logger.info( diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 6c8b17c4205d..59b5d497be68 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -37,6 +37,7 @@ Requester, RoomStreamToken, StateMap, + StreamKeyType, StreamToken, UserID, ) @@ -410,10 +411,10 @@ async def current_sync_for_user( set_tag(SynapseTags.SYNC_RESULT, bool(sync_result)) return sync_result - async def push_rules_for_user(self, user: UserID) -> JsonDict: + async def push_rules_for_user(self, user: UserID) -> Dict[str, Dict[str, list]]: user_id = user.to_string() - rules = await self.store.get_push_rules_for_user(user_id) - rules = format_push_rules_for_user(user, rules) + rules_raw = await self.store.get_push_rules_for_user(user_id) + rules = format_push_rules_for_user(user, rules_raw) return rules async def ephemeral_by_room( @@ -449,7 +450,7 @@ async def ephemeral_by_room( room_ids=room_ids, is_guest=sync_config.is_guest, ) - now_token = now_token.copy_and_replace("typing_key", typing_key) + now_token = now_token.copy_and_replace(StreamKeyType.TYPING, typing_key) ephemeral_by_room: JsonDict = {} @@ -471,7 +472,7 @@ async def ephemeral_by_room( room_ids=room_ids, is_guest=sync_config.is_guest, ) - now_token = now_token.copy_and_replace("receipt_key", receipt_key) + now_token = now_token.copy_and_replace(StreamKeyType.RECEIPT, receipt_key) for event in receipts: room_id = event["room_id"] @@ -537,7 +538,9 @@ async def _load_filtered_recents( prev_batch_token = now_token if recents: room_key = recents[0].internal_metadata.before - prev_batch_token = now_token.copy_and_replace("room_key", room_key) + prev_batch_token = now_token.copy_and_replace( + StreamKeyType.ROOM, room_key + ) return TimelineBatch( events=recents, prev_batch=prev_batch_token, limited=False @@ -611,7 +614,7 @@ async def _load_filtered_recents( recents = recents[-timeline_limit:] room_key = recents[0].internal_metadata.before - prev_batch_token = now_token.copy_and_replace("room_key", room_key) + prev_batch_token = now_token.copy_and_replace(StreamKeyType.ROOM, room_key) # Don't bother to bundle aggregations if the timeline is unlimited, # as clients will have all the necessary information. @@ -661,16 +664,15 @@ async def get_state_at( stream_position: point at which to get state state_filter: The state filter used to fetch state from the database. """ - # FIXME this claims to get the state at a stream position, but - # get_recent_events_for_room operates by topo ordering. This therefore - # does not reliably give you the state at the given stream position. - # (https://github.com/matrix-org/synapse/issues/3305) - last_events, _ = await self.store.get_recent_events_for_room( - room_id, end_token=stream_position.room_key, limit=1 + # FIXME: This gets the state at the latest event before the stream ordering, + # which might not be the same as the "current state" of the room at the time + # of the stream token if there were multiple forward extremities at the time. + last_event = await self.store.get_last_event_in_room_before_stream_ordering( + room_id, + end_token=stream_position.room_key, ) - if last_events: - last_event = last_events[-1] + if last_event: state = await self.get_state_after_event( last_event, state_filter=state_filter or StateFilter.all() ) @@ -1046,7 +1048,7 @@ async def unread_notifs_for_room_id( last_unread_event_id = await self.store.get_last_receipt_event_id_for_user( user_id=sync_config.user.to_string(), room_id=room_id, - receipt_type=ReceiptTypes.READ, + receipt_types=(ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE), ) return await self.store.get_unread_event_push_actions_by_room_for_user( @@ -1399,7 +1401,7 @@ async def _generate_sync_entry_for_to_device( now_token.to_device_key, ) sync_result_builder.now_token = now_token.copy_and_replace( - "to_device_key", stream_id + StreamKeyType.TO_DEVICE, stream_id ) sync_result_builder.to_device = messages else: @@ -1504,7 +1506,7 @@ async def _generate_sync_entry_for_presence( ) assert presence_key sync_result_builder.now_token = now_token.copy_and_replace( - "presence_key", presence_key + StreamKeyType.PRESENCE, presence_key ) extra_users_ids = set(newly_joined_or_invited_users) @@ -1827,7 +1829,7 @@ async def _get_rooms_changed( # stream token as it'll only be used in the context of this # room. (c.f. the docstring of `to_room_stream_token`). leave_token = since_token.copy_and_replace( - "room_key", leave_position.to_room_stream_token() + StreamKeyType.ROOM, leave_position.to_room_stream_token() ) # If this is an out of band message, like a remote invite @@ -1876,7 +1878,9 @@ async def _get_rooms_changed( if room_entry: events, start_key = room_entry - prev_batch_token = now_token.copy_and_replace("room_key", start_key) + prev_batch_token = now_token.copy_and_replace( + StreamKeyType.ROOM, start_key + ) entry = RoomSyncResultBuilder( room_id=room_id, @@ -1973,7 +1977,7 @@ async def _get_all_rooms( continue leave_token = now_token.copy_and_replace( - "room_key", RoomStreamToken(None, event.stream_ordering) + StreamKeyType.ROOM, RoomStreamToken(None, event.stream_ordering) ) room_entries.append( RoomSyncResultBuilder( diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 6854428b7ca5..bb00750bfd47 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -25,7 +25,7 @@ ) from synapse.replication.tcp.streams import TypingStream from synapse.streams import EventSource -from synapse.types import JsonDict, Requester, UserID, get_domain_from_id +from synapse.types import JsonDict, Requester, StreamKeyType, UserID, get_domain_from_id from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.util.metrics import Measure from synapse.util.wheel_timer import WheelTimer @@ -382,7 +382,7 @@ def _push_update_local(self, member: RoomMember, typing: bool) -> None: ) self.notifier.on_new_event( - "typing_key", self._latest_room_serial, rooms=[member.room_id] + StreamKeyType.TYPING, self._latest_room_serial, rooms=[member.room_id] ) async def get_all_typing_updates( diff --git a/synapse/handlers/ui_auth/checkers.py b/synapse/handlers/ui_auth/checkers.py index 472b029af3f6..05cebb5d4d89 100644 --- a/synapse/handlers/ui_auth/checkers.py +++ b/synapse/handlers/ui_auth/checkers.py @@ -256,7 +256,9 @@ class RegistrationTokenAuthChecker(UserInteractiveAuthChecker): def __init__(self, hs: "HomeServer"): super().__init__(hs) self.hs = hs - self._enabled = bool(hs.config.registration.registration_requires_token) + self._enabled = bool( + hs.config.registration.registration_requires_token + ) or bool(hs.config.registration.enable_registration_token_3pid_bypass) self.store = hs.get_datastores().main def is_enabled(self) -> bool: diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 048fd4bb8225..74f7fdfe6ce5 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -60,7 +60,7 @@ def __init__(self, hs: "HomeServer"): self.clock = hs.get_clock() self.notifier = hs.get_notifier() self.is_mine_id = hs.is_mine_id - self.update_user_directory = hs.config.server.update_user_directory + self.update_user_directory = hs.config.worker.should_update_user_directory self.search_all_users = hs.config.userdirectory.user_directory_search_all_users self.spam_checker = hs.get_spam_checker() # The current position in the current_state_delta stream diff --git a/synapse/http/client.py b/synapse/http/client.py index 8310fb466ac5..084d0a5b84e9 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -43,8 +43,10 @@ from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet.interfaces import ( IAddress, + IDelayedCall, IHostResolution, IReactorPluggableNameResolver, + IReactorTime, IResolutionReceiver, ITCPTransport, ) @@ -121,13 +123,15 @@ def check_against_blacklist( _EPSILON = 0.00000001 -def _make_scheduler(reactor): +def _make_scheduler( + reactor: IReactorTime, +) -> Callable[[Callable[[], object]], IDelayedCall]: """Makes a schedular suitable for a Cooperator using the given reactor. (This is effectively just a copy from `twisted.internet.task`) """ - def _scheduler(x): + def _scheduler(x: Callable[[], object]) -> IDelayedCall: return reactor.callLater(_EPSILON, x) return _scheduler @@ -348,7 +352,7 @@ def __init__( # XXX: The justification for using the cache factor here is that larger instances # will need both more cache and more connections. # Still, this should probably be a separate dial - pool.maxPersistentPerHost = max((100 * hs.config.caches.global_factor, 5)) + pool.maxPersistentPerHost = max(int(100 * hs.config.caches.global_factor), 5) pool.cachedConnectionTimeout = 2 * 60 self.agent: IAgent = ProxyAgent( @@ -775,7 +779,7 @@ async def get_file( ) -def _timeout_to_request_timed_out_error(f: Failure): +def _timeout_to_request_timed_out_error(f: Failure) -> Failure: if f.check(twisted_error.TimeoutError, twisted_error.ConnectingCancelledError): # The TCP connection has its own timeout (set by the 'connectTimeout' param # on the Agent), which raises twisted_error.TimeoutError exception. @@ -809,7 +813,7 @@ class _DiscardBodyWithMaxSizeProtocol(protocol.Protocol): def __init__(self, deferred: defer.Deferred): self.deferred = deferred - def _maybe_fail(self): + def _maybe_fail(self) -> None: """ Report a max size exceed error and disconnect the first time this is called. """ @@ -933,12 +937,12 @@ class InsecureInterceptableContextFactory(ssl.ContextFactory): Do not use this since it allows an attacker to intercept your communications. """ - def __init__(self): + def __init__(self) -> None: self._context = SSL.Context(SSL.SSLv23_METHOD) self._context.set_verify(VERIFY_NONE, lambda *_: False) def getContext(self, hostname=None, port=None): return self._context - def creatorForNetloc(self, hostname, port): + def creatorForNetloc(self, hostname: bytes, port: int): return self diff --git a/synapse/http/connectproxyclient.py b/synapse/http/connectproxyclient.py index 203e995bb77d..23a60af17184 100644 --- a/synapse/http/connectproxyclient.py +++ b/synapse/http/connectproxyclient.py @@ -14,15 +14,22 @@ import base64 import logging -from typing import Optional +from typing import Optional, Union import attr from zope.interface import implementer from twisted.internet import defer, protocol from twisted.internet.error import ConnectError -from twisted.internet.interfaces import IReactorCore, IStreamClientEndpoint +from twisted.internet.interfaces import ( + IAddress, + IConnector, + IProtocol, + IReactorCore, + IStreamClientEndpoint, +) from twisted.internet.protocol import ClientFactory, Protocol, connectionDone +from twisted.python.failure import Failure from twisted.web import http logger = logging.getLogger(__name__) @@ -81,14 +88,14 @@ def __init__( self._port = port self._proxy_creds = proxy_creds - def __repr__(self): + def __repr__(self) -> str: return "" % (self._proxy_endpoint,) # Mypy encounters a false positive here: it complains that ClientFactory # is incompatible with IProtocolFactory. But ClientFactory inherits from # Factory, which implements IProtocolFactory. So I think this is a bug # in mypy-zope. - def connect(self, protocolFactory: ClientFactory): # type: ignore[override] + def connect(self, protocolFactory: ClientFactory) -> "defer.Deferred[IProtocol]": # type: ignore[override] f = HTTPProxiedClientFactory( self._host, self._port, protocolFactory, self._proxy_creds ) @@ -125,10 +132,10 @@ def __init__( self.proxy_creds = proxy_creds self.on_connection: "defer.Deferred[None]" = defer.Deferred() - def startedConnecting(self, connector): + def startedConnecting(self, connector: IConnector) -> None: return self.wrapped_factory.startedConnecting(connector) - def buildProtocol(self, addr): + def buildProtocol(self, addr: IAddress) -> "HTTPConnectProtocol": wrapped_protocol = self.wrapped_factory.buildProtocol(addr) if wrapped_protocol is None: raise TypeError("buildProtocol produced None instead of a Protocol") @@ -141,13 +148,13 @@ def buildProtocol(self, addr): self.proxy_creds, ) - def clientConnectionFailed(self, connector, reason): + def clientConnectionFailed(self, connector: IConnector, reason: Failure) -> None: logger.debug("Connection to proxy failed: %s", reason) if not self.on_connection.called: self.on_connection.errback(reason) return self.wrapped_factory.clientConnectionFailed(connector, reason) - def clientConnectionLost(self, connector, reason): + def clientConnectionLost(self, connector: IConnector, reason: Failure) -> None: logger.debug("Connection to proxy lost: %s", reason) if not self.on_connection.called: self.on_connection.errback(reason) @@ -191,10 +198,10 @@ def __init__( ) self.http_setup_client.on_connected.addCallback(self.proxyConnected) - def connectionMade(self): + def connectionMade(self) -> None: self.http_setup_client.makeConnection(self.transport) - def connectionLost(self, reason=connectionDone): + def connectionLost(self, reason: Failure = connectionDone) -> None: if self.wrapped_protocol.connected: self.wrapped_protocol.connectionLost(reason) @@ -203,7 +210,7 @@ def connectionLost(self, reason=connectionDone): if not self.connected_deferred.called: self.connected_deferred.errback(reason) - def proxyConnected(self, _): + def proxyConnected(self, _: Union[None, "defer.Deferred[None]"]) -> None: self.wrapped_protocol.makeConnection(self.transport) self.connected_deferred.callback(self.wrapped_protocol) @@ -213,7 +220,7 @@ def proxyConnected(self, _): if buf: self.wrapped_protocol.dataReceived(buf) - def dataReceived(self, data: bytes): + def dataReceived(self, data: bytes) -> None: # if we've set up the HTTP protocol, we can send the data there if self.wrapped_protocol.connected: return self.wrapped_protocol.dataReceived(data) @@ -243,7 +250,7 @@ def __init__( self.proxy_creds = proxy_creds self.on_connected: "defer.Deferred[None]" = defer.Deferred() - def connectionMade(self): + def connectionMade(self) -> None: logger.debug("Connected to proxy, sending CONNECT") self.sendCommand(b"CONNECT", b"%s:%d" % (self.host, self.port)) @@ -257,14 +264,14 @@ def connectionMade(self): self.endHeaders() - def handleStatus(self, version: bytes, status: bytes, message: bytes): + def handleStatus(self, version: bytes, status: bytes, message: bytes) -> None: logger.debug("Got Status: %s %s %s", status, message, version) if status != b"200": raise ProxyConnectError(f"Unexpected status on CONNECT: {status!s}") - def handleEndHeaders(self): + def handleEndHeaders(self) -> None: logger.debug("End Headers") self.on_connected.callback(None) - def handleResponse(self, body): + def handleResponse(self, body: bytes) -> None: pass diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index a8a520f80944..2f0177f1e203 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -239,7 +239,7 @@ def __init__( self._srv_resolver = srv_resolver - def endpointForURI(self, parsed_uri: URI): + def endpointForURI(self, parsed_uri: URI) -> "MatrixHostnameEndpoint": return MatrixHostnameEndpoint( self._reactor, self._proxy_reactor, diff --git a/synapse/http/federation/srv_resolver.py b/synapse/http/federation/srv_resolver.py index f68646fd0dd4..de0e882b3312 100644 --- a/synapse/http/federation/srv_resolver.py +++ b/synapse/http/federation/srv_resolver.py @@ -16,7 +16,7 @@ import logging import random import time -from typing import Callable, Dict, List +from typing import Any, Callable, Dict, List import attr @@ -109,7 +109,7 @@ class SrvResolver: def __init__( self, - dns_client=client, + dns_client: Any = client, cache: Dict[bytes, List[Server]] = SERVER_CACHE, get_time: Callable[[], float] = time.time, ): diff --git a/synapse/http/federation/well_known_resolver.py b/synapse/http/federation/well_known_resolver.py index 43f2140429b5..71b685fadec9 100644 --- a/synapse/http/federation/well_known_resolver.py +++ b/synapse/http/federation/well_known_resolver.py @@ -74,9 +74,9 @@ _had_valid_well_known_cache: TTLCache[bytes, bool] = TTLCache("had-valid-well-known") -@attr.s(slots=True, frozen=True) +@attr.s(slots=True, frozen=True, auto_attribs=True) class WellKnownLookupResult: - delegated_server = attr.ib() + delegated_server: Optional[bytes] class WellKnownResolver: @@ -336,4 +336,4 @@ def _parse_cache_control(headers: Headers) -> Dict[bytes, Optional[bytes]]: class _FetchWellKnownFailure(Exception): # True if we didn't get a non-5xx HTTP response, i.e. this may or may not be # a temporary failure. - temporary = attr.ib() + temporary: bool = attr.ib() diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 5097b3ca5796..901c47f7567a 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -23,6 +23,8 @@ from io import BytesIO, StringIO from typing import ( TYPE_CHECKING, + Any, + BinaryIO, Callable, Dict, Generic, @@ -44,7 +46,7 @@ from twisted.internet import defer from twisted.internet.error import DNSLookupError from twisted.internet.interfaces import IReactorTime -from twisted.internet.task import _EPSILON, Cooperator +from twisted.internet.task import Cooperator from twisted.web.client import ResponseFailed from twisted.web.http_headers import Headers from twisted.web.iweb import IBodyProducer, IResponse @@ -58,11 +60,13 @@ RequestSendFailed, SynapseError, ) +from synapse.crypto.context_factory import FederationPolicyForHTTPS from synapse.http import QuieterFileBodyProducer from synapse.http.client import ( BlacklistingAgentWrapper, BodyExceededMaxSize, ByteWriteable, + _make_scheduler, encode_query_args, read_body_with_max_size, ) @@ -73,7 +77,7 @@ from synapse.logging.opentracing import set_tag, start_active_span, tags from synapse.types import JsonDict from synapse.util import json_decoder -from synapse.util.async_helpers import timeout_deferred +from synapse.util.async_helpers import AwakenableSleeper, timeout_deferred from synapse.util.metrics import Measure if TYPE_CHECKING: @@ -181,7 +185,7 @@ class JsonParser(ByteParser[Union[JsonDict, list]]): CONTENT_TYPE = "application/json" - def __init__(self): + def __init__(self) -> None: self._buffer = StringIO() self._binary_wrapper = BinaryIOWrapper(self._buffer) @@ -221,6 +225,7 @@ async def _handle_response( if max_response_size is None: max_response_size = MAX_RESPONSE_SIZE + finished = False try: check_content_type_is(response.headers, parser.CONTENT_TYPE) @@ -229,6 +234,7 @@ async def _handle_response( length = await make_deferred_yieldable(d) + finished = True value = parser.finish() except BodyExceededMaxSize as e: # The response was too big. @@ -279,6 +285,15 @@ async def _handle_response( e, ) raise + finally: + if not finished: + # There was an exception and we didn't `finish()` the parse. + # Let the parser know that it can free up any resources. + try: + parser.finish() + except Exception: + # Ignore any additional exceptions. + pass time_taken_secs = reactor.seconds() - start_ms / 1000 @@ -299,7 +314,9 @@ async def _handle_response( class BinaryIOWrapper: """A wrapper for a TextIO which converts from bytes on the fly.""" - def __init__(self, file: typing.TextIO, encoding="utf-8", errors="strict"): + def __init__( + self, file: typing.TextIO, encoding: str = "utf-8", errors: str = "strict" + ): self.decoder = codecs.getincrementaldecoder(encoding)(errors) self.file = file @@ -317,7 +334,11 @@ class MatrixFederationHttpClient: requests. """ - def __init__(self, hs: "HomeServer", tls_client_options_factory): + def __init__( + self, + hs: "HomeServer", + tls_client_options_factory: Optional[FederationPolicyForHTTPS], + ): self.hs = hs self.signing_key = hs.signing_key self.server_name = hs.hostname @@ -348,16 +369,20 @@ def __init__(self, hs: "HomeServer", tls_client_options_factory): self.version_string_bytes = hs.version_string.encode("ascii") self.default_timeout = 60 - def schedule(x): - self.reactor.callLater(_EPSILON, x) + self._cooperator = Cooperator(scheduler=_make_scheduler(self.reactor)) + + self._sleeper = AwakenableSleeper(self.reactor) - self._cooperator = Cooperator(scheduler=schedule) + def wake_destination(self, destination: str) -> None: + """Called when the remote server may have come back online.""" + + self._sleeper.wake(destination) async def _send_request_with_optional_trailing_slash( self, request: MatrixFederationRequest, try_trailing_slash_on_400: bool = False, - **send_request_args, + **send_request_args: Any, ) -> IResponse: """Wrapper for _send_request which can optionally retry the request upon receiving a combination of a 400 HTTP response code and a @@ -474,6 +499,8 @@ async def _send_request( self._store, backoff_on_404=backoff_on_404, ignore_backoff=ignore_backoff, + notifier=self.hs.get_notifier(), + replication_client=self.hs.get_replication_command_handler(), ) method_bytes = request.method.encode("ascii") @@ -664,7 +691,9 @@ async def _send_request( delay, ) - await self.clock.sleep(delay) + # Sleep for the calculated delay, or wake up immediately + # if we get notified that the server is back up. + await self._sleeper.sleep(request.destination, delay * 1000) retries_left -= 1 else: raise @@ -704,6 +733,9 @@ def build_auth_headers( Returns: A list of headers to be added as "Authorization:" headers """ + if destination is None and destination_is is None: + raise ValueError("destination and destination_is cannot both be None!") + request: JsonDict = { "method": method.decode("ascii"), "uri": url_bytes.decode("ascii"), @@ -726,8 +758,13 @@ def build_auth_headers( for key, sig in request["signatures"][self.server_name].items(): auth_headers.append( ( - 'X-Matrix origin=%s,key="%s",sig="%s"' - % (self.server_name, key, sig) + 'X-Matrix origin="%s",key="%s",sig="%s",destination="%s"' + % ( + self.server_name, + key, + sig, + request.get("destination") or request["destination_is"], + ) ).encode("ascii") ) return auth_headers @@ -1140,7 +1177,7 @@ async def get_file( self, destination: str, path: str, - output_stream, + output_stream: BinaryIO, args: Optional[QueryParams] = None, retry_on_dns_fail: bool = True, max_size: Optional[int] = None, @@ -1231,10 +1268,10 @@ async def get_file( return length, headers -def _flatten_response_never_received(e): +def _flatten_response_never_received(e: BaseException) -> str: if hasattr(e, "reasons"): reasons = ", ".join( - _flatten_response_never_received(f.value) for f in e.reasons + _flatten_response_never_received(f.value) for f in e.reasons # type: ignore[attr-defined] ) return "%s:[%s]" % (type(e).__name__, reasons) diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py index a16dde23807f..b2a50c910507 100644 --- a/synapse/http/proxyagent.py +++ b/synapse/http/proxyagent.py @@ -245,7 +245,7 @@ def http_proxy_endpoint( proxy: Optional[bytes], reactor: IReactorCore, tls_options_factory: Optional[IPolicyForHTTPS], - **kwargs, + **kwargs: object, ) -> Tuple[Optional[IStreamClientEndpoint], Optional[ProxyCredentials]]: """Parses an http proxy setting and returns an endpoint for the proxy diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index 4886626d5074..2b6d113544ca 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -162,7 +162,7 @@ def start(self, time_sec: float, name: str, method: str) -> None: with _in_flight_requests_lock: _in_flight_requests.add(self) - def stop(self, time_sec, response_code, sent_bytes): + def stop(self, time_sec: float, response_code: int, sent_bytes: int) -> None: with _in_flight_requests_lock: _in_flight_requests.discard(self) @@ -186,13 +186,13 @@ def stop(self, time_sec, response_code, sent_bytes): ) return - response_code = str(response_code) + response_code_str = str(response_code) - outgoing_responses_counter.labels(self.method, response_code).inc() + outgoing_responses_counter.labels(self.method, response_code_str).inc() response_count.labels(self.method, self.name, tag).inc() - response_timer.labels(self.method, self.name, tag, response_code).observe( + response_timer.labels(self.method, self.name, tag, response_code_str).observe( time_sec - self.start_ts ) @@ -221,7 +221,7 @@ def stop(self, time_sec, response_code, sent_bytes): # flight. self.update_metrics() - def update_metrics(self): + def update_metrics(self) -> None: """Updates the in flight metrics with values from this request.""" if not self.start_context: logger.error( diff --git a/synapse/http/server.py b/synapse/http/server.py index 31ca84188975..e3dcc3f3dd06 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -33,6 +33,7 @@ Optional, Pattern, Tuple, + TypeVar, Union, ) @@ -43,6 +44,7 @@ from zope.interface import implementer from twisted.internet import defer, interfaces +from twisted.internet.defer import CancelledError from twisted.python import failure from twisted.web import resource from twisted.web.server import NOT_DONE_YET, Request @@ -82,6 +84,76 @@ """ +# A fictional HTTP status code for requests where the client has disconnected and we +# successfully cancelled the request. Used only for logging purposes. Clients will never +# observe this code unless cancellations leak across requests or we raise a +# `CancelledError` ourselves. +# Analogous to nginx's 499 status code: +# https://github.com/nginx/nginx/blob/release-1.21.6/src/http/ngx_http_request.h#L128-L134 +HTTP_STATUS_REQUEST_CANCELLED = 499 + + +F = TypeVar("F", bound=Callable[..., Any]) + + +_cancellable_method_names = frozenset( + { + # `RestServlet`, `BaseFederationServlet` and `BaseFederationServerServlet` + # methods + "on_GET", + "on_PUT", + "on_POST", + "on_DELETE", + # `_AsyncResource`, `DirectServeHtmlResource` and `DirectServeJsonResource` + # methods + "_async_render_GET", + "_async_render_PUT", + "_async_render_POST", + "_async_render_DELETE", + "_async_render_OPTIONS", + # `ReplicationEndpoint` methods + "_handle_request", + } +) + + +def cancellable(method: F) -> F: + """Marks a servlet method as cancellable. + + Methods with this decorator will be cancelled if the client disconnects before we + finish processing the request. + + During cancellation, `Deferred.cancel()` will be invoked on the `Deferred` wrapping + the method. The `cancel()` call will propagate down to the `Deferred` that is + currently being waited on. That `Deferred` will raise a `CancelledError`, which will + propagate up, as per normal exception handling. + + Before applying this decorator to a new endpoint, you MUST recursively check + that all `await`s in the function are on `async` functions or `Deferred`s that + handle cancellation cleanly, otherwise a variety of bugs may occur, ranging from + premature logging context closure, to stuck requests, to database corruption. + + Usage: + class SomeServlet(RestServlet): + @cancellable + async def on_GET(self, request: SynapseRequest) -> ...: + ... + """ + if method.__name__ not in _cancellable_method_names and not any( + method.__name__.startswith(prefix) for prefix in _cancellable_method_names + ): + raise ValueError( + "@cancellable decorator can only be applied to servlet methods." + ) + + method.cancellable = True # type: ignore[attr-defined] + return method + + +def is_method_cancellable(method: Callable[..., Any]) -> bool: + """Checks whether a servlet method has the `@cancellable` flag.""" + return getattr(method, "cancellable", False) + def return_json_error(f: failure.Failure, request: SynapseRequest) -> None: """Sends a JSON error response to clients.""" @@ -93,6 +165,17 @@ def return_json_error(f: failure.Failure, request: SynapseRequest) -> None: error_dict = exc.error_dict() logger.info("%s SynapseError: %s - %s", request, error_code, exc.msg) + elif f.check(CancelledError): + error_code = HTTP_STATUS_REQUEST_CANCELLED + error_dict = {"error": "Request cancelled", "errcode": Codes.UNKNOWN} + + if not request._disconnected: + logger.error( + "Got cancellation before client disconnection from %r: %r", + request.request_metrics.name, + request, + exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore[arg-type] + ) else: error_code = 500 error_dict = {"error": "Internal server error", "errcode": Codes.UNKNOWN} @@ -155,6 +238,16 @@ def return_html_error( request, exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore[arg-type] ) + elif f.check(CancelledError): + code = HTTP_STATUS_REQUEST_CANCELLED + msg = "Request cancelled" + + if not request._disconnected: + logger.error( + "Got cancellation before client disconnection when handling request %r", + request, + exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore[arg-type] + ) else: code = HTTPStatus.INTERNAL_SERVER_ERROR msg = "Internal server error" @@ -223,6 +316,9 @@ def register_paths( If the regex contains groups these gets passed to the callback via an unpacked tuple. + The callback may be marked with the `@cancellable` decorator, which will + cause request processing to be cancelled when clients disconnect early. + Args: method: The HTTP method to listen to. path_patterns: The regex used to match requests. @@ -253,7 +349,9 @@ def __init__(self, extract_context: bool = False): def render(self, request: SynapseRequest) -> int: """This gets called by twisted every time someone sends us a request.""" - defer.ensureDeferred(self._async_render_wrapper(request)) + request.render_deferred = defer.ensureDeferred( + self._async_render_wrapper(request) + ) return NOT_DONE_YET @wrap_async_request_handler @@ -289,13 +387,15 @@ async def _async_render(self, request: SynapseRequest) -> Optional[Tuple[int, An method_handler = getattr(self, "_async_render_%s" % (request_method,), None) if method_handler: + request.is_render_cancellable = is_method_cancellable(method_handler) + raw_callback_return = method_handler(request) # Is it synchronous? We'll allow this for now. if isawaitable(raw_callback_return): callback_return = await raw_callback_return else: - callback_return = raw_callback_return # type: ignore + callback_return = raw_callback_return return callback_return @@ -449,6 +549,8 @@ def _get_handler_for_request( async def _async_render(self, request: SynapseRequest) -> Tuple[int, Any]: callback, servlet_classname, group_dict = self._get_handler_for_request(request) + request.is_render_cancellable = is_method_cancellable(callback) + # Make sure we have an appropriate name for this handler in prometheus # (rather than the default of JsonResource). request.request_metrics.name = servlet_classname @@ -469,7 +571,7 @@ async def _async_render(self, request: SynapseRequest) -> Tuple[int, Any]: if isinstance(raw_callback_return, (defer.Deferred, types.CoroutineType)): callback_return = await raw_callback_return else: - callback_return = raw_callback_return # type: ignore + callback_return = raw_callback_return return callback_return @@ -683,6 +785,9 @@ def respond_with_json( Returns: twisted.web.server.NOT_DONE_YET if the request is still active. """ + # The response code must always be set, for logging purposes. + request.setResponseCode(code) + # could alternatively use request.notifyFinish() and flip a flag when # the Deferred fires, but since the flag is RIGHT THERE it seems like # a waste. @@ -697,7 +802,6 @@ def respond_with_json( else: encoder = _encode_json_bytes - request.setResponseCode(code) request.setHeader(b"Content-Type", b"application/json") request.setHeader(b"Cache-Control", b"no-cache, no-store, must-revalidate") @@ -728,13 +832,15 @@ def respond_with_json_bytes( Returns: twisted.web.server.NOT_DONE_YET if the request is still active. """ + # The response code must always be set, for logging purposes. + request.setResponseCode(code) + if request._disconnected: logger.warning( "Not sending response to request %s, already disconnected.", request ) return None - request.setResponseCode(code) request.setHeader(b"Content-Type", b"application/json") request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),)) request.setHeader(b"Cache-Control", b"no-cache, no-store, must-revalidate") @@ -840,6 +946,9 @@ def respond_with_html_bytes(request: Request, code: int, html_bytes: bytes) -> N code: The HTTP response code. html_bytes: The HTML bytes to use as the response body. """ + # The response code must always be set, for logging purposes. + request.setResponseCode(code) + # could alternatively use request.notifyFinish() and flip a flag when # the Deferred fires, but since the flag is RIGHT THERE it seems like # a waste. @@ -849,7 +958,6 @@ def respond_with_html_bytes(request: Request, code: int, html_bytes: bytes) -> N ) return None - request.setResponseCode(code) request.setHeader(b"Content-Type", b"text/html; charset=utf-8") request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),)) diff --git a/synapse/http/site.py b/synapse/http/site.py index 40f6c0489451..eeec74b78ae5 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -19,6 +19,7 @@ import attr from zope.interface import implementer +from twisted.internet.defer import Deferred from twisted.internet.interfaces import IAddress, IReactorTime from twisted.python.failure import Failure from twisted.web.http import HTTPChannel @@ -91,6 +92,14 @@ def __init__( # we can't yet create the logcontext, as we don't know the method. self.logcontext: Optional[LoggingContext] = None + # The `Deferred` to cancel if the client disconnects early and + # `is_render_cancellable` is set. Expected to be set by `Resource.render`. + self.render_deferred: Optional["Deferred[None]"] = None + # A boolean indicating whether `render_deferred` should be cancelled if the + # client disconnects early. Expected to be set by the coroutine started by + # `Resource.render`, if rendering is asynchronous. + self.is_render_cancellable = False + global _next_request_seq self.request_seq = _next_request_seq _next_request_seq += 1 @@ -238,7 +247,7 @@ def render(self, resrc: Resource) -> None: request_id, request=ContextRequest( request_id=request_id, - ip_address=self.getClientIP(), + ip_address=self.getClientAddress().host, site_tag=self.synapse_site.site_tag, # The requester is going to be unknown at this point. requester=None, @@ -357,7 +366,21 @@ def connectionLost(self, reason: Union[Failure, Exception]) -> None: {"event": "client connection lost", "reason": str(reason.value)} ) - if not self._is_processing: + if self._is_processing: + if self.is_render_cancellable: + if self.render_deferred is not None: + # Throw a cancellation into the request processing, in the hope + # that it will finish up sooner than it normally would. + # The `self.processing()` context manager will call + # `_finished_processing()` when done. + with PreserveLoggingContext(): + self.render_deferred.cancel() + else: + logger.error( + "Connection from client lost, but have no Deferred to " + "cancel even though the request is marked as cancellable." + ) + else: self._finished_processing() def _started_processing(self, servlet_name: str) -> None: @@ -381,7 +404,7 @@ def _started_processing(self, servlet_name: str) -> None: self.synapse_site.access_logger.debug( "%s - %s - Received request: %s %s", - self.getClientIP(), + self.getClientAddress().host, self.synapse_site.site_tag, self.get_method(), self.get_redacted_uri(), @@ -429,7 +452,7 @@ def _finished_processing(self) -> None: "%s - %s - {%s}" " Processed request: %.3fsec/%.3fsec (%.3fsec, %.3fsec) (%.3fsec/%.3fsec/%d)" ' %sB %s "%s %s %s" "%s" [%d dbevts]', - self.getClientIP(), + self.getClientAddress().host, self.synapse_site.site_tag, requester, processing_time, diff --git a/synapse/logging/_remote.py b/synapse/logging/_remote.py index 475756f1db64..5a61b21eaf7e 100644 --- a/synapse/logging/_remote.py +++ b/synapse/logging/_remote.py @@ -31,7 +31,11 @@ TCP4ClientEndpoint, TCP6ClientEndpoint, ) -from twisted.internet.interfaces import IPushProducer, IStreamClientEndpoint +from twisted.internet.interfaces import ( + IPushProducer, + IReactorTCP, + IStreamClientEndpoint, +) from twisted.internet.protocol import Factory, Protocol from twisted.internet.tcp import Connection from twisted.python.failure import Failure @@ -59,14 +63,14 @@ class LogProducer: _buffer: Deque[logging.LogRecord] _paused: bool = attr.ib(default=False, init=False) - def pauseProducing(self): + def pauseProducing(self) -> None: self._paused = True - def stopProducing(self): + def stopProducing(self) -> None: self._paused = True self._buffer = deque() - def resumeProducing(self): + def resumeProducing(self) -> None: # If we're already producing, nothing to do. self._paused = False @@ -102,8 +106,8 @@ def __init__( host: str, port: int, maximum_buffer: int = 1000, - level=logging.NOTSET, - _reactor=None, + level: int = logging.NOTSET, + _reactor: Optional[IReactorTCP] = None, ): super().__init__(level=level) self.host = host @@ -118,7 +122,7 @@ def __init__( if _reactor is None: from twisted.internet import reactor - _reactor = reactor + _reactor = reactor # type: ignore[assignment] try: ip = ip_address(self.host) @@ -139,7 +143,7 @@ def __init__( self._stopping = False self._connect() - def close(self): + def close(self) -> None: self._stopping = True self._service.stopService() diff --git a/synapse/logging/context.py b/synapse/logging/context.py index 88cd8a9e1c39..fd9cb979208a 100644 --- a/synapse/logging/context.py +++ b/synapse/logging/context.py @@ -722,6 +722,11 @@ def nested_logging_context(suffix: str) -> LoggingContext: R = TypeVar("R") +async def _unwrap_awaitable(awaitable: Awaitable[R]) -> R: + """Unwraps an arbitrary awaitable by awaiting it.""" + return await awaitable + + @overload def preserve_fn( # type: ignore[misc] f: Callable[P, Awaitable[R]], @@ -802,17 +807,20 @@ def run_in_background( # type: ignore[misc] # by synchronous exceptions, so let's turn them into Failures. return defer.fail() + # `res` may be a coroutine, `Deferred`, some other kind of awaitable, or a plain + # value. Convert it to a `Deferred`. if isinstance(res, typing.Coroutine): + # Wrap the coroutine in a `Deferred`. res = defer.ensureDeferred(res) - - # At this point we should have a Deferred, if not then f was a synchronous - # function, wrap it in a Deferred for consistency. - if not isinstance(res, defer.Deferred): - # `res` is not a `Deferred` and not a `Coroutine`. - # There are no other types of `Awaitable`s we expect to encounter in Synapse. - assert not isinstance(res, Awaitable) - - return defer.succeed(res) + elif isinstance(res, defer.Deferred): + pass + elif isinstance(res, Awaitable): + # `res` is probably some kind of completed awaitable, such as a `DoneAwaitable` + # or `Future` from `make_awaitable`. + res = defer.ensureDeferred(_unwrap_awaitable(res)) + else: + # `res` is a plain value. Wrap it in a `Deferred`. + res = defer.succeed(res) if res.called and not res.paused: # The function should have maintained the logcontext, so we can diff --git a/synapse/logging/formatter.py b/synapse/logging/formatter.py index c0f12ecd15b8..c88b8ae5450f 100644 --- a/synapse/logging/formatter.py +++ b/synapse/logging/formatter.py @@ -16,6 +16,8 @@ import logging import traceback from io import StringIO +from types import TracebackType +from typing import Optional, Tuple, Type class LogFormatter(logging.Formatter): @@ -28,10 +30,14 @@ class LogFormatter(logging.Formatter): where it was caught are logged). """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def formatException(self, ei): + def formatException( + self, + ei: Tuple[ + Optional[Type[BaseException]], + Optional[BaseException], + Optional[TracebackType], + ], + ) -> str: sio = StringIO() (typ, val, tb) = ei diff --git a/synapse/logging/handlers.py b/synapse/logging/handlers.py index 478b5274942b..dec2a2c3dd1a 100644 --- a/synapse/logging/handlers.py +++ b/synapse/logging/handlers.py @@ -49,7 +49,7 @@ def __init__( ) self._flushing_thread.start() - def on_reactor_running(): + def on_reactor_running() -> None: self._reactor_started = True reactor_to_use: IReactorCore @@ -74,7 +74,7 @@ def shouldFlush(self, record: LogRecord) -> bool: else: return True - def _flush_periodically(self): + def _flush_periodically(self) -> None: """ Whilst this handler is active, flush the handler periodically. """ diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index f86ee9aac7a5..a02b5bf6bd28 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -884,7 +884,7 @@ def trace_servlet(request: "SynapseRequest", extract_context: bool = False): tags.SPAN_KIND: tags.SPAN_KIND_RPC_SERVER, tags.HTTP_METHOD: request.get_method(), tags.HTTP_URL: request.get_redacted_uri(), - tags.PEER_HOST_IPV6: request.getClientIP(), + tags.PEER_HOST_IPV6: request.getClientAddress().host, } request_name = request.request_metrics.name diff --git a/synapse/logging/scopecontextmanager.py b/synapse/logging/scopecontextmanager.py index d57e7c5324f8..a26a1a58e7d6 100644 --- a/synapse/logging/scopecontextmanager.py +++ b/synapse/logging/scopecontextmanager.py @@ -13,6 +13,8 @@ # limitations under the License.import logging import logging +from types import TracebackType +from typing import Optional, Type from opentracing import Scope, ScopeManager @@ -107,19 +109,26 @@ class _LogContextScope(Scope): and - if enter_logcontext was set - the logcontext is finished too. """ - def __init__(self, manager, span, logcontext, enter_logcontext, finish_on_close): + def __init__( + self, + manager: LogContextScopeManager, + span, + logcontext, + enter_logcontext: bool, + finish_on_close: bool, + ): """ Args: - manager (LogContextScopeManager): + manager: the manager that is responsible for this scope. span (Span): the opentracing span which this scope represents the local lifetime for. logcontext (LogContext): the logcontext to which this scope is attached. - enter_logcontext (Boolean): + enter_logcontext: if True the logcontext will be exited when the scope is finished - finish_on_close (Boolean): + finish_on_close: if True finish the span when the scope is closed """ super().__init__(manager, span) @@ -127,16 +136,21 @@ def __init__(self, manager, span, logcontext, enter_logcontext, finish_on_close) self._finish_on_close = finish_on_close self._enter_logcontext = enter_logcontext - def __exit__(self, exc_type, value, traceback): + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: if exc_type == twisted.internet.defer._DefGen_Return: # filter out defer.returnValue() calls exc_type = value = traceback = None super().__exit__(exc_type, value, traceback) - def __str__(self): + def __str__(self) -> str: return f"Scope<{self.span}>" - def close(self): + def close(self) -> None: active_scope = self.manager.active if active_scope is not self: logger.error( diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index f61396bb79a9..298809742a94 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -28,11 +28,11 @@ Type, TypeVar, Union, - cast, ) from prometheus_client import Metric from prometheus_client.core import REGISTRY, Counter, Gauge +from typing_extensions import ParamSpec from twisted.internet import defer @@ -256,24 +256,48 @@ async def run() -> Optional[R]: return defer.ensureDeferred(run()) -F = TypeVar("F", bound=Callable[..., Awaitable[Optional[Any]]]) +P = ParamSpec("P") -def wrap_as_background_process(desc: str) -> Callable[[F], F]: - """Decorator that wraps a function that gets called as a background - process. +def wrap_as_background_process( + desc: str, +) -> Callable[ + [Callable[P, Awaitable[Optional[R]]]], + Callable[P, "defer.Deferred[Optional[R]]"], +]: + """Decorator that wraps an asynchronous function `func`, returning a synchronous + decorated function. Calling the decorated version runs `func` as a background + process, forwarding all arguments verbatim. + + That is, + + @wrap_as_background_process + def func(*args): ... + func(1, 2, third=3) + + is equivalent to: + + def func(*args): ... + run_as_background_process(func, 1, 2, third=3) - Equivalent to calling the function with `run_as_background_process` + The former can be convenient if `func` needs to be run as a background process in + multiple places. """ - def wrap_as_background_process_inner(func: F) -> F: + def wrap_as_background_process_inner( + func: Callable[P, Awaitable[Optional[R]]] + ) -> Callable[P, "defer.Deferred[Optional[R]]"]: @wraps(func) def wrap_as_background_process_inner_2( - *args: Any, **kwargs: Any + *args: P.args, **kwargs: P.kwargs ) -> "defer.Deferred[Optional[R]]": - return run_as_background_process(desc, func, *args, **kwargs) + # type-ignore: mypy is confusing kwargs with the bg_start_span kwarg. + # Argument 4 to "run_as_background_process" has incompatible type + # "**P.kwargs"; expected "bool" + # See https://github.com/python/mypy/issues/8862 + return run_as_background_process(desc, func, *args, **kwargs) # type: ignore[arg-type] - return cast(F, wrap_as_background_process_inner_2) + return wrap_as_background_process_inner_2 return wrap_as_background_process_inner diff --git a/synapse/metrics/jemalloc.py b/synapse/metrics/jemalloc.py index 6bc329f04a9c..1fc8a0e888a1 100644 --- a/synapse/metrics/jemalloc.py +++ b/synapse/metrics/jemalloc.py @@ -18,6 +18,7 @@ import re from typing import Iterable, Optional, overload +import attr from prometheus_client import REGISTRY, Metric from typing_extensions import Literal @@ -27,52 +28,24 @@ logger = logging.getLogger(__name__) -def _setup_jemalloc_stats() -> None: - """Checks to see if jemalloc is loaded, and hooks up a collector to record - statistics exposed by jemalloc. - """ - - # Try to find the loaded jemalloc shared library, if any. We need to - # introspect into what is loaded, rather than loading whatever is on the - # path, as if we load a *different* jemalloc version things will seg fault. - - # We look in `/proc/self/maps`, which only exists on linux. - if not os.path.exists("/proc/self/maps"): - logger.debug("Not looking for jemalloc as no /proc/self/maps exist") - return - - # We're looking for a path at the end of the line that includes - # "libjemalloc". - regex = re.compile(r"/\S+/libjemalloc.*$") - - jemalloc_path = None - with open("/proc/self/maps") as f: - for line in f: - match = regex.search(line.strip()) - if match: - jemalloc_path = match.group() - - if not jemalloc_path: - # No loaded jemalloc was found. - logger.debug("jemalloc not found") - return - - logger.debug("Found jemalloc at %s", jemalloc_path) - - jemalloc = ctypes.CDLL(jemalloc_path) +@attr.s(slots=True, frozen=True, auto_attribs=True) +class JemallocStats: + jemalloc: ctypes.CDLL @overload def _mallctl( - name: str, read: Literal[True] = True, write: Optional[int] = None + self, name: str, read: Literal[True] = True, write: Optional[int] = None ) -> int: ... @overload - def _mallctl(name: str, read: Literal[False], write: Optional[int] = None) -> None: + def _mallctl( + self, name: str, read: Literal[False], write: Optional[int] = None + ) -> None: ... def _mallctl( - name: str, read: bool = True, write: Optional[int] = None + self, name: str, read: bool = True, write: Optional[int] = None ) -> Optional[int]: """Wrapper around `mallctl` for reading and writing integers to jemalloc. @@ -120,7 +93,7 @@ def _mallctl( # Where oldp/oldlenp is a buffer where the old value will be written to # (if not null), and newp/newlen is the buffer with the new value to set # (if not null). Note that they're all references *except* newlen. - result = jemalloc.mallctl( + result = self.jemalloc.mallctl( name.encode("ascii"), input_var_ref, input_len_ref, @@ -136,21 +109,80 @@ def _mallctl( return input_var.value - def _jemalloc_refresh_stats() -> None: + def refresh_stats(self) -> None: """Request that jemalloc updates its internal statistics. This needs to be called before querying for stats, otherwise it will return stale values. """ try: - _mallctl("epoch", read=False, write=1) + self._mallctl("epoch", read=False, write=1) except Exception as e: logger.warning("Failed to reload jemalloc stats: %s", e) + def get_stat(self, name: str) -> int: + """Request the stat of the given name at the time of the last + `refresh_stats` call. This may throw if we fail to read + the stat. + """ + return self._mallctl(f"stats.{name}") + + +_JEMALLOC_STATS: Optional[JemallocStats] = None + + +def get_jemalloc_stats() -> Optional[JemallocStats]: + """Returns an interface to jemalloc, if it is being used. + + Note that this will always return None until `setup_jemalloc_stats` has been + called. + """ + return _JEMALLOC_STATS + + +def _setup_jemalloc_stats() -> None: + """Checks to see if jemalloc is loaded, and hooks up a collector to record + statistics exposed by jemalloc. + """ + + global _JEMALLOC_STATS + + # Try to find the loaded jemalloc shared library, if any. We need to + # introspect into what is loaded, rather than loading whatever is on the + # path, as if we load a *different* jemalloc version things will seg fault. + + # We look in `/proc/self/maps`, which only exists on linux. + if not os.path.exists("/proc/self/maps"): + logger.debug("Not looking for jemalloc as no /proc/self/maps exist") + return + + # We're looking for a path at the end of the line that includes + # "libjemalloc". + regex = re.compile(r"/\S+/libjemalloc.*$") + + jemalloc_path = None + with open("/proc/self/maps") as f: + for line in f: + match = regex.search(line.strip()) + if match: + jemalloc_path = match.group() + + if not jemalloc_path: + # No loaded jemalloc was found. + logger.debug("jemalloc not found") + return + + logger.debug("Found jemalloc at %s", jemalloc_path) + + jemalloc_dll = ctypes.CDLL(jemalloc_path) + + stats = JemallocStats(jemalloc_dll) + _JEMALLOC_STATS = stats + class JemallocCollector(Collector): """Metrics for internal jemalloc stats.""" def collect(self) -> Iterable[Metric]: - _jemalloc_refresh_stats() + stats.refresh_stats() g = GaugeMetricFamily( "jemalloc_stats_app_memory_bytes", @@ -184,7 +216,7 @@ def collect(self) -> Iterable[Metric]: "metadata", ): try: - value = _mallctl(f"stats.{t}") + value = stats.get_stat(t) except Exception as e: # There was an error fetching the value, skip. logger.warning("Failed to read jemalloc stats.%s: %s", t, e) diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 8f9e62927465..c44e9da121ba 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -30,6 +30,7 @@ import attr import jinja2 +from typing_extensions import ParamSpec from twisted.internet import defer from twisted.web.resource import Resource @@ -46,12 +47,14 @@ CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK, CHECK_REGISTRATION_FOR_SPAM_CALLBACK, CHECK_USERNAME_FOR_SPAM_CALLBACK, + SHOULD_DROP_FEDERATED_EVENT_CALLBACK, USER_MAY_CREATE_ROOM_ALIAS_CALLBACK, USER_MAY_CREATE_ROOM_CALLBACK, USER_MAY_INVITE_CALLBACK, USER_MAY_JOIN_ROOM_CALLBACK, USER_MAY_PUBLISH_ROOM_CALLBACK, USER_MAY_SEND_3PID_INVITE_CALLBACK, + SpamChecker, ) from synapse.events.third_party_rules import ( CHECK_CAN_DEACTIVATE_USER_CALLBACK, @@ -82,6 +85,7 @@ ON_LOGGED_OUT_CALLBACK, AuthHandler, ) +from synapse.handlers.push_rules import RuleSpec, check_actions from synapse.http.client import SimpleHttpClient from synapse.http.server import ( DirectServeHtmlResource, @@ -109,6 +113,7 @@ from synapse.types import ( DomainSpecificString, JsonDict, + JsonMapping, Requester, StateMap, UserID, @@ -127,6 +132,7 @@ T = TypeVar("T") +P = ParamSpec("P") """ This package defines the 'stable' API which can be used by extension modules which @@ -134,6 +140,7 @@ """ PRESENCE_ALL_USERS = PresenceRouter.ALL_USERS +NOT_SPAM = SpamChecker.NOT_SPAM __all__ = [ "errors", @@ -142,6 +149,7 @@ "respond_with_html", "run_in_background", "cached", + "NOT_SPAM", "UserID", "DatabasePool", "LoggingTransaction", @@ -151,6 +159,7 @@ "PRESENCE_ALL_USERS", "LoginResponse", "JsonDict", + "JsonMapping", "EventBase", "StateMap", "ProfileInfo", @@ -193,6 +202,7 @@ def __init__(self, hs: "HomeServer", auth_handler: AuthHandler) -> None: self._clock: Clock = hs.get_clock() self._registration_handler = hs.get_registration_handler() self._send_email_handler = hs.get_send_email_handler() + self._push_rules_handler = hs.get_push_rules_handler() self.custom_template_dir = hs.config.server.custom_template_directory try: @@ -228,6 +238,9 @@ def register_spam_checker_callbacks( self, *, check_event_for_spam: Optional[CHECK_EVENT_FOR_SPAM_CALLBACK] = None, + should_drop_federated_event: Optional[ + SHOULD_DROP_FEDERATED_EVENT_CALLBACK + ] = None, user_may_join_room: Optional[USER_MAY_JOIN_ROOM_CALLBACK] = None, user_may_invite: Optional[USER_MAY_INVITE_CALLBACK] = None, user_may_send_3pid_invite: Optional[USER_MAY_SEND_3PID_INVITE_CALLBACK] = None, @@ -248,6 +261,7 @@ def register_spam_checker_callbacks( """ return self._spam_checker.register_callbacks( check_event_for_spam=check_event_for_spam, + should_drop_federated_event=should_drop_federated_event, user_may_join_room=user_may_join_room, user_may_invite=user_may_invite, user_may_send_3pid_invite=user_may_send_3pid_invite, @@ -795,9 +809,9 @@ def invalidate_access_token( def run_db_interaction( self, desc: str, - func: Callable[..., T], - *args: Any, - **kwargs: Any, + func: Callable[P, T], + *args: P.args, + **kwargs: P.kwargs, ) -> "defer.Deferred[T]": """Run a function with a database connection @@ -813,8 +827,9 @@ def run_db_interaction( Returns: Deferred[object]: result of func """ + # type-ignore: See https://github.com/python/mypy/issues/8862 return defer.ensureDeferred( - self._store.db_pool.runInteraction(desc, func, *args, **kwargs) + self._store.db_pool.runInteraction(desc, func, *args, **kwargs) # type: ignore[arg-type] ) def complete_sso_login( @@ -1292,9 +1307,9 @@ async def get_room_state( async def defer_to_thread( self, - f: Callable[..., T], - *args: Any, - **kwargs: Any, + f: Callable[P, T], + *args: P.args, + **kwargs: P.kwargs, ) -> T: """Runs the given function in a separate thread from Synapse's thread pool. @@ -1350,6 +1365,68 @@ async def store_remote_3pid_association( """ await self._store.add_user_bound_threepid(user_id, medium, address, id_server) + def check_push_rule_actions( + self, actions: List[Union[str, Dict[str, str]]] + ) -> None: + """Checks if the given push rule actions are valid according to the Matrix + specification. + + See https://spec.matrix.org/v1.2/client-server-api/#actions for the list of valid + actions. + + Added in Synapse v1.58.0. + + Args: + actions: the actions to check. + + Raises: + synapse.module_api.errors.InvalidRuleException if the actions are invalid. + """ + check_actions(actions) + + async def set_push_rule_action( + self, + user_id: str, + scope: str, + kind: str, + rule_id: str, + actions: List[Union[str, Dict[str, str]]], + ) -> None: + """Changes the actions of an existing push rule for the given user. + + See https://spec.matrix.org/v1.2/client-server-api/#push-rules for more + information about push rules and their syntax. + + Can only be called on the main process. + + Added in Synapse v1.58.0. + + Args: + user_id: the user for which to change the push rule's actions. + scope: the push rule's scope, currently only "global" is allowed. + kind: the push rule's kind. + rule_id: the push rule's identifier. + actions: the actions to run when the rule's conditions match. + + Raises: + RuntimeError if this method is called on a worker or `scope` is invalid. + synapse.module_api.errors.RuleNotFoundException if the rule being modified + can't be found. + synapse.module_api.errors.InvalidRuleException if the actions are invalid. + """ + if self.worker_app is not None: + raise RuntimeError("module tried to change push rule actions on a worker") + + if scope != "global": + raise RuntimeError( + "invalid scope %s, only 'global' is currently allowed" % scope + ) + + spec = RuleSpec(scope, kind, rule_id, "actions") + await self._push_rules_handler.set_rule_attr( + user_id, spec, {"actions": actions} + ) + class PublicRoomListManager: """Contains methods for adding to, removing from and querying whether a room @@ -1419,7 +1496,7 @@ def _validate_user_id(self, user_id: str) -> None: f"{user_id} is not local to this homeserver; can't access account data for remote users." ) - async def get_global(self, user_id: str, data_type: str) -> Optional[JsonDict]: + async def get_global(self, user_id: str, data_type: str) -> Optional[JsonMapping]: """ Gets some global account data, of a specified type, for the specified user. diff --git a/synapse/module_api/errors.py b/synapse/module_api/errors.py index 1db900e41f64..bedd045d6fe1 100644 --- a/synapse/module_api/errors.py +++ b/synapse/module_api/errors.py @@ -15,15 +15,21 @@ """Exception types which are exposed as part of the stable module API""" from synapse.api.errors import ( + Codes, InvalidClientCredentialsError, RedirectException, SynapseError, ) from synapse.config._base import ConfigError +from synapse.handlers.push_rules import InvalidRuleException +from synapse.storage.push_rule import RuleNotFoundException __all__ = [ + "Codes", "InvalidClientCredentialsError", "RedirectException", "SynapseError", "ConfigError", + "InvalidRuleException", + "RuleNotFoundException", ] diff --git a/synapse/notifier.py b/synapse/notifier.py index 16d15a1f3328..ba23257f5498 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -46,6 +46,7 @@ JsonDict, PersistedEventPosition, RoomStreamToken, + StreamKeyType, StreamToken, UserID, ) @@ -228,9 +229,7 @@ def __init__(self, hs: "HomeServer"): # Called when there are new things to stream over replication self.replication_callbacks: List[Callable[[], None]] = [] - # Called when remote servers have come back online after having been - # down. - self.remote_server_up_callbacks: List[Callable[[str], None]] = [] + self._federation_client = hs.get_federation_http_client() self._third_party_rules = hs.get_third_party_event_rules() @@ -372,7 +371,7 @@ def _notify_pending_new_room_events( if users or rooms: self.on_new_event( - "room_key", + StreamKeyType.ROOM, max_room_stream_token, users=users, rooms=rooms, @@ -442,7 +441,7 @@ def on_new_event( for room in rooms: user_streams |= self.room_to_user_streams.get(room, set()) - if stream_key == "to_device_key": + if stream_key == StreamKeyType.TO_DEVICE: issue9533_logger.debug( "to-device messages stream id %s, awaking streams for %s", new_token, @@ -731,3 +730,7 @@ def notify_remote_server_up(self, server: str) -> None: # circular dependencies. if self.federation_sender: self.federation_sender.wake_destination(server) + + # Tell the federation client about the fact the server is back up, so + # that any in flight requests can be immediately retried. + self._federation_client.wake_destination(server) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index a1b771109848..57c4d70466b6 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -12,6 +12,80 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +This module implements the push rules & notifications portion of the Matrix +specification. + +There's a few related features: + +* Push notifications (i.e. email or outgoing requests to a Push Gateway). +* Calculation of unread notifications (for /sync and /notifications). + +When Synapse receives a new event (locally, via the Client-Server API, or via +federation), the following occurs: + +1. The push rules get evaluated to generate a set of per-user actions. +2. The event is persisted into the database. +3. (In the background) The notifier is notified about the new event. + +The per-user actions are initially stored in the event_push_actions_staging table, +before getting moved into the event_push_actions table when the event is persisted. +The event_push_actions table is periodically summarised into the event_push_summary +and event_push_summary_stream_ordering tables. + +Since push actions block an event from being persisted the generation of push +actions is performance sensitive. + +The general interaction of the classes are: + + +---------------------------------------------+ + | FederationEventHandler/EventCreationHandler | + +---------------------------------------------+ + | + v + +-----------------------+ +---------------------------+ + | BulkPushRuleEvaluator |---->| PushRuleEvaluatorForEvent | + +-----------------------+ +---------------------------+ + | + v + +-----------------------------+ + | EventPushActionsWorkerStore | + +-----------------------------+ + +The notifier notifies the pusher pool of the new event, which checks for affected +users. Each user-configured pusher of the affected users then performs the +previously calculated action. + +The general interaction of the classes are: + + +----------+ + | Notifier | + +----------+ + | + v + +------------+ +--------------+ + | PusherPool |---->| PusherConfig | + +------------+ +--------------+ + | + | +---------------+ + +<--->| PusherFactory | + | +---------------+ + v + +------------------------+ +-----------------------------------------------+ + | EmailPusher/HttpPusher |---->| EventPushActionsWorkerStore/PusherWorkerStore | + +------------------------+ +-----------------------------------------------+ + | + v + +-------------------------+ + | Mailer/SimpleHttpClient | + +-------------------------+ + +The Pusher instance also calls out to various utilities for generating payloads +(or email templates), but those interactions are not detailed in this diagram +(and are specific to the type of pusher). + +""" + import abc from typing import TYPE_CHECKING, Any, Dict, Optional diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py deleted file mode 100644 index 60758df01664..000000000000 --- a/synapse/push/action_generator.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -from typing import TYPE_CHECKING - -from synapse.events import EventBase -from synapse.events.snapshot import EventContext -from synapse.push.bulk_push_rule_evaluator import BulkPushRuleEvaluator -from synapse.util.metrics import Measure - -if TYPE_CHECKING: - from synapse.server import HomeServer - -logger = logging.getLogger(__name__) - - -class ActionGenerator: - def __init__(self, hs: "HomeServer"): - self.clock = hs.get_clock() - self.bulk_evaluator = BulkPushRuleEvaluator(hs) - # really we want to get all user ids and all profile tags too, - # since we want the actions for each profile tag for every user and - # also actions for a client with no profile tag for each user. - # Currently the event stream doesn't support profile tags on an - # event stream, so we just run the rules for a client with no profile - # tag (ie. we just need all the users). - - async def handle_push_actions_for_event( - self, event: EventBase, context: EventContext - ) -> None: - with Measure(self.clock, "action_for_event_by_user"): - await self.bulk_evaluator.action_for_event_by_user(event, context) diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index f42f605f2383..a17b35a605fb 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -277,6 +277,21 @@ def make_base_prepend_rules( ], "actions": ["dont_notify"], }, + # XXX: This is an experimental rule that is only enabled if msc3786_enabled + # is enabled, if it is not the rule gets filtered out in _load_rules() in + # PushRulesWorkerStore + { + "rule_id": "global/override/.org.matrix.msc3786.rule.room.server_acl", + "conditions": [ + { + "kind": "event_match", + "key": "type", + "pattern": "m.room.server_acl", + "_cache_key": "_room_server_acl", + } + ], + "actions": ["dont_notify"], + }, ] diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index b07cf2eee705..4cc8a2ecca7a 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -20,8 +20,8 @@ from prometheus_client import Counter from synapse.api.constants import EventTypes, Membership, RelationTypes -from synapse.event_auth import get_user_power_level -from synapse.events import EventBase +from synapse.event_auth import auth_types_for_event, get_user_power_level +from synapse.events import EventBase, relation_from_event from synapse.events.snapshot import EventContext from synapse.state import POWER_KEY from synapse.storage.databases.main.roommember import EventIdMembership @@ -29,7 +29,9 @@ from synapse.util.caches import CacheMetric, register_cache from synapse.util.caches.descriptors import lru_cache from synapse.util.caches.lrucache import LruCache +from synapse.util.metrics import measure_func +from ..storage.state import StateFilter from .push_rule_evaluator import PushRuleEvaluatorForEvent if TYPE_CHECKING: @@ -77,8 +79,8 @@ def _should_count_as_unread(event: EventBase, context: EventContext) -> bool: return False # Exclude edits. - relates_to = event.content.get("m.relates_to", {}) - if relates_to.get("rel_type") == RelationTypes.REPLACE: + relates_to = relation_from_event(event) + if relates_to and relates_to.rel_type == RelationTypes.REPLACE: return False # Mark events that have a non-empty string body as unread. @@ -105,6 +107,7 @@ class BulkPushRuleEvaluator: def __init__(self, hs: "HomeServer"): self.hs = hs self.store = hs.get_datastores().main + self.clock = hs.get_clock() self._event_auth_handler = hs.get_event_auth_handler() # Used by `RulesForRoom` to ensure only one thing mutates the cache at a @@ -166,8 +169,12 @@ def _get_rules_for_room(self, room_id: str) -> "RulesForRoomData": async def _get_power_levels_and_sender_level( self, event: EventBase, context: EventContext ) -> Tuple[dict, int]: - prev_state_ids = await context.get_prev_state_ids() + event_types = auth_types_for_event(event.room_version, event) + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types(event_types) + ) pl_event_id = prev_state_ids.get(POWER_KEY) + if pl_event_id: # fastpath: if there's a power level event, that's all we need, and # not having a power level event is an extreme edge case @@ -185,6 +192,7 @@ async def _get_power_levels_and_sender_level( return pl_event.content if pl_event else {}, sender_level + @measure_func("action_for_event_by_user") async def action_for_event_by_user( self, event: EventBase, context: EventContext ) -> None: @@ -192,6 +200,10 @@ async def action_for_event_by_user( should increment the unread count, and insert the results into the event_push_actions_staging table. """ + if event.internal_metadata.is_outlier(): + # This can happen due to out of band memberships + return + count_as_unread = _should_count_as_unread(event, context) rules_by_user = await self._get_rules_for_event(event, context) @@ -208,8 +220,6 @@ async def action_for_event_by_user( event, len(room_members), sender_power_level, power_levels ) - condition_cache: Dict[str, bool] = {} - # If the event is not a state event check if any users ignore the sender. if not event.is_state(): ignorers = await self.store.ignored_by(event.sender) @@ -247,8 +257,8 @@ async def action_for_event_by_user( if "enabled" in rule and not rule["enabled"]: continue - matches = _condition_checker( - evaluator, rule["conditions"], uid, display_name, condition_cache + matches = evaluator.check_conditions( + rule["conditions"], uid, display_name ) if matches: actions = [x for x in rule["actions"] if x != "dont_notify"] @@ -267,32 +277,6 @@ async def action_for_event_by_user( ) -def _condition_checker( - evaluator: PushRuleEvaluatorForEvent, - conditions: List[dict], - uid: str, - display_name: Optional[str], - cache: Dict[str, bool], -) -> bool: - for cond in conditions: - _cache_key = cond.get("_cache_key", None) - if _cache_key: - res = cache.get(_cache_key, None) - if res is False: - return False - elif res is True: - continue - - res = evaluator.matches(cond, uid, display_name) - if _cache_key: - cache[_cache_key] = bool(res) - - if not res: - return False - - return True - - MemberMap = Dict[str, Optional[EventIdMembership]] Rule = Dict[str, dict] RulesByUser = Dict[str, List[Rule]] diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 5818344520f5..d5603596c004 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -405,7 +405,7 @@ async def dispatch_push( rejected = [] if "rejected" in resp: rejected = resp["rejected"] - else: + if not rejected: self.badge_count_last_call = badge return rejected diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index f617c759e6cf..54db6b5612a3 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -129,9 +129,55 @@ def __init__( # Maps strings of e.g. 'content.body' -> event["content"]["body"] self._value_cache = _flatten_dict(event) + # Maps cache keys to final values. + self._condition_cache: Dict[str, bool] = {} + + def check_conditions( + self, conditions: List[dict], uid: str, display_name: Optional[str] + ) -> bool: + """ + Returns true if a user's conditions/user ID/display name match the event. + + Args: + conditions: The user's conditions to match. + uid: The user's MXID. + display_name: The display name. + + Returns: + True if all conditions match the event, False otherwise. + """ + for cond in conditions: + _cache_key = cond.get("_cache_key", None) + if _cache_key: + res = self._condition_cache.get(_cache_key, None) + if res is False: + return False + elif res is True: + continue + + res = self.matches(cond, uid, display_name) + if _cache_key: + self._condition_cache[_cache_key] = bool(res) + + if not res: + return False + + return True + def matches( self, condition: Dict[str, Any], user_id: str, display_name: Optional[str] ) -> bool: + """ + Returns true if a user's condition/user ID/display name match the event. + + Args: + condition: The user's condition to match. + uid: The user's MXID. + display_name: The display name, or None if there is not one. + + Returns: + True if the condition matches the event, False otherwise. + """ if condition["kind"] == "event_match": return self._event_match(condition, user_id) elif condition["kind"] == "contains_display_name": @@ -146,6 +192,16 @@ def matches( return True def _event_match(self, condition: dict, user_id: str) -> bool: + """ + Check an "event_match" push rule condition. + + Args: + condition: The "event_match" push rule condition to match. + user_id: The user's MXID. + + Returns: + True if the condition matches the event, False otherwise. + """ pattern = condition.get("pattern", None) if not pattern: @@ -167,13 +223,22 @@ def _event_match(self, condition: dict, user_id: str) -> bool: return _glob_matches(pattern, body, word_boundary=True) else: - haystack = self._get_value(condition["key"]) + haystack = self._value_cache.get(condition["key"], None) if haystack is None: return False return _glob_matches(pattern, haystack) def _contains_display_name(self, display_name: Optional[str]) -> bool: + """ + Check an "event_match" push rule condition. + + Args: + display_name: The display name, or None if there is not one. + + Returns: + True if the display name is found in the event body, False otherwise. + """ if not display_name: return False @@ -191,9 +256,6 @@ def _contains_display_name(self, display_name: Optional[str]) -> bool: return bool(r.search(body)) - def _get_value(self, dotted_key: str) -> Optional[str]: - return self._value_cache.get(dotted_key, None) - # Caches (string, is_glob, word_boundary) -> regex for push. See _glob_matches regex_cache: LruCache[Tuple[str, bool, bool], Pattern] = LruCache( diff --git a/synapse/push/push_tools.py b/synapse/push/push_tools.py index 957c9b780b94..a1bf5b20dd42 100644 --- a/synapse/push/push_tools.py +++ b/synapse/push/push_tools.py @@ -24,7 +24,9 @@ async def get_badge_count(store: DataStore, user_id: str, group_by_room: bool) - invites = await store.get_invited_rooms_for_local_user(user_id) joins = await store.get_rooms_for_user(user_id) - my_receipts_by_room = await store.get_receipts_for_user(user_id, ReceiptTypes.READ) + my_receipts_by_room = await store.get_receipts_for_user( + user_id, (ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE) + ) badge = len(invites) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py deleted file mode 100644 index ec199a161db8..000000000000 --- a/synapse/python_dependencies.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2015, 2016 OpenMarket Ltd -# Copyright 2017 Vector Creations Ltd -# Copyright 2018 New Vector Ltd -# Copyright 2020 The Matrix.org Foundation C.I.C. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import itertools -import logging -from typing import Set - -logger = logging.getLogger(__name__) - - -# REQUIREMENTS is a simple list of requirement specifiers[1], and must be -# installed. It is passed to setup() as install_requires in setup.py. -# -# CONDITIONAL_REQUIREMENTS is the optional dependencies, represented as a dict -# of lists. The dict key is the optional dependency name and can be passed to -# pip when installing. The list is a series of requirement specifiers[1] to be -# installed when that optional dependency requirement is specified. It is passed -# to setup() as extras_require in setup.py -# -# Note that these both represent runtime dependencies (and the versions -# installed are checked at runtime). -# -# Also note that we replicate these constraints in the Synapse Dockerfile while -# pre-installing dependencies. If these constraints are updated here, the same -# change should be made in the Dockerfile. -# -# [1] https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers. - -REQUIREMENTS = [ - # we use the TYPE_CHECKER.redefine method added in jsonschema 3.0.0 - "jsonschema>=3.0.0", - # frozendict 2.1.2 is broken on Debian 10: https://github.com/Marco-Sulla/python-frozendict/issues/41 - "frozendict>=1,!=2.1.2", - "unpaddedbase64>=1.1.0", - "canonicaljson>=1.4.0", - # we use the type definitions added in signedjson 1.1. - "signedjson>=1.1.0", - "pynacl>=1.2.1", - # validating SSL certs for IP addresses requires service_identity 18.1. - "service_identity>=18.1.0", - # Twisted 18.9 introduces some logger improvements that the structured - # logger utilises - "Twisted[tls]>=18.9.0", - "treq>=15.1", - # Twisted has required pyopenssl 16.0 since about Twisted 16.6. - "pyopenssl>=16.0.0", - "pyyaml>=3.11", - "pyasn1>=0.1.9", - "pyasn1-modules>=0.0.7", - "bcrypt>=3.1.0", - "pillow>=5.4.0", - "sortedcontainers>=1.4.4", - "pymacaroons>=0.13.0", - "msgpack>=0.5.2", - "phonenumbers>=8.2.0", - # we use GaugeHistogramMetric, which was added in prom-client 0.4.0. - "prometheus_client>=0.4.0", - # we use `order`, which arrived in attrs 19.2.0. - # Note: 21.1.0 broke `/sync`, see #9936 - "attrs>=19.2.0,!=21.1.0", - "netaddr>=0.7.18", - # Jinja 2.x is incompatible with MarkupSafe>=2.1. To ensure that admins do not - # end up with a broken installation, with recent MarkupSafe but old Jinja, we - # add a lower bound to the Jinja2 dependency. - "Jinja2>=3.0", - "bleach>=1.4.3", - # We use `ParamSpec`, which was added in `typing-extensions` 3.10.0.0. - "typing-extensions>=3.10.0", - # We enforce that we have a `cryptography` version that bundles an `openssl` - # with the latest security patches. - "cryptography>=3.4.7", - # ijson 3.1.4 fixes a bug with "." in property names - "ijson>=3.1.4", - "matrix-common~=1.1.0", - # We need packaging.requirements.Requirement, added in 16.1. - "packaging>=16.1", - # At the time of writing, we only use functions from the version `importlib.metadata` - # which shipped in Python 3.8. This corresponds to version 1.4 of the backport. - "importlib_metadata>=1.4 ; python_version < '3.8'", -] - -CONDITIONAL_REQUIREMENTS = { - "matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"], - "postgres": [ - # we use execute_values with the fetch param, which arrived in psycopg 2.8. - "psycopg2>=2.8 ; platform_python_implementation != 'PyPy'", - "psycopg2cffi>=2.8 ; platform_python_implementation == 'PyPy'", - "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'", - ], - "saml2": [ - "pysaml2>=4.5.0", - ], - "oidc": ["authlib>=0.14.0"], - # systemd-python is necessary for logging to the systemd journal via - # `systemd.journal.JournalHandler`, as is documented in - # `contrib/systemd/log_config.yaml`. - "systemd": ["systemd-python>=231"], - "url_preview": ["lxml>=4.2.0"], - "sentry": ["sentry-sdk>=0.7.2"], - "opentracing": ["jaeger-client>=4.0.0", "opentracing>=2.2.0"], - "jwt": ["pyjwt>=1.6.4"], - # hiredis is not a *strict* dependency, but it makes things much faster. - # (if it is not installed, we fall back to slow code.) - "redis": ["txredisapi>=1.4.7", "hiredis"], - # Required to use experimental `caches.track_memory_usage` config option. - "cache_memory": ["pympler"], -} - -ALL_OPTIONAL_REQUIREMENTS: Set[str] = set() - -for name, optional_deps in CONDITIONAL_REQUIREMENTS.items(): - # Exclude systemd as it's a system-based requirement. - # Exclude lint as it's a dev-based requirement. - if name not in ["systemd"]: - ALL_OPTIONAL_REQUIREMENTS = set(optional_deps) | ALL_OPTIONAL_REQUIREMENTS - - -# ensure there are no double-quote characters in any of the deps (otherwise the -# 'pip install' incantation in DependencyException will break) -for dep in itertools.chain( - REQUIREMENTS, - *CONDITIONAL_REQUIREMENTS.values(), -): - if '"' in dep: - raise Exception( - "Dependency `%s` contains double-quote; use single-quotes instead" % (dep,) - ) - - -def list_requirements(): - return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS) - - -if __name__ == "__main__": - import sys - - sys.stdout.writelines(req + "\n" for req in list_requirements()) diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index 2bd244ed79df..a4ae4040c353 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -26,7 +26,8 @@ from synapse.api.errors import HttpResponseException, SynapseError from synapse.http import RequestTimedOutError -from synapse.http.server import HttpServer +from synapse.http.server import HttpServer, is_method_cancellable +from synapse.http.site import SynapseRequest from synapse.logging import opentracing from synapse.logging.opentracing import trace from synapse.types import JsonDict @@ -310,6 +311,12 @@ def register(self, http_server: HttpServer) -> None: url_args = list(self.PATH_ARGS) method = self.METHOD + if self.CACHE and is_method_cancellable(self._handle_request): + raise Exception( + f"{self.__class__.__name__} has been marked as cancellable, but CACHE " + "is set. The cancellable flag would have no effect." + ) + if self.CACHE: url_args.append("txn_id") @@ -324,7 +331,7 @@ def register(self, http_server: HttpServer) -> None: ) async def _check_auth_and_handle( - self, request: Request, **kwargs: Any + self, request: SynapseRequest, **kwargs: Any ) -> Tuple[int, JsonDict]: """Called on new incoming requests when caching is enabled. Checks if there is a cached response for the request and returns that, @@ -340,8 +347,18 @@ async def _check_auth_and_handle( if self.CACHE: txn_id = kwargs.pop("txn_id") + # We ignore the `@cancellable` flag, since cancellation wouldn't interupt + # `_handle_request` and `ResponseCache` does not handle cancellation + # correctly yet. In particular, there may be issues to do with logging + # context lifetimes. + return await self.response_cache.wrap( txn_id, self._handle_request, request, **kwargs ) + # The `@cancellable` decorator may be applied to `_handle_request`. But we + # told `HttpServer.register_paths` that our handler is `_check_auth_and_handle`, + # so we have to set up the cancellable flag ourselves. + request.is_render_cancellable = is_method_cancellable(self._handle_request) + return await self._handle_request(request, **kwargs) diff --git a/synapse/replication/tcp/__init__.py b/synapse/replication/tcp/__init__.py index 1fa60af8e6bc..2c5f5f0bf867 100644 --- a/synapse/replication/tcp/__init__.py +++ b/synapse/replication/tcp/__init__.py @@ -15,7 +15,7 @@ """This module implements the TCP replication protocol used by synapse to communicate between the master process and its workers (when they're enabled). -Further details can be found in docs/tcp_replication.rst +Further details can be found in docs/tcp_replication.md Structure of the module: diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 122892c7bca2..a52e25c1af3f 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -21,7 +21,7 @@ from twisted.internet.protocol import ReconnectingClientFactory from twisted.python.failure import Failure -from synapse.api.constants import EventTypes +from synapse.api.constants import EventTypes, ReceiptTypes from synapse.federation import send_queue from synapse.federation.sender import FederationSender from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable @@ -43,7 +43,7 @@ EventsStreamEventRow, EventsStreamRow, ) -from synapse.types import PersistedEventPosition, ReadReceipt, UserID +from synapse.types import PersistedEventPosition, ReadReceipt, StreamKeyType, UserID from synapse.util.async_helpers import Linearizer, timeout_deferred from synapse.util.metrics import Measure @@ -153,19 +153,19 @@ async def on_rdata( if stream_name == TypingStream.NAME: self._typing_handler.process_replication_rows(token, rows) self.notifier.on_new_event( - "typing_key", token, rooms=[row.room_id for row in rows] + StreamKeyType.TYPING, token, rooms=[row.room_id for row in rows] ) elif stream_name == PushRulesStream.NAME: self.notifier.on_new_event( - "push_rules_key", token, users=[row.user_id for row in rows] + StreamKeyType.PUSH_RULES, token, users=[row.user_id for row in rows] ) elif stream_name in (AccountDataStream.NAME, TagAccountDataStream.NAME): self.notifier.on_new_event( - "account_data_key", token, users=[row.user_id for row in rows] + StreamKeyType.ACCOUNT_DATA, token, users=[row.user_id for row in rows] ) elif stream_name == ReceiptsStream.NAME: self.notifier.on_new_event( - "receipt_key", token, rooms=[row.room_id for row in rows] + StreamKeyType.RECEIPT, token, rooms=[row.room_id for row in rows] ) await self._pusher_pool.on_new_receipts( token, token, {row.room_id for row in rows} @@ -173,14 +173,18 @@ async def on_rdata( elif stream_name == ToDeviceStream.NAME: entities = [row.entity for row in rows if row.entity.startswith("@")] if entities: - self.notifier.on_new_event("to_device_key", token, users=entities) + self.notifier.on_new_event( + StreamKeyType.TO_DEVICE, token, users=entities + ) elif stream_name == DeviceListsStream.NAME: all_room_ids: Set[str] = set() for row in rows: if row.entity.startswith("@"): room_ids = await self.store.get_rooms_for_user(row.entity) all_room_ids.update(room_ids) - self.notifier.on_new_event("device_list_key", token, rooms=all_room_ids) + self.notifier.on_new_event( + StreamKeyType.DEVICE_LIST, token, rooms=all_room_ids + ) elif stream_name == GroupServerStream.NAME: self.notifier.on_new_event( "groups_key", token, users=[row.user_id for row in rows] @@ -401,10 +405,8 @@ async def _on_new_receipts( # we only want to send on receipts for our own users if not self._is_mine_id(receipt.user_id): continue - if ( - receipt.data.get("hidden", False) - and self._hs.config.experimental.msc2285_enabled - ): + # Private read receipts never get sent over federation. + if receipt.receipt_type == ReceiptTypes.READ_PRIVATE: continue receipt_info = ReadReceipt( receipt.room_id, diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py index fe34948168ab..32f52e54d8c7 100644 --- a/synapse/replication/tcp/commands.py +++ b/synapse/replication/tcp/commands.py @@ -58,6 +58,15 @@ def get_logcontext_id(self) -> str: # by default, we just use the command name. return self.NAME + def redis_channel_name(self, prefix: str) -> str: + """ + Returns the Redis channel name upon which to publish this command. + + Args: + prefix: The prefix for the channel. + """ + return prefix + SC = TypeVar("SC", bound="_SimpleCommand") @@ -395,6 +404,9 @@ def __repr__(self) -> str: f"{self.user_agent!r}, {self.device_id!r}, {self.last_seen})" ) + def redis_channel_name(self, prefix: str) -> str: + return f"{prefix}/USER_IP" + class RemoteServerUpCommand(_SimpleCommand): """Sent when a worker has detected that a remote server is no longer diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index 615f1828dd73..e1cbfa50ebd2 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -1,5 +1,5 @@ # Copyright 2017 Vector Creations Ltd -# Copyright 2020 The Matrix.org Foundation C.I.C. +# Copyright 2020, 2022 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -101,6 +101,9 @@ def __init__(self, hs: "HomeServer"): self._instance_id = hs.get_instance_id() self._instance_name = hs.get_instance_name() + # Additional Redis channel suffixes to subscribe to. + self._channels_to_subscribe_to: List[str] = [] + self._is_presence_writer = ( hs.get_instance_name() in hs.config.worker.writers.presence ) @@ -243,6 +246,31 @@ def __init__(self, hs: "HomeServer"): # If we're NOT using Redis, this must be handled by the master self._should_insert_client_ips = hs.get_instance_name() == "master" + if self._is_master or self._should_insert_client_ips: + self.subscribe_to_channel("USER_IP") + + def subscribe_to_channel(self, channel_name: str) -> None: + """ + Indicates that we wish to subscribe to a Redis channel by name. + + (The name will later be prefixed with the server name; i.e. subscribing + to the 'ABC' channel actually subscribes to 'example.com/ABC' Redis-side.) + + Raises: + - If replication has already started, then it's too late to subscribe + to new channels. + """ + + if self._factory is not None: + # We don't allow subscribing after the fact to avoid the chance + # of missing an important message because we didn't subscribe in time. + raise RuntimeError( + "Cannot subscribe to more channels after replication started." + ) + + if channel_name not in self._channels_to_subscribe_to: + self._channels_to_subscribe_to.append(channel_name) + def _add_command_to_stream_queue( self, conn: IReplicationConnection, cmd: Union[RdataCommand, PositionCommand] ) -> None: @@ -321,7 +349,9 @@ def start_replication(self, hs: "HomeServer") -> None: # Now create the factory/connection for the subscription stream. self._factory = RedisDirectTcpReplicationClientFactory( - hs, outbound_redis_connection + hs, + outbound_redis_connection, + channel_names=self._channels_to_subscribe_to, ) hs.get_reactor().connectTCP( hs.config.redis.redis_host, @@ -537,7 +567,7 @@ def on_POSITION(self, conn: IReplicationConnection, cmd: PositionCommand) -> Non # Ignore POSITION that are just our own echoes return - logger.info("Handling '%s %s'", cmd.NAME, cmd.to_line()) + logger.debug("Handling '%s %s'", cmd.NAME, cmd.to_line()) self._add_command_to_stream_queue(conn, cmd) @@ -567,6 +597,11 @@ async def _process_position( # between then and now. missing_updates = cmd.prev_token != current_token while missing_updates: + # Note: There may very well not be any new updates, but we check to + # make sure. This can particularly happen for the event stream where + # event persisters continuously send `POSITION`. See `resource.py` + # for why this can happen. + logger.info( "Fetching replication rows for '%s' between %i and %i", stream_name, @@ -590,7 +625,7 @@ async def _process_position( [stream.parse_row(row) for row in rows], ) - logger.info("Caught up with stream '%s' to %i", stream_name, cmd.new_token) + logger.info("Caught up with stream '%s' to %i", stream_name, cmd.new_token) # We've now caught up to position sent to us, notify handler. await self._replication_data_handler.on_position( diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py index 989c5be0327e..fd1c0ec6afa2 100644 --- a/synapse/replication/tcp/redis.py +++ b/synapse/replication/tcp/redis.py @@ -14,7 +14,7 @@ import logging from inspect import isawaitable -from typing import TYPE_CHECKING, Any, Generic, Optional, Type, TypeVar, cast +from typing import TYPE_CHECKING, Any, Generic, List, Optional, Type, TypeVar, cast import attr import txredisapi @@ -85,14 +85,15 @@ class RedisSubscriber(txredisapi.SubscriberProtocol): Attributes: synapse_handler: The command handler to handle incoming commands. - synapse_stream_name: The *redis* stream name to subscribe to and publish + synapse_stream_prefix: The *redis* stream name to subscribe to and publish from (not anything to do with Synapse replication streams). synapse_outbound_redis_connection: The connection to redis to use to send commands. """ synapse_handler: "ReplicationCommandHandler" - synapse_stream_name: str + synapse_stream_prefix: str + synapse_channel_names: List[str] synapse_outbound_redis_connection: txredisapi.ConnectionHandler def __init__(self, *args: Any, **kwargs: Any): @@ -117,8 +118,13 @@ async def _send_subscribe(self) -> None: # it's important to make sure that we only send the REPLICATE command once we # have successfully subscribed to the stream - otherwise we might miss the # POSITION response sent back by the other end. - logger.info("Sending redis SUBSCRIBE for %s", self.synapse_stream_name) - await make_deferred_yieldable(self.subscribe(self.synapse_stream_name)) + fully_qualified_stream_names = [ + f"{self.synapse_stream_prefix}/{stream_suffix}" + for stream_suffix in self.synapse_channel_names + ] + [self.synapse_stream_prefix] + logger.info("Sending redis SUBSCRIBE for %r", fully_qualified_stream_names) + await make_deferred_yieldable(self.subscribe(fully_qualified_stream_names)) + logger.info( "Successfully subscribed to redis stream, sending REPLICATE command" ) @@ -215,10 +221,10 @@ async def _async_send_command(self, cmd: Command) -> None: # remote instances. tcp_outbound_commands_counter.labels(cmd.NAME, "redis").inc() + channel_name = cmd.redis_channel_name(self.synapse_stream_prefix) + await make_deferred_yieldable( - self.synapse_outbound_redis_connection.publish( - self.synapse_stream_name, encoded_string - ) + self.synapse_outbound_redis_connection.publish(channel_name, encoded_string) ) @@ -300,20 +306,27 @@ def format_address(address: IAddress) -> str: class RedisDirectTcpReplicationClientFactory(SynapseRedisFactory): """This is a reconnecting factory that connects to redis and immediately - subscribes to a stream. + subscribes to some streams. Args: hs outbound_redis_connection: A connection to redis that will be used to send outbound commands (this is separate to the redis connection used to subscribe). + channel_names: A list of channel names to append to the base channel name + to additionally subscribe to. + e.g. if ['ABC', 'DEF'] is specified then we'll listen to: + example.com; example.com/ABC; and example.com/DEF. """ maxDelay = 5 protocol = RedisSubscriber def __init__( - self, hs: "HomeServer", outbound_redis_connection: txredisapi.ConnectionHandler + self, + hs: "HomeServer", + outbound_redis_connection: txredisapi.ConnectionHandler, + channel_names: List[str], ): super().__init__( @@ -326,7 +339,8 @@ def __init__( ) self.synapse_handler = hs.get_replication_command_handler() - self.synapse_stream_name = hs.hostname + self.synapse_stream_prefix = hs.hostname + self.synapse_channel_names = channel_names self.synapse_outbound_redis_connection = outbound_redis_connection @@ -340,7 +354,8 @@ def buildProtocol(self, addr: IAddress) -> RedisSubscriber: # protocol. p.synapse_handler = self.synapse_handler p.synapse_outbound_redis_connection = self.synapse_outbound_redis_connection - p.synapse_stream_name = self.synapse_stream_name + p.synapse_stream_prefix = self.synapse_stream_prefix + p.synapse_channel_names = self.synapse_channel_names return p diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index c6870df8f954..99f09669f00b 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -204,6 +204,15 @@ async def _run_notifier_loop(self) -> None: # turns out that e.g. account data streams share # their "current token" with each other, meaning # that it is *not* safe to send a POSITION. + + # Note: `last_token` may not *actually* be the + # last token we sent out in a RDATA or POSITION. + # This can happen if we sent out an RDATA for + # position X when our current token was say X+1. + # Other workers will see RDATA for X and then a + # POSITION with last token of X+1, which will + # cause them to check if there were any missing + # updates between X and X+1. logger.info( "Sending position: %s -> %s", stream.NAME, diff --git a/synapse/res/templates/notif.html b/synapse/res/templates/notif.html index 0aaef97df893..7d86681fed53 100644 --- a/synapse/res/templates/notif.html +++ b/synapse/res/templates/notif.html @@ -30,7 +30,7 @@ {%- elif message.msgtype == "m.notice" %} {{ message.body_text_html }} {%- elif message.msgtype == "m.image" and message.image_url %} - + {%- elif message.msgtype == "m.file" %} {{ message.body_text_plain }} {%- else %} diff --git a/synapse/rest/client/account.py b/synapse/rest/client/account.py index 5587cae98a61..bdc4a9c0683d 100644 --- a/synapse/rest/client/account.py +++ b/synapse/rest/client/account.py @@ -882,9 +882,7 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: response = { "user_id": requester.user.to_string(), - # MSC: https://github.com/matrix-org/matrix-doc/pull/3069 # Entered spec in Matrix 1.2 - "org.matrix.msc3069.is_guest": bool(requester.is_guest), "is_guest": bool(requester.is_guest), } diff --git a/synapse/rest/client/auth.py b/synapse/rest/client/auth.py index e0b2b80e5b9b..eb77337044da 100644 --- a/synapse/rest/client/auth.py +++ b/synapse/rest/client/auth.py @@ -112,7 +112,7 @@ async def on_POST(self, request: Request, stagetype: str) -> None: try: await self.auth_handler.add_oob_auth( - LoginType.RECAPTCHA, authdict, request.getClientIP() + LoginType.RECAPTCHA, authdict, request.getClientAddress().host ) except LoginError as e: # Authentication failed, let user try again @@ -132,7 +132,7 @@ async def on_POST(self, request: Request, stagetype: str) -> None: try: await self.auth_handler.add_oob_auth( - LoginType.TERMS, authdict, request.getClientIP() + LoginType.TERMS, authdict, request.getClientAddress().host ) except LoginError as e: # Authentication failed, let user try again @@ -161,7 +161,9 @@ async def on_POST(self, request: Request, stagetype: str) -> None: try: await self.auth_handler.add_oob_auth( - LoginType.REGISTRATION_TOKEN, authdict, request.getClientIP() + LoginType.REGISTRATION_TOKEN, + authdict, + request.getClientAddress().host, ) except LoginError as e: html = self.registration_token_template.render( diff --git a/synapse/rest/client/knock.py b/synapse/rest/client/knock.py index 0152a0c66a50..ad025c8a4529 100644 --- a/synapse/rest/client/knock.py +++ b/synapse/rest/client/knock.py @@ -15,8 +15,6 @@ import logging from typing import TYPE_CHECKING, Awaitable, Dict, List, Optional, Tuple -from twisted.web.server import Request - from synapse.api.constants import Membership from synapse.api.errors import SynapseError from synapse.http.server import HttpServer @@ -97,7 +95,7 @@ async def on_POST( return 200, {"room_id": room_id} def on_PUT( - self, request: Request, room_identifier: str, txn_id: str + self, request: SynapseRequest, room_identifier: str, txn_id: str ) -> Awaitable[Tuple[int, JsonDict]]: set_tag("txn_id", txn_id) diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index c9d44c5964ce..cf4196ac0a2b 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -69,9 +69,7 @@ class LoginRestServlet(RestServlet): SSO_TYPE = "m.login.sso" TOKEN_TYPE = "m.login.token" JWT_TYPE = "org.matrix.login.jwt" - JWT_TYPE_DEPRECATED = "m.login.jwt" APPSERVICE_TYPE = "m.login.application_service" - APPSERVICE_TYPE_UNSTABLE = "uk.half-shot.msc2778.login.application_service" REFRESH_TOKEN_PARAM = "refresh_token" def __init__(self, hs: "HomeServer"): @@ -126,7 +124,6 @@ def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: flows: List[JsonDict] = [] if self.jwt_enabled: flows.append({"type": LoginRestServlet.JWT_TYPE}) - flows.append({"type": LoginRestServlet.JWT_TYPE_DEPRECATED}) if self.cas_enabled: # we advertise CAS for backwards compat, though MSC1721 renamed it @@ -156,7 +153,6 @@ def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: flows.extend({"type": t} for t in self.auth_handler.get_supported_login_types()) flows.append({"type": LoginRestServlet.APPSERVICE_TYPE}) - flows.append({"type": LoginRestServlet.APPSERVICE_TYPE_UNSTABLE}) return 200, {"flows": flows} @@ -175,15 +171,12 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, LoginResponse]: ) try: - if login_submission["type"] in ( - LoginRestServlet.APPSERVICE_TYPE, - LoginRestServlet.APPSERVICE_TYPE_UNSTABLE, - ): + if login_submission["type"] == LoginRestServlet.APPSERVICE_TYPE: appservice = self.auth.get_appservice_by_req(request) if appservice.is_rate_limited(): await self._address_ratelimiter.ratelimit( - None, request.getClientIP() + None, request.getClientAddress().host ) result = await self._do_appservice_login( @@ -191,23 +184,29 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, LoginResponse]: appservice, should_issue_refresh_token=should_issue_refresh_token, ) - elif self.jwt_enabled and ( - login_submission["type"] == LoginRestServlet.JWT_TYPE - or login_submission["type"] == LoginRestServlet.JWT_TYPE_DEPRECATED + elif ( + self.jwt_enabled + and login_submission["type"] == LoginRestServlet.JWT_TYPE ): - await self._address_ratelimiter.ratelimit(None, request.getClientIP()) + await self._address_ratelimiter.ratelimit( + None, request.getClientAddress().host + ) result = await self._do_jwt_login( login_submission, should_issue_refresh_token=should_issue_refresh_token, ) elif login_submission["type"] == LoginRestServlet.TOKEN_TYPE: - await self._address_ratelimiter.ratelimit(None, request.getClientIP()) + await self._address_ratelimiter.ratelimit( + None, request.getClientAddress().host + ) result = await self._do_token_login( login_submission, should_issue_refresh_token=should_issue_refresh_token, ) else: - await self._address_ratelimiter.ratelimit(None, request.getClientIP()) + await self._address_ratelimiter.ratelimit( + None, request.getClientAddress().host + ) result = await self._do_other_login( login_submission, should_issue_refresh_token=should_issue_refresh_token, @@ -342,6 +341,15 @@ async def _complete_login( user_id = canonical_uid device_id = login_submission.get("device_id") + + # If device_id is present, check that device_id is not longer than a reasonable 512 characters + if device_id and len(device_id) > 512: + raise LoginError( + 400, + "device_id cannot be longer than 512 characters.", + errcode=Codes.INVALID_PARAM, + ) + initial_display_name = login_submission.get("initial_device_display_name") ( device_id, diff --git a/synapse/rest/client/notifications.py b/synapse/rest/client/notifications.py index ff040de6b840..24bc7c90957f 100644 --- a/synapse/rest/client/notifications.py +++ b/synapse/rest/client/notifications.py @@ -58,7 +58,7 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: ) receipts_by_room = await self.store.get_receipts_for_user_with_orderings( - user_id, ReceiptTypes.READ + user_id, [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE] ) notif_event_ids = [pa.event_id for pa in push_actions] diff --git a/synapse/rest/client/push_rule.py b/synapse/rest/client/push_rule.py index a93f6fd5e0a8..8191b4e32c34 100644 --- a/synapse/rest/client/push_rule.py +++ b/synapse/rest/client/push_rule.py @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import TYPE_CHECKING, List, Optional, Sequence, Tuple, Union - -import attr +from typing import TYPE_CHECKING, List, Sequence, Tuple, Union from synapse.api.errors import ( NotFoundError, @@ -22,6 +20,7 @@ SynapseError, UnrecognizedRequestError, ) +from synapse.handlers.push_rules import InvalidRuleException, RuleSpec, check_actions from synapse.http.server import HttpServer from synapse.http.servlet import ( RestServlet, @@ -29,7 +28,6 @@ parse_string, ) from synapse.http.site import SynapseRequest -from synapse.push.baserules import BASE_RULE_IDS from synapse.push.clientformat import format_push_rules_for_user from synapse.push.rulekinds import PRIORITY_CLASS_MAP from synapse.rest.client._base import client_patterns @@ -40,14 +38,6 @@ from synapse.server import HomeServer -@attr.s(slots=True, frozen=True, auto_attribs=True) -class RuleSpec: - scope: str - template: str - rule_id: str - attr: Optional[str] - - class PushRuleRestServlet(RestServlet): PATTERNS = client_patterns("/(?Ppushrules/.*)$", v1=True) SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR = ( @@ -60,6 +50,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.notifier = hs.get_notifier() self._is_worker = hs.config.worker.worker_app is not None + self._push_rules_handler = hs.get_push_rules_handler() async def on_PUT(self, request: SynapseRequest, path: str) -> Tuple[int, JsonDict]: if self._is_worker: @@ -81,8 +72,13 @@ async def on_PUT(self, request: SynapseRequest, path: str) -> Tuple[int, JsonDic user_id = requester.user.to_string() if spec.attr: - await self.set_rule_attr(user_id, spec, content) - self.notify_user(user_id) + try: + await self._push_rules_handler.set_rule_attr(user_id, spec, content) + except InvalidRuleException as e: + raise SynapseError(400, "Invalid actions: %s" % e) + except RuleNotFoundException: + raise NotFoundError("Unknown rule") + return 200, {} if spec.rule_id.startswith("."): @@ -98,23 +94,23 @@ async def on_PUT(self, request: SynapseRequest, path: str) -> Tuple[int, JsonDic before = parse_string(request, "before") if before: - before = _namespaced_rule_id(spec, before) + before = f"global/{spec.template}/{before}" after = parse_string(request, "after") if after: - after = _namespaced_rule_id(spec, after) + after = f"global/{spec.template}/{after}" try: await self.store.add_push_rule( user_id=user_id, - rule_id=_namespaced_rule_id_from_spec(spec), + rule_id=f"global/{spec.template}/{spec.rule_id}", priority_class=priority_class, conditions=conditions, actions=actions, before=before, after=after, ) - self.notify_user(user_id) + self._push_rules_handler.notify_user(user_id) except InconsistentRuleException as e: raise SynapseError(400, str(e)) except RuleNotFoundException as e: @@ -133,11 +129,11 @@ async def on_DELETE( requester = await self.auth.get_user_by_req(request) user_id = requester.user.to_string() - namespaced_rule_id = _namespaced_rule_id_from_spec(spec) + namespaced_rule_id = f"global/{spec.template}/{spec.rule_id}" try: await self.store.delete_push_rule(user_id, namespaced_rule_id) - self.notify_user(user_id) + self._push_rules_handler.notify_user(user_id) return 200, {} except StoreError as e: if e.code == 404: @@ -152,9 +148,9 @@ async def on_GET(self, request: SynapseRequest, path: str) -> Tuple[int, JsonDic # we build up the full structure and then decide which bits of it # to send which means doing unnecessary work sometimes but is # is probably not going to make a whole lot of difference - rules = await self.store.get_push_rules_for_user(user_id) + rules_raw = await self.store.get_push_rules_for_user(user_id) - rules = format_push_rules_for_user(requester.user, rules) + rules = format_push_rules_for_user(requester.user, rules_raw) path_parts = path.split("/")[1:] @@ -172,55 +168,6 @@ async def on_GET(self, request: SynapseRequest, path: str) -> Tuple[int, JsonDic else: raise UnrecognizedRequestError() - def notify_user(self, user_id: str) -> None: - stream_id = self.store.get_max_push_rules_stream_id() - self.notifier.on_new_event("push_rules_key", stream_id, users=[user_id]) - - async def set_rule_attr( - self, user_id: str, spec: RuleSpec, val: Union[bool, JsonDict] - ) -> None: - if spec.attr not in ("enabled", "actions"): - # for the sake of potential future expansion, shouldn't report - # 404 in the case of an unknown request so check it corresponds to - # a known attribute first. - raise UnrecognizedRequestError() - - namespaced_rule_id = _namespaced_rule_id_from_spec(spec) - rule_id = spec.rule_id - is_default_rule = rule_id.startswith(".") - if is_default_rule: - if namespaced_rule_id not in BASE_RULE_IDS: - raise NotFoundError("Unknown rule %s" % (namespaced_rule_id,)) - if spec.attr == "enabled": - if isinstance(val, dict) and "enabled" in val: - val = val["enabled"] - if not isinstance(val, bool): - # Legacy fallback - # This should *actually* take a dict, but many clients pass - # bools directly, so let's not break them. - raise SynapseError(400, "Value for 'enabled' must be boolean") - await self.store.set_push_rule_enabled( - user_id, namespaced_rule_id, val, is_default_rule - ) - elif spec.attr == "actions": - if not isinstance(val, dict): - raise SynapseError(400, "Value must be a dict") - actions = val.get("actions") - if not isinstance(actions, list): - raise SynapseError(400, "Value for 'actions' must be dict") - _check_actions(actions) - namespaced_rule_id = _namespaced_rule_id_from_spec(spec) - rule_id = spec.rule_id - is_default_rule = rule_id.startswith(".") - if is_default_rule: - if namespaced_rule_id not in BASE_RULE_IDS: - raise SynapseError(404, "Unknown rule %r" % (namespaced_rule_id,)) - await self.store.set_push_rule_actions( - user_id, namespaced_rule_id, actions, is_default_rule - ) - else: - raise UnrecognizedRequestError() - def _rule_spec_from_path(path: Sequence[str]) -> RuleSpec: """Turn a sequence of path components into a rule spec @@ -291,24 +238,11 @@ def _rule_tuple_from_request_object( raise InvalidRuleException("No actions found") actions = req_obj["actions"] - _check_actions(actions) + check_actions(actions) return conditions, actions -def _check_actions(actions: List[Union[str, JsonDict]]) -> None: - if not isinstance(actions, list): - raise InvalidRuleException("No actions found") - - for a in actions: - if a in ["notify", "dont_notify", "coalesce"]: - pass - elif isinstance(a, dict) and "set_tweak" in a: - pass - else: - raise InvalidRuleException("Unrecognised action") - - def _filter_ruleset_with_path(ruleset: JsonDict, path: List[str]) -> JsonDict: if path == []: raise UnrecognizedRequestError( @@ -357,17 +291,5 @@ def _priority_class_from_spec(spec: RuleSpec) -> int: return pc -def _namespaced_rule_id_from_spec(spec: RuleSpec) -> str: - return _namespaced_rule_id(spec, spec.rule_id) - - -def _namespaced_rule_id(spec: RuleSpec, rule_id: str) -> str: - return "global/%s/%s" % (spec.template, rule_id) - - -class InvalidRuleException(Exception): - pass - - def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: PushRuleRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/read_marker.py b/synapse/rest/client/read_marker.py index f51be511d1f4..3644705e6abb 100644 --- a/synapse/rest/client/read_marker.py +++ b/synapse/rest/client/read_marker.py @@ -15,8 +15,7 @@ import logging from typing import TYPE_CHECKING, Tuple -from synapse.api.constants import ReadReceiptEventFields, ReceiptTypes -from synapse.api.errors import Codes, SynapseError +from synapse.api.constants import ReceiptTypes from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseRequest @@ -36,6 +35,7 @@ class ReadMarkerRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() self.auth = hs.get_auth() + self.config = hs.config self.receipts_handler = hs.get_receipts_handler() self.read_marker_handler = hs.get_read_marker_handler() self.presence_handler = hs.get_presence_handler() @@ -48,27 +48,42 @@ async def on_POST( await self.presence_handler.bump_presence_active_time(requester.user) body = parse_json_object_from_request(request) - read_event_id = body.get(ReceiptTypes.READ, None) - hidden = body.get(ReadReceiptEventFields.MSC2285_HIDDEN, False) - - if not isinstance(hidden, bool): - raise SynapseError( - 400, - "Param %s must be a boolean, if given" - % ReadReceiptEventFields.MSC2285_HIDDEN, - Codes.BAD_JSON, - ) + valid_receipt_types = { + ReceiptTypes.READ, + ReceiptTypes.FULLY_READ, + ReceiptTypes.READ_PRIVATE, + } + + unrecognized_types = set(body.keys()) - valid_receipt_types + if unrecognized_types: + # It's fine if there are unrecognized receipt types, but let's log + # it to help debug clients that have typoed the receipt type. + # + # We specifically *don't* error here, as a) it stops us processing + # the valid receipts, and b) we need to be extensible on receipt + # types. + logger.info("Ignoring unrecognized receipt types: %s", unrecognized_types) + + read_event_id = body.get(ReceiptTypes.READ, None) if read_event_id: await self.receipts_handler.received_client_receipt( room_id, ReceiptTypes.READ, user_id=requester.user.to_string(), event_id=read_event_id, - hidden=hidden, ) - read_marker_event_id = body.get("m.fully_read", None) + read_private_event_id = body.get(ReceiptTypes.READ_PRIVATE, None) + if read_private_event_id and self.config.experimental.msc2285_enabled: + await self.receipts_handler.received_client_receipt( + room_id, + ReceiptTypes.READ_PRIVATE, + user_id=requester.user.to_string(), + event_id=read_private_event_id, + ) + + read_marker_event_id = body.get(ReceiptTypes.FULLY_READ, None) if read_marker_event_id: await self.read_marker_handler.received_client_read_marker( room_id, diff --git a/synapse/rest/client/receipts.py b/synapse/rest/client/receipts.py index b24ad2d1be13..4b03eb876b75 100644 --- a/synapse/rest/client/receipts.py +++ b/synapse/rest/client/receipts.py @@ -13,12 +13,10 @@ # limitations under the License. import logging -import re from typing import TYPE_CHECKING, Tuple -from synapse.api.constants import ReadReceiptEventFields, ReceiptTypes -from synapse.api.errors import Codes, SynapseError -from synapse.http import get_request_user_agent +from synapse.api.constants import ReceiptTypes +from synapse.api.errors import SynapseError from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseRequest @@ -26,8 +24,6 @@ from ._base import client_patterns -pattern = re.compile(r"(?:Element|SchildiChat)/1\.[012]\.") - if TYPE_CHECKING: from synapse.server import HomeServer @@ -46,6 +42,7 @@ def __init__(self, hs: "HomeServer"): self.hs = hs self.auth = hs.get_auth() self.receipts_handler = hs.get_receipts_handler() + self.read_marker_handler = hs.get_read_marker_handler() self.presence_handler = hs.get_presence_handler() async def on_POST( @@ -53,35 +50,38 @@ async def on_POST( ) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request) - if receipt_type != ReceiptTypes.READ: - raise SynapseError(400, "Receipt type must be 'm.read'") - - # Do not allow older SchildiChat and Element Android clients (prior to Element/1.[012].x) to send an empty body. - user_agent = get_request_user_agent(request) - allow_empty_body = False - if "Android" in user_agent: - if pattern.match(user_agent) or "Riot" in user_agent: - allow_empty_body = True - body = parse_json_object_from_request(request, allow_empty_body) - hidden = body.get(ReadReceiptEventFields.MSC2285_HIDDEN, False) - - if not isinstance(hidden, bool): + if self.hs.config.experimental.msc2285_enabled and receipt_type not in [ + ReceiptTypes.READ, + ReceiptTypes.READ_PRIVATE, + ReceiptTypes.FULLY_READ, + ]: raise SynapseError( 400, - "Param %s must be a boolean, if given" - % ReadReceiptEventFields.MSC2285_HIDDEN, - Codes.BAD_JSON, + "Receipt type must be 'm.read', 'org.matrix.msc2285.read.private' or 'm.fully_read'", ) + elif ( + not self.hs.config.experimental.msc2285_enabled + and receipt_type != ReceiptTypes.READ + ): + raise SynapseError(400, "Receipt type must be 'm.read'") + + parse_json_object_from_request(request, allow_empty_body=False) await self.presence_handler.bump_presence_active_time(requester.user) - await self.receipts_handler.received_client_receipt( - room_id, - receipt_type, - user_id=requester.user.to_string(), - event_id=event_id, - hidden=hidden, - ) + if receipt_type == ReceiptTypes.FULLY_READ: + await self.read_marker_handler.received_client_read_marker( + room_id, + user_id=requester.user.to_string(), + event_id=event_id, + ) + else: + await self.receipts_handler.received_client_receipt( + room_id, + receipt_type, + user_id=requester.user.to_string(), + event_id=event_id, + ) return 200, {} diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index 70baf50fa473..e8e51a9c66ad 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -352,7 +352,7 @@ async def on_GET(self, request: Request) -> Tuple[int, JsonDict]: if self.inhibit_user_in_use_error: return 200, {"available": True} - ip = request.getClientIP() + ip = request.getClientAddress().host with self.ratelimiter.ratelimit(ip) as wait_deferred: await wait_deferred @@ -394,7 +394,7 @@ def __init__(self, hs: "HomeServer"): ) async def on_GET(self, request: Request) -> Tuple[int, JsonDict]: - await self.ratelimiter.ratelimit(None, (request.getClientIP(),)) + await self.ratelimiter.ratelimit(None, (request.getClientAddress().host,)) if not self.hs.config.registration.enable_registration: raise SynapseError( @@ -441,7 +441,7 @@ def __init__(self, hs: "HomeServer"): async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: body = parse_json_object_from_request(request) - client_addr = request.getClientIP() + client_addr = request.getClientAddress().host await self.ratelimiter.ratelimit(None, client_addr, update=False) @@ -929,6 +929,10 @@ def _calculate_registration_flows( # always let users provide both MSISDN & email flows.append([LoginType.MSISDN, LoginType.EMAIL_IDENTITY]) + # Add a flow that doesn't require any 3pids, if the config requests it. + if config.registration.enable_registration_token_3pid_bypass: + flows.append([LoginType.REGISTRATION_TOKEN]) + # Prepend m.login.terms to all flows if we're requiring consent if config.consent.user_consent_at_registration: for flow in flows: @@ -942,7 +946,8 @@ def _calculate_registration_flows( # Prepend registration token to all flows if we're requiring a token if config.registration.registration_requires_token: for flow in flows: - flow.insert(0, LoginType.REGISTRATION_TOKEN) + if LoginType.REGISTRATION_TOKEN not in flow: + flow.insert(0, LoginType.REGISTRATION_TOKEN) return flows diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 47e152c8cc7a..5a2361a2e691 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -21,6 +21,7 @@ from twisted.web.server import Request +from synapse import event_auth from synapse.api.constants import EventTypes, Membership from synapse.api.errors import ( AuthError, @@ -29,10 +30,11 @@ MissingClientTokenError, ShadowBanError, SynapseError, + UnredactedContentDeletedError, ) from synapse.api.filtering import Filter from synapse.events.utils import format_event_for_client_v2 -from synapse.http.server import HttpServer +from synapse.http.server import HttpServer, cancellable from synapse.http.servlet import ( ResolveRoomIdMixin, RestServlet, @@ -107,10 +109,10 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() def register(self, http_server: HttpServer) -> None: - # /room/$roomid/state/$eventtype + # /rooms/$roomid/state/$eventtype no_state_key = "/rooms/(?P[^/]*)/state/(?P[^/]*)$" - # /room/$roomid/state/$eventtype/$statekey + # /rooms/$roomid/state/$eventtype/$statekey state_key = ( "/rooms/(?P[^/]*)/state/" "(?P[^/]*)/(?P[^/]*)$" @@ -141,6 +143,7 @@ def register(self, http_server: HttpServer) -> None: self.__class__.__name__, ) + @cancellable def on_GET_no_state_key( self, request: SynapseRequest, room_id: str, event_type: str ) -> Awaitable[Tuple[int, JsonDict]]: @@ -151,6 +154,7 @@ def on_PUT_no_state_key( ) -> Awaitable[Tuple[int, JsonDict]]: return self.on_PUT(request, room_id, event_type, "") + @cancellable async def on_GET( self, request: SynapseRequest, room_id: str, event_type: str, state_key: str ) -> Tuple[int, JsonDict]: @@ -479,6 +483,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.store = hs.get_datastores().main + @cancellable async def on_GET( self, request: SynapseRequest, room_id: str ) -> Tuple[int, JsonDict]: @@ -600,6 +605,7 @@ def __init__(self, hs: "HomeServer"): self.message_handler = hs.get_message_handler() self.auth = hs.get_auth() + @cancellable async def on_GET( self, request: SynapseRequest, room_id: str ) -> Tuple[int, List[JsonDict]]: @@ -643,18 +649,55 @@ def __init__(self, hs: "HomeServer"): super().__init__() self.clock = hs.get_clock() self._store = hs.get_datastores().main + self._state = hs.get_state_handler() self.event_handler = hs.get_event_handler() self._event_serializer = hs.get_event_client_serializer() self._relations_handler = hs.get_relations_handler() self.auth = hs.get_auth() + self.content_keep_ms = hs.config.server.redaction_retention_period + self.msc2815_enabled = hs.config.experimental.msc2815_enabled async def on_GET( self, request: SynapseRequest, room_id: str, event_id: str ) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request, allow_guest=True) + + include_unredacted_content = self.msc2815_enabled and ( + parse_string( + request, + "fi.mau.msc2815.include_unredacted_content", + allowed_values=("true", "false"), + ) + == "true" + ) + if include_unredacted_content and not await self.auth.is_server_admin( + requester.user + ): + power_level_event = await self._state.get_current_state( + room_id, EventTypes.PowerLevels, "" + ) + + auth_events = {} + if power_level_event: + auth_events[(EventTypes.PowerLevels, "")] = power_level_event + + redact_level = event_auth.get_named_level(auth_events, "redact", 50) + user_level = event_auth.get_user_power_level( + requester.user.to_string(), auth_events + ) + if user_level < redact_level: + raise SynapseError( + 403, + "You don't have permission to view redacted events in this room.", + errcode=Codes.FORBIDDEN, + ) + try: event = await self.event_handler.get_event( - requester.user, room_id, event_id + requester.user, + room_id, + event_id, + show_redacted=include_unredacted_content, ) except AuthError: # This endpoint is supposed to return a 404 when the requester does @@ -663,14 +706,21 @@ async def on_GET( raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND) if event: + if include_unredacted_content and await self._store.have_censored_event( + event_id + ): + raise UnredactedContentDeletedError(self.content_keep_ms) + # Ensure there are bundled aggregations available. aggregations = await self._relations_handler.get_bundled_aggregations( [event], requester.user.to_string() ) time_now = self.clock.time_msec() + # per MSC2676, /rooms/{roomId}/event/{eventId}, should return the + # *original* event, rather than the edited version event_dict = self._event_serializer.serialize_event( - event, time_now, bundle_aggregations=aggregations + event, time_now, bundle_aggregations=aggregations, apply_edits=False ) return 200, event_dict diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 2e25e8638b9f..e8772f86e72f 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -180,13 +180,10 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: affect_presence = set_presence != PresenceState.OFFLINE - if affect_presence: - await self.presence_handler.set_state( - user, {"presence": set_presence}, True - ) - context = await self.presence_handler.user_syncing( - user.to_string(), affect_presence=affect_presence + user.to_string(), + affect_presence=affect_presence, + presence_state=set_presence, ) with context: sync_result = await self.sync_handler.wait_for_sync_for_user( diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index 914fb3acf5aa..61375651bc15 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -15,7 +15,9 @@ """This module contains logic for storing HTTP PUT transactions. This is used to ensure idempotency when performing PUTs using the REST API.""" import logging -from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Tuple +from typing import TYPE_CHECKING, Awaitable, Callable, Dict, Tuple + +from typing_extensions import ParamSpec from twisted.python.failure import Failure from twisted.web.server import Request @@ -32,6 +34,9 @@ CLEANUP_PERIOD_MS = 1000 * 60 * 30 # 30 mins +P = ParamSpec("P") + + class HttpTransactionCache: def __init__(self, hs: "HomeServer"): self.hs = hs @@ -65,9 +70,9 @@ def _get_transaction_key(self, request: Request) -> str: def fetch_or_execute_request( self, request: Request, - fn: Callable[..., Awaitable[Tuple[int, JsonDict]]], - *args: Any, - **kwargs: Any, + fn: Callable[P, Awaitable[Tuple[int, JsonDict]]], + *args: P.args, + **kwargs: P.kwargs, ) -> Awaitable[Tuple[int, JsonDict]]: """A helper function for fetch_or_execute which extracts a transaction key from the given request. @@ -82,9 +87,9 @@ def fetch_or_execute_request( def fetch_or_execute( self, txn_key: str, - fn: Callable[..., Awaitable[Tuple[int, JsonDict]]], - *args: Any, - **kwargs: Any, + fn: Callable[P, Awaitable[Tuple[int, JsonDict]]], + *args: P.args, + **kwargs: P.kwargs, ) -> Awaitable[Tuple[int, JsonDict]]: """Fetches the response for this transaction, or executes the given function to produce a response for this transaction. diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 9a65aa484360..c1bd775fece4 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -86,22 +86,23 @@ def on_GET(self, request: Request) -> Tuple[int, JsonDict]: # Implements additional endpoints as described in MSC2432 "org.matrix.msc2432": True, # Implements additional endpoints as described in MSC2666 - "uk.half-shot.msc2666": True, + "uk.half-shot.msc2666.mutual_rooms": True, # Whether new rooms will be set to encrypted or not (based on presets). "io.element.e2ee_forced.public": self.e2ee_forced_public, "io.element.e2ee_forced.private": self.e2ee_forced_private, "io.element.e2ee_forced.trusted_private": self.e2ee_forced_trusted_private, # Supports the busy presence state described in MSC3026. "org.matrix.msc3026.busy_presence": self.config.experimental.msc3026_enabled, - # Supports receiving hidden read receipts as per MSC2285 + # Supports receiving private read receipts as per MSC2285 "org.matrix.msc2285": self.config.experimental.msc2285_enabled, # Adds support for importing historical messages as per MSC2716 "org.matrix.msc2716": self.config.experimental.msc2716_enabled, # Adds support for jump to date endpoints (/timestamp_to_event) as per MSC3030 "org.matrix.msc3030": self.config.experimental.msc3030_enabled, # Adds support for thread relations, per MSC3440. - "org.matrix.msc3440": self.config.experimental.msc3440_enabled, "org.matrix.msc3440.stable": True, # TODO: remove when "v1.3" is added above + # Allows moderators to fetch redacted event content as described in MSC2815 + "fi.mau.msc2815": self.config.experimental.msc2815_enabled, }, }, ) diff --git a/synapse/rest/media/v1/preview_html.py b/synapse/rest/media/v1/preview_html.py index ca73965fc28f..0358c68a6452 100644 --- a/synapse/rest/media/v1/preview_html.py +++ b/synapse/rest/media/v1/preview_html.py @@ -246,7 +246,9 @@ def parse_html_description(tree: "etree.Element") -> Optional[str]: Grabs any text nodes which are inside the tag, unless they are within an HTML5 semantic markup tag (
,