Merge branch 'master' into switch-to-sqlalchemy-2

Author: amercader

.circleci/config.yml

@@ -37,7 +37,7 @@ defaults:
 
   redis_image: &redis_image
     # Note, this should be in sync with test-infrastructure/docker-compose.yml
-    image: redis:3
+    image: redis:7
     name: ckan-redis
 
   solr_image: &solr_image

.github/workflows/publish-pypi.yml

@@ -0,0 +1,72 @@
+name: Publish to PyPI and TestPyPI
+
+on: push
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.9"
+    - name: Install pypa/build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: python3 -m build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+
+  publish-to-pypi:
+    name: Publish Python distribution on PyPI
+    # Publish to PyPI when pushing a tag
+    if: startsWith(github.ref, 'refs/tags/')
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/ckan
+    permissions:
+      id-token: write
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+
+  publish-to-testpypi:
+    name: Publish Python distribution on TestPyPI
+    # Publish to Test PyPI when a pull request is merged
+    if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://test.pypi.org/p/ckan
+    permissions:
+      id-token: write
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        repository-url: https://test.pypi.org/legacy/

MANIFEST.in

@@ -15,6 +15,7 @@ recursive-include ckanext *.py
 recursive-include ckanext/*/i18n *
 recursive-include ckanext/*/public *
 recursive-include ckanext/*/assets *
+recursive-include ckanext/*/migration *
 recursive-include ckanext/*/templates *
 recursive-include ckanext/*/theme/public *
 recursive-include ckanext/*/theme/templates *

README.rst

@@ -71,9 +71,9 @@ future plans as well as questions and discussions about CKAN development, deploy
 Community Chat
 ~~~~~~~~~~~~~~
 
-If you want to talk about CKAN development say hi to the CKAN developers and members of
-the CKAN community on the public `CKAN chat on Gitter`_. Gitter is free and open-source;
-you can sign in with your GitHub, GitLab, or Twitter account.
+If you want to talk about CKAN development, say hi to the CKAN developers and members of
+the CKAN community on the public `CKAN chat on Gitter`_. Gitter is free and open-source, and
+you can sign in using your GitHub, GitLab, or X (formerly known as Twitter) account.
 
 The logs for the old `#ckan`_ IRC channel (2014 to 2018) can be found here:
 https://github.com/ckan/irc-logs.

changes/5468.bugfix

@@ -0,0 +1 @@
+fix Page view tracking of datasets is not working if ckan is running at a subpath

changes/5847.feature

@@ -0,0 +1,9 @@
+Allow configuring datastore full text field indexes with new
+ckan.datastore.default_fts_index_field_types config option.
+
+The default is an empty list, avoiding automatically creating
+separate full text indexes for any individual columns. The
+whole-row full text index still exists for all tables.
+
+Use the `ckan datastore fts-index` command to remove existing
+column indexes to reclaim database space.

changes/7723.changes changes/7723.misc

@@ -0,0 +1,15 @@
+Faster dataset metadata updates by detecting changes and only updating resource
+metadata when dataset fields defined by IDatasetForm.resource_validation_dependencies
+have changed (default: none)
+
+Now activities are created and metadata_modified updated only if there is a real change.
+
+metadata_modified may now be set by sysadmins which is useful for harvesting or mirroring.
+
+The allow_partial_update context parameter has been removed, now normal API
+users may call package_update without passing resources. In this case the existing
+resources will remain untouched instead of being deleted.
+
+package_update and actions that call it now report whether there was a real change by
+adding the package id to a new changed_entities context value or changed_entities
+envelope value for API calls.

changes/8407.feature

@@ -0,0 +1 @@
+Fix `tracking` extension to use ORM models and comply with new `ckan.model` models

changes/8447.bugfix

@@ -0,0 +1 @@
+Fix internal server error when viewing a deleted user.

changes/8482.bugfix

@@ -0,0 +1 @@
+Fix display of user organizations page if user belongs to no organizations.

changes/8483.bugfix

@@ -0,0 +1,3 @@
+Adds new logo and updates UI header in `midnight-blue` theme.
+Adds macro for creating navigation links.
+Adds helpers `endpoint_from_url` and `page_is_active`.

changes/8492.feature

@@ -0,0 +1 @@
+fix for upload regression from #8360

changes/8496.bugfix

@@ -0,0 +1 @@
+Avoid exceptions in tracking extension when dataset fields are missing

changes/8499.misc

@@ -0,0 +1,2 @@
+Fix error when viewing history of a deleted resource or its package before
+the deletion date.

changes/8501.bugfix

@@ -0,0 +1 @@
+Upgrade requirements to latest versions

changes/8504.misc

@@ -0,0 +1 @@
+Register pytest plugins as entrypoints to make them available to all extensions

changes/8507.misc

@@ -0,0 +1 @@
+ Don't add author email to pyproject.toml if empty when creating an extension

changes/8519.misc

@@ -0,0 +1 @@
+Automate publishing CKAN package to PyPI

changes/8520.misc

@@ -0,0 +1 @@
+Fix showing '0 members' for all groups on a dataset page.

changes/8537.bugfix

@@ -2,7 +2,7 @@
 from __future__ import annotations
 
 import datetime
-from typing import Any, Callable, Iterable
+from typing import Any, Callable, Iterable, Literal
 
 import sqlalchemy
 from sqlalchemy import Table
@@ -109,15 +109,21 @@ def get_unique_constraints(table: Table, context: Context) -> list[list[str]]:
     return list_of_constraints
 
 
-def table_dict_save(table_dict: dict[str, Any],
-                    ModelClass: Any,
-                    context: Context,
-                    extra_attrs: Iterable[str] = ()) -> Any:
+def table_dict_save(
+        table_dict: dict[str, Any],
+        ModelClass: Any,
+        context: Context,
+        extra_attrs: Iterable[str] = ()
+        ) -> tuple[Any, Literal['create', 'update', None]]:
     '''Given a dict and a model class, update or create a sqlalchemy object.
     This will use an existing object if "id" is supplied OR if any unique
     constraints are met. e.g supplying just a tag name will get out that tag
     obj.
 
+    Returns (obj, change) where change is:
+    - 'create' if this is a new object
+    - 'update' if any fields were changed or extra_attrs passed
+    - None if no change for an existing object
     '''
     session = context["session"]
 
@@ -130,7 +136,8 @@ def table_dict_save(table_dict: dict[str, Any],
     if id:
         obj = session.get(ModelClass, id)
 
-    if not obj and isinstance(table, Table):
+    new = not obj
+    if new:
         unique_constraints = get_unique_constraints(table, context)
         for constraint in unique_constraints:
             params = dict((key, table_dict.get(key)) for key in constraint)
@@ -146,11 +153,13 @@ def table_dict_save(table_dict: dict[str, Any],
     if not obj:
         obj = ModelClass()
 
-    obj.from_dict(table_dict)
+    changed, _skipped = obj.from_dict(table_dict)
     for a in extra_attrs:
         if a in table_dict:
             setattr(obj, a, table_dict[a])
 
     session.add(obj)
 
-    return obj
+    return (
+        obj, 'create' if new else 'update' if changed or extra_attrs else None
+    )