Merge remote-tracking branch 'origin/master' into action-apikey-show

.circleci/config.yml

@@ -73,7 +73,7 @@ defaults:
       CKAN_POSTGRES_USER: ckan_default
       CKAN_POSTGRES_PWD: pass
       PGPASSWORD: ckan
-      NODE_TESTS_CONTAINER: 2
+      NODE_TESTS_CONTAINER: 3
       PYTEST_COMMON_OPTIONS: -v --ckan-ini=test-core-circle-ci.ini --cov=ckan --cov=ckanext --junitxml=/root/junit/junit.xml --test-group-count 4  --test-group-random-seed 1
   pg_image: &pg_image
     image: postgres:10
@@ -112,6 +112,8 @@ jobs:
           path: ~/junit
       - <<: *start_test_server
       - <<: *run_front_tests
+      - store_artifacts:
+          path: ~/project/cypress/screenshots
   test-python-3:
     docker:
       - image: python:3-stretch
@@ -140,6 +142,8 @@ jobs:
           path: ~/junit
       - <<: *start_test_server
       - <<: *run_front_tests
+      - store_artifacts:
+          path: ~/project/cypress/screenshots
       - run: coveralls
 workflows:
   version: 2

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,22 @@
+---
+name: Bug report
+about: Report something that is broken or doesn't work as expected
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**CKAN version**
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Steps to reproduce**
+Steps to reproduce the behavior:
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Additional details**
+If possible, please provide the full stack trace of the error raised,  or add screenshots to help explain your problem.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,14 @@
+blank_issues_enabled: true
+contact_links:
+  - name: CKAN Support Mailing List
+    url: https://groups.google.com/a/ckan.org/forum/#!forum/ckan-dev
+    about: Please ask and answer questions here.
+  - name: CKAN Gitter Channel
+    url: https://gitter.im/ckan/chat
+    about: For support requests or general questions you can also reach other CKAN users and developers here.
+  - name: Security Issues
+    url: mailto:[email protected]
+    about: Please report any security related vulnerabilities here.
+  - name: Ideas Repository
+    url: https://github.com/ckan/ideas
+    about: For new feature requests or discussion, please create an issue on the ideas repository.

.gitignore

@@ -44,7 +44,6 @@ ckan_deb/DEBIAN/prerm
 
 # node.js
 node_modules/
-package-lock.json
 
 # docker
 contrib/docker/.env

CONDUCT.rst → CODE_OF_CONDUCT.md

@@ -1,3 +1,4 @@
+# CKAN Community Code of Conduct
 
 This code of conduct outlines our expectations for participants within the **CKAN** community, as well as steps to reporting unacceptable behavior. We are committed to providing a welcoming and inspiring community for all and expect our code of conduct to be honored. Anyone who violates this code of conduct may be banned from the community.
 
@@ -10,8 +11,7 @@ Our open source community strives to:
 * **Be careful in the words that we choose**: we are a community of professionals, and we conduct ourselves professionally. Be kind to others. Do not insult or put down other participants. Harassment and other exclusionary behavior aren't acceptable.
 * **Try to understand why we disagree**: Disagreements, both social and technical, happen all the time. It is important that we resolve disagreements and differing views constructively. Remember that we’re different. The strength of our community comes from its diversity, people from a wide range of backgrounds. Different people have different perspectives on issues. Being unable to understand why someone holds a viewpoint doesn’t mean that they’re wrong. Don’t forget that it is human to err and blaming each other doesn’t get us anywhere. Instead, focus on helping to resolve issues and learning from mistakes.
 
-Definitions
-----
+## Definitions
 
 Harassment includes, but is not limited to:
 
@@ -40,16 +40,14 @@ Our open source community prioritizes marginalized people’s safety over privil
 - Criticizing racist, sexist, cissexist, or otherwise oppressive behavior or assumptions
 
 
-Diversity Statement
-----
+## Diversity Statement
 
 We encourage everyone to participate and are committed to building a community for all. Although we will fail at times, we seek to treat everyone both as fairly and equally as possible. Whenever a participant has made a mistake, we expect them to take responsibility for it. If someone has been harmed or offended, it is our responsibility to listen carefully and respectfully, and do our best to right the wrong.
 
 Although this list cannot be exhaustive, we explicitly honor diversity in age, gender, gender identity or expression, culture, ethnicity, language, national origin, political beliefs, profession, race, religion, sexual orientation, socioeconomic status, and technical ability. We will not tolerate discrimination based on any of the protected
 characteristics above, including participants with disabilities.
 
-Reporting Issues
-----
+## Reporting Issues
 
 If you experience or witness unacceptable behavior—or have any other concerns—please report it by contacting us via [email protected]. All reports will be handled with discretion. In your report please include:
 
@@ -60,17 +58,15 @@ include them as well. Your account of what occurred, and if you believe the inci
 
 After filing a report, a representative will contact you personally, review the incident, follow up with any additional questions, and make a decision as to how to respond. If the person who is harassing you is part of the response team, they will recuse themselves from handling your incident. If the complaint originates from a member of the response team, it will be handled by a different member of the response team. We will respect confidentiality requests for the purpose of protecting victims of abuse.
 
-Attribution & Acknowledgements
-----
+## Attribution & Acknowledgements
 
-This document is derived on the `opencodeofconduct <https://github.com/todogroup/opencodeofconduct>`.
+This document is derived on the [opencodeofconduct](https://github.com/todogroup/opencodeofconduct).
 
 We all stand on the shoulders of giants across many open source communities.  We'd like to thank the communities and projects that established code of conducts and diversity statements as our inspiration:
 
-* `Django <https://www.djangoproject.com/conduct/reporting/>`
-* `Python <https://www.python.org/community/diversity/>`
-* `Ubuntu <http://www.ubuntu.com/about/about-ubuntu/conduct>`
-* `Contributor Covenant <http://contributor-covenant.org/>`
-* `Geek Feminism <http://geekfeminism.org/about/code-of-conduct/>`
-* `Citizen Code of Conduct <http://citizencodeofconduct.org/>`
-
+* [Django](https://www.djangoproject.com/conduct/reporting/)
+* [Python](https://www.python.org/community/diversity/)
+* [Ubuntu](http://www.ubuntu.com/about/about-ubuntu/conduct)
+* [Contributor Covenant](http://contributor-covenant.org/)
+* [Geek Feminism](http://geekfeminism.org/about/code-of-conduct/)
+* [Citizen Code of Conduct](http://citizencodeofconduct.org/)

CONTRIBUTING.md

@@ -0,0 +1,4 @@
+For contributing CKAN (whether code, bug reports, translations, documentation,
+etc.) see our contributing guidelines:
+
+http://docs.ckan.org/en/latest/contributing

README.rst

@@ -85,7 +85,7 @@ If you've figured out how to do something with CKAN and want to document it for
 others, make a new page on the `CKAN wiki`_ and tell us about it on the
 ckan-dev mailing list or on Gitter.
 
-.. _ckan-dev: http://lists.okfn.org/mailman/listinfo/ckan-dev
+.. _ckan-dev: https://groups.google.com/a/ckan.org/forum/#!forum/ckan-dev
 .. _#ckan: http://webchat.freenode.net/?channels=ckan
 .. _CKAN Wiki: https://github.com/ckan/ckan/wiki
 .. _CKAN chat on Gitter: https://gitter.im/ckan/chat

changes/4781.migration

@@ -0,0 +1 @@
+When ``ckan.cache_enabled`` is set to ``False`` (default) all requests include the ``Cache-control: private`` header. If ``ckan.cache_enabled`` is set to ``True``, when the user is not logged in and there is no session data, a ``Cache-Control: public`` header will be added. For all other requests the ``Cache-control: private`` header will be added. Note that you will also need to set the ``ckan.cache_expires`` config option to allow caching of requests.

changes/5346.feature

@@ -0,0 +1,7 @@
+Dataset collaborators: In addition to traditional organization-based permissions, CKAN instances can also enable
+the dataset collaborators feature, which allows dataset-level authorization. This provides
+more granular control over who can access and modify datasets that belong to an organization,
+or allows authorization setups not based on organizations. It works by allowing users with
+appropriate permissions to give permissions to other users over individual datasets, regardless
+of what organization they belong to. To learn more about how to enable it and the different 
+configuration options available, check the documentation on :ref:`dataset_collaborators`. 

changes/5436.bugfix

@@ -0,0 +1 @@
+`ckan.i18n_directory` config option ignored in Flask app.

changes/5458.migration

@@ -0,0 +1 @@
+The minimum PostgreSQL version required starting from this version is 9.5

ckan-uwsgi.ini

@@ -0,0 +1,14 @@
+[uwsgi]
+
+http            =  127.0.0.1:8080
+uid             =  www-data
+guid            =  www-data
+wsgi-file       =  /etc/ckan/default/wsgi.py
+virtualenv      =  /usr/lib/ckan/default
+module          =  wsgi:application
+master          =  true
+pidfile         =  /tmp/%n.pid
+harakiri        =  50
+max-requests    =  5000
+vacuum          =  true
+callable        =  application

ckan/authz.py

@@ -225,12 +225,19 @@ def is_authorized(action, context, data_dict=None):
 
 # these are the permissions that roles have
 ROLE_PERMISSIONS = OrderedDict([
-    ('admin', ['admin']),
+    ('admin', ['admin', 'membership']),
     ('editor', ['read', 'delete_dataset', 'create_dataset', 'update_dataset', 'manage_group']),
     ('member', ['read', 'manage_group']),
 ])
 
 
+def get_collaborator_capacities():
+    if check_config_permission('allow_admin_collaborators'):
+        return ('admin', 'editor', 'member')
+    else:
+        return ('editor', 'member')
+
+
 def _trans_role_admin():
     return _('Admin')
 
@@ -397,6 +404,64 @@ def get_user_id_for_username(user_name, allow_none=False):
     raise Exception('Not logged in user')
 
 
+def can_manage_collaborators(package_id, user_id):
+    '''
+    Returns True if a user is allowed to manage the collaborators of a given
+    dataset.
+
+    Currently a user can manage collaborators if:
+
+    1. Is an administrator of the organization the dataset belongs to
+    2. Is a collaborator with role "admin" (
+        assuming :ref:`ckan.auth.allow_admin_collaborators` is set to True)
+    3. Is the creator of the dataset and the dataset does not belong to an
+        organization (
+        requires :ref:`ckan.auth.create_dataset_if_not_in_organization`
+        and :ref:`ckan.auth.create_unowned_dataset`)
+    '''
+    pkg = model.Package.get(package_id)
+
+    owner_org = pkg.owner_org
+
+    if (not owner_org
+            and check_config_permission('create_dataset_if_not_in_organization')
+            and check_config_permission('create_unowned_dataset')
+            and pkg.creator_user_id == user_id):
+        # User is the creator of this unowned dataset
+        return True
+
+    if has_user_permission_for_group_or_org(
+            owner_org, user_id, 'membership'):
+        # User is an administrator of the organization the dataset belongs to
+        return True
+
+    # Check if user is a collaborator with admin role
+    return user_is_collaborator_on_dataset(user_id, pkg.id, 'admin')
+
+
+def user_is_collaborator_on_dataset(user_id, dataset_id, capacity=None):
+    '''
+    Returns True if the provided user is a collaborator on the provided
+    dataset.
+
+    If capacity is provided it restricts the check to the capacity
+    provided (eg `admin` or `editor`). Multiple capacities can be
+    provided passing a list
+
+    '''
+
+    q = model.Session.query(model.PackageMember) \
+        .filter(model.PackageMember.user_id == user_id) \
+        .filter(model.PackageMember.package_id == dataset_id)
+
+    if capacity:
+        if isinstance(capacity, six.string_types):
+            capacity = [capacity]
+        q = q.filter(model.PackageMember.capacity.in_(capacity))
+
+    return q.count() > 0
+
+
 CONFIG_PERMISSIONS_DEFAULTS = {
     # permission and default
     # these are prefixed with ckan.auth. in config to override
@@ -411,6 +476,9 @@ def get_user_id_for_username(user_name, allow_none=False):
     'create_user_via_web': True,
     'roles_that_cascade_to_sub_groups': 'admin',
     'public_activity_stream_detail': False,
+    'allow_dataset_collaborators': False,
+    'allow_admin_collaborators': False,
+    'allow_collaborators_to_change_owner_org': False,
 }
 
 

ckan/cli/__init__.py

@@ -5,7 +5,7 @@
 import click
 import logging
 from logging.config import fileConfig as loggingFileConfig
-from configparser import ConfigParser
+from six.moves.configparser import ConfigParser
 
 from ckan.exceptions import CkanConfigurationException