Rebase + improvements for #727

.travis.yml

@@ -1,31 +1,38 @@
 language: python
 matrix:
   include:
+    - os: osx
+      language: generic
+      env: PYTHON_INSTALLER=pyenv TRAVIS_PYTHON_VERSION=2.7.14
+    - os: osx
+      language: generic
+      env: PYTHON_INSTALLER=pyenv TRAVIS_PYTHON_VERSION=3.6.4
+    - os: osx
+      language: generic
+      env:
+        - PYTHON_INSTALLER=brew
+        - PIP_USER_FLAG="--user"
     - os: linux
       python: 2.7
     - os: linux
-      python: 3.4
+      python: 3.4   # When support for 3.4 is removed unpin the PyYAML version below.
     - os: linux
       python: 3.5
     - os: linux
       python: 3.6
-    - os: osx
-      language: generic
-      env: PYTHON_INSTALLER=brew
-    - os: osx
-      language: generic
-      env: PYTHON_INSTALLER=pyenv TRAVIS_PYTHON_VERSION=2.7.14
-    - os: osx
-      language: generic
-      env: PYTHON_INSTALLER=pyenv TRAVIS_PYTHON_VERSION=3.6.4
+    - os: linux
+      python: 3.7
+    - os: linux
+      python: 3.8
 
 # command to install dependencies
 install:
   - source .travis/install.sh
   - python --version
-  - python -m pip install PyYAML argparse rospkg vcstools catkin_pkg python-dateutil rosdistro
-  - python -m pip install -e .
-  - python -m pip install nose coverage flake8 mock codecov
+  - if [ $TRAVIS_PYTHON_VERSION == "3.4" ]; then python -m pip install PyYAML==5.2; fi # Forcing PyYAML 5.2 while we retain Python 3.4 support PyYAML 5.3 and higher does not support Python 3.4
+  - python -m pip install $PIP_USER_FLAG PyYAML argparse rospkg vcstools catkin_pkg python-dateutil rosdistro
+  - python -m pip install $PIP_USER_FLAG -e .
+  - python -m pip install $PIP_USER_FLAG nose coverage flake8 mock codecov
 # command to run tests
 script:
   - python -m nose --with-coverage --cover-package=rosdep2 --with-xunit test

.travis/install.sh

@@ -5,6 +5,7 @@ do_install()
     set -e
 
     if [[ $TRAVIS_OS_NAME == 'osx' && $PYTHON_INSTALLER == 'pyenv' ]]; then
+        brew update
         brew install pyenv-virtualenv
         pyenv versions
         eval "$(pyenv init -)"

CHANGELOG.rst

@@ -1,7 +1,41 @@
-0.17.1 (2019-10-22)
--------------------
-- Warn about ROS_PYTHON_VERSION only when it is needed
-  - https://github.com/ros-infrastructure/rosdep/issues/725
+0.19.0 (2020-04-03)
+-------------------
+- Only release for Python3 into focal
+  - https://github.com/ros-infrastructure/rosdep/pull/734
+- Added --rosdistro argument to rosdep-update to scope update to one rosdistro
+  - https://github.com/ros-infrastructure/rosdep/pull/738
+  - Fixes https://github.com/ros-infrastructure/rosdep/pull/723
+- Fix CI for Python 3.4 and run slower CI jobs first
+  - https://github.com/ros-infrastructure/rosdep/pull/739
+- Strip Alpine's patch version from OS codename
+  - https://github.com/ros-infrastructure/rosdep/pull/716
+  - Fixes https://github.com/ros-infrastructure/rosdep/issues/715
+- Raise a clear and specific error message for null entries
+  - https://github.com/ros-infrastructure/rosdep/pull/726
+- Use DNF as the default installer on RHEL 8 and newer
+  - https://github.com/ros-infrastructure/rosdep/pull/713
+- Updates to YUM and DNF handling
+  - https://github.com/ros-infrastructure/rosdep/pull/640
+- Fix tests so they don't assume euid != 0
+  - https://github.com/ros-infrastructure/rosdep/pull/703
+- Update openSUSE package query function and enable pip installer
+  - https://github.com/ros-infrastructure/rosdep/pull/729
+- Fix conditional dependencies when one package uses manifest.xml
+  - https://github.com/ros-infrastructure/rosdep/pull/737
+- Handle StopIteration when querying in debian platform
+  - https://github.com/ros-infrastructure/rosdep/pull/701
+- Use entry points rather than console scripts to enable usage on Windows
+  - https://github.com/ros-infrastructure/rosdep/pull/656
+- Depend on modules packages only to allow modules packages to be co-installable.
+  - https://github.com/ros-infrastructure/rosdep/pull/750
+
+
+0.18.0 (2019-11-20)
+-------------------
+- split -modules into separate Debian package
+  - https://github.com/ros-infrastructure/rosdep/pull/731
+- fix macOS CI
+  - https://github.com/ros-infrastructure/rosdep/pull/730
 
 0.17.0 (2019-10-18)
 -------------------

Makefile

@@ -27,4 +27,4 @@ testsetup:
 	echo "running rosdep tests"
 
 test: testsetup
-	nosetests --with-coverage --cover-package=rosdep2 --with-xunit test
+	nosetests3 --with-coverage --cover-package=rosdep2 --with-xunit test

doc/rosdep_yaml_format.rst

@@ -87,7 +87,8 @@ OS name identifiers and supported package managers
 
  * ``fedora``: Fedora Project
 
-   * ``yum`` (default)
+   * ``dnf`` (default)
+   * ``yum``
    * ``source``
 
  * ``freebsd``: FreeBSD

setup.py

@@ -1,26 +1,40 @@
 import os
 from setuptools import setup
 
-exec(open(os.path.join(os.path.dirname(__file__), 'src', 'rosdep2', '_version.py')).read())
-
-setup(
-    name='rosdep',
-    version=__version__,  # noqa:F821
-    packages=['rosdep2', 'rosdep2.ament_packages', 'rosdep2.platforms'],
-    package_dir={'': 'src'},
-    install_requires=['catkin_pkg >= 0.4.0', 'rospkg >= 1.1.10', 'rosdistro >= 0.7.5', 'PyYAML >= 3.1'],
-    test_suite='nose.collector',
-    test_requires=['mock', 'nose >= 1.0'],
-    scripts=['scripts/rosdep', 'scripts/rosdep-source'],
-    author='Tully Foote, Ken Conley',
-    author_email='tfoote@osrfoundation.org',
-    url='http://wiki.ros.org/rosdep',
-    keywords=['ROS'],
-    classifiers=[
+kwargs = {
+    'name': 'rosdep',
+    # same version as in:
+    # - src/rosdep2/__init__.py
+    # - stdeb.cfg
+    'version': '0.19.0',
+    'packages': ['rosdep2', 'rosdep2.ament_packages', 'rosdep2.platforms'],
+    'package_dir': {'': 'src'},
+    'install_requires': ['catkin_pkg >= 0.4.0', 'rospkg >= 1.2.7', 'rosdistro >= 0.7.5', 'PyYAML >= 3.1'],
+    'test_suite': 'nose.collector',
+    'test_requires': ['mock', 'nose >= 1.0'],
+    'author': 'Tully Foote, Ken Conley',
+    'author_email': 'tfoote@osrfoundation.org',
+    'url': 'http://wiki.ros.org/rosdep',
+    'keywords': ['ROS'],
+    'entry_points': {
+        'console_scripts': [
+            'rosdep = rosdep2.main:rosdep_main',
+            'rosdep-source = rosdep2.install:install_main'
+        ]
+    },
+    'classifiers': [
         'Programming Language :: Python',
         'License :: OSI Approved :: BSD License'],
-    description='rosdep package manager abstraction tool for ROS',
-    long_description='Command-line tool for installing system '
-                     'dependencies on a variety of platforms.',
-    license='BSD'
-)
+    'description': 'rosdep package manager abstraction tool for ROS',
+    'long_description': 'Command-line tool for installing system '
+                        'dependencies on a variety of platforms.',
+    'license': 'BSD',
+}
+if 'SKIP_PYTHON_MODULES' in os.environ:
+    kwargs['packages'] = []
+    kwargs['package_dir'] = {}
+if 'SKIP_PYTHON_SCRIPTS' in os.environ:
+    kwargs['name'] += '_modules'
+    kwargs['entry_points'] = {}
+
+setup(**kwargs)

src/rosdep2/_version.py

@@ -1 +1,4 @@
-__version__ = '0.17.1'
+# same version as in:
+# - setup.py
+# - stdeb.cfg
+__version__ = '0.19.0'

scripts/rosdep-source → src/rosdep2/install.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright (c) 2011, Willow Garage, Inc.
 # All rights reserved.
 #
@@ -39,7 +38,7 @@
 from rosdep2 import InstallFailed
 from rosdep2.platforms import source
 
-NAME='rosdep-source'
+NAME = 'rosdep-source'
 
 
 def install_main():
@@ -49,16 +48,12 @@ def install_main():
         parser.error("please specify one and only one rdmanifest url")
     if args[0] != 'install':
         parser.error("currently only support the 'install' command")
-    rdmanifest_url= args[1]
+    rdmanifest_url = args[1]
     try:
         if os.path.isfile(rdmanifest_url):
             source.install_from_file(rdmanifest_url)
         else:
             source.install_from_url(rdmanifest_url)
     except InstallFailed as e:
-        print("ERROR: installation failed:\n%s"%e, file=sys.stderr)
+        print("ERROR: installation failed:\n%s" % e, file=sys.stderr)
         sys.exit(1)
-
-
-if __name__ == '__main__':
-    install_main()

src/rosdep2/lookup.py

@@ -144,6 +144,9 @@ def get_rule_for_platform(self, os_name, os_version, installer_keys, default_ins
                                 return_key = installer_key
                                 break
 
+        # Check if the rule is null
+        if data is None:
+            raise ResolutionError(rosdep_key, self.data, os_name, os_version, '[%s] defined as "not available" for OS version [%s]' % (rosdep_key, os_version))
         if type(data) not in (dict, list, type('str')):
             raise InvalidData('rosdep OS definition for [%s:%s] must be a dictionary, string, or list: %s' % (self.rosdep_key, os_name, data), origin=self.origin)
 
@@ -171,7 +174,7 @@ def __str__(self):
 \trosdep key : %s
 \tOS name    : %s
 \tOS version : %s
-\tData: %s""" % (self.args[0], self.rosdep_key, self.os_name, self.os_version, pretty_data)
+\tData:\n%s""" % (self.args[0], self.rosdep_key, self.os_name, self.os_version, pretty_data.replace('\n', '\n\t\t'))
 
 
 class RosdepView(object):

src/rosdep2/main.py

@@ -274,7 +274,7 @@ def setup_environment_variables(ros_distro):
     """
     Set environment variables needed to find ROS packages and evaluate conditional dependencies.
 
-    :param rosdistro: The requested ROS distro passed on the CLI, or None
+    :param ros_distro: The requested ROS distro passed on the CLI, or None
     """
     if ros_distro is not None:
         if 'ROS_DISTRO' in os.environ and os.environ['ROS_DISTRO'] != ros_distro:
@@ -361,7 +361,9 @@ def _rosdep_main(args):
     parser.add_option('--rosdistro', dest='ros_distro', default=None,
                       help='Explicitly sets the ROS distro to use, overriding '
                            'the normal method of detecting the ROS distro '
-                           'using the ROS_DISTRO environment variable.')
+                           'using the ROS_DISTRO environment variable. '
+                           "When used with the 'update' verb, "
+                           'only the specified distro will be updated.')
     parser.add_option('--as-root', default=[], action='append',
                       metavar='INSTALLER_KEY:<bool>', help='Override '
                       'whether sudo is used for a specific installer, '
@@ -373,10 +375,10 @@ def _rosdep_main(args):
                            'If specified end-of-life distros are being '
                            'fetched too.')
     parser.add_option('-t', '--dependency-types', dest='dependency_types',
-                    type="choice", choices=("build", "buildtool", "run", "test"),
-                    default=[], action='append',
-                    help='Dependency types to install, can be given multiple times. '
-                        'Chose from build, buildtool, run, test. Default: all.')
+                      type="choice", choices=("build", "buildtool", "build_export", "exec", "run", "test", "doc"),
+                      default=[], action='append',
+                      help='Dependency types to install, can be given multiple times. '
+                           'Chose from build, buildtool, build_export, exec, run, test, doc. Default: all except doc.')
 
     options, args = parser.parse_args(args)
     _global_options = options
@@ -655,14 +657,18 @@ def update_error_handler(data_source, exc):
             pass
         update_sources_list(success_handler=update_success_handler,
                             error_handler=update_error_handler,
-                            skip_eol_distros=not options.include_eol_distros)
+                            skip_eol_distros=not options.include_eol_distros,
+                            ros_distro=options.ros_distro)
         print('updated cache in %s' % (sources_cache_dir))
     except InvalidData as e:
         print('ERROR: invalid sources list file:\n\t%s' % (e), file=sys.stderr)
         return 1
     except IOError as e:
         print('ERROR: error loading sources list:\n\t%s' % (e), file=sys.stderr)
         return 1
+    except ValueError as e:
+        print('ERROR: invalid argument value provided:\n\t%s' % (e), file=sys.stderr)
+        return 1
     if error_occured:
         print('ERROR: Not all sources were able to be updated.\n[[[')
         for e in error_occured:

src/rosdep2/platforms/alpine.py

@@ -48,6 +48,7 @@ def register_platforms(context):
     context.add_os_installer_key(OS_ALPINE, PIP_INSTALLER)
     context.add_os_installer_key(OS_ALPINE, SOURCE_INSTALLER)
     context.set_default_os_installer_key(OS_ALPINE, lambda self: APK_INSTALLER)
+    context.set_os_version_type(OS_ALPINE, lambda self: ".".join(self.get_version().split('.')[:2]))
 
 
 def apk_detect(pkgs, exec_fn=read_stdout):

src/rosdep2/platforms/debian.py

@@ -31,8 +31,16 @@
 import subprocess
 import sys
 
-from rospkg.os_detect import OS_DEBIAN, OS_LINARO, OS_UBUNTU, OS_ELEMENTARY, OS_MX, OsDetect, read_os_release
-
+from rospkg.os_detect import (
+    OS_DEBIAN,
+    OS_LINARO,
+    OS_UBUNTU,
+    OS_ELEMENTARY,
+    OS_MX,
+    OS_POP,
+    OsDetect,
+    read_os_release
+)
 from .pip import PIP_INSTALLER
 from .gem import GEM_INSTALLER
 from .source import SOURCE_INSTALLER
@@ -55,6 +63,7 @@ def register_platforms(context):
     register_elementary(context)
     register_linaro(context)
     register_mx(context)
+    register_pop(context)
 
 
 def register_debian(context):
@@ -98,6 +107,16 @@ def register_mx(context):
         context.set_os_override(OS_DEBIAN, version[version.find("(") + 1:version.find(")")])
 
 
+def register_pop(context):
+    # Pop! OS is an alias for Ubuntu. If Pop! is detected and it's
+    # not set as an override force ubuntu.
+    (os_name, os_version) = context.get_os_name_and_version()
+    if os_name == OS_POP and not context.os_override:
+        print('rosdep detected OS: [%s] aliasing it to: [%s]' %
+              (OS_POP, OS_UBUNTU), file=sys.stderr)
+        context.set_os_override(OS_UBUNTU, context.os_detect.get_codename())
+
+
 def register_ubuntu(context):
     context.add_os_installer_key(OS_UBUNTU, APT_INSTALLER)
     context.add_os_installer_key(OS_UBUNTU, PIP_INSTALLER)
@@ -140,20 +159,32 @@ def _read_apt_cache_showpkg(packages, exec_fn=None):
 
         header = 'Package: %s' % p
         # proceed to Package header
-        while next(lines) != header:
+        try:
+            while next(lines) != header:
+                pass
+        except StopIteration:
             pass
 
         # proceed to versions section
-        while next(lines) != 'Versions: ':
+        try:
+            while next(lines) != 'Versions: ':
+                pass
+        except StopIteration:
             pass
 
         # virtual packages don't have versions
-        if next(lines) != '':
-            yield p, False, None
-            continue
+        try:
+            if next(lines) != '':
+                yield p, False, None
+                continue
+        except StopIteration:
+            break
 
         # proceed to reserve provides section
-        while next(lines) != 'Reverse Provides: ':
+        try:
+            while next(lines) != 'Reverse Provides: ':
+                pass
+        except StopIteration:
             pass
 
         pr = [line.split(' ', 2)[0] for line in lines]