Merge pull request #1610 from grycap/boto3

Boto3

Author: micafer

IM/InfrastructureManager.py

@@ -1434,10 +1434,7 @@ def check_oidc_token(im_auth):
 
                 if Config.OIDC_GROUPS:
                     # Get user groups from any of the possible fields
-                    user_groups = userinfo.get('groups',  # Generic
-                                               userinfo.get('entitlements',  # GEANT
-                                                            userinfo.get('eduperson_entitlement',  # EGI Check-in
-                                                                         [])))
+                    user_groups = userinfo.get(Config.OIDC_GROUPS_CLAIM, [])
 
                     if not set(Config.OIDC_GROUPS).issubset(user_groups):
                         raise InvaliddUserException("Invalid InfrastructureManager credentials. " +

IM/__init__.py

@@ -19,7 +19,7 @@
            'InfrastructureInfo', 'InfrastructureManager', 'recipe', 'request', 'REST', 'retry',
            'ServiceRequests', 'SSH', 'SSHRetry', 'timedcall', 'UnixHTTPAdapter',
            'VirtualMachine', 'VMRC', 'xmlobject']
-__version__ = '1.17.1'
+__version__ = '1.18.0'
 __author__ = 'Miguel Caballer'
 
 

IM/config.py

@@ -107,6 +107,7 @@ class Config:
     OIDC_USER_INFO_PATH = "/userinfo"
     OIDC_INSTROSPECT_PATH = "/introspect"
     OIDC_GROUPS = []
+    OIDC_GROUPS_CLAIM = "groups"
     VM_NUM_USE_CTXT_DIST = 30
     DELAY_BETWEEN_VM_RETRIES = 5
     VERIFI_SSL = False

IM/connectors/CloudConnector.py

@@ -738,10 +738,6 @@ def manage_dns_entries(self, op, vm, auth_data, extra_args=None):
                 vm.dns_entries = []
             if op == "add":
                 dns_entries = [entry for entry in self.get_dns_entries(vm) if entry not in vm.dns_entries]
-                dns_entries = []
-                for entry in self.get_dns_entries(vm):
-                    if entry not in vm.dns_entries:
-                        dns_entries.append(entry)
             else:
                 dns_entries = list(vm.dns_entries)
             if dns_entries:

IM/connectors/EC2.py

@@ -21,6 +21,8 @@
 import os
 import re
 import socket
+from random import choice
+from string import ascii_lowercase, digits
 from netaddr import IPNetwork, IPAddress
 try:
     from urlparse import urlparse
@@ -370,9 +372,12 @@ def _generate_ingress_data(self, namespace, name, dns, port, apps_dns, vm):
             if dns_url[1]:
                 host = dns_url[1]
                 if apps_dns and not host.endswith(apps_dns):
-                    if not host.endswith(".") and not apps_dns.startswith("."):
-                        host += "."
-                    host += apps_dns
+                    if host.endswith("."):
+                        host = host[:-1]
+                    host += "-" + ''.join(choice(ascii_lowercase + digits) for _ in range(4))
+                    if apps_dns.startswith("."):
+                        apps_dns = apps_dns[1:]
+                    host += "." + apps_dns
             if dns_url[2]:
                 path = dns_url[2]
 

IM/connectors/Kubernetes.py

@@ -777,3 +777,7 @@ IM 1.17.1:
     * Speed up Ansible installation using newer versions.
     * Fix problem with 0 disk flavors in OpenStack.
     * Flush Inf data to DB in case of service termination.
+
+IM 1.18.0:
+    * Enable to get IM stats.
+    * Migrate EC2 conn to boto3 library.

changelog

@@ -6,7 +6,7 @@
     "@type": "SoftwareSourceCode",
     "identifier": "im",
     "name": "Infrastructure Manager",
-    "version": "1.17.1",
+    "version": "1.18.0",
     "description": "IM is a tool that deploys complex and customized virtual infrastructures on IaaS Cloud deployments",
     "license": "GNU General Public License v3.0",
     "author": [

codemeta.json

@@ -3,8 +3,8 @@
 copyright = '2023, I3M-GRyCAP'
 author = 'micafer'
 
-version = '1.17'
-release = '1.17.1'
+version = '1.18'
+release = '1.18.0'
 
 master_doc = 'index'
 

doc/source/conf.py

@@ -1,8 +1,8 @@
 # Dockerfile to create a container with the IM service
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 ARG BRANCH=devel
 LABEL maintainer="Miguel Caballer <[email protected]>"
-LABEL version="1.17.1"
+LABEL version="1.18.0"
 LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)"
 EXPOSE 8899 8800
 
@@ -12,19 +12,18 @@ RUN apt-get update && apt-get install --no-install-recommends -y patch wget pyth
 
 # Install IM
 RUN apt-get update && apt-get install --no-install-recommends -y python3-setuptools python3-pip git && \
-     pip3 install -U pip && \
-     pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \
-     pip3 install pyOpenSSL cheroot xmltodict pymongo ansible==8.7.0&& \
-     pip3 install git+https://github.com/micafer/libcloud@ost_nets_extra && \
-     pip3 install apache-libcloud==3.8.0 git+https://github.com/grycap/im@$BRANCH && \
+     pip3 install --break-system-packages msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \
+     pip3 install --break-system-packages pyOpenSSL cheroot xmltodict pymongo ansible==8.7.0&& \
+     pip3 install --break-system-packages git+https://github.com/micafer/libcloud@ost_nets_extra && \
+     pip3 install --break-system-packages apache-libcloud==3.8.0 git+https://github.com/grycap/im@$BRANCH && \
      apt-get purge -y python3-pip git && \
      apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/
 
 # Patch libcloud to add network extra
 # untill this PR is merged and released
 # https://github.com/apache/libcloud/pull/2016
 COPY ost.patch /tmp/ost.patch
-RUN patch /usr/local/lib/python3.10/dist-packages/libcloud/compute/drivers/openstack.py < /tmp/ost.patch && rm /tmp/ost.patch
+RUN patch /usr/local/lib/python3.12/dist-packages/libcloud/compute/drivers/openstack.py < /tmp/ost.patch && rm /tmp/ost.patch
 
 # Copy im configuration files
 RUN mkdir /etc/im
@@ -38,8 +37,5 @@ RUN sed -i -e 's/VM_NUM_USE_CTXT_DIST = 30/VM_NUM_USE_CTXT_DIST = 3/g' /etc/im/i
 # Copy a ansible.cfg with correct minimum values
 COPY ansible.cfg /etc/ansible/ansible.cfg
 
-# Fix boto issue https://github.com/boto/boto/issues/3783
-COPY endpoints.json /usr/local/lib/python3.10/dist-packages/boto/endpoints.json
-
 # Start IM service
 CMD /usr/local/bin/im_service

docker-devel/Dockerfile

@@ -1,5 +1,5 @@
 # Dockerfile to create a container with the IM service
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 
 ENV VERSION=1.17.1
 
@@ -14,18 +14,17 @@ RUN apt-get update && apt-get install --no-install-recommends -y patch wget pyth
 
 # Install IM
 RUN apt-get update && apt-get install --no-install-recommends -y python3-setuptools python3-pip git && \
-     pip3 install -U pip && \
-     pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \
-     pip3 install pyOpenSSL cheroot xmltodict pymongo ansible==8.7.0&& \
-     pip3 install apache-libcloud==3.8.0 IM==${VERSION} &&\
+     pip3 install --break-system-packages msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \
+     pip3 install --break-system-packages pyOpenSSL cheroot xmltodict pymongo ansible==8.7.0&& \
+     pip3 install --break-system-packages apache-libcloud==3.8.0 IM==${VERSION} &&\
      apt-get purge -y python3-pip git && \
      apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/
 
 # Patch libcloud to add network extra
 # untill this PR is merged and released
 # https://github.com/apache/libcloud/pull/2016
 COPY ost.patch /tmp/ost.patch
-RUN patch /usr/local/lib/python3.10/dist-packages/libcloud/compute/drivers/openstack.py < /tmp/ost.patch && rm /tmp/ost.patch
+RUN patch /usr/local/lib/python3.12/dist-packages/libcloud/compute/drivers/openstack.py < /tmp/ost.patch && rm /tmp/ost.patch
 
 # Copy im configuration files
 RUN mkdir /etc/im
@@ -36,8 +35,5 @@ RUN wget https://raw.githubusercontent.com/grycap/im/v${VERSION}/etc/logging.con
 # Copy a ansible.cfg with correct minimum values
 COPY ansible.cfg /etc/ansible/ansible.cfg
 
-# Fix boto issue https://github.com/boto/boto/issues/3783
-COPY endpoints.json /usr/local/lib/python3.10/dist-packages/boto/endpoints.json
-
 # Start IM service
 CMD ["/usr/local/bin/im_service"]

docker-py3/Dockerfile

@@ -40,8 +40,5 @@ RUN apk add --no-cache git &&\
 # Copy a ansible.cfg with correct minimum values
 COPY ansible.cfg /etc/ansible/ansible.cfg
 
-# Fix boto issue https://github.com/boto/boto/issues/3783
-COPY endpoints.json /usr/lib/python3.10/site-packages/boto/endpoints.json
-
 # Start IM service
 CMD im_service.py

docker-py3/Dockerfile.alp

@@ -145,6 +145,8 @@ OIDC_ISSUERS = https://aai.egi.eu/auth/realms/egi
 #OIDC_INSTROSPECT_PATH = "/introspect"
 # List of OIDC groups that will be allowed to access the IM service
 #OIDC_GROUPS =
+# Claim where the groups are stored in the OIDC token
+# OIDC_GROUPS_CLAIM = groups
 # Force the users to pass a valid OIDC token
 #FORCE_OIDC_AUTH = False
 

etc/im.cfg

@@ -48,7 +48,7 @@ dependencies = [
     "PyYAML",
     "suds-community",
     "cheroot",
-    "boto >= 2.29",
+    "boto3",
     "apache-libcloud >= 3.2.0",
     "RADL >= 1.3.3",
     "flask",

pyproject.toml

@@ -3,7 +3,7 @@ ansible-base
 paramiko >= 1.14
 PyYAML
 cheroot
-boto >= 2.29
+boto3
 apache-libcloud >= 3.3.1
 RADL >= 1.3.3
 flask