Agency: Adds import command for staka lu

TYPE: Feature
LINK: ogc-1891

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
         exclude: .pre-commit-config.yaml
       - id: pt_structure
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.1
+    rev: v0.7.4
     hooks:
       - id: ruff
         args: [ "--fix" ]
@@ -33,7 +33,7 @@ repos:
     - id: sass-lint
       files: '^src/.*\.scss'
   - repo: https://github.com/pre-commit/mirrors-eslint
-    rev: v9.14.0
+    rev: v9.15.0
     hooks:
     - id: eslint
       files: '^src/.*\.jsx?$'

src/onegov/agency/cli.py

@@ -2,7 +2,9 @@
 import transaction
 
 from onegov.agency.collections import ExtendedAgencyCollection
-from onegov.agency.data_import import import_bs_data, import_membership_titles
+from onegov.agency.data_import import (import_bs_data,
+                                       import_membership_titles,
+                                       import_lu_data)
 from onegov.agency.excel_export import export_person_xlsx
 from onegov.agency.models import ExtendedAgencyMembership, ExtendedPerson
 from onegov.core.cli import command_group
@@ -162,11 +164,11 @@ def import_bs_data_files(
     clean: bool
 ) -> 'Callable[[AgencyRequest, AgencyApp], None]':
     """
+
     Usage:
 
         onegov-agency  --select /onegov_agency/bs import-bs-data \
-        $agency_file \
-        $people_file \
+        $agency_file $people_file
     """
 
     buffer = 100
@@ -212,6 +214,61 @@ def execute(request: 'AgencyRequest', app: 'AgencyApp') -> None:
     return execute
 
 
+@cli.command('import-lu-data', context_settings={'singular': True})
+@click.argument('data-file', type=click.Path(exists=True))
+@click.option('--dry-run', is_flag=True, default=False)
+@click.option('--clean', is_flag=True, default=False)
+def import_lu_data_files(
+    data_file: str,
+    dry_run: bool,
+    clean: bool
+) -> 'Callable[[AgencyRequest, AgencyApp], None]':
+    """
+
+    Usage:
+
+        onegov-agency --select /onegov_agency/lu import-lu-data $people_file
+    """
+
+    buffer = 100
+
+    def execute(request: 'AgencyRequest', app: 'AgencyApp') -> None:
+
+        if clean:
+            session = request.session
+
+            for ix, person in enumerate(session.query(Person)):
+                session.delete(person)
+                if ix % buffer == 0:
+                    app.es_indexer.process()
+                    app.psql_indexer.bulk_process(session)
+
+            session.flush()
+            click.secho('All Persons removed', fg='green')
+
+            for ix, agency in enumerate(session.query(Agency)):
+                session.delete(agency)
+                if ix % buffer == 0:
+                    app.es_indexer.process()
+                    app.psql_indexer.bulk_process(session)
+
+            session.flush()
+            click.secho('All Agencies removed', fg='green')
+
+            click.secho('Exiting...')
+            return
+
+        agencies, people = import_lu_data(data_file, request, app)
+        click.secho(f'Imported {len(people)} persons and '
+                    f'{len(agencies)} agencies', fg='green')
+
+        if dry_run:
+            transaction.abort()
+            click.secho('Aborting transaction', fg='yellow')
+
+    return execute
+
+
 @cli.command('create-pdf')
 @pass_group_context
 @click.option('--root/--no-root', default=True)

src/onegov/agency/data_import.py

@@ -37,7 +37,8 @@ def _read(filename: 'StrOrBytesPath', *args: *Ts) -> T:
         with open(filename, 'rb') as f:
             file = CSVFile(
                 f,
-                encoding='iso-8859-1'
+                # encoding='iso-8859-1'
+                encoding='utf-8',  # lu
             )
             return func(file, *args)
 
@@ -70,6 +71,9 @@ def get_phone(string: str) -> str:
     if not string.startswith('+'):
         if len(string.replace(' ', '')) == 10:  # be sure #digits fit CH
             return string.replace('0', '+41 ', 1)
+        # lu adds country digits
+        if len(string.replace(' ', '')) == 9:
+            return f'+41{string}'
     return string
 
 
@@ -303,6 +307,179 @@ def import_bs_data(
     return agencies, persons
 
 
+def get_plz_city(plz: str | None, ort: str | None) -> str | None:
+    if plz and ort:
+        return f'{plz} {ort}'
+
+    if ort:
+        return ort
+
+    if plz:
+        return plz
+
+    return None
+
+
+def get_web_address(internet_adresse: str) -> str | None:
+    if not internet_adresse:
+        return None
+
+    if internet_adresse.startswith('http'):
+        return internet_adresse
+
+    return f'http://{internet_adresse}'
+
+
+def check_skip(line: 'DefaultRow') -> bool:
+    skip = False
+
+    if line.department == 'zNeu':
+        skip = True
+
+    if any(s in line.vorname for s in ('Zi.', 'Korr.', 'test')):
+        skip = True
+
+    if any(s in line.nachname for s in ('WG', 'WH', 'W3', 'W5',
+                                        'frei neuer MA', 'frei  neuer MA',
+                                        'AAL Picket')):
+        skip = True
+
+    if line.nachname == '' and line.vorname == '':
+        skip = True  # empty lines in file
+
+    if skip:
+        print(f'Skipping {str(line)[:120]}..')
+        return True
+
+    return False
+
+
+@with_open
+def import_lu_people(
+    csvfile: CSVFile['DefaultRow'],
+    agencies: 'Mapping[str, ExtendedAgency]',
+    session: 'Session',
+    app: 'AgencyApp'
+) -> list['ExtendedPerson']:
+
+    people = ExtendedPersonCollection(session)
+    persons = []
+
+    def parse_person(line: 'DefaultRow') -> None:
+        person_ = people.add(
+            last_name=v_(line.nachname) or ' ',
+            first_name=v_(line.vorname) or ' ',
+            salutation=None,
+            academic_title=v_(line.akad__titel),
+            function=v_(line.funktion),
+            email=v_(line.e_mail_adresse),
+            phone=get_phone(line.isdn_nummer),
+            phone_direct=get_phone(line.mobil),
+            website=v_(get_web_address(line.internet_adresse)),
+            notes=v_(line.bemerkungen),
+            location_address=v_(line.adresse),
+            location_code_city=v_(get_plz_city(line.plz, line.ort)),
+            access='public'
+        )
+        persons.append(person_)
+
+        # A person has only one membership
+        agency_id = (line.unterabteilung_2 or line.unterabteilung or
+                     line.abteilung)
+        hi_code = v_(line.hi_code)
+        order = 0 if not hi_code else int(hi_code)
+        if agency_id:
+            agency = agencies.get(agency_id)
+            if agency and order:
+                agency.add_person(person_.id,
+                                  title=person_.function or 'Mitglied',
+                                  order_within_agency=order)
+            elif agency:
+                agency.add_person(person_.id,
+                                  title=person_.function or 'Mitglied')
+            else:
+                print(f'Error agency id {agency_id} not found')
+
+    for ix, line in enumerate(csvfile.lines):
+        if ix % 100 == 0:
+            app.es_indexer.process()
+            app.psql_indexer.bulk_process(session)
+
+        if not check_skip(line):
+            parse_person(line)
+
+    return persons
+
+
+@with_open
+def import_lu_agencies(
+    csvfile: CSVFile['DefaultRow'],
+    session: 'Session',
+    app: 'AgencyApp'
+) -> dict[str, 'ExtendedAgency']:
+
+    added_agencies = {}
+    agencies = ExtendedAgencyCollection(session)
+
+    # Hierarchy: Hierarchie: Department, Dienststelle, Abteilung,
+    # Unterabteilung, Unterabteilung 2, Unterabteilung 3
+    for ix, line in enumerate(csvfile.lines):
+        if ix % 100 == 0:
+            app.es_indexer.process()
+            app.psql_indexer.bulk_process(session)
+
+        if check_skip(line):
+            continue
+
+        dienststelle, abteilung, unterabteilung, unterabteilung_2 = (
+            None, None, None, None)
+        department_name = v_(line.department) or ''
+        department = agencies.add_or_get(None, department_name)
+        added_agencies[department_name] = department
+        export_fields = ['person.title', 'person.phone']
+
+        dienststellen_name = v_(line.dienststelle)
+        if dienststellen_name:
+            dienststelle = agencies.add_or_get(
+                department, dienststellen_name, export_fields=export_fields)
+            added_agencies[dienststellen_name] = dienststelle
+
+        abteilungs_name = v_(line.abteilung)
+        if abteilungs_name:
+            abteilung = agencies.add_or_get(
+                dienststelle, abteilungs_name, export_fields=export_fields)
+            added_agencies[abteilungs_name] = abteilung
+
+        unterabteilungs_name = v_(line.unterabteilung)
+        if unterabteilungs_name:
+            unterabteilung = (
+                agencies.add_or_get(abteilung, unterabteilungs_name,
+                                    export_fields=export_fields))
+            added_agencies[unterabteilungs_name] = unterabteilung
+
+        unterabteilung_2_name = v_(line.unterabteilung_2)
+        if unterabteilung_2_name:
+            unterabteilung_2 = (
+                agencies.add_or_get(unterabteilung, unterabteilung_2_name,
+                                    export_fields=export_fields))
+            added_agencies[unterabteilung_2_name] = unterabteilung_2
+
+    return added_agencies
+
+
+def import_lu_data(
+    data_file: 'StrOrBytesPath',
+    request: 'AgencyRequest',
+    app: 'AgencyApp'
+) -> tuple[dict[str, 'ExtendedAgency'], list['ExtendedPerson']]:
+
+    session = request.session
+    agencies = import_lu_agencies(data_file, session, app)
+    people = import_lu_people(data_file, agencies, session, app)
+
+    return agencies, people
+
+
 @with_open
 def parse_agencies(csvfile: CSVFile['DefaultRow']) -> dict[str, str]:
     lines_by_id = {line.verzorgeinheitid: line for line in csvfile.lines}

src/onegov/agency/models/agency.py

@@ -165,8 +165,11 @@ def add_person(  # type:ignore[override]
         )
         self.memberships.append(membership)
 
-        for order, _membership in enumerate(self.memberships):
-            _membership.order_within_agency = order
+        # re-order all memberships cannot be done here, because the order
+        # within the agency is not yet set. do be done once all memberships
+        # are added to the agency.
+        # for order, _membership in enumerate(self.memberships):
+        #     _membership.order_within_agency = order
 
         session.flush()
 

src/onegov/core/csv.py

@@ -209,7 +209,7 @@ def __init__(
     @lru_cache(maxsize=128)
     def as_valid_identifier(value: str) -> str:
         result = normalize_header(value)
-        for invalid in '- .%/,;':
+        for invalid in '- .%/,;()':
             result = result.replace(invalid, '_')
         while result and result[0] in '_0123456789':
             result = result[1:]

tests/onegov/core/test_adjacency_list.py

@@ -88,11 +88,31 @@ def test_add_or_get_page(session):
 
     assert root.title == 'Wurzel'
     assert root.name == 'root'
+    assert family.query().count() == 1
 
     root = family.add_or_get_root(title='Wurzel', name='root')
 
     assert root.title == 'Wurzel'
     assert root.name == 'root'
+    assert family.query().count() == 1
+
+    # test case-insensitive
+    test = family.add_or_get_root(title='Test')
+    assert test.title == 'Test'
+    assert test.name == 'test'
+    assert family.query().count() == 2
+
+    test = family.add_or_get_root(title='test')
+    assert test.title == 'Test'
+    assert test.name == 'test'
+    assert family.query().count() == 2
+
+    # invalid name (not normalized)
+    with pytest.raises(AssertionError) as assertion_info:
+        family.add_or_get_root(title='Test', name='Test')
+        assert 'The given name was not normalized' in assertion_info.value
+
+    assert family.query().count() == 2
 
 
 def test_page_by_path(session):