Migrate from puppetdbquery to puppetdb::query_facts

Bug: T13025
miraheze · Dec 21, 2024 · f97deee · f97deee
1 parent 02e83c4
commit f97deee
Show file tree

Hide file tree

Showing 23 changed files with 193 additions and 349 deletions.
diff --git a/modules/base/manifests/firewall.pp b/modules/base/manifests/firewall.pp
@@ -30,48 +30,22 @@
         source => 'puppet:///modules/base/firewall/main-input-default-drop.conf',
     }
 
-    $firewall_rules_str = join(
-        query_facts('Class[Role::Icinga2]', ['networking'])
-        .map |$key, $value| {
-            if ( $value['networking']['interfaces']['he-ipv6'] ) {
-                "${value['networking']['ip']} ${value['networking']['interfaces']['he-ipv6']['ip6']}"
-            } elsif ( $value['networking']['interfaces']['ens19'] and $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens19']['ip']} ${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } elsif ( $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } else {
-                "${value['networking']['ip']} ${value['networking']['ip6']}"
-            }
-        }
-        .flatten()
-        .unique()
-        .sort(),
-        ' '
-    )
+    $subquery = @("PQL")
+    resources { type = 'Class' and title = 'Role::Icinga2' }
+    | PQL
+    $firewall_rules_str = vmlib::generate_firewall_ip($subquery)
+
     ferm::service { 'nrpe':
         proto  => 'tcp',
         port   => '5666',
         srange => "(${firewall_rules_str})",
     }
 
-    $firewall_bastion_hosts = join(
-        query_facts('Class[Base]', ['networking'])
-        .map |$key, $value| {
-            if ( $value['networking']['interfaces']['he-ipv6'] ) {
-                "${value['networking']['ip']} ${value['networking']['interfaces']['he-ipv6']['ip6']}"
-            } elsif ( $value['networking']['interfaces']['ens19'] and $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens19']['ip']} ${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } elsif ( $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } else {
-                "${value['networking']['ip']} ${value['networking']['ip6']}"
-            }
-        }
-        .flatten()
-        .unique()
-        .sort(),
-        ' '
-    )
+    $subquery = @("PQL")
+    resources { type = 'Class' and title = 'Base' }
+    | PQL
+    $firewall_bastion_hosts = vmlib::generate_firewall_ip($subquery)
+
     ferm::service { 'ssh':
         proto  => 'tcp',
         port   => '22',

diff --git a/modules/prometheus/manifests/class.pp b/modules/prometheus/manifests/class.pp
@@ -3,10 +3,15 @@
     String $module,
     Integer $port,
 ) {
-    $servers = query_nodes("Class[${module}] or Define[${module}]")
-        .flatten()
-        .unique()
-        .sort()
+
+    $pql = @("PQL")
+    nodes[certname] {
+        (resources {type =  "Class" and title = "${module}" }
+        or resources {type = "Define" and title = "${module}" })
+        order by certname
+    }
+    | PQL
+    $servers = puppetdb_query($pql).map |$resource| { $resource['certname'] }.flatten().unique().sort
 
     file { $dest:
         ensure  => present,

diff --git a/modules/prometheus/manifests/exporter/cadvisor.pp b/modules/prometheus/manifests/exporter/cadvisor.pp
@@ -8,22 +8,10 @@
         subscribe => Package['cadvisor'],
     }
 
-    $firewall_rules_str = join(
-        query_facts('Class[Prometheus] or Class[Role::Grafana]', ['networking'])
-        .map |$key, $value| {
-            if ( $value['networking']['interfaces']['ens19'] and $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens19']['ip']} ${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } elsif ( $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } else {
-                "${value['networking']['ip']} ${value['networking']['ip6']}"
-            }
-        }
-        .flatten()
-        .unique()
-        .sort(),
-        ' '
-    )
+    $subquery = @("PQL")
+    resources { type = 'Class' and title = 'Prometheus' }
+    | PQL
+    $firewall_rules_str = vmlib::generate_firewall_ip($subquery)
     ferm::service { 'prometheus cadvisor_exporter':
         proto  => 'tcp',
         port   => '4194',

diff --git a/modules/prometheus/manifests/exporter/mariadb.pp b/modules/prometheus/manifests/exporter/mariadb.pp
@@ -64,22 +64,10 @@
         ensure  => running,
     }
 
-    $firewall_rules_str = join(
-        query_facts('Class[Prometheus]', ['networking'])
-        .map |$key, $value| {
-            if ( $value['networking']['interfaces']['ens19'] and $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens19']['ip']} ${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } elsif ( $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } else {
-                "${value['networking']['ip']} ${value['networking']['ip6']}"
-            }
-        }
-        .flatten()
-        .unique()
-        .sort(),
-        ' '
-    )
+    $subquery = @("PQL")
+    resources { type = 'Class' and title = 'Prometheus' }
+    | PQL
+    $firewall_rules_str = vmlib::generate_firewall_ip($subquery)
     ferm::service { 'prometheus mysqld_exporter':
         proto  => 'tcp',
         port   => '9104',

diff --git a/modules/prometheus/manifests/exporter/openldap.pp b/modules/prometheus/manifests/exporter/openldap.pp
@@ -28,22 +28,10 @@
         }
     }
 
-    $firewall_rules_str = join(
-        query_facts('Class[Role::Prometheus]', ['networking'])
-        .map |$key, $value| {
-            if ( $value['networking']['interfaces']['ens19'] and $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens19']['ip']} ${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } elsif ( $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } else {
-                "${value['networking']['ip']} ${value['networking']['ip6']}"
-            }
-        }
-        .flatten()
-        .unique()
-        .sort(),
-        ' '
-    )
+    $subquery = @("PQL")
+    resources { type = 'Class' and title = 'Role::Prometheus' }
+    | PQL
+    $firewall_rules_str = vmlib::generate_firewall_ip($subquery)
     ferm::service { 'prometheus openldap_exporter':
         proto  => 'tcp',
         port   => '9142',

diff --git a/modules/prometheus/manifests/exporter/varnish.pp b/modules/prometheus/manifests/exporter/varnish.pp
@@ -9,24 +9,11 @@
         restart => true,
     }
 
-    $firewall_rules_str = join(
-        query_facts('Class[Role::Prometheus]', ['networking'])
-        .map |$key, $value| {
-            if ( $value['networking']['interfaces']['he-ipv6'] ) {
-                "${value['networking']['ip']} ${value['networking']['interfaces']['he-ipv6']['ip6']}"
-            } elsif ( $value['networking']['interfaces']['ens19'] and $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens19']['ip']} ${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } elsif ( $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } else {
-                "${value['networking']['ip']} ${value['networking']['ip6']}"
-            }
-        }
-        .flatten()
-        .unique()
-        .sort(),
-        ' '
-    )
+    $subquery = @("PQL")
+    resources { type = 'Class' and title = 'Role::Prometheus' }
+    | PQL
+    $firewall_rules_str = vmlib::generate_firewall_ip($subquery)
+
     ferm::service { 'prometheus varnish_exporter':
         proto  => 'tcp',
         port   => $listen_port,

diff --git a/modules/prometheus/manifests/init.pp b/modules/prometheus/manifests/init.pp
@@ -56,10 +56,13 @@
         refreshonly => true,
     }
 
-    $servers = query_nodes('Class[Base]')
-              .flatten()
-              .unique()
-              .sort()
+    $pql = @("PQL")
+    nodes[certname] {
+        resources {type =  "Class" and title = "Base" }
+        order by certname
+    }
+    | PQL
+    $servers = puppetdb_query($pql).map |$resource| { $resource['certname'] }.flatten().unique().sort
 
     file { '/etc/prometheus/targets/nodes.yaml':
         ensure  => present,

diff --git a/modules/puppetdb/functions/query_facts.pp b/modules/puppetdb/functions/query_facts.pp
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: Apache-2.0
+# @summery query for custome facts for a host and return a hash of facts values keyed to the certname
+# @param filter a hash of fact name to fetch
+# @param a pql subquery to apply to the query
+function puppetdb::query_facts(
+    Array[String[1]]    $filter,
+    Optional[String[1]] $subquery = undef,
+) >> Hash[Stdlib::Fqdn, Hash] {
+    $_subquery = $subquery ? {
+        undef   => '',
+        default => " and ${subquery}"
+    }
+    $filter_str = $filter.map |$filter| { "\"${filter}\"" }.join(',')
+    $pql = "facts[certname, name, value] { name in [${filter_str}] ${_subquery} }"
+    puppetdb::munge_facts(puppetdb_query($pql))
+}
diff --git a/modules/puppetdb/lib/puppet/functions/puppetdb/munge_facts.rb b/modules/puppetdb/lib/puppet/functions/puppetdb/munge_facts.rb
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: Apache-2.0
+Puppet::Functions.create_function(:'puppetdb::munge_facts') do
+  dispatch :munge_facts do
+    param 'Array[Hash]', :facts
+  end
+
+  def munge_facts(facts)
+    facts_out = Hash.new {|h, k| h[k] = {}}
+    facts.each do |f|
+      facts_out[f['certname']][f['name']] = f['value']
+    end
+    facts_out
+  end
+end
diff --git a/modules/role/manifests/burrow.pp b/modules/role/manifests/burrow.pp
@@ -26,24 +26,10 @@
         metrics_addr => '0.0.0.0:9500'
     }
 
-    $firewall_rules_str = join(
-        query_facts('Class[Role::Prometheus]', ['networking'])
-        .map |$key, $value| {
-            if ( $value['networking']['interfaces']['he-ipv6'] ) {
-                "${value['networking']['ip']} ${value['networking']['interfaces']['he-ipv6']['ip6']}"
-            } elsif ( $value['networking']['interfaces']['ens19'] and $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens19']['ip']} ${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } elsif ( $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } else {
-                "${value['networking']['ip']} ${value['networking']['ip6']}"
-            }
-        }
-        .flatten()
-        .unique()
-        .sort(),
-        ' '
-    )
+    $subquery = @("PQL")
+    resources { type = 'Class' and title = 'Role::Prometheus' }
+    | PQL
+    $firewall_rules_str = vmlib::generate_firewall_ip($subquery)
 
     # Burrow offers a HTTP REST API
     ferm::service { 'burrow-main':

diff --git a/modules/role/manifests/changeprop.pp b/modules/role/manifests/changeprop.pp
@@ -4,22 +4,14 @@
     include role::prometheus::statsd_exporter
 
     # TODO: Restrict beta access at some point once we get working.
-    $firewall_rules_str = join(
-        query_facts('Class[Role::Mediawiki] or Class[Role::Mediawiki_task] or Class[Role::Mediawiki_beta] or Class[Role::Icinga2]', ['networking'])
-        .map |$key, $value| {
-            if ( $value['networking']['interfaces']['ens19'] and $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens19']['ip']} ${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } elsif ( $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } else {
-                "${value['networking']['ip']} ${value['networking']['ip6']}"
-            }
-        }
-        .flatten()
-        .unique()
-        .sort(),
-        ' '
-    )
+    $subquery = @("PQL")
+    (resources { type = 'Class' and title = 'Role::Mediawik' } or
+    resources { type = 'Class' and title = 'Role::Mediawiki_task' } or
+    resources { type = 'Class' and title = 'Role::Mediawiki_beta' } or
+    resources { type = 'Class' and title = 'Role::Icinga2' })
+    | PQL
+    $firewall_rules_str = vmlib::generate_firewall_ip($subquery)
+
     ferm::service { 'changeprop':
         proto   => 'tcp',
         port    => '7200',

diff --git a/modules/role/manifests/cloud.pp b/modules/role/manifests/cloud.pp
@@ -4,20 +4,10 @@
 
     class { '::cpufrequtils': }
 
-    $firewall_rules_str = join(
-        query_facts('Class[Role::Cloud]', ['networking'])
-        .map |$key, $value| {
-            if ( $value['networking']['interfaces']['vmbr1'] ) {
-                "${value['networking']['interfaces']['vmbr1']['ip']} ${value['networking']['ip']} ${value['networking']['ip6']}"
-            } else {
-                "${value['networking']['ip']} ${value['networking']['ip6']}"
-            }
-        }
-        .flatten()
-        .unique()
-        .sort(),
-        ' '
-    )
+    $subquery = @("PQL")
+    resources { type = 'Class' and title = 'Role::Cloud' }
+    | PQL
+    $firewall_rules_str = vmlib::generate_firewall_ip($subquery)
 
     ferm::service { 'proxmox port 5900:5999':
         proto  => 'tcp',

diff --git a/modules/role/manifests/irc.pp b/modules/role/manifests/irc.pp
@@ -32,44 +32,21 @@
         udp_port     => '5071',
     }
 
-    $firewall_irc_rules_str = join(
-        query_facts('Class[Role::Mediawiki] or Class[Role::Mediawiki_task] or Class[Role::Mediawiki_beta]', ['networking'])
-        .map |$key, $value| {
-            if ( $value['networking']['interfaces']['ens19'] and $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens19']['ip']} ${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } elsif ( $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } else {
-                "${value['networking']['ip']} ${value['networking']['ip6']}"
-            }
-        }
-        .flatten()
-        .unique()
-        .sort(),
-        ' '
-    )
+    $subquery = @("PQL")
+    (resources { type = 'Class' and title = 'Role::Mediawik' } or
+    resources { type = 'Class' and title = 'Role::Mediawiki_task' } or
+    resources { type = 'Class' and title = 'Role::Mediawiki_beta' })
+    | PQL
+    $firewall_irc_rules_str = vmlib::generate_firewall_ip($subquery)
+
     ferm::service { 'ircrcbot':
         proto  => 'udp',
         port   => '5070',
         srange => "(${firewall_irc_rules_str})",
     }
 
-    $firewall_all_rules_str = join(
-        query_facts('Class[Base]', ['networking'])
-        .map |$key, $value| {
-            if ( $value['networking']['interfaces']['ens19'] and $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens19']['ip']} ${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } elsif ( $value['networking']['interfaces']['ens18'] ) {
-                "${value['networking']['interfaces']['ens18']['ip']} ${value['networking']['interfaces']['ens18']['ip6']}"
-            } else {
-                "${value['networking']['ip']} ${value['networking']['ip6']}"
-            }
-        }
-        .flatten()
-        .unique()
-        .sort(),
-        ' '
-    )
+    $firewall_all_rules_str = vmlib::generate_firewall_ip()
+
     ferm::service { 'irclogserverbot':
         proto  => 'udp',
         port   => '5071',