Merge pull request #530 from anmenaga/adapter_lookup_table

Adapter lookup table for non-list operations on adapted resources

Author: SteveL-MSFT

dsc/tests/dsc_discovery.tests.ps1

@@ -4,6 +4,12 @@
 Describe 'tests for resource discovery' {
     BeforeAll {
         $env:DSC_RESOURCE_PATH = $testdrive
+
+        $script:lookupTableFilePath = if ($IsWindows) {
+            Join-Path $env:LocalAppData "dsc\AdaptedResourcesLookupTable.json"
+        } else {
+            Join-Path $env:HOME ".dsc" "AdaptedResourcesLookupTable.json"
+        }
     }
 
     AfterEach {
@@ -96,4 +102,52 @@ Describe 'tests for resource discovery' {
             $env:DSC_RESOURCE_PATH = $oldPath
         }
     }
+
+    It 'Ensure List operation populates adapter lookup table' {
+        # remove adapter lookup table file
+        Remove-Item -Force -Path $script:lookupTableFilePath -ErrorAction SilentlyContinue
+        Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeFalse
+
+        # perform List on an adapter - this should create adapter lookup table file
+        $oldPSModulePath = $env:PSModulePath
+        $TestClassResourcePath = Resolve-Path "$PSScriptRoot/../../powershell-adapter/Tests"
+        $env:DSC_RESOURCE_PATH = $null
+        $env:PSModulePath += [System.IO.Path]::PathSeparator + $TestClassResourcePath
+        dsc resource list -a Microsoft.DSC/PowerShell | Out-Null
+        gc -raw $script:lookupTableFilePath
+        $script:lookupTableFilePath | Should -FileContentMatchExactly 'Microsoft.DSC/PowerShell'
+        Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeTrue
+        $env:PSModulePath = $oldPSModulePath
+    }
+
+    It 'Ensure non-List operation populates adapter lookup table' {
+
+        # remove adapter lookup table file
+        Remove-Item -Force -Path $script:lookupTableFilePath -ErrorAction SilentlyContinue
+        Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeFalse
+
+        # perform Get on an adapter - this should create adapter lookup table file
+        $oldPSModulePath = $env:PSModulePath
+        $TestClassResourcePath = Resolve-Path "$PSScriptRoot/../../powershell-adapter/Tests"
+        $env:DSC_RESOURCE_PATH = $null
+        $env:PSModulePath += [System.IO.Path]::PathSeparator + $TestClassResourcePath
+        "{'Name':'TestClassResource1'}" | dsc resource get -r 'TestClassResource/TestClassResource' | Out-Null
+
+        Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeTrue
+        $script:lookupTableFilePath | Should -FileContentMatchExactly 'testclassresource/testclassresource'
+        $env:PSModulePath = $oldPSModulePath
+    }
+
+    It 'Verify adapter lookup table is used on repeat invocations' {
+
+        $oldPSModulePath = $env:PSModulePath
+        $TestClassResourcePath = Resolve-Path "$PSScriptRoot/../../powershell-adapter/Tests"
+        $env:DSC_RESOURCE_PATH = $null
+        $env:PSModulePath += [System.IO.Path]::PathSeparator + $TestClassResourcePath
+        dsc resource list -a Microsoft.DSC/PowerShell | Out-Null
+        "{'Name':'TestClassResource1'}" | dsc -l trace resource get -r 'TestClassResource/TestClassResource' 2> $TestDrive/tracing.txt
+
+        "$TestDrive/tracing.txt" | Should -FileContentMatchExactly "Lookup table found resource 'testclassresource/testclassresource' in adapter 'Microsoft.DSC/PowerShell'"
+        $env:PSModulePath = $oldPSModulePath
+    }
 }

dsc_lib/Cargo.toml

@@ -10,6 +10,7 @@ clap = { version = "4.4", features = ["derive"] }
 derive_builder ="0.20.0"
 indicatif = "0.17.0"
 jsonschema = "0.18.0"
+linked-hash-map = "0.5.6"
 num-traits = "0.2.14"
 regex = "1.7.0"
 reqwest = { version = "0.12.0", features = ["rustls-tls"], default-features = false }

dsc_lib/src/discovery/command_discovery.rs

@@ -8,11 +8,13 @@ use crate::dscresources::resource_manifest::{import_manifest, validate_semver, K
 use crate::dscresources::command_resource::invoke_command;
 use crate::dscerror::DscError;
 use indicatif::ProgressStyle;
+use linked_hash_map::LinkedHashMap;
 use regex::RegexBuilder;
 use semver::Version;
-use std::collections::{BTreeMap, HashSet};
+use std::collections::{BTreeMap, HashSet, HashMap};
 use std::env;
 use std::ffi::OsStr;
+use std::fs;
 use std::fs::File;
 use std::io::BufReader;
 use std::path::{Path, PathBuf};
@@ -279,6 +281,7 @@ impl ResourceDiscovery for CommandDiscovery {
         }
 
         self.adapted_resources = adapted_resources;
+
         Ok(())
     }
 
@@ -294,6 +297,11 @@ impl ResourceDiscovery for CommandDiscovery {
         } else {
             self.discover_resources("*")?;
             self.discover_adapted_resources(type_name_filter, adapter_name_filter)?;
+            
+            // add/update found adapted resources to the lookup_table
+            add_resources_to_lookup_table(&self.adapted_resources);
+
+            // note: in next line 'BTreeMap::append' will leave self.adapted_resources empty
             resources.append(&mut self.adapted_resources);
         }
 
@@ -334,7 +342,8 @@ impl ResourceDiscovery for CommandDiscovery {
         debug!("Found {} matching non-adapter-based resources", found_resources.len());
 
         // now go through the adapters
-        for (adapter_name, adapters) in self.adapters.clone() {
+        let sorted_adapters = sort_adapters_based_on_lookup_table(&self.adapters, &remaining_required_resource_types);
+        for (adapter_name, adapters) in sorted_adapters {
             // TODO: handle version requirements
             let Some(adapter) = adapters.first() else {
                 // skip if no adapters
@@ -353,6 +362,8 @@ impl ResourceDiscovery for CommandDiscovery {
             }
 
             self.discover_adapted_resources("*", &adapter_name)?;
+            // add/update found adapted resources to the lookup_table
+            add_resources_to_lookup_table(&self.adapted_resources);
 
             // now go through the adapter resources and add them to the list of resources
             for (adapted_name, adapted_resource) in &self.adapted_resources {
@@ -496,3 +507,95 @@ fn load_manifest(path: &Path) -> Result<DscResource, DscError> {
 
     Ok(resource)
 }
+
+fn sort_adapters_based_on_lookup_table(unsorted_adapters: &BTreeMap<String, Vec<DscResource>>, needed_resource_types: &Vec<String>) -> LinkedHashMap<String, Vec<DscResource>>
+{
+    let mut result = LinkedHashMap::<String, Vec<DscResource>>::new();
+    let lookup_table = load_adapted_resources_lookup_table();
+    // first add adapters (for needed types) that can be found in the lookup table
+    for needed_resource in needed_resource_types {
+        if let Some(adapter_name) = lookup_table.get(needed_resource) {
+            if let Some(resource_vec) = unsorted_adapters.get(adapter_name) {
+                debug!("Lookup table found resource '{}' in adapter '{}'", needed_resource, adapter_name);
+                result.insert(adapter_name.to_string(), resource_vec.clone());
+            }
+        }
+    }
+
+    // now add remaining adapters
+    for (adapter_name, adapters) in unsorted_adapters {
+        if !result.contains_key(adapter_name) {
+            result.insert(adapter_name.to_string(), adapters.clone());
+        }
+    }
+
+    result
+}
+
+fn add_resources_to_lookup_table(adapted_resources: &BTreeMap<String, Vec<DscResource>>)
+{
+    let mut lookup_table = load_adapted_resources_lookup_table();
+
+    for (resource_name, res_vec) in adapted_resources {
+        if let Some(adapter_name) = &res_vec[0].require_adapter {
+            lookup_table.insert(resource_name.to_string().to_lowercase(), adapter_name.to_string());
+        } else {
+            info!("Resource '{resource_name}' in 'adapted_resources' is missing 'require_adapter' field.");
+        }
+    };
+
+    save_adapted_resources_lookup_table(&lookup_table);
+}
+
+fn save_adapted_resources_lookup_table(lookup_table: &HashMap<String, String>)
+{
+    if let Ok(lookup_table_json) = serde_json::to_string(&lookup_table) {
+        let file_path = get_lookup_table_file_path();
+        debug!("Saving lookup table with {} items to {:?}", lookup_table.len(), file_path);
+
+        let path = std::path::Path::new(&file_path);
+        if let Some(prefix) = path.parent() {
+            if fs::create_dir_all(prefix).is_ok()  {
+                if fs::write(file_path.clone(), lookup_table_json).is_err() {
+                    info!("Unable to write lookup_table file {file_path:?}");
+                }
+            } else {
+                info!("Unable to create parent directories of the lookup_table file {file_path:?}");
+            }
+        } else {
+            info!("Unable to get directory of the lookup_table file {file_path:?}");
+        }
+    } else {
+        info!("Unable to serialize lookup_table to json");
+    }
+}
+
+fn load_adapted_resources_lookup_table() -> HashMap<String, String>
+{
+    let file_path = get_lookup_table_file_path();
+    
+    let lookup_table: HashMap<String, String> = match fs::read(file_path.clone()){
+        Ok(data) => { serde_json::from_slice(&data).unwrap_or_default() },
+        Err(_) => { HashMap::new() }
+    };
+
+    debug!("Read {} items into lookup table from {:?}", lookup_table.len(), file_path);
+    lookup_table
+}
+
+#[cfg(target_os = "windows")]
+fn get_lookup_table_file_path() -> String
+{
+    // $env:LocalAppData+"dsc\AdaptedResourcesLookupTable.json"
+    let Ok(local_app_data_path) = std::env::var("LocalAppData") else { return String::new(); };
+
+    Path::new(&local_app_data_path).join("dsc").join("AdaptedResourcesLookupTable.json").display().to_string()
+}
+
+#[cfg(not(target_os = "windows"))]
+fn get_lookup_table_file_path() -> String
+{
+    // $env:HOME+".dsc/AdaptedResourcesLookupTable.json"
+    let Ok(home_path) = std::env::var("HOME") else { return String::new(); };
+    Path::new(&home_path).join(".dsc").join("AdaptedResourcesLookupTable.json").display().to_string()
+}

powershell-adapter/Tests/powershellgroup.resource.tests.ps1

@@ -270,9 +270,10 @@ Describe 'PowerShell adapter resource tests' {
         }
     }
 
-    It 'Dsc can process large resource output' -Tag z1{
+    It 'Dsc can process large resource output' {
         $env:TestClassResourceResultCount = 5000 # with sync resource invocations this was not possible
 
+        dsc resource list -a Microsoft.DSC/PowerShell | Out-Null
         $r = dsc resource export -r TestClassResource/TestClassResource
         $LASTEXITCODE | Should -Be 0
         $res = $r | ConvertFrom-Json