refactor: split config merging and improve types

Author: BlackDark

index.ts

@@ -3,7 +3,7 @@ import "dotenv/config";
 import fs from "node:fs";
 import { MergedCustomFormatResource } from "./src/__generated__/mergedTypes";
 import { configureRadarrApi, configureSonarrApi, getArrApi, unsetApi } from "./src/api";
-import { getConfig } from "./src/config";
+import { getConfig, validateConfig } from "./src/config";
 import {
   calculateCFsToManage,
   loadCFFromConfig,
@@ -29,13 +29,21 @@ import {
   transformTrashQPCFs,
   transformTrashQPToTemplate,
 } from "./src/trash-guide";
-import { ArrType, MappedMergedTemplates } from "./src/types/common.types";
-import { ConfigArrInstance, ConfigQualityProfile, YamlConfigIncludeItem } from "./src/types/config.types";
+import { ArrType, CFProcessing, MappedMergedTemplates } from "./src/types/common.types";
+import { ConfigQualityProfile, InputConfigArrInstance, InputConfigIncludeItem, MergedConfigInstance } from "./src/types/config.types";
 import { TrashQualityDefintion } from "./src/types/trashguide.types";
 import { DEBUG_CREATE_FILES, IS_DRY_RUN } from "./src/util";
 
-const pipeline = async (value: ConfigArrInstance, arrType: ArrType) => {
-  const api = getArrApi();
+/**
+ * Load data from trash, recyclarr, custom configs and merge.
+ * Afterwards do sanitize and check against required configuration.
+ * @param value
+ * @param arrType
+ */
+const mergeConfigsAndTemplates = async (
+  value: InputConfigArrInstance,
+  arrType: ArrType,
+): Promise<{ mergedCFs: CFProcessing; config: MergedConfigInstance }> => {
   const recyclarrTemplateMap = loadRecyclarrTemplates(arrType);
   const trashTemplates = await loadQPFromTrash(arrType);
 
@@ -45,9 +53,7 @@ const pipeline = async (value: ConfigArrInstance, arrType: ArrType) => {
   };
 
   if (value.include) {
-    logger.info(`Found ${value.include.length} templates to include ...`);
-
-    const mappedIncludes = value.include.reduce<{ recyclarr: YamlConfigIncludeItem[]; trash: YamlConfigIncludeItem[] }>(
+    const mappedIncludes = value.include.reduce<{ recyclarr: InputConfigIncludeItem[]; trash: InputConfigIncludeItem[] }>(
       (previous, current) => {
         switch (current.source) {
           case "TRASH":
@@ -65,14 +71,15 @@ const pipeline = async (value: ConfigArrInstance, arrType: ArrType) => {
       { recyclarr: [], trash: [] },
     );
 
-    logger.debug(mappedIncludes.recyclarr, `Included ${mappedIncludes.recyclarr.length} templates [recyclarr]`);
-    logger.debug(mappedIncludes.trash, `Included ${mappedIncludes.trash.length} templates [trash]`);
+    logger.info(
+      `Found ${value.include.length} templates to include: [recyclarr]=${mappedIncludes.recyclarr.length}, [trash]=${mappedIncludes.trash.length} ...`,
+    );
 
     mappedIncludes.recyclarr.forEach((e) => {
       const template = recyclarrTemplateMap.get(e.template);
 
       if (!template) {
-        logger.info(`Unknown recyclarr template requested: ${e.template}`);
+        logger.warn(`Unknown recyclarr template requested: ${e.template}`);
         return;
       }
 
@@ -97,7 +104,7 @@ const pipeline = async (value: ConfigArrInstance, arrType: ArrType) => {
       const template = trashTemplates.get(e.template);
 
       if (!template) {
-        logger.info(`Unknown trash template requested: ${e.template}`);
+        logger.warn(`Unknown trash template requested: ${e.template}`);
         return;
       }
 
@@ -147,10 +154,21 @@ const pipeline = async (value: ConfigArrInstance, arrType: ArrType) => {
   const configCFDefinition = loadCFFromConfig();
   const mergedCFs = mergeCfSources([trashCFs, localFileCFs, configCFDefinition]);
 
-  const idsToManage = calculateCFsToManage(recylarrMergedTemplates);
+  const validatedConfig = validateConfig(recylarrMergedTemplates);
+  return { mergedCFs: mergedCFs, config: validatedConfig };
+};
+
+const pipeline = async (value: InputConfigArrInstance, arrType: ArrType) => {
+  const api = getArrApi();
+
+  const { config, mergedCFs } = await mergeConfigsAndTemplates(value, arrType);
+
+  const idsToManage = calculateCFsToManage(config);
 
   logger.debug(Array.from(idsToManage), `CustomFormats to manage`);
 
+  // TODO here the configs should be merged -> sanitize + check
+
   const serverCFs = await loadServerCustomFormats();
   logger.info(`CustomFormats on server: ${serverCFs.length}`);
 
@@ -162,7 +180,7 @@ const pipeline = async (value: ConfigArrInstance, arrType: ArrType) => {
   await manageCf(mergedCFs, serverCFMapping, idsToManage);
   logger.info(`CustomFormats synchronized`);
 
-  const qualityDefinition = recylarrMergedTemplates.quality_definition?.type;
+  const qualityDefinition = config.quality_definition?.type;
 
   if (qualityDefinition) {
     const qdSonarr = await loadQualityDefinitionFromServer();
@@ -203,10 +221,11 @@ const pipeline = async (value: ConfigArrInstance, arrType: ArrType) => {
 
   // merge CFs of templates and custom CFs into one mapping of QualityProfile -> CFs + Score
   // TODO traversing the merged templates probably to often once should be enough. Loop once and extract a couple of different maps, arrays as needed. Future optimization.
-  const cfToQualityProfiles = mapQualityProfiles(mergedCFs, recylarrMergedTemplates.custom_formats, recylarrMergedTemplates);
+  // TODO double config
+  const cfToQualityProfiles = mapQualityProfiles(mergedCFs, config.custom_formats, config);
 
   // merge profiles from recyclarr templates into one
-  const qualityProfilesMerged = recylarrMergedTemplates.quality_profiles.reduce((p, c) => {
+  const qualityProfilesMerged = config.quality_profiles.reduce((p, c) => {
     let existingQp = p.get(c.name);
 
     if (!existingQp) {
@@ -289,42 +308,34 @@ const run = async () => {
 
   // TODO currently this has to be run sequentially because of the centrally configured api
 
-  logHeading(`Processing Sonarr ...`);
+  const sonarrConfig = applicationConfig.sonarr;
 
-  for (const instanceName in applicationConfig.sonarr) {
-    const instance = applicationConfig.sonarr[instanceName];
-    logger.info(`Processing Sonarr Instance: ${instanceName}`);
-    await configureSonarrApi(instance.base_url, instance.api_key);
-    await pipeline(instance, "SONARR");
-    unsetApi();
-  }
+  if (sonarrConfig == null || Array.isArray(sonarrConfig) || typeof sonarrConfig !== "object" || Object.keys(sonarrConfig).length <= 0) {
+    logHeading(`No Sonarr instances defined.`);
+  } else {
+    logHeading(`Processing Sonarr ...`);
 
-  if (
-    typeof applicationConfig.sonarr === "object" &&
-    !Array.isArray(applicationConfig.sonarr) &&
-    applicationConfig.sonarr !== null &&
-    Object.keys(applicationConfig.sonarr).length <= 0
-  ) {
-    logger.info(`No sonarr instances defined.`);
+    for (const [instanceName, instance] of Object.entries(sonarrConfig)) {
+      logger.info(`Processing Sonarr Instance: ${instanceName}`);
+      await configureSonarrApi(instance.base_url, instance.api_key);
+      await pipeline(instance, "SONARR");
+      unsetApi();
+    }
   }
 
-  logHeading(`Processing Radarr ...`);
+  const radarrConfig = applicationConfig.radarr;
 
-  for (const instanceName in applicationConfig.radarr) {
-    logger.info(`Processing Radarr instance: ${instanceName}`);
-    const instance = applicationConfig.radarr[instanceName];
-    await configureRadarrApi(instance.base_url, instance.api_key);
-    await pipeline(instance, "RADARR");
-    unsetApi();
-  }
+  if (radarrConfig == null || Array.isArray(radarrConfig) || typeof radarrConfig !== "object" || Object.keys(radarrConfig).length <= 0) {
+    logHeading(`No Radarr instances defined.`);
+  } else {
+    logHeading(`Processing Radarr ...`);
 
-  if (
-    typeof applicationConfig.radarr === "object" &&
-    !Array.isArray(applicationConfig.radarr) &&
-    applicationConfig.radarr !== null &&
-    Object.keys(applicationConfig.radarr).length <= 0
-  ) {
-    logger.info(`No radarr instances defined.`);
+    for (const [instanceName, instance] of Object.entries(radarrConfig)) {
+      logger.info(`Processing Radarr Instance: ${instanceName}`);
+      await configureRadarrApi(instance.base_url, instance.api_key);
+      await pipeline(instance, "RADARR");
+      unsetApi();
+    }
   }
 };
 

src/__generated__/mergedTypes.ts

@@ -43,10 +43,10 @@ radarr: {}
 
   test("should transform without error - quality_profiles instead of assign_scores_to", async () => {
     const cloned = cloneWithJSON(config);
-    const instance = cloned.sonarr["instance1"];
-    const { assign_scores_to, quality_profiles, ...rest } = instance.custom_formats[0];
+    const instance = cloned.sonarr!["instance1"];
+    const { assign_scores_to, quality_profiles, ...rest } = instance.custom_formats![0];
 
-    instance.custom_formats[0] = { ...rest, quality_profiles: assign_scores_to ?? quality_profiles };
+    instance.custom_formats![0] = { ...rest, quality_profiles: assign_scores_to ?? quality_profiles };
 
     const transformed = transformConfig(config);
     expect(transformed).not.toBeNull();

src/config.test.ts

@@ -4,11 +4,13 @@ import { logger } from "./logger";
 import {
   ConfigArrInstance,
   ConfigCustomFormat,
+  ConfigIncludeItem,
   ConfigSchema,
   InputConfigArrInstance,
   InputConfigIncludeItem,
+  InputConfigInstance,
   InputConfigSchema,
-  YamlConfigIncludeItem,
+  MergedConfigInstance,
 } from "./types/config.types";
 import { ROOT_PATH } from "./util";
 
@@ -92,11 +94,7 @@ export const getSecrets = () => {
 
 // 2024-09-30: Recyclarr assign_scores_to adjustments
 export const transformConfig = (input: InputConfigSchema): ConfigSchema => {
-  const parseIncludes = (items: InputConfigIncludeItem[] = []): YamlConfigIncludeItem[] => {
-    return items.map((e) => ({ template: e.template, source: e.source ?? "RECYCLARR" }));
-  };
-
-  const mappedCustomFormats = (arrInput: Record<string, InputConfigArrInstance>): Record<string, ConfigArrInstance> => {
+  const mappedCustomFormats = (arrInput: Record<string, InputConfigArrInstance> = {}): Record<string, ConfigArrInstance> => {
     return Object.entries(arrInput).reduce(
       (p, [key, value]) => {
         const mappedCustomFormats = (value.custom_formats || []).map<ConfigCustomFormat>((cf) => {
@@ -119,7 +117,7 @@ export const transformConfig = (input: InputConfigSchema): ConfigSchema => {
           return { ...rest, assign_scores_to: mapped_assign_scores };
         });
 
-        p[key] = { ...value, include: parseIncludes(value.include), custom_formats: mappedCustomFormats };
+        p[key] = { ...value, include: value.include?.map(parseIncludes), custom_formats: mappedCustomFormats };
         return p;
       },
       {} as Record<string, ConfigArrInstance>,
@@ -132,3 +130,19 @@ export const transformConfig = (input: InputConfigSchema): ConfigSchema => {
     sonarr: mappedCustomFormats(input.sonarr),
   };
 };
+
+export const parseIncludes = (input: InputConfigIncludeItem): ConfigIncludeItem => ({
+  template: input.template,
+  source: input.source ?? "RECYCLARR",
+});
+
+export const validateConfig = (input: InputConfigInstance): MergedConfigInstance => {
+  // TODO add validation and warnings like assign_scores. Setting default values not always the best
+  return {
+    ...input,
+    custom_formats: (input.custom_formats || []).map((e) => ({
+      trash_ids: e.trash_ids,
+      assign_scores_to: e.assign_scores_to ?? e.quality_profiles ?? [],
+    })),
+  };
+};

src/config.ts

@@ -38,7 +38,7 @@ export const manageCf = async (
 
   let updatedCFs = 0;
   let errorCFs = 0;
-  let validCFs = 0;
+  const validCFss: ConfigarrCF[] = [];
   let createCFs = 0;
 
   const manageSingle = async (carrId: string) => {
@@ -77,8 +77,7 @@ export const manageCf = async (
           throw new Error(`Failed updating CF ${tr.requestConfig.name}`);
         }
       } else {
-        logger.debug(`CF ${tr.requestConfig.name} does not need update.`);
-        validCFs++;
+        validCFss.push(tr.carrConfig);
       }
     } else {
       // Create
@@ -101,7 +100,11 @@ export const manageCf = async (
     await manageSingle(cf);
   }
 
-  logger.info(`Created CFs: ${createCFs}, Updated CFs: ${updatedCFs}, Untouched CFs: ${validCFs}, Error CFs: ${errorCFs}`);
+  logger.debug(
+    validCFss.map((e) => `${e.name}`),
+    `CFs with no update:`,
+  );
+  logger.info(`Created CFs: ${createCFs}, Updated CFs: ${updatedCFs}, Untouched CFs: ${validCFss.length}, Error CFs: ${errorCFs}`);
 };
 export const loadLocalCfs = async (): Promise<CFProcessing | null> => {
   const config = getConfig();
@@ -186,6 +189,7 @@ export const calculateCFsToManage = (yaml: ConfigCustomFormatList) => {
 
   return cfTrashToManage;
 };
+
 export const mergeCfSources = (listOfCfs: (CFProcessing | null)[]): CFProcessing => {
   return listOfCfs.reduce<CFProcessing>(
     (p, c) => {

src/custom-formats.ts

@@ -127,7 +127,7 @@ describe("QualityDefinitions", async () => {
 
   test("calculateQualityDefinitionDiff - diff min size", async ({}) => {
     const clone: TrashQualityDefintion = JSON.parse(JSON.stringify(client));
-    clone.qualities[0].min = 3;
+    clone.qualities[0]!.min = 3;
 
     const result = calculateQualityDefinitionDiff(server, clone);
 
@@ -137,7 +137,7 @@ describe("QualityDefinitions", async () => {
 
   test("calculateQualityDefinitionDiff - diff max size", async ({}) => {
     const clone: TrashQualityDefintion = JSON.parse(JSON.stringify(client));
-    clone.qualities[0].max = 3;
+    clone.qualities[0]!.max = 3;
 
     const result = calculateQualityDefinitionDiff(server, clone);
 
@@ -147,7 +147,7 @@ describe("QualityDefinitions", async () => {
 
   test("calculateQualityDefinitionDiff - diff preferred size", async ({}) => {
     const clone: TrashQualityDefintion = JSON.parse(JSON.stringify(client));
-    clone.qualities[0].preferred = 3;
+    clone.qualities[0]!.preferred = 3;
 
     const result = calculateQualityDefinitionDiff(server, clone);
 
@@ -157,7 +157,7 @@ describe("QualityDefinitions", async () => {
 
   test("calculateQualityDefinitionDiff - create new element", async ({}) => {
     const clone: TrashQualityDefintion = JSON.parse(JSON.stringify(client));
-    clone.qualities[0].quality = "New";
+    clone.qualities[0]!.quality = "New";
 
     const result = calculateQualityDefinitionDiff(server, clone);