CSV importing and exporting fixes (#8824)

Fixes issue #5793 (and duplicate #8822) - Fix importing multi-select and array fields. - Fix exporting and importing RAW_JSON fields. --------- Co-authored-by: ad-elias <[email protected]>
twentyhq · Dec 5, 2024 · f60ce38 · f60ce38
1 parent 815e5df
commit f60ce38
Show file tree

Hide file tree

Showing 10 changed files with 181 additions and 28 deletions.
diff --git a/.../src/modules/object-record/object-options-dropdown/hooks/useExportProcessRecordsForCSV.ts b/.../src/modules/object-record/object-options-dropdown/hooks/useExportProcessRecordsForCSV.ts
@@ -11,28 +11,36 @@ export const useExportProcessRecordsForCSV = (objectNameSingular: string) => {
   });
 
   const processRecordsForCSVExport = (records: ObjectRecord[]) => {
-    return records.map((record) => {
-      const currencyFields = objectMetadataItem.fields.filter(
-        (field) => field.type === FieldMetadataType.Currency,
-      );
+    return records.map((record) =>
+      objectMetadataItem.fields.reduce(
+        (processedRecord, field) => {
+          if (!isDefined(record[field.name])) {
+            return processedRecord;
+          }
 
-      const processedRecord = {
-        ...record,
-      };
-
-      for (const currencyField of currencyFields) {
-        if (isDefined(record[currencyField.name])) {
-          processedRecord[currencyField.name] = {
-            amountMicros: convertCurrencyMicrosToCurrencyAmount(
-              record[currencyField.name].amountMicros,
-            ),
-            currencyCode: record[currencyField.name].currencyCode,
-          } satisfies FieldCurrencyValue;
-        }
-      }
-
-      return processedRecord;
-    });
+          switch (field.type) {
+            case FieldMetadataType.Currency:
+              return {
+                ...processedRecord,
+                [field.name]: {
+                  amountMicros: convertCurrencyMicrosToCurrencyAmount(
+                    record[field.name].amountMicros,
+                  ),
+                  currencyCode: record[field.name].currencyCode,
+                } satisfies FieldCurrencyValue,
+              };
+            case FieldMetadataType.RawJson:
+              return {
+                ...processedRecord,
+                [field.name]: JSON.stringify(record[field.name]),
+              };
+            default:
+              return processedRecord;
+          }
+        },
+        { ...record },
+      ),
+    );
   };
 
   return { processRecordsForCSVExport };

diff --git a/...rd/spreadsheet-import/hooks/__tests__/useOpenObjectRecordsSpreadsheetImportDialog.test.ts b/...rd/spreadsheet-import/hooks/__tests__/useOpenObjectRecordsSpreadsheetImportDialog.test.ts
@@ -290,6 +290,8 @@ const companyMocks = [
             name: 'Example Company',
             id: companyId,
             visaSponsorship: false,
+            deletedAt: undefined,
+            workPolicy: [],
           },
         ],
         upsert: true,

diff --git a/...nt/src/modules/object-record/spreadsheet-import/hooks/useBuildAvailableFieldsForImport.ts b/...nt/src/modules/object-record/spreadsheet-import/hooks/useBuildAvailableFieldsForImport.ts
@@ -143,6 +143,25 @@ export const useBuildAvailableFieldsForImport = () => {
             fieldMetadataItem.label + ' (ID)',
           ),
         });
+      } else if (fieldMetadataItem.type === FieldMetadataType.MultiSelect) {
+        availableFieldsForImport.push({
+          icon: getIcon(fieldMetadataItem.icon),
+          label: fieldMetadataItem.label,
+          key: fieldMetadataItem.name,
+          fieldType: {
+            type: 'multiSelect',
+            options:
+              fieldMetadataItem.options?.map((option) => ({
+                label: option.label,
+                value: option.value,
+                color: option.color,
+              })) || [],
+          },
+          fieldValidationDefinitions: getSpreadSheetFieldValidationDefinitions(
+            fieldMetadataItem.type,
+            fieldMetadataItem.label + ' (ID)',
+          ),
+        });
       } else if (fieldMetadataItem.type === FieldMetadataType.Boolean) {
         availableFieldsForImport.push({
           icon: getIcon(fieldMetadataItem.icon),

diff --git a/...rc/modules/object-record/spreadsheet-import/utils/buildRecordFromImportedStructuredRow.ts b/...rc/modules/object-record/spreadsheet-import/utils/buildRecordFromImportedStructuredRow.ts
@@ -9,6 +9,7 @@ import { COMPOSITE_FIELD_IMPORT_LABELS } from '@/object-record/spreadsheet-impor
 import { ImportedStructuredRow } from '@/spreadsheet-import/types';
 import { isNonEmptyString } from '@sniptt/guards';
 import { isDefined } from 'twenty-ui';
+import { z } from 'zod';
 import { FieldMetadataType } from '~/generated-metadata/graphql';
 import { castToString } from '~/utils/castToString';
 import { convertCurrencyAmountToCurrencyMicros } from '~/utils/convertCurrencyToCurrencyMicros';
@@ -203,6 +204,35 @@ export const buildRecordFromImportedStructuredRow = (
           source: 'IMPORT',
         };
         break;
+      case FieldMetadataType.Array:
+      case FieldMetadataType.MultiSelect: {
+        const stringArrayJSONSchema = z
+          .preprocess((value) => {
+            try {
+              if (typeof value !== 'string') {
+                return [];
+              }
+              return JSON.parse(value);
+            } catch {
+              return [];
+            }
+          }, z.array(z.string()))
+          .catch([]);
+
+        recordToBuild[field.name] =
+          stringArrayJSONSchema.parse(importedFieldValue);
+        break;
+      }
+      case FieldMetadataType.RawJson: {
+        if (typeof importedFieldValue === 'string') {
+          try {
+            recordToBuild[field.name] = JSON.parse(importedFieldValue);
+          } catch {
+            break;
+          }
+        }
+        break;
+      }
       default:
         recordToBuild[field.name] = importedFieldValue;
         break;

diff --git a/...ont/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep.tsx b/...ont/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep.tsx
@@ -77,7 +77,7 @@ export enum ColumnType {
 
 export type MatchedOptions<T> = {
   entry: string;
-  value: T;
+  value?: T;
 };
 
 type EmptyColumn = { type: ColumnType.empty; index: number; header: string };

diff --git a/packages/twenty-front/src/modules/spreadsheet-import/types/index.ts b/packages/twenty-front/src/modules/spreadsheet-import/types/index.ts
@@ -79,6 +79,11 @@ export type Select = {
   options: SelectOption[];
 };
 
+export type MultiSelect = {
+  type: 'multiSelect';
+  options: SelectOption[];
+};
+
 export type SelectOption = {
   // Icon
   icon?: IconComponent | null;
@@ -96,7 +101,11 @@ export type Input = {
   type: 'input';
 };
 
-export type SpreadsheetImportFieldType = Checkbox | Select | Input;
+export type SpreadsheetImportFieldType =
+  | Checkbox
+  | Select
+  | MultiSelect
+  | Input;
 
 export type Field<T extends string> = {
   // Icon

diff --git a/packages/twenty-front/src/modules/spreadsheet-import/utils/generateExampleRow.ts b/packages/twenty-front/src/modules/spreadsheet-import/utils/generateExampleRow.ts
@@ -3,6 +3,7 @@ import { Field, Fields } from '@/spreadsheet-import/types';
 const titleMap: Record<Field<string>['fieldType']['type'], string> = {
   checkbox: 'Boolean',
   select: 'Options',
+  multiSelect: 'Options',
   input: 'Text',
 };
 

diff --git a/packages/twenty-front/src/modules/spreadsheet-import/utils/getFieldOptions.ts b/packages/twenty-front/src/modules/spreadsheet-import/utils/getFieldOptions.ts
@@ -8,5 +8,8 @@ export const getFieldOptions = <T extends string>(
   if (!field) {
     return [];
   }
-  return field.fieldType.type === 'select' ? field.fieldType.options : [];
+  return field.fieldType.type === 'select' ||
+    field.fieldType.type === 'multiSelect'
+    ? field.fieldType.options
+    : [];
 };
diff --git a/packages/twenty-front/src/modules/spreadsheet-import/utils/normalizeTableData.ts b/packages/twenty-front/src/modules/spreadsheet-import/utils/normalizeTableData.ts
@@ -8,6 +8,8 @@ import {
   ImportedStructuredRow,
 } from '@/spreadsheet-import/types';
 
+import { isDefined } from '@ui/utilities/isDefined';
+import { z } from 'zod';
 import { normalizeCheckboxValue } from './normalizeCheckboxValue';
 
 export const normalizeTableData = <T extends string>(
@@ -54,10 +56,45 @@ export const normalizeTableData = <T extends string>(
         }
         case ColumnType.matchedSelect:
         case ColumnType.matchedSelectOptions: {
-          const matchedOption = column.matchedOptions.find(
-            ({ entry }) => entry === curr,
-          );
-          acc[column.value] = matchedOption?.value || undefined;
+          const field = fields.find((field) => field.key === column.value);
+
+          if (!field) {
+            return acc;
+          }
+
+          if (field.fieldType.type === 'multiSelect' && isDefined(curr)) {
+            const currentOptionsSchema = z.preprocess(
+              (value) => JSON.parse(z.string().parse(value)),
+              z.array(z.unknown()),
+            );
+
+            const rawCurrentOptions = currentOptionsSchema.safeParse(curr).data;
+
+            const matchedOptionValues = [
+              ...new Set(
+                rawCurrentOptions
+                  ?.map(
+                    (option) =>
+                      column.matchedOptions.find(
+                        (matchedOption) => matchedOption.entry === option,
+                      )?.value,
+                  )
+                  .filter(isDefined),
+              ),
+            ];
+
+            const fieldValue =
+              matchedOptionValues && matchedOptionValues.length > 0
+                ? JSON.stringify(matchedOptionValues)
+                : undefined;
+
+            acc[column.value] = fieldValue;
+          } else {
+            const matchedOption = column.matchedOptions.find(
+              ({ entry }) => entry === curr,
+            );
+            acc[column.value] = matchedOption?.value || undefined;
+          }
           return acc;
         }
         case ColumnType.empty:

diff --git a/packages/twenty-front/src/modules/spreadsheet-import/utils/setColumn.ts b/packages/twenty-front/src/modules/spreadsheet-import/utils/setColumn.ts
@@ -6,6 +6,7 @@ import {
 } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep';
 import { Field } from '@/spreadsheet-import/types';
 
+import { z } from 'zod';
 import { uniqueEntries } from './uniqueEntries';
 
 export const setColumn = <T extends string>(
@@ -19,6 +20,7 @@ export const setColumn = <T extends string>(
       data || [],
       oldColumn.index,
     ) as MatchedOptions<T>[];
+
     const matchedOptions = uniqueData.map((record) => {
       const value = fieldOptions.find(
         (fieldOption) =>
@@ -42,6 +44,48 @@ export const setColumn = <T extends string>(
     };
   }
 
+  if (field?.fieldType.type === 'multiSelect') {
+    const fieldOptions = field.fieldType.options;
+
+    const entries = [
+      ...new Set(
+        data
+          ?.flatMap((row) => {
+            try {
+              const value = row[oldColumn.index];
+              const options = JSON.parse(z.string().parse(value));
+              return z.array(z.string()).parse(options);
+            } catch {
+              return [];
+            }
+          })
+          .filter((entry) => typeof entry === 'string'),
+      ),
+    ];
+
+    const matchedOptions = entries.map((entry) => {
+      const value = fieldOptions.find(
+        (fieldOption) =>
+          fieldOption.value === entry || fieldOption.label === entry,
+      )?.value;
+      return value
+        ? ({ entry, value } as MatchedOptions<T>)
+        : ({ entry } as MatchedOptions<T>);
+    });
+    const areAllMatched =
+      matchedOptions.filter((option) => option.value).length ===
+      entries?.length;
+
+    return {
+      ...oldColumn,
+      type: areAllMatched
+        ? ColumnType.matchedSelectOptions
+        : ColumnType.matchedSelect,
+      value: field.key,
+      matchedOptions,
+    };
+  }
+
   if (field?.fieldType.type === 'checkbox') {
     return {
       index: oldColumn.index,