Fix import recordID mapping on UI

client/web/compose/src/components/Public/Record/Importer/ErrorReport.vue

@@ -80,11 +80,11 @@
 
     <div slot="footer">
       <b-button
-        variant="dark"
+        variant="light"
         class="float-right"
         @click="$emit('close')"
       >
-        {{ $t('general.label.ok') }}
+        {{ $t('general:label.close') }}
       </b-button>
     </div>
   </b-card>
@@ -133,12 +133,12 @@ export default {
         {
           key: 'k',
           label: this.$t('recordList.import.report.error'),
-          tdClass: 'border-top text-truncate pointer',
+          tdClass: 'border-top',
         },
         {
           key: 'v',
           label: this.$t('recordList.import.report.count'),
-          tdClass: 'border-top text-truncate pointer',
+          tdClass: 'border-top',
         },
       ]
     },

client/web/compose/src/components/Public/Record/Importer/FieldMatch.vue

@@ -1,23 +1,24 @@
 <template>
-  <b-card>
-    <b-form-group>
-      <div class="mb-4">
-        <label>
-          {{ $t('recordList.import.matchFields') }}
-        </label>
-        <br>
-        <small
-          v-if="hasRequiredFileFields"
-        >
-          {{ $t('recordList.import.hasRequiredFileFields') }}: {{ showRequiredFields }}
-        </small>
+  <div class="d-flex flex-column h-100 overflow-hidden">
+    <b-form-group
+      :label="$t('recordList.import.matchFields')"
+      label-class="text-primary p-3"
+      class="flex-fill overflow-auto mb-0"
+    >
+      <div
+        v-if="hasRequiredFileFields"
+        class="small px-3 mb-3"
+      >
+        {{ $t('recordList.import.hasRequiredFileFields') }}: {{ showRequiredFields }}
       </div>
 
       <b-table
         :fields="tableFields"
         :items="rows"
         head-variant="light"
-        class="field-table"
+        sticky-header
+        style="max-height: 70vh;"
+        class="field-table mb-0"
       >
         <template #head(selected)>
           <b-form-checkbox
@@ -34,27 +35,27 @@
           />
         </template>
         <template #cell(moduleField)="data">
-          <b-form-select
+          <c-input-select
             v-model="data.item.moduleField"
             :options="moduleFields"
-            @change="moduleChanged(data)"
+            :reduce="o => o.name"
+            :placeholder="$t('recordList.import.pickModuleField')"
+            :class="{ 'border border-danger': data.item.selected && !data.item.moduleField }"
+            @input="moduleChanged(data)"
+          />
+          <span
+            v-if="data.item.fileColumn === 'id'"
+            class="small text-muted"
           >
-            <template slot="first">
-              <option
-                :value="undefined"
-                disabled
-              >
-                {{ $t('recordList.import.pickModuleField') }}
-              </option>
-            </template>
-          </b-form-select>
+            {{ $t('recordList.import.idFieldDescription') }}
+          </span>
         </template>
       </b-table>
     </b-form-group>
 
-    <div slot="footer">
+    <div class="mt-auto p-3">
       <b-button
-        variant="outline-dark"
+        variant="light"
         class="float-left"
         @click="$emit('back')"
       >
@@ -70,7 +71,7 @@
         {{ $t('general.label.import') }}
       </b-button>
     </div>
-  </b-card>
+  </div>
 </template>
 
 <script>
@@ -139,16 +140,22 @@ export default {
         {
           key: 'moduleField',
           label: this.$t('recordList.import.moduleFields'),
+          thStyle: 'width: 25rem',
         },
       ]
     },
 
     moduleFields () {
-      return this.module.fields
-        .filter(({ kind }) => !['File'].includes(kind))
+      return [
+        ...this.module.fields,
+        ...this.module.systemFields().map(({ name }) => {
+          return {
+            label: this.$t(`field:system.${name}`),
+            name,
+          }
+        }),
+      ].filter(({ kind }) => !['File'].includes(kind))
         .map(field => field.isRequired === true ? { ...field, label: field.label + '*' } : field)
-        .map(({ name: value, label }) => ({ value, text: label || value }))
-        .sort((a, b) => a.text.localeCompare(b.text))
     },
 
     requiredFields () {
@@ -187,24 +194,39 @@ export default {
   created () {
     // Prep row object for us to alter
     const { fields = {} } = this.session
-    const moduleFields = {}
+
+    const moduleFields = {
+      'id': 'recordID',
+    }
+
     this.module.fields.forEach(({ name }) => {
       moduleFields[name] = name
     })
 
+    this.module.systemFields().forEach(({ name }) => {
+      moduleFields[name] = name
+    })
+
     this.rows = Object.entries(fields)
-      .map(([fileColumn, moduleField]) => ({
-        selected: false,
-        fileColumn,
-        moduleField: moduleField || moduleFields[fileColumn],
-      }))
+      .map(([fileColumn, moduleField]) => {
+        moduleField = moduleField || moduleFields[fileColumn]
+
+        return {
+          selected: false,
+          fileColumn,
+          moduleField,
+        }
+      })
   },
 
   methods: {
     moduleChanged (data) {
-      const result = this.rows.find(row => row.fileColumn === data.item.fileColumn)
-      result.selected = true
+      if (data.item.moduleField) {
+        const result = this.rows.find(row => row.moduleField === data.item.moduleField)
+        result.selected = true
+      }
     },
+
     nextStep () {
       if (!this.canContinue) {
         return

client/web/compose/src/components/Public/Record/Importer/FileUpload.vue

@@ -37,6 +37,7 @@
       class="text-right"
     >
       <b-button
+        variant="primary"
         :disabled="!canContinue"
         @click="fileUploaded"
       >

client/web/compose/src/components/Public/Record/Importer/Progress.vue

@@ -1,44 +1,73 @@
 <template>
   <b-card>
-    <b-form-group class="my-4 mx-4">
-      <b-progress
-        :max="progress.entryCount"
-        show-value
-        show-progress
-        variant="primary"
-        height="80px"
-        class="bg-light"
-      >
-        <b-progress-bar
-          :value="progress.completed"
-          class="progress-label"
-          variant="primary"
-        >
-          <span class="font-weight-bold">{{ $t('recordList.import.progressRatio', progress) }}</span>
-        </b-progress-bar>
-      </b-progress>
-    </b-form-group>
-
-    <b-form-group class="mx-4 mb-0">
-      <span
-        v-if="progress.finishedAt && !progress.failed"
-        class="text-success"
+    <c-progress
+      :value="progress.completed"
+      :max="progress.entryCount"
+      labeled
+      progress
+      :animated="!progress.finishedAt"
+      :relative="false"
+      variant="success"
+      text-style="font-size: 1.5rem;"
+      style="height: 4rem;"
+      class="mb-4"
+    />
+
+    <div
+      v-if="!progress.finishedAt"
+      class="d-flex"
+    >
+      <span class="text-secondary">
+        <b-spinner
+          variant="secondary"
+          small
+        />
+        {{ $t('recordList.import.importing') }}
+      </span>
+
+      <b-button
+        variant="light"
+        class="ml-auto"
+        @click="$emit('close')"
       >
+        {{ $t('general:label.cancel') }}
+      </b-button>
+    </div>
 
+    <div
+      v-if="progress.finishedAt && !progress.failed"
+      class="d-flex"
+    >
+      <span class="text-success">
         {{ $t('recordList.import.success') }}
       </span>
-    </b-form-group>
+
+      <b-button
+        variant="light"
+        class="ml-auto"
+        @click="$emit('close')"
+      >
+        {{ $t('general:label.close') }}
+      </b-button>
+    </div>
   </b-card>
 </template>
 
 <script>
+import { components } from '@cortezaproject/corteza-vue'
+const { CProgress } = components
+
 let toHandle = null
 
 export default {
   i18nOptions: {
     namespaces: 'block',
   },
 
+  components: {
+    CProgress,
+  },
+
   props: {
     session: {
       type: Object,

lib/vue/src/components/progress/CProgress.vue

@@ -12,6 +12,7 @@
       <strong
         :class="textVariant"
         class="d-flex align-items-center justify-content-center position-absolute mb-0 w-100"
+        :style="textStyle"
       >
         {{ progressLabel }}
       </strong>
@@ -67,6 +68,11 @@ export default {
       type: Array,
       default: () => [],
     },
+
+    textStyle: {
+      type: String,
+      default: '',
+    },
   },
 
   computed: {

locale/en/corteza-webapp-compose/block.yaml

@@ -401,13 +401,17 @@ recordList:
     failed: 'Something went wrong during the import. Please try again: {{failReason}}'
     fileColumns: File columns
     hasRequiredFileFields: 'This module has required file upload fields'
-    matchFields: 'Match imported columns with existing ones:'
+    matchFields: 'Match imported columns with existing ones'
     moduleFields: Module fields
     onError: If any record fails to import
     onErrorFail: Cancel import
     onErrorSkip: Skip record
     pickModuleField: Pick a module field
-    progressRatio: '{{completed}} / {{entryCount}} rows'
+    importing: Importing records
+    success: Import successful
+    to: Import to {{modulename}}
+    uploadFile: Upload the file you want to import (.csv or JSON format)
+    idFieldDescription: Records with matching ID's will be updated.
     report:
       count: Number of records
       detectedErrors: Detected errors
@@ -421,9 +425,6 @@ recordList:
       startedAt: Started At
       title: Record import error report
       totalRecords: Total records
-    success: Import successful
-    to: Import to {{modulename}}
-    uploadFile: Upload the file you want to import (.csv or JSON format)
   label: Record list
   moduleFootnote: Modules without a {{0}} can only be used in a record list as an inline editor
   moduleFieldsFootnote : If no fields are selected, the default fields will be shown

server/compose/envoy/record_datasource.go

@@ -17,6 +17,8 @@ type (
 		Mapping  envoyx.DatasourceMapping
 		Provider envoyx.Provider
 
+		CheckExisting func(ctx context.Context, ref ...[]string) ([]uint64, error)
+
 		currentIndex int
 
 		// Reusable buffer for reading records
@@ -25,6 +27,8 @@ type (
 		// Index to map from ref to ID
 		// @todo we might need to flush these to the disc in case a huge dataset is passed in
 		refToID map[string]uint64
+		// @todo might be worth putting both into one map; not sure how much space we'd save up
+		existingIDs map[uint64]bool
 	}
 
 	// iteratorProvider is a wrapper around the dal.Iterator to conform to the

server/compose/envoy/store_decode.go

@@ -240,8 +240,9 @@ func (d StoreDecoder) decodeRecordDatasource(ctx context.Context, s store.Storer
 	}
 
 	ou := &RecordDatasource{
-		Provider: &iteratorProvider{iter: iter},
-		refToID:  make(map[string]uint64),
+		Provider:    &iteratorProvider{iter: iter},
+		refToID:     make(map[string]uint64),
+		existingIDs: make(map[uint64]bool),
 		// @todo consider providing defaults from the outside
 		Mapping: envoyx.DatasourceMapping{
 			KeyField:    []string{"id"},

server/compose/envoy/store_encode.go

@@ -153,9 +153,14 @@ func (e StoreEncoder) validateRecord(*types.Record) (err error) {
 }
 
 func (e StoreEncoder) prepare(ctx context.Context, p envoyx.EncodeParams, s store.Storer, rt string, nn envoyx.NodeSet) (err error) {
+	dl, err := e.grabDal(p)
+	if err != nil {
+		return
+	}
+
 	switch rt {
 	case ComposeRecordDatasourceAuxType:
-		return e.prepareRecordDatasource(ctx, p, s, nn)
+		return e.prepareRecordDatasource(ctx, p, s, dl, nn)
 	}
 
 	return