fixing conflicts

.babelrc

@@ -3,5 +3,11 @@
     "@babel/preset-react",
     "@babel/preset-env"
   ],
-  "plugins": ["@babel/plugin-proposal-object-rest-spread"]
+  "plugins": ["@babel/plugin-proposal-object-rest-spread",
+     ["@babel/plugin-transform-runtime",
+      {
+        "regenerator": true
+      }
+    ]
+  ]
 }

.eslintignore

@@ -5,9 +5,6 @@ modules/*/js/*
 modules/*/js/*/*
 htdocs/js/components/*
 
-# Ignore until ESLint is run
-modules/dataquery/
-
 # Ignore external libs
 htdocs/js/flot/*
 htdocs/js/jquery/*

.eslintrc.json

@@ -40,26 +40,14 @@
   "overrides": [
     {
       "files": [
-        "modules/behavioural_qc/**",
-        "modules/bvl_feedback/**",
-        "modules/candidate_list/**",
-        "modules/candidate_parameters/**",
-        "modules/conflict_resolver/**",
-        "modules/data_release/**",
-        "modules/document_repository/**",
-        "modules/electrophysiology_browser/**",
-        "modules/examiner/**",
-        "modules/genomic_browser/**",
-        "modules/imaging_browser/**",
-        "modules/imaging_uploader/**",
-        "modules/instrument_builder/**",
-        "modules/instrument_manager/**",
-        "modules/issue_tracker/**",
-        "modules/media/**",
-        "modules/publication/**"
+        "modules/dataquery/**"
       ],
       "rules": {
-        "require-jsdoc": "warn"
+        "require-jsdoc": "warn",
+	"max-len": "warn",
+	"no-unused-vars": "warn",
+	"camelcase": "warn",
+	"guard-for-in": "warn"
       }
     }
   ],

CHANGELOG.md

@@ -11,7 +11,9 @@ changes in the following format: PR #1234***
 ##LORIS 24.0 (Release Date: ??)
 ### Core
 #### Features
-- *Add item here*
+- Data tables may now stream data as they're loading rather than waiting
+  until all data has loaded. (PR #6853)
+
 #### Updates and Improvements
 - Module-specific permissions added for Survey Accounts, Imaging Behavioural
 Quality Control, and Behavioural Quality Control. (PR #6041)

SQL/9999-99-99-drop_tables.sql

@@ -149,6 +149,7 @@ DROP TABLE IF EXISTS `mri_scan_type`;
 DROP TABLE IF EXISTS `mri_scanner`;
 DROP TABLE IF EXISTS `mri_processing_protocol`;
 DROP TABLE IF EXISTS `ImagingFileTypes`;
+DROP TABLE IF EXISTS `mri_upload_rel`;
 
 DROP TABLE IF EXISTS `tarchive_files`;
 DROP TABLE IF EXISTS `tarchive_series`;
@@ -173,5 +174,8 @@ DROP TABLE IF EXISTS `psc`;
 DROP TABLE IF EXISTS `visit_project_subproject_rel`;
 DROP TABLE IF EXISTS `visit`;
 DROP TABLE IF EXISTS `project_subproject_rel`;
+DROP TABLE IF EXISTS `consent_group`;
+DROP TABLE IF EXISTS `hrrt_archive_files`;
+DROP TABLE IF EXISTS `hrrt_archive`;
 DROP TABLE IF EXISTS `Project`;
 DROP TABLE IF EXISTS `subproject`;

htdocs/index.php

@@ -61,4 +61,17 @@
 }
 
 // Include the body.
-print $response->getBody();
+$bodystream = $response->getBody();
+
+// First we need to disable any output buffering so that
+// it streams to the output instead of into the buffer
+// and uses up all the memory for large chunks of data.
+for ($i = ob_get_level(); $i != 0; $i = ob_get_level()) {
+    ob_end_clean();
+}
+ob_implicit_flush();
+
+while ($bodystream->eof() == false) {
+    // 64k oughta be enough for anybody.
+    print $bodystream->read(1024*64);
+}

jslib/fetchDataStream.js

@@ -0,0 +1,96 @@
+/**
+ * Process as many rows as possible from a data stream.
+ *
+ * @param {binary} data - a chunk of data read from the data stream
+ * @param {function} rowcb - The row callback function
+ * @param {function} endstreamcb - The stream termination callback
+ * function
+ *
+ * @return {object} An object containing keys "remainder" which is
+ * a slice of any unprocessed data, and a key "eos" which is a boolean
+ * indicating whether the end of the stream has been reached.
+ */
+async function processLines(data, rowcb, endstreamcb) {
+    const utf8Decoder = new TextDecoder('utf-8');
+    let row = [];
+    let colStart = -1;
+    let rowStart = 0;
+    for (let i = 0; i < data.length; i++) {
+        switch (data[i]) {
+            case 0x1e: // end of column
+                const rowdata = data.slice(colStart+1, i);
+                const encoded = utf8Decoder.decode(rowdata);
+                colStart = i;
+                row.push(encoded);
+                continue;
+            case 0x1f: // end of row
+                const rowdata2 = data.slice(colStart+1, i);
+                const encoded2 = utf8Decoder.decode(rowdata2);
+                row.push(encoded2);
+
+                rowcb(row);
+
+                rowStart = i+1;
+                colStart = i;
+                row = [];
+                continue;
+            case 0x04: // end of stream
+                rowcb(row);
+                endstreamcb(row);
+                return {remainder: [], eos: true};
+        }
+    }
+    return {remainder: data.slice(rowStart), eos: false};
+};
+
+/**
+ * fetchDataStream fetches a data stream from dataURL where
+ * rows are denoted by the byte 0x1f, columns by 0x1e, and
+ * the stream is terminated by 0x04.
+ *
+ * @param {string} dataURL   - the URL of the stream to fetch
+ * @param {function} rowcb   - A callback to call for each row
+ * @param {function} chunkcb - A callback to call for each chunk
+ *                             read from the stream.
+ * @param {function} endstreamcb - A callback to call when the final
+ *                             byte is read from the stream.
+ */
+async function fetchDataStream(dataURL, rowcb, chunkcb, endstreamcb) {
+    const response = await fetch(
+        dataURL,
+        {credentials: 'same-origin'},
+    );
+
+    const reader = response.body.getReader();
+
+    let remainder = [];
+    let doneLoop = false;
+    while (!doneLoop) {
+        await reader.read().then(({done, value}) => {
+            let combined;
+            if (remainder.length == 0) {
+                combined = value;
+            } else {
+                combined = new Uint8Array(
+                    value.length + remainder.length
+                );
+                for (let i = 0; i < remainder.length; i++) {
+                    combined[i] = remainder[i];
+                }
+                for (let i = 0; i < value.length; i++) {
+                    combined[i+remainder.length] = value[i];
+                }
+            }
+            return processLines(combined, rowcb, endstreamcb);
+        }).then(({remainder: rem, eos}) => {
+            chunkcb(eos);
+            doneLoop = eos;
+            remainder = rem;
+        }).catch((err) => {
+            console.error(err);
+            doneLoop = true;
+        });
+    };
+};
+
+export default fetchDataStream;

jsx/DataTable.js

@@ -540,6 +540,8 @@ class DataTable extends Component {
       </select>
     );
 
+    const loading = this.props.loading ? 'Loading...' : '';
+
     let header = this.props.hide.rowsPerPage === true ? '' : (
       <div className="table-header">
         <div className="row">
@@ -556,6 +558,7 @@ class DataTable extends Component {
             }}>
               {rows.length} rows displayed of {filteredCount}.
               (Maximum rows per page: {rowsPerPageDropdown})
+              {loading}
             </div>
             <div style={{
               order: '2',

jsx/FilterableDataTable.js

@@ -118,6 +118,7 @@ class FilterableDataTable extends Component {
         fields={this.props.fields}
         filter={filters}
         actions={this.props.actions}
+        loading={this.props.loading}
         getFormattedCell={this.props.getFormattedCell}
         getMappedCell={this.props.getMappedCell}
         folder={this.props.folder}

modules/api/php/endpoints/candidate/visit/image/format/raw.class.inc

@@ -115,6 +115,13 @@ class Raw extends Endpoint implements \LORIS\Middleware\ETagCalculator
         $mincpath = \NDB_Factory::singleton()->config()
             ->getSetting('MINCToolsPath');
 
+        if ($mincpath === null) {
+            return new \LORIS\Http\Response\JSON\InternalServerError(
+                'Invalid MINCToolsPath configuration setting: ' .
+                'MINCToolsPath is null'
+            );
+        }
+
         $minctooldir = new \SPLFileInfo($mincpath);
         if (!$minctooldir->isDir() || !$minctooldir->isReadable()) {
             return new \LORIS\Http\Response\JSON\InternalServerError(

modules/api/php/endpoints/candidates.class.inc

@@ -152,7 +152,8 @@ class Candidates extends Endpoint implements \LORIS\Middleware\ETagCalculator
         );
 
         $provisioner = (new \LORIS\api\Provisioners\CandidatesProvisioner())
-            ->filter($filter);
+            ->filter($filter)
+            ->filter(new \LORIS\Data\Filters\UserProjectMatch());
 
         $candidates = (new \LORIS\Data\Table())
             ->withDataFrom($provisioner)

modules/api/php/models/candidatesrow.class.inc

@@ -24,6 +24,7 @@ class CandidatesRow implements \LORIS\Data\DataInstance
 {
     private $_candid;
     private $_projectname;
+    private $_projectid;
     private $_pscid;
     private $_sitename;
     private $_edc;
@@ -40,6 +41,7 @@ class CandidatesRow implements \LORIS\Data\DataInstance
     {
         $this->_candid      = $row['CandID'] ?? null;
         $this->_projectname = $row['ProjectName'] ?? null;
+        $this->_projectid   = $row['ProjectID'] ?? null;
         $this->_pscid       = $row['PSCID'] ?? null;
         $this->_sitename    = $row['SiteName'] ?? null;
         $this->_edc         = $row['EDC'] ?? null;
@@ -56,13 +58,14 @@ class CandidatesRow implements \LORIS\Data\DataInstance
     public function jsonSerialize() : array
     {
         $obj = [
-            'CandID'  => $this->_candid,
-            'Project' => $this->_projectname,
-            'PSCID'   => $this->_pscid,
-            'Site'    => $this->_sitename,
-            'EDC'     => $this->_edc,
-            'DoB'     => $this->_dob,
-            'Sex'     => $this->_sex,
+            'CandID'    => $this->_candid,
+            'Project'   => $this->_projectname,
+            'ProjectID' => $this->_projectid,
+            'PSCID'     => $this->_pscid,
+            'Site'      => $this->_sitename,
+            'EDC'       => $this->_edc,
+            'DoB'       => $this->_dob,
+            'Sex'       => $this->_sex,
         ];
 
         return $obj;
@@ -81,4 +84,18 @@ class CandidatesRow implements \LORIS\Data\DataInstance
         }
         return intval($this->_centerid);
     }
+
+    /**
+     * Returns the ProjectID for this row, for filters such as
+     * \LORIS\Data\Filters\UserProjectMatch to match against.
+     *
+     * @return integer ProjectID
+     */
+    public function getProjectID(): int
+    {
+        if ($this->_projectid === null) {
+            throw new \Exception('ProjectID is null');
+        }
+        return intval($this->_projectid);
+    }
 }

modules/api/php/provisioners/candidatesprovisioner.class.inc

@@ -41,6 +41,7 @@ class CandidatesProvisioner extends DBRowProvisioner
              SELECT 
                c.CandID as CandID,
                p.Name as ProjectName,
+               c.RegistrationProjectID as ProjectID,
                c.PSCID as PSCID,
                c.RegistrationCenterID as CenterID,
                s.Name as SiteName,